fix #2519 remove dataformat plugin, and rename getFessEsClient method

plugin.xml

@@ -39,15 +39,6 @@
 			<param name="plugin.version" value="7.10.0" />
 			<param name="plugin.zip.version" value="7.10.0" />
 		</antcall>
-		<!-- dataformat -->
-		<antcall target="install.plugin">
-			<param name="repo.url" value="${maven.release.repo.url}" />
-			<param name="plugin.groupId" value="org/codelibs${use.fesen}" />
-			<param name="plugin.name.prefix" value="elasticsearch-" />
-			<param name="plugin.name" value="dataformat" />
-			<param name="plugin.version" value="7.10.0" />
-			<param name="plugin.zip.version" value="7.10.0" />
-		</antcall>
 		<!-- minhash -->
 		<antcall target="install.plugin">
 			<param name="repo.url" value="${maven.release.repo.url}" />

src/main/java/org/codelibs/fess/api/json/JsonApiManager.java

@@ -187,7 +187,7 @@ public class JsonApiManager extends BaseJsonApiManager {
     }
 
     protected void processPingRequest(final HttpServletRequest request, final HttpServletResponse response, final FilterChain chain) {
-        final SearchEngineClient searchEngineClient = ComponentUtil.getFessEsClient();
+        final SearchEngineClient searchEngineClient = ComponentUtil.getSearchEngineClient();
         int status;
         Exception err = null;
         try {

src/main/java/org/codelibs/fess/app/web/admin/backup/AdminBackupAction.java

@@ -67,6 +67,7 @@ import org.codelibs.fess.util.ComponentUtil;
 import org.codelibs.fess.util.GsaConfigParser;
 import org.codelibs.fess.util.RenderDataUtil;
 import org.codelibs.fess.util.ResourceUtil;
+import org.codelibs.fess.util.SearchEngineUtil;
 import org.lastaflute.core.magic.async.AsyncManager;
 import org.lastaflute.web.Execute;
 import org.lastaflute.web.response.ActionResponse;
@@ -332,9 +333,19 @@ public class AdminBackupAction extends FessAdminAction {
                     filename = id + ".bulk";
                 }
                 return asStream(filename).contentTypeOctetStream().stream(out -> {
-                    try (CurlResponse response = ComponentUtil.getCurlHelper().get("/" + index + "/_data").param("format", "json")
-                            .param("scroll", fessConfig.getIndexScrollSearchTimeout()).execute()) {
-                        out.write(response.getContentAsStream());
+                    try (final BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(out.stream(), Constants.CHARSET_UTF_8))) {
+                        SearchEngineUtil.scroll(index, hit -> {
+                            try {
+                                writer.write("{\"index\":{\"_index\":\"" + index + "\",\"_id\":\""
+                                        + StringEscapeUtils.escapeJson(hit.getId()) + "\"}}\n");
+                                writer.write(hit.getSourceAsString());
+                                writer.write("\n");
+                            } catch (IOException e) {
+                                throw new IORuntimeException(e);
+                            }
+                            return true;
+                        });
+                        writer.flush();
                     }
                 });
             }

src/main/java/org/codelibs/fess/app/web/admin/maintenance/AdminMaintenanceAction.java

@@ -15,7 +15,9 @@
  */
 package org.codelibs.fess.app.web.admin.maintenance;
 
+import java.io.BufferedWriter;
 import java.io.IOException;
+import java.io.OutputStreamWriter;
 import java.net.InetAddress;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -30,8 +32,10 @@ import java.util.zip.ZipOutputStream;
 
 import javax.annotation.Resource;
 
+import org.apache.commons.text.StringEscapeUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.codelibs.core.exception.IORuntimeException;
 import org.codelibs.core.io.CopyUtil;
 import org.codelibs.core.lang.StringUtil;
 import org.codelibs.curl.CurlResponse;
@@ -42,6 +46,7 @@ import org.codelibs.fess.app.web.base.FessAdminAction;
 import org.codelibs.fess.es.client.SearchEngineClient;
 import org.codelibs.fess.mylasta.direction.FessConfig.SimpleImpl;
 import org.codelibs.fess.util.ComponentUtil;
+import org.codelibs.fess.util.SearchEngineUtil;
 import org.lastaflute.web.Execute;
 import org.lastaflute.web.response.ActionResponse;
 import org.lastaflute.web.response.HtmlResponse;
@@ -218,9 +223,20 @@ public class AdminMaintenanceAction extends FessAdminAction {
         final ZipEntry entry = new ZipEntry(id + "/fess_basic_config.bulk");
         try {
             zos.putNextEntry(entry);
-            try (CurlResponse response = ComponentUtil.getCurlHelper().get("/.fess_basic_config/_data").param("format", "json")
-                    .param("scroll", fessConfig.getIndexScrollSearchTimeout()).execute()) {
-                CopyUtil.copy(response.getContentAsStream(), zos);
+            final String index = ".fess_basic_config";
+            try (final BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(zos, Constants.CHARSET_UTF_8))) {
+                SearchEngineUtil.scroll(index, hit -> {
+                    try {
+                        writer.write("{\"index\":{\"_index\":\"" + index + "\",\"_id\":\"" + StringEscapeUtils.escapeJson(hit.getId())
+                                + "\"}}\n");
+                        writer.write(hit.getSourceAsString());
+                        writer.write("\n");
+                    } catch (IOException e) {
+                        throw new IORuntimeException(e);
+                    }
+                    return true;
+                });
+                writer.flush();
             }
         } catch (final IOException e) {
             logger.warn("Failed to access system.properties.", e);

src/main/java/org/codelibs/fess/app/web/admin/searchlist/AdminSearchlistAction.java

@@ -385,7 +385,7 @@ public class AdminSearchlistAction extends FessAdminAction {
     public static OptionalEntity<Map<String, Object>> getDoc(final CreateForm form) {
         final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
         final FessConfig fessConfig = ComponentUtil.getFessConfig();
-        final SearchEngineClient searchEngineClient = ComponentUtil.getFessEsClient();
+        final SearchEngineClient searchEngineClient = ComponentUtil.getSearchEngineClient();
         switch (form.crudMode) {
         case CrudMode.CREATE:
             final Map<String, Object> entity = new HashMap<>();

src/main/java/org/codelibs/fess/app/web/api/admin/backup/ApiAdminBackupAction.java

@@ -26,6 +26,7 @@ import static org.codelibs.fess.app.web.admin.backup.AdminBackupAction.getUserIn
 import java.io.BufferedWriter;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
+import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStreamWriter;
 import java.io.Writer;
@@ -33,13 +34,15 @@ import java.util.List;
 import java.util.Map;
 import java.util.function.Consumer;
 
-import org.codelibs.curl.CurlResponse;
+import org.apache.commons.text.StringEscapeUtils;
+import org.codelibs.core.exception.IORuntimeException;
 import org.codelibs.fess.Constants;
 import org.codelibs.fess.app.web.api.ApiResult;
 import org.codelibs.fess.app.web.api.ApiResult.ApiBackupFilesResponse;
 import org.codelibs.fess.app.web.api.admin.FessApiAdminAction;
 import org.codelibs.fess.mylasta.direction.FessConfig;
 import org.codelibs.fess.util.ComponentUtil;
+import org.codelibs.fess.util.SearchEngineUtil;
 import org.lastaflute.web.Execute;
 import org.lastaflute.web.response.JsonResponse;
 import org.lastaflute.web.response.StreamResponse;
@@ -92,9 +95,19 @@ public class ApiAdminBackupAction extends FessApiAdminAction {
                     filename = id + ".bulk";
                 }
                 return asStream(filename).contentTypeOctetStream().stream(out -> {
-                    try (CurlResponse response =
-                            ComponentUtil.getCurlHelper().get("/" + index + "/_data").param("format", "json").execute()) {
-                        out.write(response.getContentAsStream());
+                    try (final BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(out.stream(), Constants.CHARSET_UTF_8))) {
+                        SearchEngineUtil.scroll(index, hit -> {
+                            try {
+                                writer.write("{\"index\":{\"_index\":\"" + index + "\",\"_id\":\""
+                                        + StringEscapeUtils.escapeJson(hit.getId()) + "\"}}\n");
+                                writer.write(hit.getSourceAsString());
+                                writer.write("\n");
+                            } catch (IOException e) {
+                                throw new IORuntimeException(e);
+                            }
+                            return true;
+                        });
+                        writer.flush();
                     }
                 });
             }

src/main/java/org/codelibs/fess/crawler/FessCrawlerThread.java

@@ -64,7 +64,7 @@ public class FessCrawlerThread extends CrawlerThread {
             final CrawlingConfigHelper crawlingConfigHelper = ComponentUtil.getCrawlingConfigHelper();
             final CrawlingInfoHelper crawlingInfoHelper = ComponentUtil.getCrawlingInfoHelper();
             final IndexingHelper indexingHelper = ComponentUtil.getIndexingHelper();
-            final SearchEngineClient searchEngineClient = ComponentUtil.getFessEsClient();
+            final SearchEngineClient searchEngineClient = ComponentUtil.getSearchEngineClient();
 
             final String url = urlQueue.getUrl();
             ResponseData responseData = null;

src/main/java/org/codelibs/fess/ds/callback/FileListIndexUpdateCallbackImpl.java

@@ -254,7 +254,7 @@ public class FileListIndexUpdateCallbackImpl implements IndexUpdateCallback {
             final long maxAccessCount = getMaxAccessCount(paramMap, dataMap);
             final String url = dataMap.get(fessConfig.getIndexFieldUrl()).toString();
             if (maxAccessCount != 1L) {
-                final SearchEngineClient searchEngineClient = ComponentUtil.getFessEsClient();
+                final SearchEngineClient searchEngineClient = ComponentUtil.getSearchEngineClient();
                 final IndexingHelper indexingHelper = ComponentUtil.getIndexingHelper();
                 final long count = indexingHelper.deleteDocumentByQuery(searchEngineClient,
                         QueryBuilders.prefixQuery(fessConfig.getIndexFieldUrl(), url));
@@ -295,7 +295,7 @@ public class FileListIndexUpdateCallbackImpl implements IndexUpdateCallback {
     }
 
     protected void deleteDocuments() {
-        final SearchEngineClient searchEngineClient = ComponentUtil.getFessEsClient();
+        final SearchEngineClient searchEngineClient = ComponentUtil.getSearchEngineClient();
         final IndexingHelper indexingHelper = ComponentUtil.getIndexingHelper();
         for (final String url : deleteUrlList) {
             indexingHelper.deleteDocumentByUrl(searchEngineClient, url);

src/main/java/org/codelibs/fess/ds/callback/IndexUpdateCallbackImpl.java

@@ -76,7 +76,7 @@ public class IndexUpdateCallbackImpl implements IndexUpdateCallback {
 
         final long startTime = System.currentTimeMillis();
         final FessConfig fessConfig = ComponentUtil.getFessConfig();
-        final SearchEngineClient searchEngineClient = ComponentUtil.getFessEsClient();
+        final SearchEngineClient searchEngineClient = ComponentUtil.getSearchEngineClient();
 
         if (logger.isDebugEnabled()) {
             logger.debug("Adding {}", dataMap);
@@ -162,7 +162,7 @@ public class IndexUpdateCallbackImpl implements IndexUpdateCallback {
         synchronized (docList) {
             if (!docList.isEmpty()) {
                 final IndexingHelper indexingHelper = ComponentUtil.getIndexingHelper();
-                final SearchEngineClient searchEngineClient = ComponentUtil.getFessEsClient();
+                final SearchEngineClient searchEngineClient = ComponentUtil.getSearchEngineClient();
                 indexingHelper.sendDocuments(searchEngineClient, docList);
             }
         }

src/main/java/org/codelibs/fess/es/client/SearchEngineClient.java

@@ -803,25 +803,22 @@ public class SearchEngineClient implements Client {
                     for (final SearchHit hit : hits) {
                         count++;
                         if (!cursor.apply(creator.build(response, hit))) {
-                            if (scrollId != null) {
-                                client.prepareClearScroll().addScrollId(scrollId)
-                                        .execute(ActionListener.wrap(res -> {}, e1 -> logger.warn("Failed to clear scrollId.", e1)));
-                            }
                             break;
                         }
                     }
 
                     response = client.prepareSearchScroll(scrollId).setScroll(scrollForDelete).execute()
                             .actionGet(fessConfig.getIndexBulkTimeout());
+                    if (!scrollId.equals(response.getScrollId())) {
+                        deleteScrollContext(scrollId);
+                    }
                     scrollId = response.getScrollId();
                 }
             } catch (final SearchPhaseExecutionException e) {
-                if (scrollId != null) {
-                    client.prepareClearScroll().addScrollId(scrollId)
-                            .execute(ActionListener.wrap(res -> {}, e1 -> logger.warn("Failed to clear scrollId.", e1)));
-                }
                 throw new InvalidQueryException(messages -> messages.addErrorsInvalidQueryParseError(UserMessages.GLOBAL_PROPERTY_KEY),
                         "Invalid query: " + searchRequestBuilder, e);
+            } finally {
+                deleteScrollContext(scrollId);
             }
         }
 

src/main/java/org/codelibs/fess/helper/DataIndexHelper.java

@@ -241,7 +241,7 @@ public class DataIndexHelper {
                                     .mustNot(QueryBuilders.existsQuery(fessConfig.getIndexFieldExpires())))
                             .mustNot(QueryBuilders.termQuery(fessConfig.getIndexFieldSegment(), sessionId));
             try {
-                final SearchEngineClient searchEngineClient = ComponentUtil.getFessEsClient();
+                final SearchEngineClient searchEngineClient = ComponentUtil.getSearchEngineClient();
                 final String index = fessConfig.getIndexDocumentUpdateIndex();
                 searchEngineClient.admin().indices().prepareRefresh(index).execute().actionGet();
                 final long numOfDeleted = searchEngineClient.deleteByQuery(index, queryBuilder);

src/main/java/org/codelibs/fess/helper/KeyMatchHelper.java

@@ -119,7 +119,7 @@ public class KeyMatchHelper {
     }
 
     protected List<Map<String, Object>> getDocumentList(final KeyMatch keyMatch) {
-        final SearchEngineClient searchEngineClient = ComponentUtil.getFessEsClient();
+        final SearchEngineClient searchEngineClient = ComponentUtil.getSearchEngineClient();
         final FessConfig fessConfig = ComponentUtil.getFessConfig();
         return searchEngineClient.getDocumentList(fessConfig.getIndexDocumentSearchIndex(),
                 searchRequestBuilder -> SearchConditionBuilder
@@ -155,7 +155,7 @@ public class KeyMatchHelper {
     }
 
     public List<Map<String, Object>> getBoostedDocumentList(final String term, final int size) {
-        final SearchEngineClient searchEngineClient = ComponentUtil.getFessEsClient();
+        final SearchEngineClient searchEngineClient = ComponentUtil.getSearchEngineClient();
         final Pair<QueryBuilder, ScoreFunctionBuilder<?>> pair = getQueryMap().get(toLowerCase(term));
         if (pair == null) {
             return Collections.emptyList();

src/main/java/org/codelibs/fess/helper/SearchHelper.java

@@ -93,7 +93,7 @@ public class SearchHelper {
         final FessConfig fessConfig = ComponentUtil.getFessConfig();
         final QueryHelper queryHelper = ComponentUtil.getQueryHelper();
         final List<Map<String, Object>> documentItems =
-                ComponentUtil.getFessEsClient().search(fessConfig.getIndexDocumentSearchIndex(), searchRequestBuilder -> {
+                ComponentUtil.getSearchEngineClient().search(fessConfig.getIndexDocumentSearchIndex(), searchRequestBuilder -> {
                     queryHelper.processSearchPreference(searchRequestBuilder, userBean, query);
                     return SearchConditionBuilder.builder(searchRequestBuilder).query(query).offset(pageStart).size(pageSize)
                             .facetInfo(params.getFacetInfo()).geoInfo(params.getGeoInfo()).highlightInfo(params.getHighlightInfo())
@@ -189,7 +189,7 @@ public class SearchHelper {
             query = ComponentUtil.getQueryStringBuilder().params(params).build() + " sort:" + sortField;
         }
         final FessConfig fessConfig = ComponentUtil.getFessConfig();
-        return ComponentUtil.getFessEsClient().<Map<String, Object>> scrollSearch(fessConfig.getIndexDocumentSearchIndex(),
+        return ComponentUtil.getSearchEngineClient().<Map<String, Object>> scrollSearch(fessConfig.getIndexDocumentSearchIndex(),
                 searchRequestBuilder -> {
                     final QueryHelper queryHelper = ComponentUtil.getQueryHelper();
                     queryHelper.processSearchPreference(searchRequestBuilder, userBean, query);
@@ -233,7 +233,7 @@ public class SearchHelper {
         final QueryContext queryContext = ComponentUtil.getQueryHelper().build(params.getType(), query, context -> {
             context.skipRoleQuery();
         });
-        return ComponentUtil.getFessEsClient().deleteByQuery(ComponentUtil.getFessConfig().getIndexDocumentUpdateIndex(),
+        return ComponentUtil.getSearchEngineClient().deleteByQuery(ComponentUtil.getFessConfig().getIndexDocumentUpdateIndex(),
                 queryContext.getQueryBuilder());
     }
 
@@ -281,7 +281,7 @@ public class SearchHelper {
     public OptionalEntity<Map<String, Object>> getDocumentByDocId(final String docId, final String[] fields,
             final OptionalThing<FessUserBean> userBean) {
         final FessConfig fessConfig = ComponentUtil.getFessConfig();
-        return ComponentUtil.getFessEsClient().getDocument(fessConfig.getIndexDocumentSearchIndex(), builder -> {
+        return ComponentUtil.getSearchEngineClient().getDocument(fessConfig.getIndexDocumentSearchIndex(), builder -> {
             final BoolQueryBuilder boolQuery =
                     QueryBuilders.boolQuery().must(QueryBuilders.termQuery(fessConfig.getIndexFieldDocId(), docId));
             final Set<String> roleSet = ComponentUtil.getRoleQueryHelper().build(SearchRequestType.JSON); // TODO SearchRequestType?
@@ -300,7 +300,7 @@ public class SearchHelper {
     public List<Map<String, Object>> getDocumentListByDocIds(final String[] docIds, final String[] fields,
             final OptionalThing<FessUserBean> userBean, final SearchRequestType searchRequestType) {
         final FessConfig fessConfig = ComponentUtil.getFessConfig();
-        return ComponentUtil.getFessEsClient().getDocumentList(fessConfig.getIndexDocumentSearchIndex(), builder -> {
+        return ComponentUtil.getSearchEngineClient().getDocumentList(fessConfig.getIndexDocumentSearchIndex(), builder -> {
             final BoolQueryBuilder boolQuery =
                     QueryBuilders.boolQuery().must(QueryBuilders.termsQuery(fessConfig.getIndexFieldDocId(), docIds));
             final QueryHelper queryHelper = ComponentUtil.getQueryHelper();
@@ -319,14 +319,14 @@ public class SearchHelper {
     }
 
     public boolean update(final String id, final String field, final Object value) {
-        return ComponentUtil.getFessEsClient().update(ComponentUtil.getFessConfig().getIndexDocumentUpdateIndex(), id, field, value);
+        return ComponentUtil.getSearchEngineClient().update(ComponentUtil.getFessConfig().getIndexDocumentUpdateIndex(), id, field, value);
     }
 
     public boolean update(final String id, final Consumer<UpdateRequestBuilder> builderLambda) {
         try {
             final FessConfig fessConfig = ComponentUtil.getFessConfig();
             final UpdateRequestBuilder builder =
-                    ComponentUtil.getFessEsClient().prepareUpdate().setIndex(fessConfig.getIndexDocumentUpdateIndex()).setId(id);
+                    ComponentUtil.getSearchEngineClient().prepareUpdate().setIndex(fessConfig.getIndexDocumentUpdateIndex()).setId(id);
             builderLambda.accept(builder);
             final UpdateResponse response = builder.execute().actionGet(fessConfig.getIndexIndexTimeout());
             return response.getResult() == Result.CREATED || response.getResult() == Result.UPDATED;
@@ -336,7 +336,7 @@ public class SearchHelper {
     }
 
     public boolean bulkUpdate(final Consumer<BulkRequestBuilder> consumer) {
-        final BulkRequestBuilder builder = ComponentUtil.getFessEsClient().prepareBulk();
+        final BulkRequestBuilder builder = ComponentUtil.getSearchEngineClient().prepareBulk();
         consumer.accept(builder);
         try {
             final BulkResponse response = builder.execute().get();

src/main/java/org/codelibs/fess/helper/SuggestHelper.java

@@ -96,7 +96,7 @@ public class SuggestHelper {
         split(fessConfig.getSuggestFieldRoles(), ",").of(stream -> stream.filter(StringUtil::isNotBlank).forEach(roleFieldNameSet::add));
         contentFieldList = Arrays.asList(stream(fessConfig.getSuggestFieldContents()).get(stream -> stream.toArray(n -> new String[n])));
 
-        final SearchEngineClient searchEngineClient = ComponentUtil.getFessEsClient();
+        final SearchEngineClient searchEngineClient = ComponentUtil.getSearchEngineClient();
         searchEngineClient.admin().cluster().prepareHealth().setWaitForYellowStatus().execute()
                 .actionGet(fessConfig.getIndexHealthTimeout());
 
@@ -208,7 +208,7 @@ public class SuggestHelper {
         final int docPerReq = fessConfig.getSuggestUpdateDocPerRequestAsInteger();
         final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
         suggester.indexer().indexFromDocument(() -> {
-            final ESSourceReader reader = new ESSourceReader(ComponentUtil.getFessEsClient(), suggester.settings(),
+            final ESSourceReader reader = new ESSourceReader(ComponentUtil.getSearchEngineClient(), suggester.settings(),
                     fessConfig.getIndexDocumentSearchIndex(), "_doc"); // TODO remove type
             reader.setScrollSize(fessConfig.getSuggestSourceReaderScrollSizeAsInteger());
             reader.setLimitDocNumPercentage(fessConfig.getSuggestUpdateContentsLimitNumPercentage());
@@ -243,7 +243,7 @@ public class SuggestHelper {
         boolQueryBuilder.mustNot(QueryBuilders.termQuery(FieldNames.KINDS, SuggestItem.Kind.QUERY.toString()));
         boolQueryBuilder.mustNot(QueryBuilders.termQuery(FieldNames.KINDS, SuggestItem.Kind.USER.toString()));
 
-        SuggestUtil.deleteByQuery(ComponentUtil.getFessEsClient(), suggester.settings(), suggester.getIndex(), boolQueryBuilder);
+        SuggestUtil.deleteByQuery(ComponentUtil.getSearchEngineClient(), suggester.settings(), suggester.getIndex(), boolQueryBuilder);
     }
 
     public void purgeSearchlogSuggest(final LocalDateTime time) {
@@ -255,7 +255,7 @@ public class SuggestHelper {
         boolQueryBuilder.must(QueryBuilders.termQuery(FieldNames.KINDS, SuggestItem.Kind.QUERY.toString()));
         boolQueryBuilder.mustNot(QueryBuilders.termQuery(FieldNames.KINDS, SuggestItem.Kind.USER.toString()));
 
-        SuggestUtil.deleteByQuery(ComponentUtil.getFessEsClient(), suggester.settings(), suggester.getIndex(), boolQueryBuilder);
+        SuggestUtil.deleteByQuery(ComponentUtil.getSearchEngineClient(), suggester.settings(), suggester.getIndex(), boolQueryBuilder);
     }
 
     public long getAllWordsNum() {

src/main/java/org/codelibs/fess/helper/SystemHelper.java

@@ -413,7 +413,7 @@ public class SystemHelper {
     }
 
     public void setupAdminHtmlData(final TypicalAction action, final ActionRuntime runtime) {
-        runtime.registerData("developmentMode", ComponentUtil.getFessEsClient().isEmbedded());
+        runtime.registerData("developmentMode", ComponentUtil.getSearchEngineClient().isEmbedded());
         final FessConfig fessConfig = ComponentUtil.getFessConfig();
         final String installationLink = fessConfig.getOnlineHelpInstallation();
         runtime.registerData("installationLink", getHelpUrl(installationLink));
@@ -428,7 +428,7 @@ public class SystemHelper {
     }
 
     public void setupSearchHtmlData(final TypicalAction action, final ActionRuntime runtime) {
-        runtime.registerData("developmentMode", ComponentUtil.getFessEsClient().isEmbedded());
+        runtime.registerData("developmentMode", ComponentUtil.getSearchEngineClient().isEmbedded());
         final FessConfig fessConfig = ComponentUtil.getFessConfig();
         final String installationLink = fessConfig.getOnlineHelpInstallation();
         runtime.registerData("installationLink", getHelpUrl(installationLink));
@@ -465,7 +465,7 @@ public class SystemHelper {
     }
 
     public void reloadConfiguration() {
-        ComponentUtil.getFessEsClient().refresh();
+        ComponentUtil.getSearchEngineClient().refresh();
         ComponentUtil.getLabelTypeHelper().init();
         ComponentUtil.getPathMappingHelper().init();
         ComponentUtil.getSuggestHelper().init();

src/main/java/org/codelibs/fess/job/PingSearchEngineJob.java

@@ -35,7 +35,7 @@ public class PingSearchEngineJob {
     private static final Logger logger = LogManager.getLogger(PingSearchEngineJob.class);
 
     public String execute() {
-        final SearchEngineClient searchEngineClient = ComponentUtil.getFessEsClient();
+        final SearchEngineClient searchEngineClient = ComponentUtil.getSearchEngineClient();
         final FessConfig fessConfig = ComponentUtil.getFessConfig();
         final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
 

src/main/java/org/codelibs/fess/job/PurgeDocJob.java

@@ -28,7 +28,7 @@ public class PurgeDocJob {
     private static final Logger logger = LogManager.getLogger(PurgeDocJob.class);
 
     public String execute() {
-        final SearchEngineClient searchEngineClient = ComponentUtil.getFessEsClient();
+        final SearchEngineClient searchEngineClient = ComponentUtil.getSearchEngineClient();
         final FessConfig fessConfig = ComponentUtil.getFessConfig();
 
         final StringBuilder resultBuf = new StringBuilder();

src/main/java/org/codelibs/fess/job/UpdateLabelJob.java

@@ -38,7 +38,7 @@ public class UpdateLabelJob {
     protected QueryBuilder queryBuilder = null;
 
     public String execute() {
-        final SearchEngineClient searchEngineClient = ComponentUtil.getFessEsClient();
+        final SearchEngineClient searchEngineClient = ComponentUtil.getSearchEngineClient();
         final FessConfig fessConfig = ComponentUtil.getFessConfig();
         final LabelTypeHelper labelTypeHelper = ComponentUtil.getLabelTypeHelper();
         final LanguageHelper languageHelper = ComponentUtil.getLanguageHelper();

src/main/java/org/codelibs/fess/score/ScoreBooster.java

@@ -51,7 +51,7 @@ public abstract class ScoreBooster {
 
     protected Function<Map<String, Object>, String[]> idFinder = params -> {
         final FessConfig fessConfig = ComponentUtil.getFessConfig();
-        final SearchEngineClient client = ComponentUtil.getFessEsClient();
+        final SearchEngineClient client = ComponentUtil.getSearchEngineClient();
         final String index = fessConfig.getIndexDocumentUpdateIndex();
         final Object url = params.get("url");
         if (url == null) {
@@ -68,7 +68,7 @@ public abstract class ScoreBooster {
         if (ids.length == 0) {
             return 0L;
         }
-        final SearchEngineClient client = ComponentUtil.getFessEsClient();
+        final SearchEngineClient client = ComponentUtil.getSearchEngineClient();
         if (bulkRequestBuilder == null) {
             bulkRequestBuilder = client.prepareBulk();
         }
@@ -96,7 +96,7 @@ public abstract class ScoreBooster {
 
     protected UpdateRequestBuilder createUpdateRequestBuilder() {
         final FessConfig fessConfig = ComponentUtil.getFessConfig();
-        return ComponentUtil.getFessEsClient().prepareUpdate().setIndex(fessConfig.getIndexDocumentSearchIndex());
+        return ComponentUtil.getSearchEngineClient().prepareUpdate().setIndex(fessConfig.getIndexDocumentSearchIndex());
     }
 
     protected void flush() {

src/main/java/org/codelibs/fess/thumbnail/ThumbnailManager.java

@@ -367,7 +367,7 @@ public class ThumbnailManager {
             this.expiry = expiry;
             this.fessConfig = ComponentUtil.getFessConfig();
             this.maxPurgeSize = fessConfig.getPageThumbnailPurgeMaxFetchSizeAsInteger();
-            this.searchEngineClient = ComponentUtil.getFessEsClient();
+            this.searchEngineClient = ComponentUtil.getSearchEngineClient();
         }
 
         protected void deleteFiles() {

src/main/java/org/codelibs/fess/thumbnail/impl/BaseThumbnailGenerator.java

@@ -151,7 +151,7 @@ public abstract class BaseThumbnailGenerator implements ThumbnailGenerator {
         // TODO bulk
         final FessConfig fessConfig = ComponentUtil.getFessConfig();
         try {
-            ComponentUtil.getIndexingHelper().updateDocument(ComponentUtil.getFessEsClient(), thumbnailId,
+            ComponentUtil.getIndexingHelper().updateDocument(ComponentUtil.getSearchEngineClient(), thumbnailId,
                     fessConfig.getIndexFieldThumbnail(), value);
         } catch (final Exception e) {
             logger.warn("Failed to update thumbnail field at {}", thumbnailId, e);
@@ -160,7 +160,7 @@ public abstract class BaseThumbnailGenerator implements ThumbnailGenerator {
 
     protected boolean process(final String id, final BiPredicate<String, String> consumer) {
         final FessConfig fessConfig = ComponentUtil.getFessConfig();
-        final SearchEngineClient searchEngineClient = ComponentUtil.getFessEsClient();
+        final SearchEngineClient searchEngineClient = ComponentUtil.getSearchEngineClient();
         final IndexingHelper indexingHelper = ComponentUtil.getIndexingHelper();
         try {
             final Map<String, Object> doc = indexingHelper.getDocument(searchEngineClient, id,

src/main/java/org/codelibs/fess/timer/SystemMonitorTarget.java

@@ -178,7 +178,7 @@ public class SystemMonitorTarget implements TimeoutTarget {
     private void appendFesenStats(final StringBuilder buf) {
         String stats = null;
         try {
-            final SearchEngineClient esClient = ComponentUtil.getFessEsClient();
+            final SearchEngineClient esClient = ComponentUtil.getSearchEngineClient();
             final NodesStatsResponse response = esClient.admin().cluster().prepareNodesStats().all().execute().actionGet(10000L);
             final XContentBuilder builder = XContentFactory.jsonBuilder();
             builder.startObject();

src/main/java/org/codelibs/fess/util/ComponentUtil.java

@@ -357,7 +357,7 @@ public final class ComponentUtil {
         return getComponent(DATA_SERVICE);
     }
 
-    public static SearchEngineClient getFessEsClient() {
+    public static SearchEngineClient getSearchEngineClient() {
         return getComponent(FESS_ES_CLIENT);
     }
 

src/main/java/org/codelibs/fess/util/SearchEngineUtil.java

@@ -18,6 +18,7 @@ package org.codelibs.fess.util;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.OutputStream;
+import java.util.function.Function;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -25,6 +26,8 @@ import org.codelibs.fesen.common.xcontent.ToXContent;
 import org.codelibs.fesen.common.xcontent.XContentBuilder;
 import org.codelibs.fesen.common.xcontent.XContentFactory;
 import org.codelibs.fesen.common.xcontent.XContentType;
+import org.codelibs.fesen.search.SearchHit;
+import org.codelibs.fess.es.client.SearchEngineClient;
 
 public final class SearchEngineUtil {
 
@@ -49,7 +52,15 @@ public final class SearchEngineUtil {
         return getXContentBuilderOutputStream((builder, params) -> xContent.toXContent(builder, params), xContentType);
     }
 
+    public static long scroll(String index, Function<SearchHit, Boolean> callback) {
+        SearchEngineClient client = ComponentUtil.getSearchEngineClient();
+        return client.<SearchHit> scrollSearch(index, searchRequestBuilder -> true, (searchResponse, hit) -> hit, hit -> {
+            return callback.apply(hit);
+        });
+    }
+
     public interface XContentBuilderCallback {
         XContentBuilder apply(XContentBuilder builder, ToXContent.Params params) throws IOException;
     }
+
 }