diff --git a/src/main/java/org/codelibs/fess/api/json/JsonApiManager.java b/src/main/java/org/codelibs/fess/api/json/JsonApiManager.java index 04f82c9b1..61013f35b 100644 --- a/src/main/java/org/codelibs/fess/api/json/JsonApiManager.java +++ b/src/main/java/org/codelibs/fess/api/json/JsonApiManager.java @@ -29,6 +29,7 @@ import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import org.codelibs.core.exception.IORuntimeException; import org.codelibs.core.lang.StringUtil; import org.codelibs.fess.Constants; import org.codelibs.fess.api.BaseJsonApiManager; @@ -113,12 +114,71 @@ public class JsonApiManager extends BaseJsonApiManager { case PING: processPingRequest(request, response, chain); break; + case SCROLL: + processScrollSearchRequest(request, response, chain); + break; default: writeJsonResponse(99, StringUtil.EMPTY, "Not found."); break; } } + protected void processScrollSearchRequest(final HttpServletRequest request, final HttpServletResponse response, final FilterChain chain) { + final SearchService searchService = ComponentUtil.getComponent(SearchService.class); + final FessConfig fessConfig = ComponentUtil.getFessConfig(); + + if (!fessConfig.isApiSearchScroll()) { + writeJsonResponse(99, StringUtil.EMPTY, "Scroll Search is not available."); + return; + } + + final StringBuilder buf = new StringBuilder(1000); + request.setAttribute(Constants.SEARCH_LOG_ACCESS_TYPE, Constants.SEARCH_LOG_ACCESS_TYPE_JSON); + final JsonRequestParams params = new JsonRequestParams(request, fessConfig); + try { + response.setContentType("application/x-ndjson; charset=UTF-8"); + final long count = + searchService.scrollSearch(params, doc -> { + buf.setLength(0); + buf.append('{'); + boolean first2 = true; + for (final Map.Entry entry : doc.entrySet()) { + final String name = entry.getKey(); + if (StringUtil.isNotBlank(name) && entry.getValue() != null + && ComponentUtil.getQueryHelper().isApiResponseField(name)) { + if (!first2) { + buf.append(','); + } else { + first2 = false; + } + buf.append(escapeJson(name)); + buf.append(':'); + buf.append(escapeJson(entry.getValue())); + } + } + buf.append('}'); + buf.append('\n'); + try { + response.getWriter().print(buf.toString()); + } catch (IOException e) { + throw new IORuntimeException(e); + } + return true; + }, OptionalThing.empty()); + response.flushBuffer(); + if (logger.isDebugEnabled()) { + logger.debug("Loaded " + count + " docs"); + } + } catch (Exception e) { + int status = 9; + if (logger.isDebugEnabled()) { + logger.debug("Failed to process a ping request.", e); + } + writeJsonResponse(status, null, e); + } + + } + protected void processPingRequest(final HttpServletRequest request, final HttpServletResponse response, final FilterChain chain) { final FessEsClient fessEsClient = ComponentUtil.getFessEsClient(); int status; diff --git a/src/main/java/org/codelibs/fess/app/service/SearchService.java b/src/main/java/org/codelibs/fess/app/service/SearchService.java index 5a3897f9a..f62e067ce 100644 --- a/src/main/java/org/codelibs/fess/app/service/SearchService.java +++ b/src/main/java/org/codelibs/fess/app/service/SearchService.java @@ -16,7 +16,9 @@ package org.codelibs.fess.app.service; import java.text.NumberFormat; +import java.util.Collections; import java.util.Enumeration; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Locale; @@ -24,6 +26,8 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.function.Consumer; +import java.util.function.Function; +import java.util.stream.Collectors; import javax.annotation.Resource; import javax.servlet.http.HttpServletRequest; @@ -53,6 +57,7 @@ import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.update.UpdateRequestBuilder; import org.elasticsearch.action.update.UpdateResponse; +import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.lastaflute.taglib.function.LaFunctions; @@ -173,7 +178,50 @@ public class SearchService { } - public int deleteByQuery(final HttpServletRequest request, final SearchRequestParams params) { + public long scrollSearch(final SearchRequestParams params, final Function, Boolean> cursor, + final OptionalThing userBean) { + final FessConfig fessConfig = ComponentUtil.getFessConfig(); + LaRequestUtil.getOptionalRequest().ifPresent(request -> { + request.setAttribute(Constants.REQUEST_LANGUAGES, params.getLanguages()); + request.setAttribute(Constants.REQUEST_QUERIES, params.getQuery()); + }); + + final String query = + QueryStringBuilder.query(params.getQuery()).extraQueries(params.getExtraQueries()).fields(params.getFields()).build(); + + final int pageSize = params.getPageSize(); + final String sortField = params.getSort(); + return fessEsClient.> scrollSearch( + fessConfig.getIndexDocumentSearchIndex(), + fessConfig.getIndexDocumentType(), + searchRequestBuilder -> { + fessConfig.processSearchPreference(searchRequestBuilder, userBean); + return SearchConditionBuilder.builder(searchRequestBuilder) + .query(StringUtil.isBlank(sortField) ? query : query + " sort:" + sortField).size(pageSize) + .responseFields(queryHelper.getResponseFields()).searchRequestType(params.getType()).build(); + }, + (searchResponse, hit) -> { + final Map source = hit.getSourceAsMap(); + if (source != null) { + final Map docMap = new HashMap<>(source); + docMap.put(fessConfig.getIndexFieldId(), hit.getId()); + docMap.put(fessConfig.getIndexFieldVersion(), hit.getVersion()); + return docMap; + } + final Map fields = hit.getFields(); + if (fields != null) { + final Map docMap = + fields.entrySet().stream() + .collect(Collectors.toMap(e -> e.getKey(), e -> (Object) e.getValue().getValues())); + docMap.put(fessConfig.getIndexFieldId(), hit.getId()); + docMap.put(fessConfig.getIndexFieldVersion(), hit.getVersion()); + return docMap; + } + return Collections.emptyMap(); + }, cursor); + } + + public long deleteByQuery(final HttpServletRequest request, final SearchRequestParams params) { final String query = QueryStringBuilder.query(params.getQuery()).extraQueries(params.getExtraQueries()).fields(params.getFields()).build(); diff --git a/src/main/java/org/codelibs/fess/app/web/api/admin/searchlist/ApiAdminSearchlistAction.java b/src/main/java/org/codelibs/fess/app/web/api/admin/searchlist/ApiAdminSearchlistAction.java index bfc42123c..2bb155a62 100644 --- a/src/main/java/org/codelibs/fess/app/web/api/admin/searchlist/ApiAdminSearchlistAction.java +++ b/src/main/java/org/codelibs/fess/app/web/api/admin/searchlist/ApiAdminSearchlistAction.java @@ -209,7 +209,7 @@ public class ApiAdminSearchlistAction extends FessApiAdminAction { throwValidationErrorApi(messages -> messages.addErrorsInvalidQueryUnknown(GLOBAL)); } try { - final int count = searchService.deleteByQuery(request, body); + final long count = searchService.deleteByQuery(request, body); return asJson(new ApiDeleteResponse().count(count).status(Status.OK).result()); } catch (final InvalidQueryException e) { if (logger.isDebugEnabled()) { diff --git a/src/main/java/org/codelibs/fess/es/client/FessEsClient.java b/src/main/java/org/codelibs/fess/es/client/FessEsClient.java index 85ef280fe..4831a7661 100644 --- a/src/main/java/org/codelibs/fess/es/client/FessEsClient.java +++ b/src/main/java/org/codelibs/fess/es/client/FessEsClient.java @@ -31,6 +31,7 @@ import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.function.Function; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -177,6 +178,8 @@ public class FessEsClient implements Client { protected Map> configListMap = new HashMap<>(); + protected String scrollForSearch = "1m"; + protected int sizeForDelete = 100; protected String scrollForDelete = "1m"; @@ -630,7 +633,7 @@ public class FessEsClient implements Client { } } - public int deleteByQuery(final String index, final String type, final QueryBuilder queryBuilder) { + public long deleteByQuery(final String index, final String type, final QueryBuilder queryBuilder) { final FessConfig fessConfig = ComponentUtil.getFessConfig(); SearchResponse response = @@ -652,8 +655,8 @@ public class FessEsClient implements Client { final BulkRequestBuilder bulkRequest = client.prepareBulk(); for (final SearchHit hit : hits) { bulkRequest.add(client.prepareDelete(index, type, hit.getId())); + count++; } - count += hits.length; final BulkResponse bulkResponse = bulkRequest.execute().actionGet(fessConfig.getIndexBulkTimeout()); if (bulkResponse.hasFailures()) { throw new IllegalBehaviorStateException(bulkResponse.buildFailureMessage()); @@ -709,6 +712,51 @@ public class FessEsClient implements Client { return searchResult.build(searchRequestBuilder, execTime, OptionalEntity.ofNullable(searchResponse, () -> {})); } + public long scrollSearch(final String index, final String type, final SearchCondition condition, + final EntityCreator creator, final Function cursor) { + long count = 0; + + final SearchRequestBuilder searchRequestBuilder = client.prepareSearch(index).setTypes(type).setScroll(scrollForSearch); + if (condition.build(searchRequestBuilder)) { + final FessConfig fessConfig = ComponentUtil.getFessConfig(); + + try { + if (logger.isDebugEnabled()) { + logger.debug("Query DSL:\n" + searchRequestBuilder.toString()); + } + SearchResponse response = searchRequestBuilder.execute().actionGet(ComponentUtil.getFessConfig().getIndexSearchTimeout()); + + String scrollId = response.getScrollId(); + while (scrollId != null) { + final SearchHits searchHits = response.getHits(); + final SearchHit[] hits = searchHits.getHits(); + if (hits.length == 0) { + scrollId = null; + break; + } + + for (final SearchHit hit : hits) { + count++; + if (!cursor.apply(creator.build(response, hit))) { + scrollId = null; + break; + } + } + + response = + client.prepareSearchScroll(scrollId).setScroll(scrollForDelete).execute() + .actionGet(fessConfig.getIndexBulkTimeout()); + scrollId = response.getScrollId(); + } + } catch (final SearchPhaseExecutionException e) { + throw new InvalidQueryException(messages -> messages.addErrorsInvalidQueryParseError(UserMessages.GLOBAL_PROPERTY_KEY), + "Invalid query: " + searchRequestBuilder, e); + } + } + + return count; + } + public OptionalEntity> getDocument(final String index, final String type, final SearchCondition condition) { return getDocument( @@ -1378,6 +1426,10 @@ public class FessEsClient implements Client { this.scrollForDelete = scrollForDelete; } + public void setScrollForSearch(String scrollForSearch) { + this.scrollForSearch = scrollForSearch; + } + @Override public Client filterWithHeader(final Map headers) { return client.filterWithHeader(headers); diff --git a/src/main/java/org/codelibs/fess/helper/DataIndexHelper.java b/src/main/java/org/codelibs/fess/helper/DataIndexHelper.java index e9f35096a..b684d894b 100644 --- a/src/main/java/org/codelibs/fess/helper/DataIndexHelper.java +++ b/src/main/java/org/codelibs/fess/helper/DataIndexHelper.java @@ -266,7 +266,7 @@ public class DataIndexHelper { final FessEsClient fessEsClient = ComponentUtil.getFessEsClient(); final String index = fessConfig.getIndexDocumentUpdateIndex(); fessEsClient.admin().indices().prepareRefresh(index).execute().actionGet(); - final int numOfDeleted = fessEsClient.deleteByQuery(index, fessConfig.getIndexDocumentType(), queryBuilder); + final long numOfDeleted = fessEsClient.deleteByQuery(index, fessConfig.getIndexDocumentType(), queryBuilder); logger.info("Deleted {} old docs.", numOfDeleted); } catch (final Exception e) { logger.error("Could not delete old docs at " + dataConfig, e); diff --git a/src/main/java/org/codelibs/fess/helper/IndexingHelper.java b/src/main/java/org/codelibs/fess/helper/IndexingHelper.java index 9eba7c015..b7ce42030 100644 --- a/src/main/java/org/codelibs/fess/helper/IndexingHelper.java +++ b/src/main/java/org/codelibs/fess/helper/IndexingHelper.java @@ -140,13 +140,13 @@ public class IndexingHelper { return fessEsClient.delete(fessConfig.getIndexDocumentUpdateIndex(), fessConfig.getIndexDocumentType(), id, 0); } - public int deleteDocumentByUrl(final FessEsClient fessEsClient, final String url) { + public long deleteDocumentByUrl(final FessEsClient fessEsClient, final String url) { final FessConfig fessConfig = ComponentUtil.getFessConfig(); return fessEsClient.deleteByQuery(fessConfig.getIndexDocumentUpdateIndex(), fessConfig.getIndexDocumentType(), QueryBuilders.termQuery(fessConfig.getIndexFieldUrl(), url)); } - public int deleteDocumentsByDocId(final FessEsClient fessEsClient, final List docIdList) { + public long deleteDocumentsByDocId(final FessEsClient fessEsClient, final List docIdList) { final FessConfig fessConfig = ComponentUtil.getFessConfig(); return fessEsClient.deleteByQuery(fessConfig.getIndexDocumentUpdateIndex(), fessConfig.getIndexDocumentType(), QueryBuilders .idsQuery(fessConfig.getIndexDocumentType()).addIds(docIdList.stream().toArray(n -> new String[n]))); diff --git a/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java b/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java index 53a09637b..a6ad484fa 100644 --- a/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java +++ b/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java @@ -160,6 +160,9 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction /** The key of the configuration. e.g. */ String API_SEARCH_ACCEPT_REFERERS = "api.search.accept.referers"; + /** The key of the configuration. e.g. false */ + String API_SEARCH_SCROLL = "api.search.scroll"; + /** The key of the configuration. e.g. */ String VIRTUAL_HOST_HEADERS = "virtual.host.headers"; @@ -1648,6 +1651,20 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction */ Integer getApiSearchAcceptReferersAsInteger(); + /** + * Get the value for the key 'api.search.scroll'.
+ * The value is, e.g. false
+ * @return The value of found property. (NotNull: if not found, exception but basically no way) + */ + String getApiSearchScroll(); + + /** + * Is the property for the key 'api.search.scroll' true?
+ * The value is, e.g. false
+ * @return The determination, true or false. (if not found, exception but basically no way) + */ + boolean isApiSearchScroll(); + /** * Get the value for the key 'virtual.host.headers'.
* The value is, e.g.
@@ -5688,6 +5705,14 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction return getAsInteger(FessConfig.API_SEARCH_ACCEPT_REFERERS); } + public String getApiSearchScroll() { + return get(FessConfig.API_SEARCH_SCROLL); + } + + public boolean isApiSearchScroll() { + return is(FessConfig.API_SEARCH_SCROLL); + } + public String getVirtualHostHeaders() { return get(FessConfig.VIRTUAL_HOST_HEADERS); } @@ -7813,6 +7838,7 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction defaultMap.put(FessConfig.API_ACCESS_TOKEN_REQUEST_PARAMETER, ""); defaultMap.put(FessConfig.API_ADMIN_ACCESS_PERMISSIONS, "Radmin-api"); defaultMap.put(FessConfig.API_SEARCH_ACCEPT_REFERERS, ""); + defaultMap.put(FessConfig.API_SEARCH_SCROLL, "false"); defaultMap.put(FessConfig.VIRTUAL_HOST_HEADERS, ""); defaultMap.put(FessConfig.HTTP_PROXY_HOST, ""); defaultMap.put(FessConfig.HTTP_PROXY_PORT, "8080"); diff --git a/src/main/resources/fess_config.properties b/src/main/resources/fess_config.properties index 29742d5b1..c1d068df6 100644 --- a/src/main/resources/fess_config.properties +++ b/src/main/resources/fess_config.properties @@ -99,6 +99,7 @@ api.access.token.required=false api.access.token.request.parameter= api.admin.access.permissions=Radmin-api api.search.accept.referers= +api.search.scroll=false # Virtual Host: Host:fess.codelibs.org=fess virtual.host.headers=