diff --git a/src/main/java/org/codelibs/fess/api/json/SearchApiManager.java b/src/main/java/org/codelibs/fess/api/json/SearchApiManager.java index 55871b4d0..17e577e78 100644 --- a/src/main/java/org/codelibs/fess/api/json/SearchApiManager.java +++ b/src/main/java/org/codelibs/fess/api/json/SearchApiManager.java @@ -848,6 +848,8 @@ public class SearchApiManager extends BaseApiManager { private int startPosition = -1; + private int offset = -1; + private int pageSize = -1; protected JsonRequestParams(final HttpServletRequest request, final FessConfig fessConfig) { @@ -935,6 +937,25 @@ public class SearchApiManager extends BaseApiManager { return startPosition; } + @Override + public int getOffset() { + if (offset != -1) { + return offset; + } + + final String value = request.getParameter("offset"); + if (StringUtil.isBlank(value)) { + offset = 0; + } else { + try { + offset = Integer.parseInt(value); + } catch (final NumberFormatException e) { + offset = 0; + } + } + return offset; + } + @Override public int getPageSize() { if (pageSize != -1) { @@ -1071,6 +1092,11 @@ public class SearchApiManager extends BaseApiManager { throw new UnsupportedOperationException(); } + @Override + public int getOffset() { + throw new UnsupportedOperationException(); + } + @Override public int getPageSize() { throw new UnsupportedOperationException(); diff --git a/src/main/java/org/codelibs/fess/app/web/admin/searchlist/ListForm.java b/src/main/java/org/codelibs/fess/app/web/admin/searchlist/ListForm.java index 326f2bcb5..2fac0ee4c 100644 --- a/src/main/java/org/codelibs/fess/app/web/admin/searchlist/ListForm.java +++ b/src/main/java/org/codelibs/fess/app/web/admin/searchlist/ListForm.java @@ -47,6 +47,9 @@ public class ListForm extends SearchRequestParams { @ValidateTypeFailure public Integer start; + @ValidateTypeFailure + public Integer offset; + @ValidateTypeFailure public Integer pn; @@ -91,6 +94,14 @@ public class ListForm extends SearchRequestParams { return start; } + @Override + public int getOffset() { + if (offset == null) { + offset = 0; + } + return offset; + } + @Override public int getPageSize() { final FessConfig fessConfig = ComponentUtil.getFessConfig(); diff --git a/src/main/java/org/codelibs/fess/app/web/base/SearchForm.java b/src/main/java/org/codelibs/fess/app/web/base/SearchForm.java index 14b088b87..901adba46 100644 --- a/src/main/java/org/codelibs/fess/app/web/base/SearchForm.java +++ b/src/main/java/org/codelibs/fess/app/web/base/SearchForm.java @@ -55,6 +55,9 @@ public class SearchForm extends SearchRequestParams { @ValidateTypeFailure public Integer start; + @ValidateTypeFailure + public Integer offset; + @ValidateTypeFailure public Integer pn; @@ -68,13 +71,20 @@ public class SearchForm extends SearchRequestParams { @Override public int getStartPosition() { - final FessConfig fessConfig = ComponentUtil.getFessConfig(); if (start == null) { - start = fessConfig.getPagingSearchPageStartAsInteger(); + start = ComponentUtil.getFessConfig().getPagingSearchPageStartAsInteger(); } return start; } + @Override + public int getOffset() { + if (offset == null) { + offset = 0; + } + return offset; + } + @Override public int getPageSize() { final FessConfig fessConfig = ComponentUtil.getFessConfig(); diff --git a/src/main/java/org/codelibs/fess/entity/GeoInfo.java b/src/main/java/org/codelibs/fess/entity/GeoInfo.java index 160a76893..4cb921624 100644 --- a/src/main/java/org/codelibs/fess/entity/GeoInfo.java +++ b/src/main/java/org/codelibs/fess/entity/GeoInfo.java @@ -15,8 +15,6 @@ */ package org.codelibs.fess.entity; -import static org.codelibs.core.stream.StreamUtil.stream; - import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -25,6 +23,7 @@ import java.util.Map; import javax.servlet.http.HttpServletRequest; import org.codelibs.core.lang.StringUtil; +import org.codelibs.core.stream.StreamUtil; import org.codelibs.fess.exception.InvalidQueryException; import org.codelibs.fess.mylasta.direction.FessConfig; import org.codelibs.fess.util.ComponentUtil; @@ -43,7 +42,7 @@ public class GeoInfo { final String[] geoFields = fessConfig.getQueryGeoFieldsAsArray(); final Map> geoMap = new HashMap<>(); - stream(request.getParameterMap()) + StreamUtil.stream(request.getParameterMap()) .of(stream -> stream.filter(e -> e.getKey().startsWith("geo.") && e.getKey().endsWith(".point")).forEach(e -> { final String key = e.getKey(); for (final String geoField : geoFields) { @@ -51,7 +50,7 @@ public class GeoInfo { final String distanceKey = key.replaceFirst(".point$", ".distance"); final String distance = request.getParameter(distanceKey); if (StringUtil.isNotBlank(distance)) { - stream(e.getValue()).of(s -> s.forEach(pt -> { + StreamUtil.stream(e.getValue()).of(s -> s.forEach(pt -> { List list = geoMap.get(geoField); if (list == null) { list = new ArrayList<>(); @@ -95,7 +94,7 @@ public class GeoInfo { builder = queryBuilders[0]; } else if (queryBuilders.length > 1) { final BoolQueryBuilder boolQuery = QueryBuilders.boolQuery(); - stream(queryBuilders).of(stream -> stream.forEach(boolQuery::must)); + StreamUtil.stream(queryBuilders).of(stream -> stream.forEach(boolQuery::must)); builder = boolQuery; } diff --git a/src/main/java/org/codelibs/fess/entity/SearchRequestParams.java b/src/main/java/org/codelibs/fess/entity/SearchRequestParams.java index d5006a065..f751033d4 100644 --- a/src/main/java/org/codelibs/fess/entity/SearchRequestParams.java +++ b/src/main/java/org/codelibs/fess/entity/SearchRequestParams.java @@ -63,6 +63,8 @@ public abstract class SearchRequestParams { public abstract int getPageSize(); + public abstract int getOffset(); + public abstract String[] getExtraQueries(); public abstract Object getAttribute(String name); diff --git a/src/main/java/org/codelibs/fess/es/client/SearchEngineClient.java b/src/main/java/org/codelibs/fess/es/client/SearchEngineClient.java index 9279480da..04abfe3e0 100644 --- a/src/main/java/org/codelibs/fess/es/client/SearchEngineClient.java +++ b/src/main/java/org/codelibs/fess/es/client/SearchEngineClient.java @@ -18,6 +18,7 @@ package org.codelibs.fess.es.client; import static org.codelibs.core.stream.StreamUtil.split; import static org.codelibs.core.stream.StreamUtil.stream; import static org.codelibs.opensearch.runner.OpenSearchRunner.newConfigs; +import static org.opensearch.action.ActionListener.wrap; import java.io.File; import java.io.IOException; @@ -918,7 +919,7 @@ public class SearchEngineClient implements Client { protected void deleteScrollContext(final String scrollId) { if (scrollId != null) { client.prepareClearScroll().addScrollId(scrollId) - .execute(ActionListener.wrap(res -> {}, e -> logger.warn("Failed to clear the scroll context.", e))); + .execute(wrap(res -> {}, e -> logger.warn("Failed to clear the scroll context.", e))); } } diff --git a/src/main/java/org/codelibs/fess/helper/SearchHelper.java b/src/main/java/org/codelibs/fess/helper/SearchHelper.java index b8c6ffb7d..ea782c3db 100644 --- a/src/main/java/org/codelibs/fess/helper/SearchHelper.java +++ b/src/main/java/org/codelibs/fess/helper/SearchHelper.java @@ -159,33 +159,7 @@ public class SearchHelper { protected List> searchInternal(final String query, final SearchRequestParams params, final OptionalThing userBean) { - final FessConfig fessConfig = ComponentUtil.getFessConfig(); - final QueryHelper queryHelper = ComponentUtil.getQueryHelper(); - final int pageSize = params.getPageSize(); - LaRequestUtil.getOptionalRequest().ifPresent(request -> { - request.setAttribute(Constants.REQUEST_PAGE_SIZE, pageSize); - }); - return ComponentUtil.getSearchEngineClient().search(fessConfig.getIndexDocumentSearchIndex(), searchRequestBuilder -> { - queryHelper.processSearchPreference(searchRequestBuilder, userBean, query); - return SearchConditionBuilder.builder(searchRequestBuilder).query(query).offset(params.getStartPosition()).size(pageSize) - .facetInfo(params.getFacetInfo()).geoInfo(params.getGeoInfo()).highlightInfo(params.getHighlightInfo()) - .similarDocHash(params.getSimilarDocHash()).responseFields(params.getResponseFields()) - .searchRequestType(params.getType()).trackTotalHits(params.getTrackTotalHits()).build(); - }, (searchRequestBuilder, execTime, searchResponse) -> { - searchResponse.ifPresent(r -> { - if (r.getTotalShards() != r.getSuccessfulShards() && fessConfig.isQueryTimeoutLogging()) { - // partial results - final StringBuilder buf = new StringBuilder(1000); - buf.append("[SEARCH TIMEOUT] {\"exec_time\":").append(execTime)// - .append(",\"request\":").append(searchRequestBuilder.toString())// - .append(",\"response\":").append(r.toString()).append('}'); - logger.warn(buf.toString()); - } - }); - final QueryResponseList queryResponseList = ComponentUtil.getQueryResponseList(); - queryResponseList.init(searchResponse, params.getStartPosition(), params.getPageSize()); - return queryResponseList; - }); + return ComponentUtil.getRankFusionProcessor().search(query, params, userBean); } public long scrollSearch(final SearchRequestParams params, final BooleanFunction> cursor, diff --git a/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java b/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java index bf37fff69..f04be0aa8 100644 --- a/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java +++ b/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java @@ -1140,6 +1140,18 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction * */ String QUERY_FACET_QUERIES = "query.facet.queries"; + /** The key of the configuration. e.g. 200 */ + String RANK_FUSION_window_size = "rank.fusion.window_size"; + + /** The key of the configuration. e.g. 20 */ + String RANK_FUSION_rank_constant = "rank.fusion.rank_constant"; + + /** The key of the configuration. e.g. -1 */ + String RANK_FUSION_THREADS = "rank.fusion.threads"; + + /** The key of the configuration. e.g. rf_score */ + String RANK_FUSION_score_field = "rank.fusion.score_field"; + /** The key of the configuration. e.g. true */ String SMB_ROLE_FROM_FILE = "smb.role.from.file"; @@ -5345,6 +5357,60 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction */ String getQueryFacetQueries(); + /** + * Get the value for the key 'rank.fusion.window_size'.
+ * The value is, e.g. 200
+ * comment: ranking + * @return The value of found property. (NotNull: if not found, exception but basically no way) + */ + String getRankFusionWindowSize(); + + /** + * Get the value for the key 'rank.fusion.window_size' as {@link Integer}.
+ * The value is, e.g. 200
+ * comment: ranking + * @return The value of found property. (NotNull: if not found, exception but basically no way) + * @throws NumberFormatException When the property is not integer. + */ + Integer getRankFusionWindowSizeAsInteger(); + + /** + * Get the value for the key 'rank.fusion.rank_constant'.
+ * The value is, e.g. 20
+ * @return The value of found property. (NotNull: if not found, exception but basically no way) + */ + String getRankFusionRankConstant(); + + /** + * Get the value for the key 'rank.fusion.rank_constant' as {@link Integer}.
+ * The value is, e.g. 20
+ * @return The value of found property. (NotNull: if not found, exception but basically no way) + * @throws NumberFormatException When the property is not integer. + */ + Integer getRankFusionRankConstantAsInteger(); + + /** + * Get the value for the key 'rank.fusion.threads'.
+ * The value is, e.g. -1
+ * @return The value of found property. (NotNull: if not found, exception but basically no way) + */ + String getRankFusionThreads(); + + /** + * Get the value for the key 'rank.fusion.threads' as {@link Integer}.
+ * The value is, e.g. -1
+ * @return The value of found property. (NotNull: if not found, exception but basically no way) + * @throws NumberFormatException When the property is not integer. + */ + Integer getRankFusionThreadsAsInteger(); + + /** + * Get the value for the key 'rank.fusion.score_field'.
+ * The value is, e.g. rf_score
+ * @return The value of found property. (NotNull: if not found, exception but basically no way) + */ + String getRankFusionScoreField(); + /** * Get the value for the key 'smb.role.from.file'.
* The value is, e.g. true
@@ -9448,6 +9514,34 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction return get(FessConfig.QUERY_FACET_QUERIES); } + public String getRankFusionWindowSize() { + return get(FessConfig.RANK_FUSION_window_size); + } + + public Integer getRankFusionWindowSizeAsInteger() { + return getAsInteger(FessConfig.RANK_FUSION_window_size); + } + + public String getRankFusionRankConstant() { + return get(FessConfig.RANK_FUSION_rank_constant); + } + + public Integer getRankFusionRankConstantAsInteger() { + return getAsInteger(FessConfig.RANK_FUSION_rank_constant); + } + + public String getRankFusionThreads() { + return get(FessConfig.RANK_FUSION_THREADS); + } + + public Integer getRankFusionThreadsAsInteger() { + return getAsInteger(FessConfig.RANK_FUSION_THREADS); + } + + public String getRankFusionScoreField() { + return get(FessConfig.RANK_FUSION_score_field); + } + public String getSmbRoleFromFile() { return get(FessConfig.SMB_ROLE_FROM_FILE); } @@ -11019,6 +11113,10 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction defaultMap.put(FessConfig.QUERY_FACET_FIELDS_MISSING, ""); defaultMap.put(FessConfig.QUERY_FACET_QUERIES, "labels.facet_timestamp_title:labels.facet_timestamp_1day=timestamp:[now/d-1d TO *]\tlabels.facet_timestamp_1week=timestamp:[now/d-7d TO *]\tlabels.facet_timestamp_1month=timestamp:[now/d-1M TO *]\tlabels.facet_timestamp_1year=timestamp:[now/d-1y TO *]\nlabels.facet_contentLength_title:labels.facet_contentLength_10k=content_length:[0 TO 9999]\tlabels.facet_contentLength_10kto100k=content_length:[10000 TO 99999]\tlabels.facet_contentLength_100kto500k=content_length:[100000 TO 499999]\tlabels.facet_contentLength_500kto1m=content_length:[500000 TO 999999]\tlabels.facet_contentLength_1m=content_length:[1000000 TO *]\nlabels.facet_filetype_title:labels.facet_filetype_html=filetype:html\tlabels.facet_filetype_word=filetype:word\tlabels.facet_filetype_excel=filetype:excel\tlabels.facet_filetype_powerpoint=filetype:powerpoint\tlabels.facet_filetype_odt=filetype:odt\tlabels.facet_filetype_ods=filetype:ods\tlabels.facet_filetype_odp=filetype:odp\tlabels.facet_filetype_pdf=filetype:pdf\tlabels.facet_filetype_txt=filetype:txt\tlabels.facet_filetype_others=filetype:others\n"); + defaultMap.put(FessConfig.RANK_FUSION_window_size, "200"); + defaultMap.put(FessConfig.RANK_FUSION_rank_constant, "20"); + defaultMap.put(FessConfig.RANK_FUSION_THREADS, "-1"); + defaultMap.put(FessConfig.RANK_FUSION_score_field, "rf_score"); defaultMap.put(FessConfig.SMB_ROLE_FROM_FILE, "true"); defaultMap.put(FessConfig.SMB_AVAILABLE_SID_TYPES, "1,2,4:2,5:1"); defaultMap.put(FessConfig.FILE_ROLE_FROM_FILE, "true"); diff --git a/src/main/java/org/codelibs/fess/rank/fusion/DefaultSearcher.java b/src/main/java/org/codelibs/fess/rank/fusion/DefaultSearcher.java new file mode 100644 index 000000000..48284d1b4 --- /dev/null +++ b/src/main/java/org/codelibs/fess/rank/fusion/DefaultSearcher.java @@ -0,0 +1,186 @@ +/* + * Copyright 2012-2023 CodeLibs Project and the Others. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ +package org.codelibs.fess.rank.fusion; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.codelibs.core.stream.StreamUtil; +import org.codelibs.fess.Constants; +import org.codelibs.fess.entity.SearchRequestParams; +import org.codelibs.fess.es.client.SearchEngineClient.SearchConditionBuilder; +import org.codelibs.fess.helper.ViewHelper; +import org.codelibs.fess.mylasta.action.FessUserBean; +import org.codelibs.fess.mylasta.direction.FessConfig; +import org.codelibs.fess.rank.fusion.SearchResult.SearchResultBuilder; +import org.codelibs.fess.util.ComponentUtil; +import org.codelibs.fess.util.FacetResponse; +import org.dbflute.optional.OptionalEntity; +import org.dbflute.optional.OptionalThing; +import org.lastaflute.web.util.LaRequestUtil; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.common.document.DocumentField; +import org.opensearch.search.SearchHit; +import org.opensearch.search.SearchHits; +import org.opensearch.search.aggregations.Aggregations; +import org.opensearch.search.fetch.subphase.highlight.HighlightField; + +public class DefaultSearcher extends RankFusionSearcher { + + private static final Logger logger = LogManager.getLogger(DefaultSearcher.class); + + @Override + protected SearchResult search(final String query, final SearchRequestParams params, final OptionalThing userBean) { + final int pageSize = params.getPageSize(); + LaRequestUtil.getOptionalRequest().ifPresent(request -> { + request.setAttribute(Constants.REQUEST_PAGE_SIZE, pageSize); + }); + final OptionalEntity searchResponseOpt = sendRequest(query, params, userBean); + return processResponse(searchResponseOpt); + } + + protected SearchResult processResponse(final OptionalEntity searchResponseOpt) { + final FessConfig fessConfig = ComponentUtil.getFessConfig(); + final SearchResultBuilder builder = SearchResult.create(); + searchResponseOpt.ifPresent(searchResponse -> { + final SearchHits searchHits = searchResponse.getHits(); + builder.allRecordCount(searchHits.getTotalHits().value); + builder.allRecordCountRelation(searchHits.getTotalHits().relation.toString()); + builder.queryTime(searchResponse.getTook().millis()); + + if (searchResponse.getTotalShards() != searchResponse.getSuccessfulShards()) { + builder.partialResults(true); + } + + // build highlighting fields + final String hlPrefix = ComponentUtil.getQueryHelper().getHighlightPrefix(); + for (final SearchHit searchHit : searchHits.getHits()) { + final Map docMap = parseSearchHit(fessConfig, hlPrefix, searchHit); + + if (fessConfig.isResultCollapsed()) { + final Map innerHits = searchHit.getInnerHits(); + if (innerHits != null) { + final SearchHits innerSearchHits = innerHits.get(fessConfig.getQueryCollapseInnerHitsName()); + if (innerSearchHits != null) { + final long totalHits = innerSearchHits.getTotalHits().value; + if (totalHits > 1) { + docMap.put(fessConfig.getQueryCollapseInnerHitsName() + "_count", totalHits); + final DocumentField bitsField = searchHit.getFields().get(fessConfig.getIndexFieldContentMinhashBits()); + if (bitsField != null && !bitsField.getValues().isEmpty()) { + docMap.put(fessConfig.getQueryCollapseInnerHitsName() + "_hash", bitsField.getValues().get(0)); + } + docMap.put(fessConfig.getQueryCollapseInnerHitsName(), StreamUtil.stream(innerSearchHits.getHits()) + .get(stream -> stream.map(v -> parseSearchHit(fessConfig, hlPrefix, v)).toArray(n -> new Map[n]))); + } + } + } + } + + builder.addDocument(docMap); + } + + // facet + final Aggregations aggregations = searchResponse.getAggregations(); + if (aggregations != null) { + builder.facetResponse(new FacetResponse(aggregations)); + } + + }); + return builder.build(); + } + + protected OptionalEntity sendRequest(final String query, final SearchRequestParams params, + final OptionalThing userBean) { + final FessConfig fessConfig = ComponentUtil.getFessConfig(); + final int pageSize = params.getPageSize(); + return ComponentUtil.getSearchEngineClient().search(fessConfig.getIndexDocumentSearchIndex(), searchRequestBuilder -> { + ComponentUtil.getQueryHelper().processSearchPreference(searchRequestBuilder, userBean, query); + return SearchConditionBuilder.builder(searchRequestBuilder).query(query).offset(params.getStartPosition()).size(pageSize) + .facetInfo(params.getFacetInfo()).geoInfo(params.getGeoInfo()).highlightInfo(params.getHighlightInfo()) + .similarDocHash(params.getSimilarDocHash()).responseFields(params.getResponseFields()) + .searchRequestType(params.getType()).trackTotalHits(params.getTrackTotalHits()).build(); + }, (searchRequestBuilder, execTime, searchResponse) -> { + searchResponse.ifPresent(r -> { + if (r.getTotalShards() != r.getSuccessfulShards() && fessConfig.isQueryTimeoutLogging()) { + // partial results + final StringBuilder buf = new StringBuilder(1000); + buf.append("[SEARCH TIMEOUT] {\"exec_time\":").append(execTime)// + .append(",\"request\":").append(searchRequestBuilder.toString())// + .append(",\"response\":").append(r.toString()).append('}'); + logger.warn(buf.toString()); + } + }); + return searchResponse; + }); + } + + protected Map parseSearchHit(final FessConfig fessConfig, final String hlPrefix, final SearchHit searchHit) { + final Map docMap = new HashMap<>(32); + if (searchHit.getSourceAsMap() == null) { + searchHit.getFields().forEach((key, value) -> { + docMap.put(key, value.getValue()); + }); + } else { + docMap.putAll(searchHit.getSourceAsMap()); + } + + final ViewHelper viewHelper = ComponentUtil.getViewHelper(); + + final Map highlightFields = searchHit.getHighlightFields(); + try { + if (highlightFields != null) { + highlightFields.values().stream().forEach(highlightField -> { + final String text = viewHelper.createHighlightText(highlightField); + if (text != null) { + docMap.put(hlPrefix + highlightField.getName(), text); + } + }); + if (Constants.TEXT_FRAGMENT_TYPE_HIGHLIGHT.equals(fessConfig.getQueryHighlightTextFragmentType())) { + docMap.put(Constants.TEXT_FRAGMENTS, + viewHelper.createTextFragmentsByHighlight(highlightFields.values().toArray(n -> new HighlightField[n]))); + } + } + } catch (final Exception e) { + if (logger.isDebugEnabled()) { + logger.debug("Could not create a highlighting value: {}", docMap, e); + } + } + + if (Constants.TEXT_FRAGMENT_TYPE_QUERY.equals(fessConfig.getQueryHighlightTextFragmentType())) { + docMap.put(Constants.TEXT_FRAGMENTS, viewHelper.createTextFragmentsByQuery()); + } + + // ContentTitle + if (viewHelper != null) { + docMap.put(fessConfig.getResponseFieldContentTitle(), viewHelper.getContentTitle(docMap)); + docMap.put(fessConfig.getResponseFieldContentDescription(), viewHelper.getContentDescription(docMap)); + docMap.put(fessConfig.getResponseFieldUrlLink(), viewHelper.getUrlLink(docMap)); + docMap.put(fessConfig.getResponseFieldSitePath(), viewHelper.getSitePath(docMap)); + } + + if (!docMap.containsKey(Constants.SCORE)) { + docMap.put(Constants.SCORE, searchHit.getScore()); + } + + if (!docMap.containsKey(fessConfig.getIndexFieldId())) { + docMap.put(fessConfig.getIndexFieldId(), searchHit.getId()); + } + return docMap; + } + +} diff --git a/src/main/java/org/codelibs/fess/rank/fusion/RankFusionProcessor.java b/src/main/java/org/codelibs/fess/rank/fusion/RankFusionProcessor.java new file mode 100644 index 000000000..57b1601e6 --- /dev/null +++ b/src/main/java/org/codelibs/fess/rank/fusion/RankFusionProcessor.java @@ -0,0 +1,311 @@ +/* + * Copyright 2012-2023 CodeLibs Project and the Others. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ +package org.codelibs.fess.rank.fusion; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.lucene.search.TotalHits.Relation; +import org.codelibs.fess.entity.FacetInfo; +import org.codelibs.fess.entity.GeoInfo; +import org.codelibs.fess.entity.HighlightInfo; +import org.codelibs.fess.entity.SearchRequestParams; +import org.codelibs.fess.mylasta.action.FessUserBean; +import org.codelibs.fess.mylasta.direction.FessConfig; +import org.codelibs.fess.util.ComponentUtil; +import org.codelibs.fess.util.QueryResponseList; +import org.dbflute.optional.OptionalThing; + +public class RankFusionProcessor implements AutoCloseable { + + private static final Logger logger = LogManager.getLogger(RankFusionProcessor.class); + + protected RankFusionSearcher[] searchers = new RankFusionSearcher[1]; + + protected ExecutorService executorService; + + protected int windowSize; + + @PostConstruct + public void init() { + final FessConfig fessConfig = ComponentUtil.getFessConfig(); + final int maxPageSize = fessConfig.getPagingSearchPageMaxSizeAsInteger(); + final int windowSize = fessConfig.getRankFusionWindowSizeAsInteger(); + if (maxPageSize * 2 < windowSize) { + logger.warn("rank.fusion.window_size is lower than paging.search.page.max.size. " + + "The window size should be 2x more than the page size. ({} * 2 <= {})", maxPageSize, windowSize); + this.windowSize = 2 * maxPageSize; + } else { + this.windowSize = windowSize; + } + } + + @PreDestroy + public void close() throws Exception { + if (executorService != null) { + try { + executorService.shutdown(); + executorService.awaitTermination(60, TimeUnit.SECONDS); + } catch (final InterruptedException e) { + if (logger.isDebugEnabled()) { + logger.debug("Interrupted.", e); + } + } finally { + executorService.shutdownNow(); + } + } + } + + public List> search(final String query, final SearchRequestParams params, + final OptionalThing userBean) { + final int pageSize = params.getPageSize(); + if (searchers.length == 1) { + final SearchResult searchResult = searchers[0].search(query, params, userBean); + return new QueryResponseList(searchResult.getDocumentList(), searchResult.getAllRecordCount(), + searchResult.getAllRecordCountRelation(), searchResult.getQueryTime(), searchResult.isPartialResults(), + searchResult.getFacetResponse(), params.getStartPosition(), pageSize, 0); + } + + final int startPosition = params.getStartPosition(); + if (startPosition * 2 >= windowSize) { + int offset = params.getOffset(); + if (offset < 0) { + offset = 0; + } else if (offset > windowSize / 2) { + offset = windowSize / 2; + } + int start = startPosition - offset; + if (start < 0) { + start = 0; + } + final SearchRequestParams reqParams = new SearchRequestParamsWrapper(params, start, pageSize); + final SearchResult searchResult = searchers[0].search(query, reqParams, userBean); + long allRecordCount = searchResult.getAllRecordCount(); + if (Relation.EQUAL_TO.toString().equals(searchResult.getAllRecordCountRelation())) { + allRecordCount += offset; + } + return new QueryResponseList(searchResult.getDocumentList(), allRecordCount, searchResult.getAllRecordCountRelation(), + searchResult.getQueryTime(), searchResult.isPartialResults(), searchResult.getFacetResponse(), + params.getStartPosition(), pageSize, offset); + } + + final FessConfig fessConfig = ComponentUtil.getFessConfig(); + final int rankConstant = fessConfig.getRankFusionRankConstantAsInteger(); + final int size = windowSize / searchers.length; + final List> resultList = new ArrayList<>(); + for (int i = 0; i < searchers.length; i++) { + final SearchRequestParams reqParams = new SearchRequestParamsWrapper(params, 0, i == 0 ? windowSize : size); + final RankFusionSearcher searcher = searchers[i]; + resultList.add(executorService.submit(() -> searcher.search(query, reqParams, userBean))); + } + final SearchResult[] results = resultList.stream().map(f -> { + try { + return f.get(); + } catch (InterruptedException | ExecutionException e) { + if (logger.isDebugEnabled()) { + logger.debug("Failed to process a search result.", e); + } + return SearchResult.create().build(); + } + }).toArray(n -> new SearchResult[n]); + + final String scoreField = fessConfig.getRankFusionScoreField(); + final Map> scoreDocMap = new HashMap<>(); + final String idField = fessConfig.getIndexFieldId(); + final Set mainIdSet = new HashSet<>(); + for (int i = 0; i < results.length; i++) { + final List> docList = results[i].getDocumentList(); + for (int j = 0; j < docList.size(); j++) { + final Map doc = docList.get(j); + if (doc.get(idField) instanceof final String id) { + final float score = 1.0f / (rankConstant + j); + if (scoreDocMap.containsKey(id)) { + Map baseDoc = scoreDocMap.get(id); + float oldScore = toFloat(baseDoc.get(scoreField)); + baseDoc.put(scoreField, oldScore + score); + } else { + doc.put(scoreField, Float.valueOf(score)); + scoreDocMap.put(id, doc); + } + if (i == 0 && j < windowSize / 2) { + mainIdSet.add(id); + } + } + } + } + + final var docs = scoreDocMap.values().stream() + .sorted((e1, e2) -> Float.compare(toFloat(e2.get(scoreField)), toFloat(e1.get(scoreField)))).toList(); + int offset = 0; + for (int i = 0; i < windowSize / 2; i++) { + if (!mainIdSet.contains(docs.get(i).get(idField))) { + offset++; + } + } + final SearchResult mainResult = results[0]; + long allRecordCount = mainResult.getAllRecordCount(); + if (Relation.EQUAL_TO.toString().equals(mainResult.getAllRecordCountRelation())) { + allRecordCount += offset; + } + return new QueryResponseList(docs.subList(startPosition, startPosition + pageSize), allRecordCount, + mainResult.getAllRecordCountRelation(), mainResult.getQueryTime(), mainResult.isPartialResults(), + mainResult.getFacetResponse(), startPosition, pageSize, offset); + } + + protected float toFloat(final Object value) { + if (value instanceof final Float f) { + return f; + } + if (value instanceof final String s) { + return Float.parseFloat(s); + } + return 0.0f; + } + + protected static class SearchRequestParamsWrapper extends SearchRequestParams { + private final SearchRequestParams parent; + private final int startPosition; + private final int pageSize; + + SearchRequestParamsWrapper(final SearchRequestParams parent, final int startPosition, final int pageSize) { + this.parent = parent; + this.startPosition = startPosition; + this.pageSize = pageSize; + } + + public SearchRequestParams getParent() { + return parent; + } + + @Override + public String getQuery() { + return parent.getQuery(); + } + + @Override + public Map getFields() { + return parent.getFields(); + } + + @Override + public Map getConditions() { + return parent.getConditions(); + } + + @Override + public String[] getLanguages() { + return parent.getLanguages(); + } + + @Override + public GeoInfo getGeoInfo() { + return parent.getGeoInfo(); + } + + @Override + public FacetInfo getFacetInfo() { + return parent.getFacetInfo(); + } + + @Override + public HighlightInfo getHighlightInfo() { + return parent.getHighlightInfo(); + } + + @Override + public String getSort() { + return parent.getSort(); + } + + @Override + public int getStartPosition() { + return startPosition; + } + + @Override + public int getOffset() { + return 0; + } + + @Override + public int getPageSize() { + return pageSize; + } + + @Override + public String[] getExtraQueries() { + return parent.getExtraQueries(); + } + + @Override + public Object getAttribute(final String name) { + return parent.getAttribute(name); + } + + @Override + public Locale getLocale() { + return parent.getLocale(); + } + + @Override + public SearchRequestType getType() { + return parent.getType(); + } + + @Override + public String getSimilarDocHash() { + return parent.getSimilarDocHash(); + } + } + + public void setSeacher(final RankFusionSearcher searcher) { + this.searchers[0] = searcher; + } + + public void register(final RankFusionSearcher searcher) { + if (logger.isDebugEnabled()) { + logger.debug("Load {}", searcher.getClass().getSimpleName()); + } + final RankFusionSearcher[] newSearchers = Arrays.copyOf(searchers, searchers.length + 1); + newSearchers[newSearchers.length - 1] = searcher; + searchers = newSearchers; + synchronized (this) { + if (executorService == null) { + int numThreads = ComponentUtil.getFessConfig().getRankFusionThreadsAsInteger(); + if (numThreads <= 0) { + numThreads = (Runtime.getRuntime().availableProcessors() * 3) / 2 + 1; + } + executorService = Executors.newFixedThreadPool(numThreads); + } + } + } +} diff --git a/src/main/java/org/codelibs/fess/rank/fusion/RankFusionSearcher.java b/src/main/java/org/codelibs/fess/rank/fusion/RankFusionSearcher.java new file mode 100644 index 000000000..ee821ca7e --- /dev/null +++ b/src/main/java/org/codelibs/fess/rank/fusion/RankFusionSearcher.java @@ -0,0 +1,26 @@ +/* + * Copyright 2012-2023 CodeLibs Project and the Others. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ +package org.codelibs.fess.rank.fusion; + +import org.codelibs.fess.entity.SearchRequestParams; +import org.codelibs.fess.mylasta.action.FessUserBean; +import org.dbflute.optional.OptionalThing; + +public abstract class RankFusionSearcher { + + protected abstract SearchResult search(String query, SearchRequestParams params, OptionalThing userBean); + +} diff --git a/src/main/java/org/codelibs/fess/rank/fusion/SearchResult.java b/src/main/java/org/codelibs/fess/rank/fusion/SearchResult.java new file mode 100644 index 000000000..c322c9973 --- /dev/null +++ b/src/main/java/org/codelibs/fess/rank/fusion/SearchResult.java @@ -0,0 +1,120 @@ +/* + * Copyright 2012-2023 CodeLibs Project and the Others. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ +package org.codelibs.fess.rank.fusion; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.apache.lucene.search.TotalHits.Relation; +import org.codelibs.fess.util.FacetResponse; + +public class SearchResult { + + protected final List> documentList; + protected final long allRecordCount; + protected final String allRecordCountRelation; + protected final long queryTime; + protected final boolean partialResults; + protected final FacetResponse facetResponse; + + SearchResult(final List> documentList, final long allRecordCount, final String allRecordCountRelation, + final long queryTime, final boolean partialResults, final FacetResponse facetResponse) { + this.documentList = documentList; + this.allRecordCount = allRecordCount; + this.allRecordCountRelation = allRecordCountRelation; + this.queryTime = queryTime; + this.partialResults = partialResults; + this.facetResponse = facetResponse; + } + + public List> getDocumentList() { + return documentList; + } + + public long getAllRecordCount() { + return allRecordCount; + } + + public String getAllRecordCountRelation() { + return allRecordCountRelation; + } + + public long getQueryTime() { + return queryTime; + } + + public boolean isPartialResults() { + return partialResults; + } + + public FacetResponse getFacetResponse() { + return facetResponse; + } + + public static SearchResultBuilder create() { + return new SearchResultBuilder(); + } + + static class SearchResultBuilder { + + private long allRecordCount; + private String allRecordCountRelation = Relation.GREATER_THAN_OR_EQUAL_TO.toString(); + private long queryTime; + private boolean partialResults; + private FacetResponse facetResponse; + private final List> documentList = new ArrayList<>(); + + public SearchResultBuilder allRecordCount(final long allRecordCount) { + this.allRecordCount = allRecordCount; + return this; + } + + public SearchResultBuilder allRecordCountRelation(final String allRecordCountRelation) { + this.allRecordCountRelation = allRecordCountRelation; + return this; + } + + public SearchResultBuilder queryTime(final long queryTime) { + this.queryTime = queryTime; + return this; + } + + public SearchResultBuilder partialResults(final boolean partialResults) { + this.partialResults = partialResults; + return this; + } + + public SearchResultBuilder addDocument(final Map doc) { + documentList.add(doc); + return this; + } + + public SearchResultBuilder facetResponse(final FacetResponse facetResponse) { + this.facetResponse = facetResponse; + return this; + } + + public SearchResult build() { + return new SearchResult(documentList, // + allRecordCount, // + allRecordCountRelation, // + queryTime, // + partialResults, // + facetResponse); + } + } +} diff --git a/src/main/java/org/codelibs/fess/util/ComponentUtil.java b/src/main/java/org/codelibs/fess/util/ComponentUtil.java index 84db3a71a..2afcc36e1 100644 --- a/src/main/java/org/codelibs/fess/util/ComponentUtil.java +++ b/src/main/java/org/codelibs/fess/util/ComponentUtil.java @@ -81,6 +81,7 @@ import org.codelibs.fess.mylasta.direction.FessProp; import org.codelibs.fess.query.QueryFieldConfig; import org.codelibs.fess.query.QueryProcessor; import org.codelibs.fess.query.parser.QueryParser; +import org.codelibs.fess.rank.fusion.RankFusionProcessor; import org.codelibs.fess.script.ScriptEngineFactory; import org.codelibs.fess.sso.SsoManager; import org.codelibs.fess.thumbnail.ThumbnailManager; @@ -196,8 +197,6 @@ public final class ComponentUtil { private static final String CRAWLER_PROPERTIES = "systemProperties"; - private static final String QUERY_RESPONSE_LIST = "queryResponseList"; - private static final String JOB_EXECUTOR_SUFFIX = "JobExecutor"; private static final String KEY_MATCH_HELPER = "keyMatchHelper"; @@ -214,6 +213,8 @@ public final class ComponentUtil { private static final String CORS_HANDLER_FACTORY = "corsHandlerFactory"; + private static final String RANK_FUSION_PROCESSOR = "rankFusionProcessor"; + private static IndexingHelper indexingHelper; private static CrawlingConfigHelper crawlingConfigHelper; @@ -247,10 +248,6 @@ public final class ComponentUtil { return getComponent(cipherName); } - public static QueryResponseList getQueryResponseList() { - return getComponent(QUERY_RESPONSE_LIST); - } - public static DynamicProperties getSystemProperties() { return getComponent(CRAWLER_PROPERTIES); } @@ -519,6 +516,10 @@ public final class ComponentUtil { return getComponent(CORS_HANDLER_FACTORY); } + public static RankFusionProcessor getRankFusionProcessor() { + return getComponent(RANK_FUSION_PROCESSOR); + } + public static T getComponent(final Class clazz) { try { return SingletonLaContainer.getComponent(clazz); diff --git a/src/main/java/org/codelibs/fess/util/FacetResponse.java b/src/main/java/org/codelibs/fess/util/FacetResponse.java index 5627cde3f..c0c2d4709 100644 --- a/src/main/java/org/codelibs/fess/util/FacetResponse.java +++ b/src/main/java/org/codelibs/fess/util/FacetResponse.java @@ -94,4 +94,9 @@ public class FacetResponse { return fieldList; } + @Override + public String toString() { + return "FacetResponse [queryCountMap=" + queryCountMap + ", fieldList=" + fieldList + "]"; + } + } diff --git a/src/main/java/org/codelibs/fess/util/QueryResponseList.java b/src/main/java/org/codelibs/fess/util/QueryResponseList.java index 4068868d2..14f7415d9 100644 --- a/src/main/java/org/codelibs/fess/util/QueryResponseList.java +++ b/src/main/java/org/codelibs/fess/util/QueryResponseList.java @@ -17,35 +17,21 @@ package org.codelibs.fess.util; import java.util.ArrayList; import java.util.Collection; -import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.ListIterator; import java.util.Map; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.codelibs.core.stream.StreamUtil; -import org.codelibs.fess.Constants; -import org.codelibs.fess.helper.QueryHelper; -import org.codelibs.fess.helper.ViewHelper; -import org.codelibs.fess.mylasta.direction.FessConfig; -import org.dbflute.optional.OptionalEntity; -import org.opensearch.action.search.SearchResponse; -import org.opensearch.common.document.DocumentField; -import org.opensearch.search.SearchHit; -import org.opensearch.search.SearchHits; -import org.opensearch.search.aggregations.Aggregations; -import org.opensearch.search.fetch.subphase.highlight.HighlightField; - public class QueryResponseList implements List> { - private static final Logger logger = LogManager.getLogger(QueryResponseList.class); - protected final List> parent; + protected final int start; + + protected final int offset; + /** The value of current page number. */ - protected int pageSize; + protected final int pageSize; /** The value of current page number. */ protected int currentPageNumber; @@ -76,127 +62,36 @@ public class QueryResponseList implements List> { protected long queryTime; - public QueryResponseList() { - parent = new ArrayList<>(); - } - // for testing - protected QueryResponseList(final List> parent) { - this.parent = parent; + protected QueryResponseList(final List> documentList, final int start, final int pageSize, final int offset) { + this.parent = documentList; + this.offset = offset; + this.start = start; + this.pageSize = pageSize; } - public void init(final OptionalEntity searchResponseOpt, final int start, final int pageSize) { - searchResponseOpt.ifPresent(searchResponse -> { - final FessConfig fessConfig = ComponentUtil.getFessConfig(); - final SearchHits searchHits = searchResponse.getHits(); - allRecordCount = searchHits.getTotalHits().value; - allRecordCountRelation = searchHits.getTotalHits().relation.toString(); - queryTime = searchResponse.getTook().millis(); - - if (searchResponse.getTotalShards() != searchResponse.getSuccessfulShards()) { - partialResults = true; - } - - // build highlighting fields - final QueryHelper queryHelper = ComponentUtil.getQueryHelper(); - final String hlPrefix = queryHelper.getHighlightPrefix(); - for (final SearchHit searchHit : searchHits.getHits()) { - final Map docMap = parseSearchHit(fessConfig, hlPrefix, searchHit); - - if (fessConfig.isResultCollapsed()) { - final Map innerHits = searchHit.getInnerHits(); - if (innerHits != null) { - final SearchHits innerSearchHits = innerHits.get(fessConfig.getQueryCollapseInnerHitsName()); - if (innerSearchHits != null) { - final long totalHits = innerSearchHits.getTotalHits().value; - if (totalHits > 1) { - docMap.put(fessConfig.getQueryCollapseInnerHitsName() + "_count", totalHits); - final DocumentField bitsField = searchHit.getFields().get(fessConfig.getIndexFieldContentMinhashBits()); - if (bitsField != null && !bitsField.getValues().isEmpty()) { - docMap.put(fessConfig.getQueryCollapseInnerHitsName() + "_hash", bitsField.getValues().get(0)); - } - docMap.put(fessConfig.getQueryCollapseInnerHitsName(), StreamUtil.stream(innerSearchHits.getHits()) - .get(stream -> stream.map(v -> parseSearchHit(fessConfig, hlPrefix, v)).toArray(n -> new Map[n]))); - } - } - } - } - - parent.add(docMap); - } - - // facet - final Aggregations aggregations = searchResponse.getAggregations(); - if (aggregations != null) { - facetResponse = new FacetResponse(aggregations); - } - - }); - + public QueryResponseList(final List> documentList, final long allRecordCount, final String allRecordCountRelation, + final long queryTime, final boolean partialResults, final FacetResponse facetResponse, final int start, final int pageSize, + final int offset) { + this(documentList, start, pageSize, offset); + this.allRecordCount = allRecordCount; + this.allRecordCountRelation = allRecordCountRelation; + this.queryTime = queryTime; + this.partialResults = partialResults; + this.facetResponse = facetResponse; if (pageSize > 0) { - calculatePageInfo(start, pageSize); + calculatePageInfo(); } } - protected Map parseSearchHit(final FessConfig fessConfig, final String hlPrefix, final SearchHit searchHit) { - final Map docMap = new HashMap<>(32); - if (searchHit.getSourceAsMap() == null) { - searchHit.getFields().forEach((key, value) -> { - docMap.put(key, value.getValue()); - }); - } else { - docMap.putAll(searchHit.getSourceAsMap()); + protected void calculatePageInfo() { + int startWithOffset = start - offset; + if (startWithOffset < 0) { + startWithOffset = 0; } - - final ViewHelper viewHelper = ComponentUtil.getViewHelper(); - - final Map highlightFields = searchHit.getHighlightFields(); - try { - if (highlightFields != null) { - highlightFields.values().stream().forEach(highlightField -> { - final String text = viewHelper.createHighlightText(highlightField); - if (text != null) { - docMap.put(hlPrefix + highlightField.getName(), text); - } - }); - if (Constants.TEXT_FRAGMENT_TYPE_HIGHLIGHT.equals(fessConfig.getQueryHighlightTextFragmentType())) { - docMap.put(Constants.TEXT_FRAGMENTS, - viewHelper.createTextFragmentsByHighlight(highlightFields.values().toArray(n -> new HighlightField[n]))); - } - } - } catch (final Exception e) { - if (logger.isDebugEnabled()) { - logger.debug("Could not create a highlighting value: {}", docMap, e); - } - } - - if (Constants.TEXT_FRAGMENT_TYPE_QUERY.equals(fessConfig.getQueryHighlightTextFragmentType())) { - docMap.put(Constants.TEXT_FRAGMENTS, viewHelper.createTextFragmentsByQuery()); - } - - // ContentTitle - if (viewHelper != null) { - docMap.put(fessConfig.getResponseFieldContentTitle(), viewHelper.getContentTitle(docMap)); - docMap.put(fessConfig.getResponseFieldContentDescription(), viewHelper.getContentDescription(docMap)); - docMap.put(fessConfig.getResponseFieldUrlLink(), viewHelper.getUrlLink(docMap)); - docMap.put(fessConfig.getResponseFieldSitePath(), viewHelper.getSitePath(docMap)); - } - - if (!docMap.containsKey(Constants.SCORE)) { - docMap.put(Constants.SCORE, searchHit.getScore()); - } - - if (!docMap.containsKey(fessConfig.getIndexFieldId())) { - docMap.put(fessConfig.getIndexFieldId(), searchHit.getId()); - } - return docMap; - } - - protected void calculatePageInfo(final int start, final int size) { - pageSize = size; allPageCount = (int) ((allRecordCount - 1) / pageSize) + 1; - existPrevPage = start > 0; - existNextPage = start < (long) (allPageCount - 1) * (long) pageSize; + existPrevPage = startWithOffset > 0; + existNextPage = startWithOffset < (long) (allPageCount - 1) * (long) pageSize; currentPageNumber = start / pageSize + 1; if (existNextPage && size() < pageSize) { // collapsing @@ -347,6 +242,14 @@ public class QueryResponseList implements List> { return parent.toArray(a); } + public int getStart() { + return start; + } + + public int getOffset() { + return offset; + } + public int getPageSize() { return pageSize; } @@ -415,4 +318,14 @@ public class QueryResponseList implements List> { return queryTime; } + @Override + public String toString() { + return "QueryResponseList [parent=" + parent + ", start=" + start + ", offset=" + offset + ", pageSize=" + pageSize + + ", currentPageNumber=" + currentPageNumber + ", allRecordCount=" + allRecordCount + ", allRecordCountRelation=" + + allRecordCountRelation + ", allPageCount=" + allPageCount + ", existNextPage=" + existNextPage + ", existPrevPage=" + + existPrevPage + ", currentStartRecordNumber=" + currentStartRecordNumber + ", currentEndRecordNumber=" + + currentEndRecordNumber + ", pageNumberList=" + pageNumberList + ", searchQuery=" + searchQuery + ", execTime=" + execTime + + ", facetResponse=" + facetResponse + ", partialResults=" + partialResults + ", queryTime=" + queryTime + "]"; + } + } diff --git a/src/main/resources/app.xml b/src/main/resources/app.xml index 95a8ec0db..f3c244d36 100644 --- a/src/main/resources/app.xml +++ b/src/main/resources/app.xml @@ -7,15 +7,16 @@ - - - - + + + + + @@ -103,8 +104,6 @@ - - diff --git a/src/main/resources/fess_config.properties b/src/main/resources/fess_config.properties index 80b51ddf4..f4539d830 100644 --- a/src/main/resources/fess_config.properties +++ b/src/main/resources/fess_config.properties @@ -604,6 +604,12 @@ labels.facet_filetype_pdf=filetype:pdf\t\ labels.facet_filetype_txt=filetype:txt\t\ labels.facet_filetype_others=filetype:others\n\ +# ranking +rank.fusion.window_size=200 +rank.fusion.rank_constant=20 +rank.fusion.threads=-1 +rank.fusion.score_field=rf_score + # acl smb.role.from.file=true smb.available.sid.types=1,2,4:2,5:1 diff --git a/src/main/resources/fess_rankfusion.xml b/src/main/resources/fess_rankfusion.xml new file mode 100644 index 000000000..a4986cff1 --- /dev/null +++ b/src/main/resources/fess_rankfusion.xml @@ -0,0 +1,15 @@ + + + + + + + + + + + + diff --git a/src/test/java/org/codelibs/fess/rank/fusion/RankFusionProcessorTest.java b/src/test/java/org/codelibs/fess/rank/fusion/RankFusionProcessorTest.java new file mode 100644 index 000000000..f348e5858 --- /dev/null +++ b/src/test/java/org/codelibs/fess/rank/fusion/RankFusionProcessorTest.java @@ -0,0 +1,371 @@ +/* + * Copyright 2012-2023 CodeLibs Project and the Others. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ +package org.codelibs.fess.rank.fusion; + +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; + +import org.apache.lucene.search.TotalHits.Relation; +import org.codelibs.fess.entity.FacetInfo; +import org.codelibs.fess.entity.GeoInfo; +import org.codelibs.fess.entity.HighlightInfo; +import org.codelibs.fess.entity.SearchRequestParams; +import org.codelibs.fess.mylasta.action.FessUserBean; +import org.codelibs.fess.rank.fusion.SearchResult.SearchResultBuilder; +import org.codelibs.fess.unit.UnitFessTestCase; +import org.codelibs.fess.util.QueryResponseList; +import org.dbflute.optional.OptionalThing; + +public class RankFusionProcessorTest extends UnitFessTestCase { + + private static final String ID_FIELD = "_id"; + + public void test_default_1000docs_10size() throws Exception { + String query = "*"; + int allRecordCount = 1000; + int pageSize = 10; + try (RankFusionProcessor rankFusionProcessor = new RankFusionProcessor()) { + rankFusionProcessor.setSeacher(new TestMainSearcher(allRecordCount)); + rankFusionProcessor.init(); + + if (rankFusionProcessor.search(query, new TestSearchRequestParams(0, pageSize, 0), + OptionalThing.empty()) instanceof QueryResponseList list) { + assertEquals(pageSize, list.size()); + assertEquals(allRecordCount, list.getAllRecordCount()); + assertEquals(100, list.getAllPageCount()); + assertEquals(10, list.getCurrentEndRecordNumber()); + assertEquals(1, list.getCurrentPageNumber()); + assertEquals(1, list.getCurrentStartRecordNumber()); + assertEquals(0, list.getOffset()); + assertEquals(10, list.getPageSize()); + assertEquals(0, list.getStart()); + assertEquals("0", list.get(0).get(ID_FIELD)); + assertEquals("9", list.get(9).get(ID_FIELD)); + } else { + fail(); + } + + if (rankFusionProcessor.search(query, new TestSearchRequestParams(10, pageSize, 0), + OptionalThing.empty()) instanceof QueryResponseList list) { + assertEquals(pageSize, list.size()); + assertEquals(allRecordCount, list.getAllRecordCount()); + assertEquals(100, list.getAllPageCount()); + assertEquals(20, list.getCurrentEndRecordNumber()); + assertEquals(2, list.getCurrentPageNumber()); + assertEquals(11, list.getCurrentStartRecordNumber()); + assertEquals(0, list.getOffset()); + assertEquals(10, list.getPageSize()); + assertEquals(10, list.getStart()); + assertEquals("10", list.get(0).get(ID_FIELD)); + assertEquals("19", list.get(9).get(ID_FIELD)); + } else { + fail(); + } + + if (rankFusionProcessor.search(query, new TestSearchRequestParams(990, pageSize, 0), + OptionalThing.empty()) instanceof QueryResponseList list) { + assertEquals(pageSize, list.size()); + assertEquals(allRecordCount, list.getAllRecordCount()); + assertEquals(100, list.getAllPageCount()); + assertEquals(1000, list.getCurrentEndRecordNumber()); + assertEquals(100, list.getCurrentPageNumber()); + assertEquals(991, list.getCurrentStartRecordNumber()); + assertEquals(0, list.getOffset()); + assertEquals(10, list.getPageSize()); + assertEquals(990, list.getStart()); + assertEquals("990", list.get(0).get(ID_FIELD)); + assertEquals("999", list.get(9).get(ID_FIELD)); + } else { + fail(); + } + } + } + + public void test_1searcher10_1000docs_100size() throws Exception { + String query = "*"; + int allRecordCount = 1000; + int pageSize = 100; + int offset = 45; + try (RankFusionProcessor rankFusionProcessor = new RankFusionProcessor()) { + rankFusionProcessor.setSeacher(new TestMainSearcher(allRecordCount)); + rankFusionProcessor.register(new TestSubSearcher(10, 45, 45)); + rankFusionProcessor.init(); + + if (rankFusionProcessor.search(query, new TestSearchRequestParams(0, pageSize, 0), + OptionalThing.empty()) instanceof QueryResponseList list) { + assertEquals(pageSize, list.size()); + assertEquals(allRecordCount + offset, list.getAllRecordCount()); + assertEquals(11, list.getAllPageCount()); + assertEquals(100, list.getCurrentEndRecordNumber()); + assertEquals(1, list.getCurrentPageNumber()); + assertEquals(1, list.getCurrentStartRecordNumber()); + assertEquals(offset, list.getOffset()); + assertEquals(100, list.getPageSize()); + assertEquals(0, list.getStart()); + assertEquals("0", list.get(0).get(ID_FIELD)); + assertEquals("9", list.get(1).get(ID_FIELD)); + assertEquals("1", list.get(2).get(ID_FIELD)); + assertEquals("8", list.get(3).get(ID_FIELD)); + assertEquals("2", list.get(4).get(ID_FIELD)); + assertEquals("7", list.get(5).get(ID_FIELD)); + assertEquals("3", list.get(6).get(ID_FIELD)); + assertEquals("6", list.get(7).get(ID_FIELD)); + assertEquals("4", list.get(8).get(ID_FIELD)); + assertEquals("5", list.get(9).get(ID_FIELD)); + } else { + fail(); + } + + if (rankFusionProcessor.search(query, new TestSearchRequestParams(0, pageSize, offset), + OptionalThing.empty()) instanceof QueryResponseList list) { + assertEquals(pageSize, list.size()); + assertEquals(allRecordCount + offset, list.getAllRecordCount()); + assertEquals(11, list.getAllPageCount()); + assertEquals(100, list.getCurrentEndRecordNumber()); + assertEquals(1, list.getCurrentPageNumber()); + assertEquals(1, list.getCurrentStartRecordNumber()); + assertEquals(offset, list.getOffset()); + assertEquals(100, list.getPageSize()); + assertEquals(0, list.getStart()); + assertEquals("0", list.get(0).get(ID_FIELD)); + assertEquals("9", list.get(1).get(ID_FIELD)); + assertEquals("1", list.get(2).get(ID_FIELD)); + assertEquals("8", list.get(3).get(ID_FIELD)); + assertEquals("2", list.get(4).get(ID_FIELD)); + assertEquals("7", list.get(5).get(ID_FIELD)); + assertEquals("3", list.get(6).get(ID_FIELD)); + assertEquals("6", list.get(7).get(ID_FIELD)); + assertEquals("4", list.get(8).get(ID_FIELD)); + assertEquals("5", list.get(9).get(ID_FIELD)); + } else { + fail(); + } + + if (rankFusionProcessor.search(query, new TestSearchRequestParams(100, pageSize, offset), + OptionalThing.empty()) instanceof QueryResponseList list) { + assertEquals(pageSize, list.size()); + assertEquals(allRecordCount + offset, list.getAllRecordCount()); + assertEquals(11, list.getAllPageCount()); + assertEquals(200, list.getCurrentEndRecordNumber()); + assertEquals(2, list.getCurrentPageNumber()); + assertEquals(101, list.getCurrentStartRecordNumber()); + assertEquals(offset, list.getOffset()); + assertEquals(100, list.getPageSize()); + assertEquals(100, list.getStart()); + assertEquals("55", list.get(0).get(ID_FIELD)); + assertEquals("154", list.get(99).get(ID_FIELD)); + } else { + fail(); + } + + if (rankFusionProcessor.search(query, new TestSearchRequestParams(900, pageSize, offset), + OptionalThing.empty()) instanceof QueryResponseList list) { + assertEquals(pageSize, list.size()); + assertEquals(allRecordCount + offset, list.getAllRecordCount()); + assertEquals(11, list.getAllPageCount()); + assertEquals(1000, list.getCurrentEndRecordNumber()); + assertEquals(10, list.getCurrentPageNumber()); + assertEquals(901, list.getCurrentStartRecordNumber()); + assertEquals(offset, list.getOffset()); + assertEquals(100, list.getPageSize()); + assertEquals(900, list.getStart()); + assertEquals("855", list.get(0).get(ID_FIELD)); + assertEquals("954", list.get(99).get(ID_FIELD)); + } else { + fail(); + } + + if (rankFusionProcessor.search(query, new TestSearchRequestParams(1000, pageSize, offset), + OptionalThing.empty()) instanceof QueryResponseList list) { + assertEquals(pageSize, list.size()); + assertEquals(allRecordCount + offset, list.getAllRecordCount()); + assertEquals(11, list.getAllPageCount()); + assertEquals(1045, list.getCurrentEndRecordNumber()); + assertEquals(11, list.getCurrentPageNumber()); + assertEquals(1001, list.getCurrentStartRecordNumber()); + assertEquals(offset, list.getOffset()); + assertEquals(100, list.getPageSize()); + assertEquals(1000, list.getStart()); + assertEquals("955", list.get(0).get(ID_FIELD)); + assertEquals("999", list.get(44).get(ID_FIELD)); + } else { + fail(); + } + } + } + + static class TestMainSearcher extends RankFusionSearcher { + + private long allRecordCount; + + TestMainSearcher(int allRecordCount) { + this.allRecordCount = allRecordCount; + } + + @Override + protected SearchResult search(String query, SearchRequestParams params, OptionalThing userBean) { + int start = params.getStartPosition(); + int size = params.getPageSize(); + SearchResultBuilder builder = SearchResult.create(); + for (int i = start; i < start + size; i++) { + Map doc = new HashMap<>(); + doc.put(ID_FIELD, Integer.toString(i)); + doc.put("score", 1.0f / (i + 1)); + builder.addDocument(doc); + } + builder.allRecordCount(allRecordCount); + if (allRecordCount < 10000) { + builder.allRecordCountRelation(Relation.EQUAL_TO.toString()); + } + return builder.build(); + } + } + + static class TestSubSearcher extends RankFusionSearcher { + + private int mainSize; + private int inSize; + private int outSize; + + TestSubSearcher(int mainSize, int inSize, int outSize) { + this.mainSize = mainSize; + this.inSize = inSize; + this.outSize = outSize; + } + + @Override + protected SearchResult search(String query, SearchRequestParams params, OptionalThing userBean) { + SearchResultBuilder builder = SearchResult.create(); + for (int i = 0; i < mainSize; i++) { + Map doc = new HashMap<>(); + doc.put(ID_FIELD, Integer.toString(mainSize - i - 1)); + doc.put("score", 1.0f / (i + 2)); + builder.addDocument(doc); + } + for (int i = 100; i < inSize + 100; i++) { + Map doc = new HashMap<>(); + doc.put(ID_FIELD, Integer.toString(i)); + doc.put("score", 1.0f / (mainSize + i + 2)); + builder.addDocument(doc); + } + for (int i = 200; i < outSize + 200; i++) { + Map doc = new HashMap<>(); + doc.put(ID_FIELD, Integer.toString(i)); + doc.put("score", 1.0f / (mainSize + i + 3)); + builder.addDocument(doc); + } + builder.allRecordCount(mainSize + inSize + outSize); + return builder.build(); + } + } + + static class TestSearchRequestParams extends SearchRequestParams { + + private int startPosition; + + private int pageSize; + + private int offset; + + TestSearchRequestParams(int startPosition, int pageSize, int offset) { + this.startPosition = startPosition; + this.pageSize = pageSize; + this.offset = offset; + } + + @Override + public String getQuery() { + return null; + } + + @Override + public Map getFields() { + return null; + } + + @Override + public Map getConditions() { + return null; + } + + @Override + public String[] getLanguages() { + return null; + } + + @Override + public GeoInfo getGeoInfo() { + return null; + } + + @Override + public FacetInfo getFacetInfo() { + return null; + } + + @Override + public HighlightInfo getHighlightInfo() { + return null; + } + + @Override + public String getSort() { + return null; + } + + @Override + public int getStartPosition() { + return startPosition; + } + + @Override + public int getOffset() { + return offset; + } + + @Override + public int getPageSize() { + return pageSize; + } + + @Override + public String[] getExtraQueries() { + return null; + } + + @Override + public Object getAttribute(String name) { + return null; + } + + @Override + public Locale getLocale() { + return null; + } + + @Override + public SearchRequestType getType() { + return null; + } + + @Override + public String getSimilarDocHash() { + return null; + } + + } +} diff --git a/src/test/java/org/codelibs/fess/util/QueryResponseListTest.java b/src/test/java/org/codelibs/fess/util/QueryResponseListTest.java index 0a72818c5..9eac7b278 100644 --- a/src/test/java/org/codelibs/fess/util/QueryResponseListTest.java +++ b/src/test/java/org/codelibs/fess/util/QueryResponseListTest.java @@ -15,9 +15,7 @@ */ package org.codelibs.fess.util; -import java.util.ArrayList; import java.util.List; -import java.util.Map; import org.codelibs.fess.unit.UnitFessTestCase; @@ -25,13 +23,13 @@ public class QueryResponseListTest extends UnitFessTestCase { public void test_calculatePageInfo_page0() { QueryResponseList qrList; - qrList = new QueryResponseList(new ArrayList>()) { + qrList = new QueryResponseList(null, 0, 20, 0) { @Override public int size() { return 0; } }; - qrList.calculatePageInfo(0, 20); + qrList.calculatePageInfo(); assertEquals(20, qrList.getPageSize()); assertEquals(1, qrList.getCurrentPageNumber()); assertEquals(0, qrList.getAllRecordCount()); @@ -45,14 +43,14 @@ public class QueryResponseListTest extends UnitFessTestCase { public void test_calculatePageInfo_page1() { QueryResponseList qrList; - qrList = new QueryResponseList(new ArrayList>()) { + qrList = new QueryResponseList(null, 0, 20, 0) { @Override public int size() { return 10; } }; qrList.allRecordCount = 10; - qrList.calculatePageInfo(0, 20); + qrList.calculatePageInfo(); assertEquals(20, qrList.getPageSize()); assertEquals(1, qrList.getCurrentPageNumber()); assertEquals(10, qrList.getAllRecordCount()); @@ -62,14 +60,14 @@ public class QueryResponseListTest extends UnitFessTestCase { assertEquals(1, qrList.getCurrentStartRecordNumber()); assertEquals(10, qrList.getCurrentEndRecordNumber()); - qrList = new QueryResponseList(new ArrayList>()) { + qrList = new QueryResponseList(null, 0, 20, 0) { @Override public int size() { return 20; } }; qrList.allRecordCount = 20; - qrList.calculatePageInfo(0, 20); + qrList.calculatePageInfo(); assertEquals(20, qrList.getPageSize()); assertEquals(1, qrList.getCurrentPageNumber()); assertEquals(20, qrList.getAllRecordCount()); @@ -79,14 +77,14 @@ public class QueryResponseListTest extends UnitFessTestCase { assertEquals(1, qrList.getCurrentStartRecordNumber()); assertEquals(20, qrList.getCurrentEndRecordNumber()); - qrList = new QueryResponseList(new ArrayList>()) { + qrList = new QueryResponseList(null, 0, 20, 0) { @Override public int size() { return 20; } }; qrList.allRecordCount = 21; - qrList.calculatePageInfo(0, 20); + qrList.calculatePageInfo(); assertEquals(20, qrList.getPageSize()); assertEquals(1, qrList.getCurrentPageNumber()); assertEquals(21, qrList.getAllRecordCount()); @@ -96,14 +94,14 @@ public class QueryResponseListTest extends UnitFessTestCase { assertEquals(1, qrList.getCurrentStartRecordNumber()); assertEquals(20, qrList.getCurrentEndRecordNumber()); - qrList = new QueryResponseList(new ArrayList>()) { + qrList = new QueryResponseList(null, 0, 20, 0) { @Override public int size() { return 20; } }; qrList.allRecordCount = 40; - qrList.calculatePageInfo(0, 20); + qrList.calculatePageInfo(); assertEquals(20, qrList.getPageSize()); assertEquals(1, qrList.getCurrentPageNumber()); assertEquals(40, qrList.getAllRecordCount()); @@ -113,14 +111,14 @@ public class QueryResponseListTest extends UnitFessTestCase { assertEquals(1, qrList.getCurrentStartRecordNumber()); assertEquals(20, qrList.getCurrentEndRecordNumber()); - qrList = new QueryResponseList(new ArrayList>()) { + qrList = new QueryResponseList(null, 0, 20, 0) { @Override public int size() { return 20; } }; qrList.allRecordCount = 41; - qrList.calculatePageInfo(0, 20); + qrList.calculatePageInfo(); assertEquals(20, qrList.getPageSize()); assertEquals(1, qrList.getCurrentPageNumber()); assertEquals(41, qrList.getAllRecordCount()); @@ -134,14 +132,14 @@ public class QueryResponseListTest extends UnitFessTestCase { public void test_calculatePageInfo_page2() { QueryResponseList qrList; - qrList = new QueryResponseList(new ArrayList>()) { + qrList = new QueryResponseList(null, 20, 20, 0) { @Override public int size() { return 1; } }; qrList.allRecordCount = 21; - qrList.calculatePageInfo(20, 20); + qrList.calculatePageInfo(); assertEquals(20, qrList.getPageSize()); assertEquals(2, qrList.getCurrentPageNumber()); assertEquals(21, qrList.getAllRecordCount()); @@ -151,14 +149,14 @@ public class QueryResponseListTest extends UnitFessTestCase { assertEquals(21, qrList.getCurrentStartRecordNumber()); assertEquals(21, qrList.getCurrentEndRecordNumber()); - qrList = new QueryResponseList(new ArrayList>()) { + qrList = new QueryResponseList(null, 20, 20, 0) { @Override public int size() { return 20; } }; qrList.allRecordCount = 40; - qrList.calculatePageInfo(20, 20); + qrList.calculatePageInfo(); assertEquals(20, qrList.getPageSize()); assertEquals(2, qrList.getCurrentPageNumber()); assertEquals(40, qrList.getAllRecordCount()); @@ -168,14 +166,14 @@ public class QueryResponseListTest extends UnitFessTestCase { assertEquals(21, qrList.getCurrentStartRecordNumber()); assertEquals(40, qrList.getCurrentEndRecordNumber()); - qrList = new QueryResponseList(new ArrayList>()) { + qrList = new QueryResponseList(null, 20, 20, 0) { @Override public int size() { return 20; } }; qrList.allRecordCount = 41; - qrList.calculatePageInfo(20, 20); + qrList.calculatePageInfo(); assertEquals(20, qrList.getPageSize()); assertEquals(2, qrList.getCurrentPageNumber()); assertEquals(41, qrList.getAllRecordCount()); @@ -185,14 +183,14 @@ public class QueryResponseListTest extends UnitFessTestCase { assertEquals(21, qrList.getCurrentStartRecordNumber()); assertEquals(40, qrList.getCurrentEndRecordNumber()); - qrList = new QueryResponseList(new ArrayList>()) { + qrList = new QueryResponseList(null, 20, 20, 0) { @Override public int size() { return 20; } }; qrList.allRecordCount = 61; - qrList.calculatePageInfo(20, 20); + qrList.calculatePageInfo(); assertEquals(20, qrList.getPageSize()); assertEquals(2, qrList.getCurrentPageNumber()); assertEquals(61, qrList.getAllRecordCount()); @@ -207,26 +205,26 @@ public class QueryResponseListTest extends UnitFessTestCase { QueryResponseList qrList; List pnList; - qrList = new QueryResponseList(new ArrayList>()) { + qrList = new QueryResponseList(null, 0, 20, 0) { @Override public int size() { return 20; } }; qrList.allRecordCount = 20; - qrList.calculatePageInfo(0, 20); + qrList.calculatePageInfo(); pnList = qrList.getPageNumberList(); assertEquals(1, pnList.size()); assertEquals("1", pnList.get(0)); - qrList = new QueryResponseList(new ArrayList>()) { + qrList = new QueryResponseList(null, 0, 20, 0) { @Override public int size() { return 20; } }; qrList.allRecordCount = 61; - qrList.calculatePageInfo(0, 20); + qrList.calculatePageInfo(); pnList = qrList.getPageNumberList(); assertEquals(4, pnList.size()); assertEquals("1", pnList.get(0)); @@ -234,14 +232,14 @@ public class QueryResponseListTest extends UnitFessTestCase { assertEquals("3", pnList.get(2)); assertEquals("4", pnList.get(3)); - qrList = new QueryResponseList(new ArrayList>()) { + qrList = new QueryResponseList(null, 0, 20, 0) { @Override public int size() { return 20; } }; qrList.allRecordCount = 200; - qrList.calculatePageInfo(0, 20); + qrList.calculatePageInfo(); pnList = qrList.getPageNumberList(); assertEquals(6, pnList.size()); assertEquals("1", pnList.get(0)); @@ -251,27 +249,27 @@ public class QueryResponseListTest extends UnitFessTestCase { assertEquals("5", pnList.get(4)); assertEquals("6", pnList.get(5)); - qrList = new QueryResponseList(new ArrayList>()) { + qrList = new QueryResponseList(null, 20, 20, 0) { @Override public int size() { return 1; } }; qrList.allRecordCount = 21; - qrList.calculatePageInfo(20, 20); + qrList.calculatePageInfo(); pnList = qrList.getPageNumberList(); assertEquals(2, pnList.size()); assertEquals("1", pnList.get(0)); assertEquals("2", pnList.get(1)); - qrList = new QueryResponseList(new ArrayList>()) { + qrList = new QueryResponseList(null, 20, 20, 0) { @Override public int size() { return 20; } }; qrList.allRecordCount = 61; - qrList.calculatePageInfo(20, 20); + qrList.calculatePageInfo(); pnList = qrList.getPageNumberList(); assertEquals(4, pnList.size()); assertEquals("1", pnList.get(0)); @@ -279,14 +277,14 @@ public class QueryResponseListTest extends UnitFessTestCase { assertEquals("3", pnList.get(2)); assertEquals("4", pnList.get(3)); - qrList = new QueryResponseList(new ArrayList>()) { + qrList = new QueryResponseList(null, 20, 20, 0) { @Override public int size() { return 20; } }; qrList.allRecordCount = 200; - qrList.calculatePageInfo(20, 20); + qrList.calculatePageInfo(); pnList = qrList.getPageNumberList(); assertEquals(7, pnList.size()); assertEquals("1", pnList.get(0)); @@ -302,13 +300,13 @@ public class QueryResponseListTest extends UnitFessTestCase { public void test_calculatePageInfo_collapse() { QueryResponseList qrList; - qrList = new QueryResponseList(new ArrayList>()) { + qrList = new QueryResponseList(null, 0, 20, 0) { @Override public int size() { return 0; } }; - qrList.calculatePageInfo(0, 20); + qrList.calculatePageInfo(); assertEquals(20, qrList.getPageSize()); assertEquals(1, qrList.getCurrentPageNumber()); assertEquals(0, qrList.getAllRecordCount()); @@ -318,14 +316,14 @@ public class QueryResponseListTest extends UnitFessTestCase { assertEquals(0, qrList.getCurrentStartRecordNumber()); assertEquals(0, qrList.getCurrentEndRecordNumber()); - qrList = new QueryResponseList(new ArrayList>()) { + qrList = new QueryResponseList(null, 0, 20, 0) { @Override public int size() { return 10; } }; qrList.allRecordCount = 20; - qrList.calculatePageInfo(0, 20); + qrList.calculatePageInfo(); assertEquals(20, qrList.getPageSize()); assertEquals(1, qrList.getCurrentPageNumber()); assertEquals(20, qrList.getAllRecordCount()); @@ -335,14 +333,14 @@ public class QueryResponseListTest extends UnitFessTestCase { assertEquals(1, qrList.getCurrentStartRecordNumber()); assertEquals(20, qrList.getCurrentEndRecordNumber()); - qrList = new QueryResponseList(new ArrayList>()) { + qrList = new QueryResponseList(null, 0, 20, 0) { @Override public int size() { return 10; } }; qrList.allRecordCount = 21; - qrList.calculatePageInfo(0, 20); + qrList.calculatePageInfo(); assertEquals(20, qrList.getPageSize()); assertEquals(1, qrList.getCurrentPageNumber()); assertEquals(21, qrList.getAllRecordCount()); @@ -352,14 +350,14 @@ public class QueryResponseListTest extends UnitFessTestCase { assertEquals(1, qrList.getCurrentStartRecordNumber()); assertEquals(20, qrList.getCurrentEndRecordNumber()); - qrList = new QueryResponseList(new ArrayList>()) { + qrList = new QueryResponseList(null, 0, 20, 0) { @Override public int size() { return 21; } }; qrList.allRecordCount = 41; - qrList.calculatePageInfo(0, 20); + qrList.calculatePageInfo(); assertEquals(20, qrList.getPageSize()); assertEquals(1, qrList.getCurrentPageNumber()); assertEquals(41, qrList.getAllRecordCount()); @@ -369,14 +367,14 @@ public class QueryResponseListTest extends UnitFessTestCase { assertEquals(1, qrList.getCurrentStartRecordNumber()); assertEquals(20, qrList.getCurrentEndRecordNumber()); - qrList = new QueryResponseList(new ArrayList>()) { + qrList = new QueryResponseList(null, 20, 20, 0) { @Override public int size() { return 1; } }; qrList.allRecordCount = 41; - qrList.calculatePageInfo(20, 20); + qrList.calculatePageInfo(); assertEquals(20, qrList.getPageSize()); assertEquals(2, qrList.getCurrentPageNumber()); assertEquals(41, qrList.getAllRecordCount()); diff --git a/src/test/java/org/codelibs/fess/util/QueryStringBuilderTest.java b/src/test/java/org/codelibs/fess/util/QueryStringBuilderTest.java index 7abc9b82a..c571759cd 100644 --- a/src/test/java/org/codelibs/fess/util/QueryStringBuilderTest.java +++ b/src/test/java/org/codelibs/fess/util/QueryStringBuilderTest.java @@ -131,6 +131,11 @@ public class QueryStringBuilderTest extends UnitFessTestCase { return 0; } + @Override + public int getOffset() { + return 0; + } + @Override public int getPageSize() { return 0;