diff --git a/plugin.xml b/plugin.xml index 3f267228b..b713daca0 100644 --- a/plugin.xml +++ b/plugin.xml @@ -67,6 +67,15 @@ + + + + + + + + + @@ -105,6 +114,7 @@ + diff --git a/src/main/java/org/codelibs/fess/Constants.java b/src/main/java/org/codelibs/fess/Constants.java index ee4a0899d..0203687a2 100644 --- a/src/main/java/org/codelibs/fess/Constants.java +++ b/src/main/java/org/codelibs/fess/Constants.java @@ -113,6 +113,8 @@ public class Constants extends CoreLibConstants { public static final String LOGIN_REQUIRED_PROPERTY = "login.required"; + public static final String RESULT_COLLAPSED_PROPERTY = "result.collapsed"; + public static final String LOGIN_LINK_ENALBED_PROPERTY = "login.link.enabled"; public static final String THUMBNAIL_ENALBED_PROPERTY = "thumbnail.enabled"; diff --git a/src/main/java/org/codelibs/fess/api/gsa/GsaApiManager.java b/src/main/java/org/codelibs/fess/api/gsa/GsaApiManager.java index 836726e4d..edd069cf5 100644 --- a/src/main/java/org/codelibs/fess/api/gsa/GsaApiManager.java +++ b/src/main/java/org/codelibs/fess/api/gsa/GsaApiManager.java @@ -526,5 +526,10 @@ public class GsaApiManager extends BaseApiManager implements WebApiManager { return SearchRequestType.GSA; } + @Override + public String getSimilarHash() { + return request.getParameter("sh"); + } + } } diff --git a/src/main/java/org/codelibs/fess/api/json/JsonApiManager.java b/src/main/java/org/codelibs/fess/api/json/JsonApiManager.java index 99ffebab4..5698dd2ce 100644 --- a/src/main/java/org/codelibs/fess/api/json/JsonApiManager.java +++ b/src/main/java/org/codelibs/fess/api/json/JsonApiManager.java @@ -667,5 +667,10 @@ public class JsonApiManager extends BaseJsonApiManager { return SearchRequestType.JSON; } + @Override + public String getSimilarHash() { + return request.getParameter("sh"); + } + } } diff --git a/src/main/java/org/codelibs/fess/api/suggest/SuggestApiManager.java b/src/main/java/org/codelibs/fess/api/suggest/SuggestApiManager.java index ec2634304..24e9088bd 100644 --- a/src/main/java/org/codelibs/fess/api/suggest/SuggestApiManager.java +++ b/src/main/java/org/codelibs/fess/api/suggest/SuggestApiManager.java @@ -244,5 +244,10 @@ public class SuggestApiManager extends BaseJsonApiManager { public SearchRequestType getType() { return SearchRequestType.SUGGEST; } + + @Override + public String getSimilarHash() { + throw new UnsupportedOperationException(); + } } } diff --git a/src/main/java/org/codelibs/fess/app/service/SearchService.java b/src/main/java/org/codelibs/fess/app/service/SearchService.java index ddcd474f0..e652e0177 100644 --- a/src/main/java/org/codelibs/fess/app/service/SearchService.java +++ b/src/main/java/org/codelibs/fess/app/service/SearchService.java @@ -104,7 +104,8 @@ public class SearchService { return SearchConditionBuilder.builder(searchRequestBuilder) .query(StringUtil.isBlank(sortField) ? query : query + " sort:" + sortField).offset(pageStart) .size(pageSize).facetInfo(params.getFacetInfo()).geoInfo(params.getGeoInfo()) - .responseFields(queryHelper.getResponseFields()).searchRequestType(params.getType()).build(); + .similarHash(params.getSimilarHash()).responseFields(queryHelper.getResponseFields()) + .searchRequestType(params.getType()).build(); }, (searchRequestBuilder, execTime, searchResponse) -> { final QueryResponseList queryResponseList = ComponentUtil.getQueryResponseList(); queryResponseList.init(searchResponse, pageStart, pageSize); diff --git a/src/main/java/org/codelibs/fess/app/web/admin/general/AdminGeneralAction.java b/src/main/java/org/codelibs/fess/app/web/admin/general/AdminGeneralAction.java index 51c3db535..aa1295be7 100644 --- a/src/main/java/org/codelibs/fess/app/web/admin/general/AdminGeneralAction.java +++ b/src/main/java/org/codelibs/fess/app/web/admin/general/AdminGeneralAction.java @@ -128,6 +128,9 @@ public class AdminGeneralAction extends FessAdminAction { if (form.loginRequired != null) { fessConfig.setLoginRequired(Constants.ON.equalsIgnoreCase(form.loginRequired)); } + if (form.resultCollapsed != null) { + fessConfig.setResultCollapsed(Constants.ON.equalsIgnoreCase(form.resultCollapsed)); + } if (form.loginLink != null) { fessConfig.setLoginLinkEnabled(Constants.ON.equalsIgnoreCase(form.loginLink)); } @@ -235,6 +238,7 @@ public class AdminGeneralAction extends FessAdminAction { public static void updateForm(final FessConfig fessConfig, final EditForm form) { form.loginRequired = fessConfig.isLoginRequired() ? Constants.TRUE : Constants.FALSE; + form.resultCollapsed = fessConfig.isResultCollapsed() ? Constants.TRUE : Constants.FALSE; form.loginLink = fessConfig.isLoginLinkEnabled() ? Constants.TRUE : Constants.FALSE; form.thumbnail = fessConfig.isThumbnailEnabled() ? Constants.TRUE : Constants.FALSE; form.incrementalCrawling = fessConfig.isIncrementalCrawling() ? Constants.TRUE : Constants.FALSE; diff --git a/src/main/java/org/codelibs/fess/app/web/admin/general/EditForm.java b/src/main/java/org/codelibs/fess/app/web/admin/general/EditForm.java index 7dc8b5011..6b2dbe774 100644 --- a/src/main/java/org/codelibs/fess/app/web/admin/general/EditForm.java +++ b/src/main/java/org/codelibs/fess/app/web/admin/general/EditForm.java @@ -67,6 +67,9 @@ public class EditForm { @Size(max = 10) public String loginRequired; + @Size(max = 10) + public String resultCollapsed; + @Size(max = 10) public String loginLink; diff --git a/src/main/java/org/codelibs/fess/app/web/admin/searchlist/ListForm.java b/src/main/java/org/codelibs/fess/app/web/admin/searchlist/ListForm.java index 7ec68507d..fef2274f4 100644 --- a/src/main/java/org/codelibs/fess/app/web/admin/searchlist/ListForm.java +++ b/src/main/java/org/codelibs/fess/app/web/admin/searchlist/ListForm.java @@ -58,6 +58,8 @@ public class ListForm implements SearchRequestParams { public String[] ex_q; + public String sh; + @Override public String getQuery() { return q; @@ -139,4 +141,9 @@ public class ListForm implements SearchRequestParams { public SearchRequestType getType() { return SearchRequestType.ADMIN_SEARCH; } + + @Override + public String getSimilarHash() { + return sh; + } } diff --git a/src/main/java/org/codelibs/fess/app/web/api/admin/elevateword/ApiAdminElevatewordAction.java b/src/main/java/org/codelibs/fess/app/web/api/admin/elevateword/ApiAdminElevatewordAction.java index 9bd2074d6..60d0254be 100644 --- a/src/main/java/org/codelibs/fess/app/web/api/admin/elevateword/ApiAdminElevatewordAction.java +++ b/src/main/java/org/codelibs/fess/app/web/api/admin/elevateword/ApiAdminElevatewordAction.java @@ -15,6 +15,23 @@ */ package org.codelibs.fess.app.web.api.admin.elevateword; +import static org.codelibs.core.stream.StreamUtil.stream; +import static org.codelibs.fess.app.web.admin.elevateword.AdminElevatewordAction.getElevateWord; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; +import java.io.Reader; +import java.io.Writer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.stream.Collectors; + +import javax.annotation.Resource; + import org.codelibs.core.lang.StringUtil; import org.codelibs.fess.app.pager.ElevateWordPager; import org.codelibs.fess.app.service.ElevateWordService; @@ -31,16 +48,6 @@ import org.lastaflute.web.Execute; import org.lastaflute.web.response.JsonResponse; import org.lastaflute.web.response.StreamResponse; -import javax.annotation.Resource; -import java.io.*; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.List; -import java.util.stream.Collectors; - -import static org.codelibs.core.stream.StreamUtil.stream; -import static org.codelibs.fess.app.web.admin.elevateword.AdminElevatewordAction.*; - public class ApiAdminElevatewordAction extends FessApiAdminAction { @Resource diff --git a/src/main/java/org/codelibs/fess/app/web/api/admin/group/ApiAdminGroupAction.java b/src/main/java/org/codelibs/fess/app/web/api/admin/group/ApiAdminGroupAction.java index 55bc8ac33..ec3817b1e 100644 --- a/src/main/java/org/codelibs/fess/app/web/api/admin/group/ApiAdminGroupAction.java +++ b/src/main/java/org/codelibs/fess/app/web/api/admin/group/ApiAdminGroupAction.java @@ -15,6 +15,8 @@ */ package org.codelibs.fess.app.web.api.admin.group; +import static org.codelibs.fess.app.web.admin.group.AdminGroupAction.getGroup; + import java.util.List; import java.util.stream.Collectors; @@ -31,8 +33,6 @@ import org.codelibs.fess.es.user.exentity.User; import org.lastaflute.web.Execute; import org.lastaflute.web.response.JsonResponse; -import static org.codelibs.fess.app.web.admin.group.AdminGroupAction.*; - public class ApiAdminGroupAction extends FessApiAdminAction { @Resource diff --git a/src/main/java/org/codelibs/fess/app/web/api/admin/role/ApiAdminRoleAction.java b/src/main/java/org/codelibs/fess/app/web/api/admin/role/ApiAdminRoleAction.java index 83df5ffc3..dbde051be 100644 --- a/src/main/java/org/codelibs/fess/app/web/api/admin/role/ApiAdminRoleAction.java +++ b/src/main/java/org/codelibs/fess/app/web/api/admin/role/ApiAdminRoleAction.java @@ -15,6 +15,8 @@ */ package org.codelibs.fess.app.web.api.admin.role; +import static org.codelibs.fess.app.web.admin.role.AdminRoleAction.getRole; + import java.util.List; import java.util.stream.Collectors; @@ -30,8 +32,6 @@ import org.codelibs.fess.es.user.exentity.Role; import org.lastaflute.web.Execute; import org.lastaflute.web.response.JsonResponse; -import static org.codelibs.fess.app.web.admin.role.AdminRoleAction.*; - public class ApiAdminRoleAction extends FessApiAdminAction { @Resource diff --git a/src/main/java/org/codelibs/fess/app/web/api/admin/user/ApiAdminUserAction.java b/src/main/java/org/codelibs/fess/app/web/api/admin/user/ApiAdminUserAction.java index 6720b20e3..7007ae7a1 100644 --- a/src/main/java/org/codelibs/fess/app/web/api/admin/user/ApiAdminUserAction.java +++ b/src/main/java/org/codelibs/fess/app/web/api/admin/user/ApiAdminUserAction.java @@ -15,6 +15,8 @@ */ package org.codelibs.fess.app.web.api.admin.user; +import static org.codelibs.fess.app.web.admin.user.AdminUserAction.getUser; + import java.util.List; import java.util.stream.Collectors; @@ -30,8 +32,6 @@ import org.codelibs.fess.es.user.exentity.User; import org.lastaflute.web.Execute; import org.lastaflute.web.response.JsonResponse; -import static org.codelibs.fess.app.web.admin.user.AdminUserAction.*; - public class ApiAdminUserAction extends FessApiAdminAction { @Resource diff --git a/src/main/java/org/codelibs/fess/app/web/base/SearchForm.java b/src/main/java/org/codelibs/fess/app/web/base/SearchForm.java index 55a850ccd..c98ed8597 100644 --- a/src/main/java/org/codelibs/fess/app/web/base/SearchForm.java +++ b/src/main/java/org/codelibs/fess/app/web/base/SearchForm.java @@ -55,6 +55,9 @@ public class SearchForm implements SearchRequestParams { @ValidateTypeFailure public Integer pn; + @Size(max = 1000) + public String sh; + // advance @Override @@ -136,4 +139,9 @@ public class SearchForm implements SearchRequestParams { public SearchRequestType getType() { return SearchRequestType.SEARCH; } + + @Override + public String getSimilarHash() { + return sh; + } } diff --git a/src/main/java/org/codelibs/fess/entity/SearchRequestParams.java b/src/main/java/org/codelibs/fess/entity/SearchRequestParams.java index 580fc52f8..ec0373574 100644 --- a/src/main/java/org/codelibs/fess/entity/SearchRequestParams.java +++ b/src/main/java/org/codelibs/fess/entity/SearchRequestParams.java @@ -50,6 +50,8 @@ public interface SearchRequestParams { SearchRequestType getType(); + String getSimilarHash(); + public default String[] simplifyArray(final String[] values) { return stream(values).get(stream -> stream.filter(StringUtil::isNotBlank).distinct().toArray(n -> new String[n])); } diff --git a/src/main/java/org/codelibs/fess/es/client/FessEsClient.java b/src/main/java/org/codelibs/fess/es/client/FessEsClient.java index 321436c1c..73bd1a33f 100644 --- a/src/main/java/org/codelibs/fess/es/client/FessEsClient.java +++ b/src/main/java/org/codelibs/fess/es/client/FessEsClient.java @@ -137,7 +137,9 @@ import org.elasticsearch.common.transport.InetSocketTransportAddress; import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.query.InnerHitBuilder; import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHitField; import org.elasticsearch.search.SearchHits; @@ -145,6 +147,7 @@ import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.terms.Terms.Order; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; +import org.elasticsearch.search.collapse.CollapseBuilder; import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.client.PreBuiltTransportClient; @@ -800,6 +803,7 @@ public class FessEsClient implements Client { private int size = Constants.DEFAULT_PAGE_SIZE; private GeoInfo geoInfo; private FacetInfo facetInfo; + private String similarHash; private SearchRequestType searchRequestType = SearchRequestType.SEARCH; public static SearchConditionBuilder builder(final SearchRequestBuilder searchRequestBuilder) { @@ -840,6 +844,13 @@ public class FessEsClient implements Client { return this; } + public SearchConditionBuilder similarHash(final String similarHash) { + if (StringUtil.isNotBlank(similarHash)) { + this.similarHash = similarHash; + } + return this; + } + public SearchConditionBuilder facetInfo(final FacetInfo facetInfo) { this.facetInfo = facetInfo; return this; @@ -860,14 +871,18 @@ public class FessEsClient implements Client { final QueryContext queryContext = queryHelper.build(searchRequestType, query, context -> { if (SearchRequestType.ADMIN_SEARCH.equals(searchRequestType)) { context.skipRoleQuery(); + } else if (similarHash != null) { + context.addQuery(boolQuery -> { + boolQuery.filter(QueryBuilders.termQuery(fessConfig.getIndexFieldContentMinhashBits(), similarHash)); + }); } - // geo - if (geoInfo != null && geoInfo.toQueryBuilder() != null) { - context.addQuery(boolQuery -> { - boolQuery.filter(geoInfo.toQueryBuilder()); - }); - } - }); + + if (geoInfo != null && geoInfo.toQueryBuilder() != null) { + context.addQuery(boolQuery -> { + boolQuery.filter(geoInfo.toQueryBuilder()); + }); + } + }); searchRequestBuilder.setFrom(offset).setSize(size); @@ -924,9 +939,23 @@ public class FessEsClient implements Client { })); } + if (!SearchRequestType.ADMIN_SEARCH.equals(searchRequestType) && fessConfig.isResultCollapsed() && similarHash == null) { + searchRequestBuilder.setCollapse(getCollapseBuilder(fessConfig)); + } + searchRequestBuilder.setQuery(queryContext.getQueryBuilder()); return true; } + + protected CollapseBuilder getCollapseBuilder(final FessConfig fessConfig) { + final InnerHitBuilder innerHitBuilder = + new InnerHitBuilder().setName(fessConfig.getQueryCollapseInnerHitsName()).setSize( + fessConfig.getQueryCollapseInnerHitsSizeAsInteger()); + fessConfig.getQueryCollapseInnerHitsSortBuilders().ifPresent( + builders -> stream(builders).of(stream -> stream.forEach(innerHitBuilder::addSort))); + return new CollapseBuilder(fessConfig.getIndexFieldContentMinhashBits()).setMaxConcurrentGroupRequests( + fessConfig.getQueryCollapseMaxConcurrentGroupResultsAsInteger()).setInnerHits(innerHitBuilder); + } } public boolean store(final String index, final String type, final Object obj) { diff --git a/src/main/java/org/codelibs/fess/helper/IndexingHelper.java b/src/main/java/org/codelibs/fess/helper/IndexingHelper.java index c35a76dac..73fc244ff 100644 --- a/src/main/java/org/codelibs/fess/helper/IndexingHelper.java +++ b/src/main/java/org/codelibs/fess/helper/IndexingHelper.java @@ -45,6 +45,11 @@ public class IndexingHelper { return; } final FessConfig fessConfig = ComponentUtil.getFessConfig(); + if (fessConfig.isResultCollapsed()) { + docList.forEach(doc -> { + doc.put("content_minhash", doc.get(fessConfig.getIndexFieldContent())); + }); + } final long execTime = System.currentTimeMillis(); if (logger.isDebugEnabled()) { logger.debug("Sending " + docList.size() + " documents to a server."); diff --git a/src/main/java/org/codelibs/fess/mylasta/action/FessLabels.java b/src/main/java/org/codelibs/fess/mylasta/action/FessLabels.java index 5953822c4..899795a88 100644 --- a/src/main/java/org/codelibs/fess/mylasta/action/FessLabels.java +++ b/src/main/java/org/codelibs/fess/mylasta/action/FessLabels.java @@ -680,6 +680,9 @@ public class FessLabels extends UserMessages { /** The key of the message: Cache */ public static final String LABELS_search_result_cache = "{labels.search_result_cache}"; + /** The key of the message: Similar Results ({0}) */ + public static final String LABELS_search_result_similar = "{labels.search_result_similar}"; + /** The key of the message: Label */ public static final String LABELS_facet_label_title = "{labels.facet_label_title}"; @@ -1127,6 +1130,9 @@ public class FessLabels extends UserMessages { /** The key of the message: Login Required */ public static final String LABELS_login_required = "{labels.login_required}"; + /** The key of the message: Similar Result Collapsed */ + public static final String LABELS_result_collapsed = "{labels.result_collapsed}"; + /** The key of the message: Login Link */ public static final String LABELS_login_link = "{labels.login_link}"; diff --git a/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java b/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java index d6d9c869b..c53bdfd08 100644 --- a/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java +++ b/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java @@ -342,6 +342,12 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction /** The key of the configuration. e.g. content */ String INDEX_FIELD_CONTENT = "index.field.content"; + /** The key of the configuration. e.g. content_minhash */ + String INDEX_FIELD_content_minhash = "index.field.content_minhash"; + + /** The key of the configuration. e.g. content_minhash_bits */ + String INDEX_FIELD_content_minhash_bits = "index.field.content_minhash_bits"; + /** The key of the configuration. e.g. cache */ String INDEX_FIELD_CACHE = "index.field.cache"; @@ -483,6 +489,18 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction /** The key of the configuration. e.g. */ String QUERY_ADDITIONAL_NOT_ANALYZED_FIELDS = "query.additional.not.analyzed.fields"; + /** The key of the configuration. e.g. 4 */ + String QUERY_COLLAPSE_MAX_CONCURRENT_GROUP_RESULTS = "query.collapse.max.concurrent.group.results"; + + /** The key of the configuration. e.g. similar_docs */ + String QUERY_COLLAPSE_INNER_HITS_NAME = "query.collapse.inner.hits.name"; + + /** The key of the configuration. e.g. 0 */ + String QUERY_COLLAPSE_INNER_HITS_SIZE = "query.collapse.inner.hits.size"; + + /** The key of the configuration. e.g. */ + String QUERY_COLLAPSE_INNER_HITS_SORTS = "query.collapse.inner.hits.sorts"; + /** The key of the configuration. e.g. */ String QUERY_DEFAULT_LANGUAGES = "query.default.languages"; @@ -2140,6 +2158,20 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction */ String getIndexFieldContent(); + /** + * Get the value for the key 'index.field.content_minhash'.
+ * The value is, e.g. content_minhash
+ * @return The value of found property. (NotNull: if not found, exception but basically no way) + */ + String getIndexFieldContentMinhash(); + + /** + * Get the value for the key 'index.field.content_minhash_bits'.
+ * The value is, e.g. content_minhash_bits
+ * @return The value of found property. (NotNull: if not found, exception but basically no way) + */ + String getIndexFieldContentMinhashBits(); + /** * Get the value for the key 'index.field.cache'.
* The value is, e.g. cache
@@ -2593,6 +2625,58 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction */ Integer getQueryAdditionalNotAnalyzedFieldsAsInteger(); + /** + * Get the value for the key 'query.collapse.max.concurrent.group.results'.
+ * The value is, e.g. 4
+ * @return The value of found property. (NotNull: if not found, exception but basically no way) + */ + String getQueryCollapseMaxConcurrentGroupResults(); + + /** + * Get the value for the key 'query.collapse.max.concurrent.group.results' as {@link Integer}.
+ * The value is, e.g. 4
+ * @return The value of found property. (NotNull: if not found, exception but basically no way) + * @throws NumberFormatException When the property is not integer. + */ + Integer getQueryCollapseMaxConcurrentGroupResultsAsInteger(); + + /** + * Get the value for the key 'query.collapse.inner.hits.name'.
+ * The value is, e.g. similar_docs
+ * @return The value of found property. (NotNull: if not found, exception but basically no way) + */ + String getQueryCollapseInnerHitsName(); + + /** + * Get the value for the key 'query.collapse.inner.hits.size'.
+ * The value is, e.g. 0
+ * @return The value of found property. (NotNull: if not found, exception but basically no way) + */ + String getQueryCollapseInnerHitsSize(); + + /** + * Get the value for the key 'query.collapse.inner.hits.size' as {@link Integer}.
+ * The value is, e.g. 0
+ * @return The value of found property. (NotNull: if not found, exception but basically no way) + * @throws NumberFormatException When the property is not integer. + */ + Integer getQueryCollapseInnerHitsSizeAsInteger(); + + /** + * Get the value for the key 'query.collapse.inner.hits.sorts'.
+ * The value is, e.g.
+ * @return The value of found property. (NotNull: if not found, exception but basically no way) + */ + String getQueryCollapseInnerHitsSorts(); + + /** + * Get the value for the key 'query.collapse.inner.hits.sorts' as {@link Integer}.
+ * The value is, e.g.
+ * @return The value of found property. (NotNull: if not found, exception but basically no way) + * @throws NumberFormatException When the property is not integer. + */ + Integer getQueryCollapseInnerHitsSortsAsInteger(); + /** * Get the value for the key 'query.default.languages'.
* The value is, e.g.
@@ -5261,6 +5345,14 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction return get(FessConfig.INDEX_FIELD_CONTENT); } + public String getIndexFieldContentMinhash() { + return get(FessConfig.INDEX_FIELD_content_minhash); + } + + public String getIndexFieldContentMinhashBits() { + return get(FessConfig.INDEX_FIELD_content_minhash_bits); + } + public String getIndexFieldCache() { return get(FessConfig.INDEX_FIELD_CACHE); } @@ -5509,6 +5601,34 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction return getAsInteger(FessConfig.QUERY_ADDITIONAL_NOT_ANALYZED_FIELDS); } + public String getQueryCollapseMaxConcurrentGroupResults() { + return get(FessConfig.QUERY_COLLAPSE_MAX_CONCURRENT_GROUP_RESULTS); + } + + public Integer getQueryCollapseMaxConcurrentGroupResultsAsInteger() { + return getAsInteger(FessConfig.QUERY_COLLAPSE_MAX_CONCURRENT_GROUP_RESULTS); + } + + public String getQueryCollapseInnerHitsName() { + return get(FessConfig.QUERY_COLLAPSE_INNER_HITS_NAME); + } + + public String getQueryCollapseInnerHitsSize() { + return get(FessConfig.QUERY_COLLAPSE_INNER_HITS_SIZE); + } + + public Integer getQueryCollapseInnerHitsSizeAsInteger() { + return getAsInteger(FessConfig.QUERY_COLLAPSE_INNER_HITS_SIZE); + } + + public String getQueryCollapseInnerHitsSorts() { + return get(FessConfig.QUERY_COLLAPSE_INNER_HITS_SORTS); + } + + public Integer getQueryCollapseInnerHitsSortsAsInteger() { + return getAsInteger(FessConfig.QUERY_COLLAPSE_INNER_HITS_SORTS); + } + public String getQueryDefaultLanguages() { return get(FessConfig.QUERY_DEFAULT_LANGUAGES); } diff --git a/src/main/java/org/codelibs/fess/mylasta/direction/FessProp.java b/src/main/java/org/codelibs/fess/mylasta/direction/FessProp.java index eebd7e8bb..0ceb34d06 100644 --- a/src/main/java/org/codelibs/fess/mylasta/direction/FessProp.java +++ b/src/main/java/org/codelibs/fess/mylasta/direction/FessProp.java @@ -47,6 +47,9 @@ import org.codelibs.fess.util.ComponentUtil; import org.codelibs.fess.util.PrunedTag; import org.dbflute.optional.OptionalThing; import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.search.sort.SortBuilder; +import org.elasticsearch.search.sort.SortBuilders; +import org.elasticsearch.search.sort.SortOrder; import org.lastaflute.job.LaJob; import org.lastaflute.job.subsidiary.JobConcurrentExec; import org.lastaflute.web.util.LaRequestUtil; @@ -58,6 +61,8 @@ import org.lastaflute.web.validation.theme.typed.LongTypeValidator; public interface FessProp { + public static final String QUERY_COLLAPSE_INNER_HITS_SORTS = "queryCollapseInnerHitsSorts"; + public static final String USER_CODE_PATTERN = "userCodePattern"; public static final String API_ADMIN_ACCESS_PERMISSION_SET = "apiAdminAccessPermissionSet"; @@ -269,6 +274,14 @@ public interface FessProp { return getSystemPropertyAsBoolean(Constants.LOGIN_REQUIRED_PROPERTY, false); } + public default void setResultCollapsed(final boolean value) { + setSystemPropertyAsBoolean(Constants.RESULT_COLLAPSED_PROPERTY, value); + } + + public default boolean isResultCollapsed() { + return getSystemPropertyAsBoolean(Constants.RESULT_COLLAPSED_PROPERTY, false); + } + public default void setLoginLinkEnabled(final boolean value) { setSystemPropertyAsBoolean(Constants.LOGIN_LINK_ENALBED_PROPERTY, value); } @@ -1504,4 +1517,36 @@ public interface FessProp { } return pattern.matcher(userCode).matches(); } + + String getQueryCollapseInnerHitsSorts(); + + @SuppressWarnings("rawtypes") + public default OptionalThing getQueryCollapseInnerHitsSortBuilders() { + @SuppressWarnings("unchecked") + OptionalThing ot = (OptionalThing) propMap.get(QUERY_COLLAPSE_INNER_HITS_SORTS); + if (ot == null) { + String sorts = getQueryCollapseInnerHitsSorts(); + if (StringUtil.isBlank(sorts)) { + ot = OptionalThing.empty(); + } else { + SortBuilder[] sortBuilders = + split(sorts, ",").get( + stream -> stream + .filter(StringUtil::isNotBlank) + .map(s -> { + final String[] values = s.split(":"); + if (values.length > 1) { + return SortBuilders.fieldSort(values[0]).order( + values[0].equalsIgnoreCase("desc") ? SortOrder.DESC : SortOrder.ASC); + } else { + return SortBuilders.fieldSort(values[0]).order(SortOrder.ASC); + } + }).toArray(n -> new SortBuilder[n])); + ot = OptionalThing.of(sortBuilders); + } + propMap.put(QUERY_COLLAPSE_INNER_HITS_SORTS, ot); + } + return ot; + } + } diff --git a/src/main/java/org/codelibs/fess/util/QueryResponseList.java b/src/main/java/org/codelibs/fess/util/QueryResponseList.java index a9d76fdc2..faed2f8a7 100644 --- a/src/main/java/org/codelibs/fess/util/QueryResponseList.java +++ b/src/main/java/org/codelibs/fess/util/QueryResponseList.java @@ -24,6 +24,7 @@ import java.util.ListIterator; import java.util.Map; import org.apache.commons.lang3.StringUtils; +import org.codelibs.core.stream.StreamUtil; import org.codelibs.fess.helper.QueryHelper; import org.codelibs.fess.helper.ViewHelper; import org.codelibs.fess.mylasta.direction.FessConfig; @@ -31,6 +32,7 @@ import org.dbflute.optional.OptionalEntity; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.text.Text; import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchHitField; import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.aggregations.Aggregations; import org.elasticsearch.search.fetch.subphase.highlight.HighlightField; @@ -84,6 +86,7 @@ public class QueryResponseList implements List> { public void init(final OptionalEntity searchResponseOpt, final int start, final int pageSize) { searchResponseOpt.ifPresent(searchResponse -> { + final FessConfig fessConfig = ComponentUtil.getFessConfig(); final SearchHits searchHits = searchResponse.getHits(); allRecordCount = searchHits.getTotalHits(); queryTime = searchResponse.getTookInMillis(); @@ -96,45 +99,28 @@ public class QueryResponseList implements List> { final QueryHelper queryHelper = ComponentUtil.getQueryHelper(); final String hlPrefix = queryHelper.getHighlightPrefix(); for (final SearchHit searchHit : searchHits.getHits()) { - final Map docMap = new HashMap<>(); - if (searchHit.getSource() == null) { - searchHit.getFields().forEach((key, value) -> { - docMap.put(key, value.getValue()); - }); - } else { - docMap.putAll(searchHit.getSource()); - } + final Map docMap = parseSearchHit(fessConfig, hlPrefix, searchHit); - final Map highlightFields = searchHit.getHighlightFields(); - try { - if (highlightFields != null) { - for (final Map.Entry entry : highlightFields.entrySet()) { - final HighlightField highlightField = entry.getValue(); - final Text[] fragments = highlightField.fragments(); - if (fragments != null && fragments.length != 0) { - final String[] texts = new String[fragments.length]; - for (int i = 0; i < fragments.length; i++) { - texts[i] = fragments[i].string(); + if (fessConfig.isResultCollapsed()) { + final Map innerHits = searchHit.getInnerHits(); + if (innerHits != null) { + final SearchHits innerSearchHits = innerHits.get(fessConfig.getQueryCollapseInnerHitsName()); + if (innerSearchHits != null) { + long totalHits = innerSearchHits.getTotalHits(); + if (totalHits > 0) { + docMap.put(fessConfig.getQueryCollapseInnerHitsName() + "_count", totalHits); + SearchHitField bitsField = searchHit.getFields().get(fessConfig.getIndexFieldContentMinhashBits()); + if (bitsField != null && !bitsField.getValues().isEmpty()) { + docMap.put(fessConfig.getQueryCollapseInnerHitsName() + "_hash", bitsField.getValues().get(0)); } - final String value = StringUtils.join(texts, "..."); - docMap.put(hlPrefix + highlightField.getName(), value); + docMap.put( + fessConfig.getQueryCollapseInnerHitsName(), + StreamUtil.stream(innerSearchHits.getHits()).get( + stream -> stream.map(v -> parseSearchHit(fessConfig, hlPrefix, v)).toArray( + n -> new Map[n]))); } } } - } catch (final Exception e) { - if (logger.isDebugEnabled()) { - logger.debug("Could not create a highlighting value: " + docMap, e); - } - } - - // ContentTitle - final ViewHelper viewHelper = ComponentUtil.getViewHelper(); - if (viewHelper != null) { - final FessConfig fessConfig = ComponentUtil.getFessConfig(); - docMap.put(fessConfig.getResponseFieldContentTitle(), viewHelper.getContentTitle(docMap)); - docMap.put(fessConfig.getResponseFieldContentDescription(), viewHelper.getContentDescription(docMap)); - docMap.put(fessConfig.getResponseFieldUrlLink(), viewHelper.getUrlLink(docMap)); - docMap.put(fessConfig.getResponseFieldSitePath(), viewHelper.getSitePath(docMap)); } parent.add(docMap); @@ -151,6 +137,49 @@ public class QueryResponseList implements List> { calculatePageInfo(start, pageSize); } + private Map parseSearchHit(final FessConfig fessConfig, final String hlPrefix, final SearchHit searchHit) { + final Map docMap = new HashMap<>(); + if (searchHit.getSource() == null) { + searchHit.getFields().forEach((key, value) -> { + docMap.put(key, value.getValue()); + }); + } else { + docMap.putAll(searchHit.getSource()); + } + + final Map highlightFields = searchHit.getHighlightFields(); + try { + if (highlightFields != null) { + for (final Map.Entry entry : highlightFields.entrySet()) { + final HighlightField highlightField = entry.getValue(); + final Text[] fragments = highlightField.fragments(); + if (fragments != null && fragments.length != 0) { + final String[] texts = new String[fragments.length]; + for (int i = 0; i < fragments.length; i++) { + texts[i] = fragments[i].string(); + } + final String value = StringUtils.join(texts, "..."); + docMap.put(hlPrefix + highlightField.getName(), value); + } + } + } + } catch (final Exception e) { + if (logger.isDebugEnabled()) { + logger.debug("Could not create a highlighting value: " + docMap, e); + } + } + + // ContentTitle + final ViewHelper viewHelper = ComponentUtil.getViewHelper(); + if (viewHelper != null) { + docMap.put(fessConfig.getResponseFieldContentTitle(), viewHelper.getContentTitle(docMap)); + docMap.put(fessConfig.getResponseFieldContentDescription(), viewHelper.getContentDescription(docMap)); + docMap.put(fessConfig.getResponseFieldUrlLink(), viewHelper.getUrlLink(docMap)); + docMap.put(fessConfig.getResponseFieldSitePath(), viewHelper.getSitePath(docMap)); + } + return docMap; + } + protected void calculatePageInfo(final int start, final int size) { pageSize = size; allPageCount = (int) ((allRecordCount - 1) / pageSize) + 1; diff --git a/src/main/resources/fess_config.properties b/src/main/resources/fess_config.properties index 17fc83a3a..26b7b880f 100644 --- a/src/main/resources/fess_config.properties +++ b/src/main/resources/fess_config.properties @@ -166,6 +166,8 @@ index.field.mimetype=mimetype index.field.parent_id=parent_id index.field.important_content=important_content index.field.content=content +index.field.content_minhash=content_minhash +index.field.content_minhash_bits=content_minhash_bits index.field.cache=cache index.field.digest=digest index.field.title=title @@ -221,6 +223,10 @@ query.additional.search.fields= query.additional.facet.fields= query.additional.sort.fields= query.additional.not.analyzed.fields= +query.collapse.max.concurrent.group.results=4 +query.collapse.inner.hits.name=similar_docs +query.collapse.inner.hits.size=0 +query.collapse.inner.hits.sorts= query.default.languages= query.language.mapping=\ ar=ar\n\ diff --git a/src/main/resources/fess_indices/fess.json b/src/main/resources/fess_indices/fess.json index b9244ff2b..388220b13 100644 --- a/src/main/resources/fess_indices/fess.json +++ b/src/main/resources/fess_indices/fess.json @@ -406,6 +406,12 @@ "alphanum_word_filter" : { "type" : "alphanum_word", "max_token_length" : 20 + }, + "minhash_filter" : { + "type" : "minhash", + "seed" : 1, + "bit" : 2, + "size" : 64 } }, "tokenizer": { @@ -756,6 +762,21 @@ "lowercase", "stemmer_en_filter" ] + }, + "minhash_analyzer": { + "type": "custom", + "char_filter": [ + "mapping_ja_filter" + ], + "tokenizer": "unigram_synonym_tokenizer", + "filter": [ + "alphanum_word_filter", + "cjk_bigram", + "stopword_en_filter", + "lowercase", + "stemmer_en_filter", + "minhash_filter" + ] } } } diff --git a/src/main/resources/fess_indices/fess/doc.json b/src/main/resources/fess_indices/fess/doc.json index 8adc945f1..c9c4f961d 100644 --- a/src/main/resources/fess_indices/fess/doc.json +++ b/src/main/resources/fess_indices/fess/doc.json @@ -456,6 +456,14 @@ "analyzer": "standard_analyzer", "term_vector": "with_positions_offsets" }, + "content_minhash": { + "type": "minhash", + "minhash_analyzer": "minhash_analyzer", + "copy_bits_to": "content_minhash_bits" + }, + "content_minhash_bits": { + "type": "keyword" + }, "content_length": { "type": "long" }, diff --git a/src/main/resources/fess_label.properties b/src/main/resources/fess_label.properties index 3a012ae44..3fb57a596 100644 --- a/src/main/resources/fess_label.properties +++ b/src/main/resources/fess_label.properties @@ -217,6 +217,7 @@ labels.search_result_favorited=Liked labels.search_click_count=Viewed ({0}) labels.search_result_more=more.. labels.search_result_cache=Cache +labels.search_result_similar=Similar Results ({0}) labels.facet_label_title=Label labels.facet_timestamp_title=Date Range labels.facet_timestamp_1day=Past 24 Hours @@ -366,6 +367,7 @@ labels.default_label_value=Default Label Value labels.default_sort_value=Default Sort Value labels.append_query_param_enabled=Append Params to URL labels.login_required=Login Required +labels.result_collapsed=Similar Result Collapsed labels.login_link=Login Link labels.thumbnail=Thumbnail View labels.ignore_failure_type=Excluded Failure Type diff --git a/src/main/resources/fess_label_en.properties b/src/main/resources/fess_label_en.properties index f079b7b47..27d76a3c0 100644 --- a/src/main/resources/fess_label_en.properties +++ b/src/main/resources/fess_label_en.properties @@ -217,6 +217,7 @@ labels.search_result_favorited=Liked labels.search_click_count=Viewed ({0}) labels.search_result_more=more.. labels.search_result_cache=Cache +labels.search_result_similar=Similar Results ({0}) labels.facet_label_title=Label labels.facet_timestamp_title=Date Range labels.facet_timestamp_1day=Past 24 Hours @@ -366,6 +367,7 @@ labels.default_label_value=Default Label Value labels.default_sort_value=Default Sort Value labels.append_query_param_enabled=Append Params to URL labels.login_required=Login Required +labels.result_collapsed=Similar Result Collapsed labels.login_link=Login Link labels.thumbnail=Thumbnail View labels.ignore_failure_type=Excluded Failure Type diff --git a/src/main/resources/fess_label_ja.properties b/src/main/resources/fess_label_ja.properties index 1a956908c..231ccfd19 100644 --- a/src/main/resources/fess_label_ja.properties +++ b/src/main/resources/fess_label_ja.properties @@ -210,6 +210,7 @@ labels.search_result_favorited=Liked labels.search_click_count=\u30af\u30ea\u30c3\u30af\u6570 ({0}) labels.search_result_more=\u8a73\u7d30.. labels.search_result_cache=\u30ad\u30e3\u30c3\u30b7\u30e5 +labels.search_result_similar=\u985e\u4f3c\u7d50\u679c ({0}) labels.facet_label_title=\u30e9\u30d9\u30eb labels.facet_timestamp_title=\u671f\u9593 labels.facet_timestamp_1day=24\u6642\u9593\u4ee5\u5185 @@ -359,6 +360,7 @@ labels.default_label_value=\u30c7\u30d5\u30a9\u30eb\u30c8\u306e\u30e9\u30d9\u30e labels.default_sort_value=\u30c7\u30d5\u30a9\u30eb\u30c8\u306e\u30bd\u30fc\u30c8\u5024 labels.append_query_param_enabled=\u691c\u7d22\u30d1\u30e9\u30e1\u30fc\u30bf\u306e\u8ffd\u52a0 labels.login_required=\u30ed\u30b0\u30a4\u30f3\u304c\u5fc5\u8981 +labels.result_collapsed=\u91cd\u8907\u7d50\u679c\u306e\u6298\u308a\u7573\u307f labels.login_link=\u30ed\u30b0\u30a4\u30f3\u30ea\u30f3\u30af\u8868\u793a labels.thumbnail=\u30b5\u30e0\u30cd\u30a4\u30eb\u8868\u793a labels.ignore_failure_type=\u9664\u5916\u3059\u308b\u30a8\u30e9\u30fc\u306e\u7a2e\u985e diff --git a/src/main/webapp/WEB-INF/view/admin/general/admin_general.jsp b/src/main/webapp/WEB-INF/view/admin/general/admin_general.jsp index 6ea5fa837..c694ca043 100644 --- a/src/main/webapp/WEB-INF/view/admin/general/admin_general.jsp +++ b/src/main/webapp/WEB-INF/view/admin/general/admin_general.jsp @@ -69,6 +69,18 @@ +
+ +
+ +
+ +
+
+
diff --git a/src/main/webapp/WEB-INF/view/searchResults.jsp b/src/main/webapp/WEB-INF/view/searchResults.jsp index cb26045be..f524adf64 100644 --- a/src/main/webapp/WEB-INF/view/searchResults.jsp +++ b/src/main/webapp/WEB-INF/view/searchResults.jsp @@ -56,6 +56,14 @@ + + + + + +
  • + href="/search?q=${f:u(q)}&ex_q=label%3a${f:u(countEntry.key)}&sh=${f:u(sh)}${fe:pagingQuery(null)}${fe:facetQuery()}${fe:geoQuery()}"> ${f:h(fe:label(countEntry.key))} ${f:h(countEntry.value)}
    • @@ -147,7 +155,7 @@
  • + href="/search?q=${f:u(q)}&ex_q=${f:u(queryEntry.value)}&sh=${f:u(sh)}${fe:pagingQuery(queryEntry.value)}${fe:facetQuery()}${fe:geoQuery()}"> ${f:h(facetResponse.queryCountMap[queryEntry.value])}