fix #955 collapse similar results

plugin.xml

@@ -67,6 +67,15 @@
 			<param name="plugin.version" value="5.3.0-SNAPSHOT" />
 			<param name="plugin.zip.version" value="5.3.0-20170330.022144-1" />
 		</antcall>
+		<!-- minhash -->
+		<antcall target="install.plugin">
+			<param name="repo.url" value="${maven.snapshot.repo.url}" />
+			<param name="plugin.groupId" value="org/codelibs" />
+			<param name="plugin.name.prefix" value="elasticsearch-" />
+			<param name="plugin.name" value="minhash" />
+			<param name="plugin.version" value="5.3.0-SNAPSHOT" />
+			<param name="plugin.zip.version" value="5.3.0-20170330.072432-2" />
+		</antcall>
 		<!-- kopf -->
 		<get dest="${target.dir}/kopf.zip">
 			<url url="https://github.com/codelibs/elasticsearch-kopf/archive/${kopf.branch}.zip" />
@@ -105,6 +114,7 @@
 				<include name="dataformat/stax-api-*" />
 				<include name="dataformat/xmlbeans-*" />
 				<include name="langfield/jackson-*" />
+				<include name="minhash/guava-*" />
 			</fileset>
 		</delete>
 	</target>

src/main/java/org/codelibs/fess/Constants.java

@@ -113,6 +113,8 @@ public class Constants extends CoreLibConstants {
 
     public static final String LOGIN_REQUIRED_PROPERTY = "login.required";
 
+    public static final String RESULT_COLLAPSED_PROPERTY = "result.collapsed";
+
     public static final String LOGIN_LINK_ENALBED_PROPERTY = "login.link.enabled";
 
     public static final String THUMBNAIL_ENALBED_PROPERTY = "thumbnail.enabled";

src/main/java/org/codelibs/fess/api/gsa/GsaApiManager.java

@@ -526,5 +526,10 @@ public class GsaApiManager extends BaseApiManager implements WebApiManager {
             return SearchRequestType.GSA;
         }
 
+        @Override
+        public String getSimilarHash() {
+            return request.getParameter("sh");
+        }
+
     }
 }

src/main/java/org/codelibs/fess/api/json/JsonApiManager.java

@@ -667,5 +667,10 @@ public class JsonApiManager extends BaseJsonApiManager {
             return SearchRequestType.JSON;
         }
 
+        @Override
+        public String getSimilarHash() {
+            return request.getParameter("sh");
+        }
+
     }
 }

src/main/java/org/codelibs/fess/api/suggest/SuggestApiManager.java

@@ -244,5 +244,10 @@ public class SuggestApiManager extends BaseJsonApiManager {
         public SearchRequestType getType() {
             return SearchRequestType.SUGGEST;
         }
+
+        @Override
+        public String getSimilarHash() {
+            throw new UnsupportedOperationException();
+        }
     }
 }

src/main/java/org/codelibs/fess/app/service/SearchService.java

@@ -104,7 +104,8 @@ public class SearchService {
                             return SearchConditionBuilder.builder(searchRequestBuilder)
                                     .query(StringUtil.isBlank(sortField) ? query : query + " sort:" + sortField).offset(pageStart)
                                     .size(pageSize).facetInfo(params.getFacetInfo()).geoInfo(params.getGeoInfo())
-                                    .responseFields(queryHelper.getResponseFields()).searchRequestType(params.getType()).build();
+                                    .similarHash(params.getSimilarHash()).responseFields(queryHelper.getResponseFields())
+                                    .searchRequestType(params.getType()).build();
                         }, (searchRequestBuilder, execTime, searchResponse) -> {
                             final QueryResponseList queryResponseList = ComponentUtil.getQueryResponseList();
                             queryResponseList.init(searchResponse, pageStart, pageSize);

src/main/java/org/codelibs/fess/app/web/admin/general/AdminGeneralAction.java

@@ -128,6 +128,9 @@ public class AdminGeneralAction extends FessAdminAction {
         if (form.loginRequired != null) {
             fessConfig.setLoginRequired(Constants.ON.equalsIgnoreCase(form.loginRequired));
         }
+        if (form.resultCollapsed != null) {
+            fessConfig.setResultCollapsed(Constants.ON.equalsIgnoreCase(form.resultCollapsed));
+        }
         if (form.loginLink != null) {
             fessConfig.setLoginLinkEnabled(Constants.ON.equalsIgnoreCase(form.loginLink));
         }
@@ -235,6 +238,7 @@ public class AdminGeneralAction extends FessAdminAction {
 
     public static void updateForm(final FessConfig fessConfig, final EditForm form) {
         form.loginRequired = fessConfig.isLoginRequired() ? Constants.TRUE : Constants.FALSE;
+        form.resultCollapsed = fessConfig.isResultCollapsed() ? Constants.TRUE : Constants.FALSE;
         form.loginLink = fessConfig.isLoginLinkEnabled() ? Constants.TRUE : Constants.FALSE;
         form.thumbnail = fessConfig.isThumbnailEnabled() ? Constants.TRUE : Constants.FALSE;
         form.incrementalCrawling = fessConfig.isIncrementalCrawling() ? Constants.TRUE : Constants.FALSE;

src/main/java/org/codelibs/fess/app/web/admin/general/EditForm.java

@@ -67,6 +67,9 @@ public class EditForm {
     @Size(max = 10)
     public String loginRequired;
 
+    @Size(max = 10)
+    public String resultCollapsed;
+
     @Size(max = 10)
     public String loginLink;
 

src/main/java/org/codelibs/fess/app/web/admin/searchlist/ListForm.java

@@ -58,6 +58,8 @@ public class ListForm implements SearchRequestParams {
 
     public String[] ex_q;
 
+    public String sh;
+
     @Override
     public String getQuery() {
         return q;
@@ -139,4 +141,9 @@ public class ListForm implements SearchRequestParams {
     public SearchRequestType getType() {
         return SearchRequestType.ADMIN_SEARCH;
     }
+
+    @Override
+    public String getSimilarHash() {
+        return sh;
+    }
 }

src/main/java/org/codelibs/fess/app/web/api/admin/elevateword/ApiAdminElevatewordAction.java

@@ -15,6 +15,23 @@
  */
 package org.codelibs.fess.app.web.api.admin.elevateword;
 
+import static org.codelibs.core.stream.StreamUtil.stream;
+import static org.codelibs.fess.app.web.admin.elevateword.AdminElevatewordAction.getElevateWord;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.io.Reader;
+import java.io.Writer;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import javax.annotation.Resource;
+
 import org.codelibs.core.lang.StringUtil;
 import org.codelibs.fess.app.pager.ElevateWordPager;
 import org.codelibs.fess.app.service.ElevateWordService;
@@ -31,16 +48,6 @@ import org.lastaflute.web.Execute;
 import org.lastaflute.web.response.JsonResponse;
 import org.lastaflute.web.response.StreamResponse;
 
-import javax.annotation.Resource;
-import java.io.*;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.List;
-import java.util.stream.Collectors;
-
-import static org.codelibs.core.stream.StreamUtil.stream;
-import static org.codelibs.fess.app.web.admin.elevateword.AdminElevatewordAction.*;
-
 public class ApiAdminElevatewordAction extends FessApiAdminAction {
 
     @Resource

src/main/java/org/codelibs/fess/app/web/api/admin/group/ApiAdminGroupAction.java

@@ -15,6 +15,8 @@
  */
 package org.codelibs.fess.app.web.api.admin.group;
 
+import static org.codelibs.fess.app.web.admin.group.AdminGroupAction.getGroup;
+
 import java.util.List;
 import java.util.stream.Collectors;
 
@@ -31,8 +33,6 @@ import org.codelibs.fess.es.user.exentity.User;
 import org.lastaflute.web.Execute;
 import org.lastaflute.web.response.JsonResponse;
 
-import static org.codelibs.fess.app.web.admin.group.AdminGroupAction.*;
-
 public class ApiAdminGroupAction extends FessApiAdminAction {
 
     @Resource

src/main/java/org/codelibs/fess/app/web/api/admin/role/ApiAdminRoleAction.java

@@ -15,6 +15,8 @@
  */
 package org.codelibs.fess.app.web.api.admin.role;
 
+import static org.codelibs.fess.app.web.admin.role.AdminRoleAction.getRole;
+
 import java.util.List;
 import java.util.stream.Collectors;
 
@@ -30,8 +32,6 @@ import org.codelibs.fess.es.user.exentity.Role;
 import org.lastaflute.web.Execute;
 import org.lastaflute.web.response.JsonResponse;
 
-import static org.codelibs.fess.app.web.admin.role.AdminRoleAction.*;
-
 public class ApiAdminRoleAction extends FessApiAdminAction {
 
     @Resource

src/main/java/org/codelibs/fess/app/web/api/admin/user/ApiAdminUserAction.java

@@ -15,6 +15,8 @@
  */
 package org.codelibs.fess.app.web.api.admin.user;
 
+import static org.codelibs.fess.app.web.admin.user.AdminUserAction.getUser;
+
 import java.util.List;
 import java.util.stream.Collectors;
 
@@ -30,8 +32,6 @@ import org.codelibs.fess.es.user.exentity.User;
 import org.lastaflute.web.Execute;
 import org.lastaflute.web.response.JsonResponse;
 
-import static org.codelibs.fess.app.web.admin.user.AdminUserAction.*;
-
 public class ApiAdminUserAction extends FessApiAdminAction {
 
     @Resource

src/main/java/org/codelibs/fess/app/web/base/SearchForm.java

@@ -55,6 +55,9 @@ public class SearchForm implements SearchRequestParams {
     @ValidateTypeFailure
     public Integer pn;
 
+    @Size(max = 1000)
+    public String sh;
+
     // advance
 
     @Override
@@ -136,4 +139,9 @@ public class SearchForm implements SearchRequestParams {
     public SearchRequestType getType() {
         return SearchRequestType.SEARCH;
     }
+
+    @Override
+    public String getSimilarHash() {
+        return sh;
+    }
 }

src/main/java/org/codelibs/fess/entity/SearchRequestParams.java

@@ -50,6 +50,8 @@ public interface SearchRequestParams {
 
     SearchRequestType getType();
 
+    String getSimilarHash();
+
     public default String[] simplifyArray(final String[] values) {
         return stream(values).get(stream -> stream.filter(StringUtil::isNotBlank).distinct().toArray(n -> new String[n]));
     }

src/main/java/org/codelibs/fess/es/client/FessEsClient.java

@@ -137,7 +137,9 @@ import org.elasticsearch.common.transport.InetSocketTransportAddress;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.index.query.InnerHitBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.SearchHitField;
 import org.elasticsearch.search.SearchHits;
@@ -145,6 +147,7 @@ import org.elasticsearch.search.aggregations.AggregationBuilders;
 import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms.Order;
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
+import org.elasticsearch.search.collapse.CollapseBuilder;
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.client.PreBuiltTransportClient;
@@ -800,6 +803,7 @@ public class FessEsClient implements Client {
         private int size = Constants.DEFAULT_PAGE_SIZE;
         private GeoInfo geoInfo;
         private FacetInfo facetInfo;
+        private String similarHash;
         private SearchRequestType searchRequestType = SearchRequestType.SEARCH;
 
         public static SearchConditionBuilder builder(final SearchRequestBuilder searchRequestBuilder) {
@@ -840,6 +844,13 @@ public class FessEsClient implements Client {
             return this;
         }
 
+        public SearchConditionBuilder similarHash(final String similarHash) {
+            if (StringUtil.isNotBlank(similarHash)) {
+                this.similarHash = similarHash;
+            }
+            return this;
+        }
+
         public SearchConditionBuilder facetInfo(final FacetInfo facetInfo) {
             this.facetInfo = facetInfo;
             return this;
@@ -860,14 +871,18 @@ public class FessEsClient implements Client {
             final QueryContext queryContext = queryHelper.build(searchRequestType, query, context -> {
                 if (SearchRequestType.ADMIN_SEARCH.equals(searchRequestType)) {
                     context.skipRoleQuery();
+                } else if (similarHash != null) {
+                    context.addQuery(boolQuery -> {
+                        boolQuery.filter(QueryBuilders.termQuery(fessConfig.getIndexFieldContentMinhashBits(), similarHash));
+                    });
                 }
-                // geo
-                    if (geoInfo != null && geoInfo.toQueryBuilder() != null) {
-                        context.addQuery(boolQuery -> {
-                            boolQuery.filter(geoInfo.toQueryBuilder());
-                        });
-                    }
-                });
+
+                if (geoInfo != null && geoInfo.toQueryBuilder() != null) {
+                    context.addQuery(boolQuery -> {
+                        boolQuery.filter(geoInfo.toQueryBuilder());
+                    });
+                }
+            });
 
             searchRequestBuilder.setFrom(offset).setSize(size);
 
@@ -924,9 +939,23 @@ public class FessEsClient implements Client {
                         }));
             }
 
+            if (!SearchRequestType.ADMIN_SEARCH.equals(searchRequestType) && fessConfig.isResultCollapsed() && similarHash == null) {
+                searchRequestBuilder.setCollapse(getCollapseBuilder(fessConfig));
+            }
+
             searchRequestBuilder.setQuery(queryContext.getQueryBuilder());
             return true;
         }
+
+        protected CollapseBuilder getCollapseBuilder(final FessConfig fessConfig) {
+            final InnerHitBuilder innerHitBuilder =
+                    new InnerHitBuilder().setName(fessConfig.getQueryCollapseInnerHitsName()).setSize(
+                            fessConfig.getQueryCollapseInnerHitsSizeAsInteger());
+            fessConfig.getQueryCollapseInnerHitsSortBuilders().ifPresent(
+                    builders -> stream(builders).of(stream -> stream.forEach(innerHitBuilder::addSort)));
+            return new CollapseBuilder(fessConfig.getIndexFieldContentMinhashBits()).setMaxConcurrentGroupRequests(
+                    fessConfig.getQueryCollapseMaxConcurrentGroupResultsAsInteger()).setInnerHits(innerHitBuilder);
+        }
     }
 
     public boolean store(final String index, final String type, final Object obj) {

src/main/java/org/codelibs/fess/helper/IndexingHelper.java

@@ -45,6 +45,11 @@ public class IndexingHelper {
             return;
         }
         final FessConfig fessConfig = ComponentUtil.getFessConfig();
+        if (fessConfig.isResultCollapsed()) {
+            docList.forEach(doc -> {
+                doc.put("content_minhash", doc.get(fessConfig.getIndexFieldContent()));
+            });
+        }
         final long execTime = System.currentTimeMillis();
         if (logger.isDebugEnabled()) {
             logger.debug("Sending " + docList.size() + " documents to a server.");

src/main/java/org/codelibs/fess/mylasta/action/FessLabels.java

@@ -680,6 +680,9 @@ public class FessLabels extends UserMessages {
     /** The key of the message: Cache */
     public static final String LABELS_search_result_cache = "{labels.search_result_cache}";
 
+    /** The key of the message: Similar Results ({0}) */
+    public static final String LABELS_search_result_similar = "{labels.search_result_similar}";
+
     /** The key of the message: Label */
     public static final String LABELS_facet_label_title = "{labels.facet_label_title}";
 
@@ -1127,6 +1130,9 @@ public class FessLabels extends UserMessages {
     /** The key of the message: Login Required */
     public static final String LABELS_login_required = "{labels.login_required}";
 
+    /** The key of the message: Similar Result Collapsed */
+    public static final String LABELS_result_collapsed = "{labels.result_collapsed}";
+
     /** The key of the message: Login Link */
     public static final String LABELS_login_link = "{labels.login_link}";
 

src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java

@@ -342,6 +342,12 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
     /** The key of the configuration. e.g. content */
     String INDEX_FIELD_CONTENT = "index.field.content";
 
+    /** The key of the configuration. e.g. content_minhash */
+    String INDEX_FIELD_content_minhash = "index.field.content_minhash";
+
+    /** The key of the configuration. e.g. content_minhash_bits */
+    String INDEX_FIELD_content_minhash_bits = "index.field.content_minhash_bits";
+
     /** The key of the configuration. e.g. cache */
     String INDEX_FIELD_CACHE = "index.field.cache";
 
@@ -483,6 +489,18 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
     /** The key of the configuration. e.g.  */
     String QUERY_ADDITIONAL_NOT_ANALYZED_FIELDS = "query.additional.not.analyzed.fields";
 
+    /** The key of the configuration. e.g. 4 */
+    String QUERY_COLLAPSE_MAX_CONCURRENT_GROUP_RESULTS = "query.collapse.max.concurrent.group.results";
+
+    /** The key of the configuration. e.g. similar_docs */
+    String QUERY_COLLAPSE_INNER_HITS_NAME = "query.collapse.inner.hits.name";
+
+    /** The key of the configuration. e.g. 0 */
+    String QUERY_COLLAPSE_INNER_HITS_SIZE = "query.collapse.inner.hits.size";
+
+    /** The key of the configuration. e.g.  */
+    String QUERY_COLLAPSE_INNER_HITS_SORTS = "query.collapse.inner.hits.sorts";
+
     /** The key of the configuration. e.g.  */
     String QUERY_DEFAULT_LANGUAGES = "query.default.languages";
 
@@ -2140,6 +2158,20 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
      */
     String getIndexFieldContent();
 
+    /**
+     * Get the value for the key 'index.field.content_minhash'. <br>
+     * The value is, e.g. content_minhash <br>
+     * @return The value of found property. (NotNull: if not found, exception but basically no way)
+     */
+    String getIndexFieldContentMinhash();
+
+    /**
+     * Get the value for the key 'index.field.content_minhash_bits'. <br>
+     * The value is, e.g. content_minhash_bits <br>
+     * @return The value of found property. (NotNull: if not found, exception but basically no way)
+     */
+    String getIndexFieldContentMinhashBits();
+
     /**
      * Get the value for the key 'index.field.cache'. <br>
      * The value is, e.g. cache <br>
@@ -2593,6 +2625,58 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
      */
     Integer getQueryAdditionalNotAnalyzedFieldsAsInteger();
 
+    /**
+     * Get the value for the key 'query.collapse.max.concurrent.group.results'. <br>
+     * The value is, e.g. 4 <br>
+     * @return The value of found property. (NotNull: if not found, exception but basically no way)
+     */
+    String getQueryCollapseMaxConcurrentGroupResults();
+
+    /**
+     * Get the value for the key 'query.collapse.max.concurrent.group.results' as {@link Integer}. <br>
+     * The value is, e.g. 4 <br>
+     * @return The value of found property. (NotNull: if not found, exception but basically no way)
+     * @throws NumberFormatException When the property is not integer.
+     */
+    Integer getQueryCollapseMaxConcurrentGroupResultsAsInteger();
+
+    /**
+     * Get the value for the key 'query.collapse.inner.hits.name'. <br>
+     * The value is, e.g. similar_docs <br>
+     * @return The value of found property. (NotNull: if not found, exception but basically no way)
+     */
+    String getQueryCollapseInnerHitsName();
+
+    /**
+     * Get the value for the key 'query.collapse.inner.hits.size'. <br>
+     * The value is, e.g. 0 <br>
+     * @return The value of found property. (NotNull: if not found, exception but basically no way)
+     */
+    String getQueryCollapseInnerHitsSize();
+
+    /**
+     * Get the value for the key 'query.collapse.inner.hits.size' as {@link Integer}. <br>
+     * The value is, e.g. 0 <br>
+     * @return The value of found property. (NotNull: if not found, exception but basically no way)
+     * @throws NumberFormatException When the property is not integer.
+     */
+    Integer getQueryCollapseInnerHitsSizeAsInteger();
+
+    /**
+     * Get the value for the key 'query.collapse.inner.hits.sorts'. <br>
+     * The value is, e.g.  <br>
+     * @return The value of found property. (NotNull: if not found, exception but basically no way)
+     */
+    String getQueryCollapseInnerHitsSorts();
+
+    /**
+     * Get the value for the key 'query.collapse.inner.hits.sorts' as {@link Integer}. <br>
+     * The value is, e.g.  <br>
+     * @return The value of found property. (NotNull: if not found, exception but basically no way)
+     * @throws NumberFormatException When the property is not integer.
+     */
+    Integer getQueryCollapseInnerHitsSortsAsInteger();
+
     /**
      * Get the value for the key 'query.default.languages'. <br>
      * The value is, e.g.  <br>
@@ -5261,6 +5345,14 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
             return get(FessConfig.INDEX_FIELD_CONTENT);
         }
 
+        public String getIndexFieldContentMinhash() {
+            return get(FessConfig.INDEX_FIELD_content_minhash);
+        }
+
+        public String getIndexFieldContentMinhashBits() {
+            return get(FessConfig.INDEX_FIELD_content_minhash_bits);
+        }
+
         public String getIndexFieldCache() {
             return get(FessConfig.INDEX_FIELD_CACHE);
         }
@@ -5509,6 +5601,34 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
             return getAsInteger(FessConfig.QUERY_ADDITIONAL_NOT_ANALYZED_FIELDS);
         }
 
+        public String getQueryCollapseMaxConcurrentGroupResults() {
+            return get(FessConfig.QUERY_COLLAPSE_MAX_CONCURRENT_GROUP_RESULTS);
+        }
+
+        public Integer getQueryCollapseMaxConcurrentGroupResultsAsInteger() {
+            return getAsInteger(FessConfig.QUERY_COLLAPSE_MAX_CONCURRENT_GROUP_RESULTS);
+        }
+
+        public String getQueryCollapseInnerHitsName() {
+            return get(FessConfig.QUERY_COLLAPSE_INNER_HITS_NAME);
+        }
+
+        public String getQueryCollapseInnerHitsSize() {
+            return get(FessConfig.QUERY_COLLAPSE_INNER_HITS_SIZE);
+        }
+
+        public Integer getQueryCollapseInnerHitsSizeAsInteger() {
+            return getAsInteger(FessConfig.QUERY_COLLAPSE_INNER_HITS_SIZE);
+        }
+
+        public String getQueryCollapseInnerHitsSorts() {
+            return get(FessConfig.QUERY_COLLAPSE_INNER_HITS_SORTS);
+        }
+
+        public Integer getQueryCollapseInnerHitsSortsAsInteger() {
+            return getAsInteger(FessConfig.QUERY_COLLAPSE_INNER_HITS_SORTS);
+        }
+
         public String getQueryDefaultLanguages() {
             return get(FessConfig.QUERY_DEFAULT_LANGUAGES);
         }

src/main/java/org/codelibs/fess/mylasta/direction/FessProp.java

@@ -47,6 +47,9 @@ import org.codelibs.fess.util.ComponentUtil;
 import org.codelibs.fess.util.PrunedTag;
 import org.dbflute.optional.OptionalThing;
 import org.elasticsearch.action.search.SearchRequestBuilder;
+import org.elasticsearch.search.sort.SortBuilder;
+import org.elasticsearch.search.sort.SortBuilders;
+import org.elasticsearch.search.sort.SortOrder;
 import org.lastaflute.job.LaJob;
 import org.lastaflute.job.subsidiary.JobConcurrentExec;
 import org.lastaflute.web.util.LaRequestUtil;
@@ -58,6 +61,8 @@ import org.lastaflute.web.validation.theme.typed.LongTypeValidator;
 
 public interface FessProp {
 
+    public static final String QUERY_COLLAPSE_INNER_HITS_SORTS = "queryCollapseInnerHitsSorts";
+
     public static final String USER_CODE_PATTERN = "userCodePattern";
 
     public static final String API_ADMIN_ACCESS_PERMISSION_SET = "apiAdminAccessPermissionSet";
@@ -269,6 +274,14 @@ public interface FessProp {
         return getSystemPropertyAsBoolean(Constants.LOGIN_REQUIRED_PROPERTY, false);
     }
 
+    public default void setResultCollapsed(final boolean value) {
+        setSystemPropertyAsBoolean(Constants.RESULT_COLLAPSED_PROPERTY, value);
+    }
+
+    public default boolean isResultCollapsed() {
+        return getSystemPropertyAsBoolean(Constants.RESULT_COLLAPSED_PROPERTY, false);
+    }
+
     public default void setLoginLinkEnabled(final boolean value) {
         setSystemPropertyAsBoolean(Constants.LOGIN_LINK_ENALBED_PROPERTY, value);
     }
@@ -1504,4 +1517,36 @@ public interface FessProp {
         }
         return pattern.matcher(userCode).matches();
     }
+
+    String getQueryCollapseInnerHitsSorts();
+
+    @SuppressWarnings("rawtypes")
+    public default OptionalThing<SortBuilder[]> getQueryCollapseInnerHitsSortBuilders() {
+        @SuppressWarnings("unchecked")
+        OptionalThing<SortBuilder[]> ot = (OptionalThing<SortBuilder[]>) propMap.get(QUERY_COLLAPSE_INNER_HITS_SORTS);
+        if (ot == null) {
+            String sorts = getQueryCollapseInnerHitsSorts();
+            if (StringUtil.isBlank(sorts)) {
+                ot = OptionalThing.empty();
+            } else {
+                SortBuilder[] sortBuilders =
+                        split(sorts, ",").get(
+                                stream -> stream
+                                        .filter(StringUtil::isNotBlank)
+                                        .map(s -> {
+                                            final String[] values = s.split(":");
+                                            if (values.length > 1) {
+                                                return SortBuilders.fieldSort(values[0]).order(
+                                                        values[0].equalsIgnoreCase("desc") ? SortOrder.DESC : SortOrder.ASC);
+                                            } else {
+                                                return SortBuilders.fieldSort(values[0]).order(SortOrder.ASC);
+                                            }
+                                        }).toArray(n -> new SortBuilder[n]));
+                ot = OptionalThing.of(sortBuilders);
+            }
+            propMap.put(QUERY_COLLAPSE_INNER_HITS_SORTS, ot);
+        }
+        return ot;
+    }
+
 }

src/main/java/org/codelibs/fess/util/QueryResponseList.java

@@ -24,6 +24,7 @@ import java.util.ListIterator;
 import java.util.Map;
 
 import org.apache.commons.lang3.StringUtils;
+import org.codelibs.core.stream.StreamUtil;
 import org.codelibs.fess.helper.QueryHelper;
 import org.codelibs.fess.helper.ViewHelper;
 import org.codelibs.fess.mylasta.direction.FessConfig;
@@ -31,6 +32,7 @@ import org.dbflute.optional.OptionalEntity;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.text.Text;
 import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.SearchHitField;
 import org.elasticsearch.search.SearchHits;
 import org.elasticsearch.search.aggregations.Aggregations;
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightField;
@@ -84,6 +86,7 @@ public class QueryResponseList implements List<Map<String, Object>> {
 
     public void init(final OptionalEntity<SearchResponse> searchResponseOpt, final int start, final int pageSize) {
         searchResponseOpt.ifPresent(searchResponse -> {
+            final FessConfig fessConfig = ComponentUtil.getFessConfig();
             final SearchHits searchHits = searchResponse.getHits();
             allRecordCount = searchHits.getTotalHits();
             queryTime = searchResponse.getTookInMillis();
@@ -96,45 +99,28 @@ public class QueryResponseList implements List<Map<String, Object>> {
                 final QueryHelper queryHelper = ComponentUtil.getQueryHelper();
                 final String hlPrefix = queryHelper.getHighlightPrefix();
                 for (final SearchHit searchHit : searchHits.getHits()) {
-                    final Map<String, Object> docMap = new HashMap<>();
-                    if (searchHit.getSource() == null) {
-                        searchHit.getFields().forEach((key, value) -> {
-                            docMap.put(key, value.getValue());
-                        });
-                    } else {
-                        docMap.putAll(searchHit.getSource());
-                    }
-
-                    final Map<String, HighlightField> highlightFields = searchHit.getHighlightFields();
-                    try {
-                        if (highlightFields != null) {
-                            for (final Map.Entry<String, HighlightField> entry : highlightFields.entrySet()) {
-                                final HighlightField highlightField = entry.getValue();
-                                final Text[] fragments = highlightField.fragments();
-                                if (fragments != null && fragments.length != 0) {
-                                    final String[] texts = new String[fragments.length];
-                                    for (int i = 0; i < fragments.length; i++) {
-                                        texts[i] = fragments[i].string();
+                    final Map<String, Object> docMap = parseSearchHit(fessConfig, hlPrefix, searchHit);
+
+                    if (fessConfig.isResultCollapsed()) {
+                        final Map<String, SearchHits> innerHits = searchHit.getInnerHits();
+                        if (innerHits != null) {
+                            final SearchHits innerSearchHits = innerHits.get(fessConfig.getQueryCollapseInnerHitsName());
+                            if (innerSearchHits != null) {
+                                long totalHits = innerSearchHits.getTotalHits();
+                                if (totalHits > 0) {
+                                    docMap.put(fessConfig.getQueryCollapseInnerHitsName() + "_count", totalHits);
+                                    SearchHitField bitsField = searchHit.getFields().get(fessConfig.getIndexFieldContentMinhashBits());
+                                    if (bitsField != null && !bitsField.getValues().isEmpty()) {
+                                        docMap.put(fessConfig.getQueryCollapseInnerHitsName() + "_hash", bitsField.getValues().get(0));
                                     }
-                                    final String value = StringUtils.join(texts, "...");
-                                    docMap.put(hlPrefix + highlightField.getName(), value);
+                                    docMap.put(
+                                            fessConfig.getQueryCollapseInnerHitsName(),
+                                            StreamUtil.stream(innerSearchHits.getHits()).get(
+                                                    stream -> stream.map(v -> parseSearchHit(fessConfig, hlPrefix, v)).toArray(
+                                                            n -> new Map[n])));
                                 }
                             }
                         }
-                    } catch (final Exception e) {
-                        if (logger.isDebugEnabled()) {
-                            logger.debug("Could not create a highlighting value: " + docMap, e);
-                        }
-                    }
-
-                    // ContentTitle
-                    final ViewHelper viewHelper = ComponentUtil.getViewHelper();
-                    if (viewHelper != null) {
-                        final FessConfig fessConfig = ComponentUtil.getFessConfig();
-                        docMap.put(fessConfig.getResponseFieldContentTitle(), viewHelper.getContentTitle(docMap));
-                        docMap.put(fessConfig.getResponseFieldContentDescription(), viewHelper.getContentDescription(docMap));
-                        docMap.put(fessConfig.getResponseFieldUrlLink(), viewHelper.getUrlLink(docMap));
-                        docMap.put(fessConfig.getResponseFieldSitePath(), viewHelper.getSitePath(docMap));
                     }
 
                     parent.add(docMap);
@@ -151,6 +137,49 @@ public class QueryResponseList implements List<Map<String, Object>> {
         calculatePageInfo(start, pageSize);
     }
 
+    private Map<String, Object> parseSearchHit(final FessConfig fessConfig, final String hlPrefix, final SearchHit searchHit) {
+        final Map<String, Object> docMap = new HashMap<>();
+        if (searchHit.getSource() == null) {
+            searchHit.getFields().forEach((key, value) -> {
+                docMap.put(key, value.getValue());
+            });
+        } else {
+            docMap.putAll(searchHit.getSource());
+        }
+
+        final Map<String, HighlightField> highlightFields = searchHit.getHighlightFields();
+        try {
+            if (highlightFields != null) {
+                for (final Map.Entry<String, HighlightField> entry : highlightFields.entrySet()) {
+                    final HighlightField highlightField = entry.getValue();
+                    final Text[] fragments = highlightField.fragments();
+                    if (fragments != null && fragments.length != 0) {
+                        final String[] texts = new String[fragments.length];
+                        for (int i = 0; i < fragments.length; i++) {
+                            texts[i] = fragments[i].string();
+                        }
+                        final String value = StringUtils.join(texts, "...");
+                        docMap.put(hlPrefix + highlightField.getName(), value);
+                    }
+                }
+            }
+        } catch (final Exception e) {
+            if (logger.isDebugEnabled()) {
+                logger.debug("Could not create a highlighting value: " + docMap, e);
+            }
+        }
+
+        // ContentTitle
+        final ViewHelper viewHelper = ComponentUtil.getViewHelper();
+        if (viewHelper != null) {
+            docMap.put(fessConfig.getResponseFieldContentTitle(), viewHelper.getContentTitle(docMap));
+            docMap.put(fessConfig.getResponseFieldContentDescription(), viewHelper.getContentDescription(docMap));
+            docMap.put(fessConfig.getResponseFieldUrlLink(), viewHelper.getUrlLink(docMap));
+            docMap.put(fessConfig.getResponseFieldSitePath(), viewHelper.getSitePath(docMap));
+        }
+        return docMap;
+    }
+
     protected void calculatePageInfo(final int start, final int size) {
         pageSize = size;
         allPageCount = (int) ((allRecordCount - 1) / pageSize) + 1;

src/main/resources/fess_config.properties

@@ -166,6 +166,8 @@ index.field.mimetype=mimetype
 index.field.parent_id=parent_id
 index.field.important_content=important_content
 index.field.content=content
+index.field.content_minhash=content_minhash
+index.field.content_minhash_bits=content_minhash_bits
 index.field.cache=cache
 index.field.digest=digest
 index.field.title=title
@@ -221,6 +223,10 @@ query.additional.search.fields=
 query.additional.facet.fields=
 query.additional.sort.fields=
 query.additional.not.analyzed.fields=
+query.collapse.max.concurrent.group.results=4
+query.collapse.inner.hits.name=similar_docs
+query.collapse.inner.hits.size=0
+query.collapse.inner.hits.sorts=
 query.default.languages=
 query.language.mapping=\
 ar=ar\n\

src/main/resources/fess_indices/fess.json

@@ -406,6 +406,12 @@
         "alphanum_word_filter" : {
           "type" : "alphanum_word",
           "max_token_length" : 20
+        },
+        "minhash_filter" : {
+          "type" : "minhash",
+          "seed" : 1,
+          "bit" : 2,
+          "size" : 64
         }
       },
       "tokenizer": {
@@ -756,6 +762,21 @@
             "lowercase",
             "stemmer_en_filter"
           ]
+        },
+        "minhash_analyzer": {
+          "type": "custom",
+          "char_filter": [
+            "mapping_ja_filter"
+          ],
+          "tokenizer": "unigram_synonym_tokenizer",
+          "filter": [
+            "alphanum_word_filter",
+            "cjk_bigram",
+            "stopword_en_filter",
+            "lowercase",
+            "stemmer_en_filter",
+            "minhash_filter"
+          ]
         }
       }
     }

src/main/resources/fess_indices/fess/doc.json

@@ -456,6 +456,14 @@
         "analyzer": "standard_analyzer",
         "term_vector": "with_positions_offsets"
       },
+      "content_minhash": {
+        "type": "minhash",
+        "minhash_analyzer": "minhash_analyzer",
+        "copy_bits_to": "content_minhash_bits"
+      },
+      "content_minhash_bits": {
+        "type": "keyword"
+      },
       "content_length": {
         "type": "long"
       },

src/main/resources/fess_label.properties

@@ -217,6 +217,7 @@ labels.search_result_favorited=Liked
 labels.search_click_count=Viewed ({0})
 labels.search_result_more=more..
 labels.search_result_cache=Cache
+labels.search_result_similar=Similar Results ({0})
 labels.facet_label_title=Label
 labels.facet_timestamp_title=Date Range
 labels.facet_timestamp_1day=Past 24 Hours
@@ -366,6 +367,7 @@ labels.default_label_value=Default Label Value
 labels.default_sort_value=Default Sort Value
 labels.append_query_param_enabled=Append Params to URL
 labels.login_required=Login Required
+labels.result_collapsed=Similar Result Collapsed
 labels.login_link=Login Link
 labels.thumbnail=Thumbnail View
 labels.ignore_failure_type=Excluded Failure Type

src/main/resources/fess_label_en.properties

@@ -217,6 +217,7 @@ labels.search_result_favorited=Liked
 labels.search_click_count=Viewed ({0})
 labels.search_result_more=more..
 labels.search_result_cache=Cache
+labels.search_result_similar=Similar Results ({0})
 labels.facet_label_title=Label
 labels.facet_timestamp_title=Date Range
 labels.facet_timestamp_1day=Past 24 Hours
@@ -366,6 +367,7 @@ labels.default_label_value=Default Label Value
 labels.default_sort_value=Default Sort Value
 labels.append_query_param_enabled=Append Params to URL
 labels.login_required=Login Required
+labels.result_collapsed=Similar Result Collapsed
 labels.login_link=Login Link
 labels.thumbnail=Thumbnail View
 labels.ignore_failure_type=Excluded Failure Type

src/main/resources/fess_label_ja.properties

@@ -210,6 +210,7 @@ labels.search_result_favorited=Liked
 labels.search_click_count=\u30af\u30ea\u30c3\u30af\u6570 ({0})
 labels.search_result_more=\u8a73\u7d30..
 labels.search_result_cache=\u30ad\u30e3\u30c3\u30b7\u30e5
+labels.search_result_similar=\u985e\u4f3c\u7d50\u679c ({0})
 labels.facet_label_title=\u30e9\u30d9\u30eb
 labels.facet_timestamp_title=\u671f\u9593
 labels.facet_timestamp_1day=24\u6642\u9593\u4ee5\u5185
@@ -359,6 +360,7 @@ labels.default_label_value=\u30c7\u30d5\u30a9\u30eb\u30c8\u306e\u30e9\u30d9\u30e
 labels.default_sort_value=\u30c7\u30d5\u30a9\u30eb\u30c8\u306e\u30bd\u30fc\u30c8\u5024
 labels.append_query_param_enabled=\u691c\u7d22\u30d1\u30e9\u30e1\u30fc\u30bf\u306e\u8ffd\u52a0
 labels.login_required=\u30ed\u30b0\u30a4\u30f3\u304c\u5fc5\u8981
+labels.result_collapsed=\u91cd\u8907\u7d50\u679c\u306e\u6298\u308a\u7573\u307f
 labels.login_link=\u30ed\u30b0\u30a4\u30f3\u30ea\u30f3\u30af\u8868\u793a
 labels.thumbnail=\u30b5\u30e0\u30cd\u30a4\u30eb\u8868\u793a
 labels.ignore_failure_type=\u9664\u5916\u3059\u308b\u30a8\u30e9\u30fc\u306e\u7a2e\u985e

src/main/webapp/WEB-INF/view/admin/general/admin_general.jsp

@@ -69,6 +69,18 @@
 											</div>
 										</div>
 									</div>
+									<div class="form-group">
+										<label for="resultCollapsed" class="col-sm-3 control-label"><la:message
+												key="labels.result_collapsed" /></label>
+										<div class="col-sm-9">
+											<la:errors property="resultCollapsed" />
+											<div class="checkbox">
+												<label> <la:checkbox property="resultCollapsed" /> <la:message
+														key="labels.enabled" />
+												</label>
+											</div>
+										</div>
+									</div>
 									<div class="form-group">
 										<label for="thumbnail" class="col-sm-3 control-label"><la:message
 												key="labels.thumbnail" /></label>

src/main/webapp/WEB-INF/view/searchResults.jsp

@@ -56,6 +56,14 @@
 											</la:link>
 										</small>
 									</c:if>
+									<c:if test="${doc.similar_docs_count!=null}">
+										<small class="hidden-md-down"> <la:link
+												href="/search?q=${f:u(q)}&ex_q=${f:u(queryEntry.value)}&sh=${f:u(doc.similar_docs_hash)}${fe:facetQuery()}${fe:geoQuery()}">
+												<la:message key="labels.search_result_similar"
+															arg0="${fe:formatNumber(doc.similar_docs_count)}" />
+											</la:link>
+										</small>
+									</c:if>
 								</div>
 								<div class="more hidden-md-up">
 									<a href="#result${s.index}"><la:message
@@ -130,7 +138,7 @@
 							<c:if
 								test="${countEntry.value != 0 && fe:labelexists(countEntry.key)}">
 								<li class="list-group-item"><la:link
-										href="/search?q=${f:u(q)}&ex_q=label%3a${f:u(countEntry.key)}${fe:pagingQuery(null)}${fe:facetQuery()}${fe:geoQuery()}">
+										href="/search?q=${f:u(q)}&ex_q=label%3a${f:u(countEntry.key)}&sh=${f:u(sh)}${fe:pagingQuery(null)}${fe:facetQuery()}${fe:geoQuery()}">
 											${f:h(fe:label(countEntry.key))} 
 											<span class="label label-default label-pill pull-right">${f:h(countEntry.value)}</span>
 									</la:link></li>
@@ -147,7 +155,7 @@
 					<c:forEach var="queryEntry" items="${facetQueryView.queryMap}">
 						<c:if test="${facetResponse.queryCountMap[queryEntry.value] != 0}">
 							<li class="list-group-item p-l-md"><la:link
-									href="/search?q=${f:u(q)}&ex_q=${f:u(queryEntry.value)}${fe:pagingQuery(queryEntry.value)}${fe:facetQuery()}${fe:geoQuery()}">
+									href="/search?q=${f:u(q)}&ex_q=${f:u(queryEntry.value)}&sh=${f:u(sh)}${fe:pagingQuery(queryEntry.value)}${fe:facetQuery()}${fe:geoQuery()}">
 									<la:message key="${queryEntry.key}" />
 									<span class="label label-default label-pill pull-right">${f:h(facetResponse.queryCountMap[queryEntry.value])}</span>
 								</la:link></li>