0ct0pu5/fess
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix #955 collapse similar results

Browse source
This commit is contained in:
Shinsuke Sugaya 2017-03-30 22:46:48 +09:00
parent b2029c99b3
commit bd90fde4da
29 changed files with 414 additions and 60 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
plugin.xml
View file

@ -67,6 +67,15 @@
<param name="plugin.version" value="5.3.0-SNAPSHOT" />
<param name="plugin.zip.version" value="5.3.0-20170330.022144-1" />
</antcall>
<!-- minhash -->
<antcall target="install.plugin">
<param name="repo.url" value="${maven.snapshot.repo.url}" />
<param name="plugin.groupId" value="org/codelibs" />
<param name="plugin.name.prefix" value="elasticsearch-" />
<param name="plugin.name" value="minhash" />
<param name="plugin.version" value="5.3.0-SNAPSHOT" />
<param name="plugin.zip.version" value="5.3.0-20170330.072432-2" />
</antcall>
<!-- kopf -->
<get dest="${target.dir}/kopf.zip">
<url url="https://github.com/codelibs/elasticsearch-kopf/archive/${kopf.branch}.zip" />
@ -105,6 +114,7 @@
<include name="dataformat/stax-api-*" />
<include name="dataformat/xmlbeans-*" />
<include name="langfield/jackson-*" />
<include name="minhash/guava-*" />
</fileset>
</delete>
</target>

2
src/main/java/org/codelibs/fess/Constants.java
View file

@ -113,6 +113,8 @@ public class Constants extends CoreLibConstants {
public static final String LOGIN_REQUIRED_PROPERTY = "login.required";
public static final String RESULT_COLLAPSED_PROPERTY = "result.collapsed";
public static final String LOGIN_LINK_ENALBED_PROPERTY = "login.link.enabled";
public static final String THUMBNAIL_ENALBED_PROPERTY = "thumbnail.enabled";

5
src/main/java/org/codelibs/fess/api/gsa/GsaApiManager.java
View file

@ -526,5 +526,10 @@ public class GsaApiManager extends BaseApiManager implements WebApiManager {
return SearchRequestType.GSA;
}
@Override
public String getSimilarHash() {
return request.getParameter("sh");
}
}
}

5
src/main/java/org/codelibs/fess/api/json/JsonApiManager.java
View file

@ -667,5 +667,10 @@ public class JsonApiManager extends BaseJsonApiManager {
return SearchRequestType.JSON;
}
@Override
public String getSimilarHash() {
return request.getParameter("sh");
}
}
}

5
src/main/java/org/codelibs/fess/api/suggest/SuggestApiManager.java
View file

@ -244,5 +244,10 @@ public class SuggestApiManager extends BaseJsonApiManager {
public SearchRequestType getType() {
return SearchRequestType.SUGGEST;
}
@Override
public String getSimilarHash() {
throw new UnsupportedOperationException();
}
}
}

3
src/main/java/org/codelibs/fess/app/service/SearchService.java
View file

@ -104,7 +104,8 @@ public class SearchService {
return SearchConditionBuilder.builder(searchRequestBuilder)
.query(StringUtil.isBlank(sortField) ? query : query + " sort:" + sortField).offset(pageStart)
.size(pageSize).facetInfo(params.getFacetInfo()).geoInfo(params.getGeoInfo())
.responseFields(queryHelper.getResponseFields()).searchRequestType(params.getType()).build();
.similarHash(params.getSimilarHash()).responseFields(queryHelper.getResponseFields())
.searchRequestType(params.getType()).build();
}, (searchRequestBuilder, execTime, searchResponse) -> {
final QueryResponseList queryResponseList = ComponentUtil.getQueryResponseList();
queryResponseList.init(searchResponse, pageStart, pageSize);

4
src/main/java/org/codelibs/fess/app/web/admin/general/AdminGeneralAction.java
View file

@ -128,6 +128,9 @@ public class AdminGeneralAction extends FessAdminAction {
if (form.loginRequired != null) {
fessConfig.setLoginRequired(Constants.ON.equalsIgnoreCase(form.loginRequired));
}
if (form.resultCollapsed != null) {
fessConfig.setResultCollapsed(Constants.ON.equalsIgnoreCase(form.resultCollapsed));
}
if (form.loginLink != null) {
fessConfig.setLoginLinkEnabled(Constants.ON.equalsIgnoreCase(form.loginLink));
}
@ -235,6 +238,7 @@ public class AdminGeneralAction extends FessAdminAction {
public static void updateForm(final FessConfig fessConfig, final EditForm form) {
form.loginRequired = fessConfig.isLoginRequired() ? Constants.TRUE : Constants.FALSE;
form.resultCollapsed = fessConfig.isResultCollapsed() ? Constants.TRUE : Constants.FALSE;
form.loginLink = fessConfig.isLoginLinkEnabled() ? Constants.TRUE : Constants.FALSE;
form.thumbnail = fessConfig.isThumbnailEnabled() ? Constants.TRUE : Constants.FALSE;
form.incrementalCrawling = fessConfig.isIncrementalCrawling() ? Constants.TRUE : Constants.FALSE;

3
src/main/java/org/codelibs/fess/app/web/admin/general/EditForm.java
View file

@ -67,6 +67,9 @@ public class EditForm {
@Size(max = 10)
public String loginRequired;
@Size(max = 10)
public String resultCollapsed;
@Size(max = 10)
public String loginLink;

7
src/main/java/org/codelibs/fess/app/web/admin/searchlist/ListForm.java
View file

@ -58,6 +58,8 @@ public class ListForm implements SearchRequestParams {
public String[] ex_q;
public String sh;
@Override
public String getQuery() {
return q;
@ -139,4 +141,9 @@ public class ListForm implements SearchRequestParams {
public SearchRequestType getType() {
return SearchRequestType.ADMIN_SEARCH;
}
@Override
public String getSimilarHash() {
return sh;
}
}

27
src/main/java/org/codelibs/fess/app/web/api/admin/elevateword/ApiAdminElevatewordAction.java
View file

@ -15,6 +15,23 @@
*/
package org.codelibs.fess.app.web.api.admin.elevateword;
import static org.codelibs.core.stream.StreamUtil.stream;
import static org.codelibs.fess.app.web.admin.elevateword.AdminElevatewordAction.getElevateWord;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.Reader;
import java.io.Writer;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import java.util.stream.Collectors;
import javax.annotation.Resource;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.fess.app.pager.ElevateWordPager;
import org.codelibs.fess.app.service.ElevateWordService;
@ -31,16 +48,6 @@ import org.lastaflute.web.Execute;
import org.lastaflute.web.response.JsonResponse;
import org.lastaflute.web.response.StreamResponse;
import javax.annotation.Resource;
import java.io.*;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import java.util.stream.Collectors;
import static org.codelibs.core.stream.StreamUtil.stream;
import static org.codelibs.fess.app.web.admin.elevateword.AdminElevatewordAction.*;
public class ApiAdminElevatewordAction extends FessApiAdminAction {
@Resource

4
src/main/java/org/codelibs/fess/app/web/api/admin/group/ApiAdminGroupAction.java
View file

@ -15,6 +15,8 @@
*/
package org.codelibs.fess.app.web.api.admin.group;
import static org.codelibs.fess.app.web.admin.group.AdminGroupAction.getGroup;
import java.util.List;
import java.util.stream.Collectors;
@ -31,8 +33,6 @@ import org.codelibs.fess.es.user.exentity.User;
import org.lastaflute.web.Execute;
import org.lastaflute.web.response.JsonResponse;
import static org.codelibs.fess.app.web.admin.group.AdminGroupAction.*;
public class ApiAdminGroupAction extends FessApiAdminAction {
@Resource

4
src/main/java/org/codelibs/fess/app/web/api/admin/role/ApiAdminRoleAction.java
View file

@ -15,6 +15,8 @@
*/
package org.codelibs.fess.app.web.api.admin.role;
import static org.codelibs.fess.app.web.admin.role.AdminRoleAction.getRole;
import java.util.List;
import java.util.stream.Collectors;
@ -30,8 +32,6 @@ import org.codelibs.fess.es.user.exentity.Role;
import org.lastaflute.web.Execute;
import org.lastaflute.web.response.JsonResponse;
import static org.codelibs.fess.app.web.admin.role.AdminRoleAction.*;
public class ApiAdminRoleAction extends FessApiAdminAction {
@Resource

4
src/main/java/org/codelibs/fess/app/web/api/admin/user/ApiAdminUserAction.java
View file

@ -15,6 +15,8 @@
*/
package org.codelibs.fess.app.web.api.admin.user;
import static org.codelibs.fess.app.web.admin.user.AdminUserAction.getUser;
import java.util.List;
import java.util.stream.Collectors;
@ -30,8 +32,6 @@ import org.codelibs.fess.es.user.exentity.User;
import org.lastaflute.web.Execute;
import org.lastaflute.web.response.JsonResponse;
import static org.codelibs.fess.app.web.admin.user.AdminUserAction.*;
public class ApiAdminUserAction extends FessApiAdminAction {
@Resource

8
src/main/java/org/codelibs/fess/app/web/base/SearchForm.java
View file

@ -55,6 +55,9 @@ public class SearchForm implements SearchRequestParams {
@ValidateTypeFailure
public Integer pn;
@Size(max = 1000)
public String sh;
// advance
@Override
@ -136,4 +139,9 @@ public class SearchForm implements SearchRequestParams {
public SearchRequestType getType() {
return SearchRequestType.SEARCH;
}
@Override
public String getSimilarHash() {
return sh;
}
}

2
src/main/java/org/codelibs/fess/entity/SearchRequestParams.java
View file

@ -50,6 +50,8 @@ public interface SearchRequestParams {
SearchRequestType getType();
String getSimilarHash();
public default String[] simplifyArray(final String[] values) {
return stream(values).get(stream -> stream.filter(StringUtil::isNotBlank).distinct().toArray(n -> new String[n]));
}

43
src/main/java/org/codelibs/fess/es/client/FessEsClient.java
View file

@ -137,7 +137,9 @@ import org.elasticsearch.common.transport.InetSocketTransportAddress;
import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.query.InnerHitBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHitField;
import org.elasticsearch.search.SearchHits;
@ -145,6 +147,7 @@ import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder;
import org.elasticsearch.search.aggregations.bucket.terms.Terms.Order;
import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
import org.elasticsearch.search.collapse.CollapseBuilder;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.client.PreBuiltTransportClient;
@ -800,6 +803,7 @@ public class FessEsClient implements Client {
private int size = Constants.DEFAULT_PAGE_SIZE;
private GeoInfo geoInfo;
private FacetInfo facetInfo;
private String similarHash;
private SearchRequestType searchRequestType = SearchRequestType.SEARCH;
public static SearchConditionBuilder builder(final SearchRequestBuilder searchRequestBuilder) {
@ -840,6 +844,13 @@ public class FessEsClient implements Client {
return this;
}
public SearchConditionBuilder similarHash(final String similarHash) {
if (StringUtil.isNotBlank(similarHash)) {
this.similarHash = similarHash;
}
return this;
}
public SearchConditionBuilder facetInfo(final FacetInfo facetInfo) {
this.facetInfo = facetInfo;
return this;
@ -860,14 +871,18 @@ public class FessEsClient implements Client {
final QueryContext queryContext = queryHelper.build(searchRequestType, query, context -> {
if (SearchRequestType.ADMIN_SEARCH.equals(searchRequestType)) {
context.skipRoleQuery();
} else if (similarHash != null) {
context.addQuery(boolQuery -> {
boolQuery.filter(QueryBuilders.termQuery(fessConfig.getIndexFieldContentMinhashBits(), similarHash));
});
}
// geo
if (geoInfo != null && geoInfo.toQueryBuilder() != null) {
context.addQuery(boolQuery -> {
boolQuery.filter(geoInfo.toQueryBuilder());
});
}
});
if (geoInfo != null && geoInfo.toQueryBuilder() != null) {
context.addQuery(boolQuery -> {
boolQuery.filter(geoInfo.toQueryBuilder());
});
}
});
searchRequestBuilder.setFrom(offset).setSize(size);
@ -924,9 +939,23 @@ public class FessEsClient implements Client {
}));
}
if (!SearchRequestType.ADMIN_SEARCH.equals(searchRequestType) && fessConfig.isResultCollapsed() && similarHash == null) {
searchRequestBuilder.setCollapse(getCollapseBuilder(fessConfig));
}
searchRequestBuilder.setQuery(queryContext.getQueryBuilder());
return true;
}
protected CollapseBuilder getCollapseBuilder(final FessConfig fessConfig) {
final InnerHitBuilder innerHitBuilder =
new InnerHitBuilder().setName(fessConfig.getQueryCollapseInnerHitsName()).setSize(
fessConfig.getQueryCollapseInnerHitsSizeAsInteger());
fessConfig.getQueryCollapseInnerHitsSortBuilders().ifPresent(
builders -> stream(builders).of(stream -> stream.forEach(innerHitBuilder::addSort)));
return new CollapseBuilder(fessConfig.getIndexFieldContentMinhashBits()).setMaxConcurrentGroupRequests(
fessConfig.getQueryCollapseMaxConcurrentGroupResultsAsInteger()).setInnerHits(innerHitBuilder);
}
}
public boolean store(final String index, final String type, final Object obj) {

5
src/main/java/org/codelibs/fess/helper/IndexingHelper.java
View file

@ -45,6 +45,11 @@ public class IndexingHelper {
return;
}
final FessConfig fessConfig = ComponentUtil.getFessConfig();
if (fessConfig.isResultCollapsed()) {
docList.forEach(doc -> {
doc.put("content_minhash", doc.get(fessConfig.getIndexFieldContent()));
});
}
final long execTime = System.currentTimeMillis();
if (logger.isDebugEnabled()) {
logger.debug("Sending " + docList.size() + " documents to a server.");

6
src/main/java/org/codelibs/fess/mylasta/action/FessLabels.java
View file

@ -680,6 +680,9 @@ public class FessLabels extends UserMessages {
/** The key of the message: Cache */
public static final String LABELS_search_result_cache = "{labels.search_result_cache}";
/** The key of the message: Similar Results ({0}) */
public static final String LABELS_search_result_similar = "{labels.search_result_similar}";
/** The key of the message: Label */
public static final String LABELS_facet_label_title = "{labels.facet_label_title}";
@ -1127,6 +1130,9 @@ public class FessLabels extends UserMessages {
/** The key of the message: Login Required */
public static final String LABELS_login_required = "{labels.login_required}";
/** The key of the message: Similar Result Collapsed */
public static final String LABELS_result_collapsed = "{labels.result_collapsed}";
/** The key of the message: Login Link */
public static final String LABELS_login_link = "{labels.login_link}";

120
src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java
View file

@ -342,6 +342,12 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
/** The key of the configuration. e.g. content */
String INDEX_FIELD_CONTENT = "index.field.content";
/** The key of the configuration. e.g. content_minhash */
String INDEX_FIELD_content_minhash = "index.field.content_minhash";
/** The key of the configuration. e.g. content_minhash_bits */
String INDEX_FIELD_content_minhash_bits = "index.field.content_minhash_bits";
/** The key of the configuration. e.g. cache */
String INDEX_FIELD_CACHE = "index.field.cache";
@ -483,6 +489,18 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
/** The key of the configuration. e.g. */
String QUERY_ADDITIONAL_NOT_ANALYZED_FIELDS = "query.additional.not.analyzed.fields";
/** The key of the configuration. e.g. 4 */
String QUERY_COLLAPSE_MAX_CONCURRENT_GROUP_RESULTS = "query.collapse.max.concurrent.group.results";
/** The key of the configuration. e.g. similar_docs */
String QUERY_COLLAPSE_INNER_HITS_NAME = "query.collapse.inner.hits.name";
/** The key of the configuration. e.g. 0 */
String QUERY_COLLAPSE_INNER_HITS_SIZE = "query.collapse.inner.hits.size";
/** The key of the configuration. e.g. */
String QUERY_COLLAPSE_INNER_HITS_SORTS = "query.collapse.inner.hits.sorts";
/** The key of the configuration. e.g. */
String QUERY_DEFAULT_LANGUAGES = "query.default.languages";
@ -2140,6 +2158,20 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
*/
String getIndexFieldContent();
/**
* Get the value for the key 'index.field.content_minhash'. <br>
* The value is, e.g. content_minhash <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getIndexFieldContentMinhash();
/**
* Get the value for the key 'index.field.content_minhash_bits'. <br>
* The value is, e.g. content_minhash_bits <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getIndexFieldContentMinhashBits();
/**
* Get the value for the key 'index.field.cache'. <br>
* The value is, e.g. cache <br>
@ -2593,6 +2625,58 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
*/
Integer getQueryAdditionalNotAnalyzedFieldsAsInteger();
/**
* Get the value for the key 'query.collapse.max.concurrent.group.results'. <br>
* The value is, e.g. 4 <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getQueryCollapseMaxConcurrentGroupResults();
/**
* Get the value for the key 'query.collapse.max.concurrent.group.results' as {@link Integer}. <br>
* The value is, e.g. 4 <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
* @throws NumberFormatException When the property is not integer.
*/
Integer getQueryCollapseMaxConcurrentGroupResultsAsInteger();
/**
* Get the value for the key 'query.collapse.inner.hits.name'. <br>
* The value is, e.g. similar_docs <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getQueryCollapseInnerHitsName();
/**
* Get the value for the key 'query.collapse.inner.hits.size'. <br>
* The value is, e.g. 0 <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getQueryCollapseInnerHitsSize();
/**
* Get the value for the key 'query.collapse.inner.hits.size' as {@link Integer}. <br>
* The value is, e.g. 0 <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
* @throws NumberFormatException When the property is not integer.
*/
Integer getQueryCollapseInnerHitsSizeAsInteger();
/**
* Get the value for the key 'query.collapse.inner.hits.sorts'. <br>
* The value is, e.g. <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getQueryCollapseInnerHitsSorts();
/**
* Get the value for the key 'query.collapse.inner.hits.sorts' as {@link Integer}. <br>
* The value is, e.g. <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
* @throws NumberFormatException When the property is not integer.
*/
Integer getQueryCollapseInnerHitsSortsAsInteger();
/**
* Get the value for the key 'query.default.languages'. <br>
* The value is, e.g. <br>
@ -5261,6 +5345,14 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
return get(FessConfig.INDEX_FIELD_CONTENT);
}
public String getIndexFieldContentMinhash() {
return get(FessConfig.INDEX_FIELD_content_minhash);
}
public String getIndexFieldContentMinhashBits() {
return get(FessConfig.INDEX_FIELD_content_minhash_bits);
}
public String getIndexFieldCache() {
return get(FessConfig.INDEX_FIELD_CACHE);
}
@ -5509,6 +5601,34 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
return getAsInteger(FessConfig.QUERY_ADDITIONAL_NOT_ANALYZED_FIELDS);
}
public String getQueryCollapseMaxConcurrentGroupResults() {
return get(FessConfig.QUERY_COLLAPSE_MAX_CONCURRENT_GROUP_RESULTS);
}
public Integer getQueryCollapseMaxConcurrentGroupResultsAsInteger() {
return getAsInteger(FessConfig.QUERY_COLLAPSE_MAX_CONCURRENT_GROUP_RESULTS);
}
public String getQueryCollapseInnerHitsName() {
return get(FessConfig.QUERY_COLLAPSE_INNER_HITS_NAME);
}
public String getQueryCollapseInnerHitsSize() {
return get(FessConfig.QUERY_COLLAPSE_INNER_HITS_SIZE);
}
public Integer getQueryCollapseInnerHitsSizeAsInteger() {
return getAsInteger(FessConfig.QUERY_COLLAPSE_INNER_HITS_SIZE);
}
public String getQueryCollapseInnerHitsSorts() {
return get(FessConfig.QUERY_COLLAPSE_INNER_HITS_SORTS);
}
public Integer getQueryCollapseInnerHitsSortsAsInteger() {
return getAsInteger(FessConfig.QUERY_COLLAPSE_INNER_HITS_SORTS);
}
public String getQueryDefaultLanguages() {
return get(FessConfig.QUERY_DEFAULT_LANGUAGES);
}

45
src/main/java/org/codelibs/fess/mylasta/direction/FessProp.java
View file

@ -47,6 +47,9 @@ import org.codelibs.fess.util.ComponentUtil;
import org.codelibs.fess.util.PrunedTag;
import org.dbflute.optional.OptionalThing;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.search.sort.SortBuilder;
import org.elasticsearch.search.sort.SortBuilders;
import org.elasticsearch.search.sort.SortOrder;
import org.lastaflute.job.LaJob;
import org.lastaflute.job.subsidiary.JobConcurrentExec;
import org.lastaflute.web.util.LaRequestUtil;
@ -58,6 +61,8 @@ import org.lastaflute.web.validation.theme.typed.LongTypeValidator;
public interface FessProp {
public static final String QUERY_COLLAPSE_INNER_HITS_SORTS = "queryCollapseInnerHitsSorts";
public static final String USER_CODE_PATTERN = "userCodePattern";
public static final String API_ADMIN_ACCESS_PERMISSION_SET = "apiAdminAccessPermissionSet";
@ -269,6 +274,14 @@ public interface FessProp {
return getSystemPropertyAsBoolean(Constants.LOGIN_REQUIRED_PROPERTY, false);
}
public default void setResultCollapsed(final boolean value) {
setSystemPropertyAsBoolean(Constants.RESULT_COLLAPSED_PROPERTY, value);
}
public default boolean isResultCollapsed() {
return getSystemPropertyAsBoolean(Constants.RESULT_COLLAPSED_PROPERTY, false);
}
public default void setLoginLinkEnabled(final boolean value) {
setSystemPropertyAsBoolean(Constants.LOGIN_LINK_ENALBED_PROPERTY, value);
}
@ -1504,4 +1517,36 @@ public interface FessProp {
}
return pattern.matcher(userCode).matches();
}
String getQueryCollapseInnerHitsSorts();
@SuppressWarnings("rawtypes")
public default OptionalThing<SortBuilder[]> getQueryCollapseInnerHitsSortBuilders() {
@SuppressWarnings("unchecked")
OptionalThing<SortBuilder[]> ot = (OptionalThing<SortBuilder[]>) propMap.get(QUERY_COLLAPSE_INNER_HITS_SORTS);
if (ot == null) {
String sorts = getQueryCollapseInnerHitsSorts();
if (StringUtil.isBlank(sorts)) {
ot = OptionalThing.empty();
} else {
SortBuilder[] sortBuilders =
split(sorts, ",").get(
stream -> stream
.filter(StringUtil::isNotBlank)
.map(s -> {
final String[] values = s.split(":");
if (values.length > 1) {
return SortBuilders.fieldSort(values[0]).order(
values[0].equalsIgnoreCase("desc") ? SortOrder.DESC : SortOrder.ASC);
} else {
return SortBuilders.fieldSort(values[0]).order(SortOrder.ASC);
}
}).toArray(n -> new SortBuilder[n]));
ot = OptionalThing.of(sortBuilders);
}
propMap.put(QUERY_COLLAPSE_INNER_HITS_SORTS, ot);
}
return ot;
}
}

97
src/main/java/org/codelibs/fess/util/QueryResponseList.java
View file

@ -24,6 +24,7 @@ import java.util.ListIterator;
import java.util.Map;
import org.apache.commons.lang3.StringUtils;
import org.codelibs.core.stream.StreamUtil;
import org.codelibs.fess.helper.QueryHelper;
import org.codelibs.fess.helper.ViewHelper;
import org.codelibs.fess.mylasta.direction.FessConfig;
@ -31,6 +32,7 @@ import org.dbflute.optional.OptionalEntity;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.common.text.Text;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHitField;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightField;
@ -84,6 +86,7 @@ public class QueryResponseList implements List<Map<String, Object>> {
public void init(final OptionalEntity<SearchResponse> searchResponseOpt, final int start, final int pageSize) {
searchResponseOpt.ifPresent(searchResponse -> {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final SearchHits searchHits = searchResponse.getHits();
allRecordCount = searchHits.getTotalHits();
queryTime = searchResponse.getTookInMillis();
@ -96,45 +99,28 @@ public class QueryResponseList implements List<Map<String, Object>> {
final QueryHelper queryHelper = ComponentUtil.getQueryHelper();
final String hlPrefix = queryHelper.getHighlightPrefix();
for (final SearchHit searchHit : searchHits.getHits()) {
final Map<String, Object> docMap = new HashMap<>();
if (searchHit.getSource() == null) {
searchHit.getFields().forEach((key, value) -> {
docMap.put(key, value.getValue());
});
} else {
docMap.putAll(searchHit.getSource());
}
final Map<String, Object> docMap = parseSearchHit(fessConfig, hlPrefix, searchHit);
final Map<String, HighlightField> highlightFields = searchHit.getHighlightFields();
try {
if (highlightFields != null) {
for (final Map.Entry<String, HighlightField> entry : highlightFields.entrySet()) {
final HighlightField highlightField = entry.getValue();
final Text[] fragments = highlightField.fragments();
if (fragments != null && fragments.length != 0) {
final String[] texts = new String[fragments.length];
for (int i = 0; i < fragments.length; i++) {
texts[i] = fragments[i].string();
if (fessConfig.isResultCollapsed()) {
final Map<String, SearchHits> innerHits = searchHit.getInnerHits();
if (innerHits != null) {
final SearchHits innerSearchHits = innerHits.get(fessConfig.getQueryCollapseInnerHitsName());
if (innerSearchHits != null) {
long totalHits = innerSearchHits.getTotalHits();
if (totalHits > 0) {
docMap.put(fessConfig.getQueryCollapseInnerHitsName() + "_count", totalHits);
SearchHitField bitsField = searchHit.getFields().get(fessConfig.getIndexFieldContentMinhashBits());
if (bitsField != null && !bitsField.getValues().isEmpty()) {
docMap.put(fessConfig.getQueryCollapseInnerHitsName() + "_hash", bitsField.getValues().get(0));
}
final String value = StringUtils.join(texts, "...");
docMap.put(hlPrefix + highlightField.getName(), value);
docMap.put(
fessConfig.getQueryCollapseInnerHitsName(),
StreamUtil.stream(innerSearchHits.getHits()).get(
stream -> stream.map(v -> parseSearchHit(fessConfig, hlPrefix, v)).toArray(
n -> new Map[n])));
}
}
}
} catch (final Exception e) {
if (logger.isDebugEnabled()) {
logger.debug("Could not create a highlighting value: " + docMap, e);
}
}
// ContentTitle
final ViewHelper viewHelper = ComponentUtil.getViewHelper();
if (viewHelper != null) {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
docMap.put(fessConfig.getResponseFieldContentTitle(), viewHelper.getContentTitle(docMap));
docMap.put(fessConfig.getResponseFieldContentDescription(), viewHelper.getContentDescription(docMap));
docMap.put(fessConfig.getResponseFieldUrlLink(), viewHelper.getUrlLink(docMap));
docMap.put(fessConfig.getResponseFieldSitePath(), viewHelper.getSitePath(docMap));
}
parent.add(docMap);
@ -151,6 +137,49 @@ public class QueryResponseList implements List<Map<String, Object>> {
calculatePageInfo(start, pageSize);
}
private Map<String, Object> parseSearchHit(final FessConfig fessConfig, final String hlPrefix, final SearchHit searchHit) {
final Map<String, Object> docMap = new HashMap<>();
if (searchHit.getSource() == null) {
searchHit.getFields().forEach((key, value) -> {
docMap.put(key, value.getValue());
});
} else {
docMap.putAll(searchHit.getSource());
}
final Map<String, HighlightField> highlightFields = searchHit.getHighlightFields();
try {
if (highlightFields != null) {
for (final Map.Entry<String, HighlightField> entry : highlightFields.entrySet()) {
final HighlightField highlightField = entry.getValue();
final Text[] fragments = highlightField.fragments();
if (fragments != null && fragments.length != 0) {
final String[] texts = new String[fragments.length];
for (int i = 0; i < fragments.length; i++) {
texts[i] = fragments[i].string();
}
final String value = StringUtils.join(texts, "...");
docMap.put(hlPrefix + highlightField.getName(), value);
}
}
}
} catch (final Exception e) {
if (logger.isDebugEnabled()) {
logger.debug("Could not create a highlighting value: " + docMap, e);
}
}
// ContentTitle
final ViewHelper viewHelper = ComponentUtil.getViewHelper();
if (viewHelper != null) {
docMap.put(fessConfig.getResponseFieldContentTitle(), viewHelper.getContentTitle(docMap));
docMap.put(fessConfig.getResponseFieldContentDescription(), viewHelper.getContentDescription(docMap));
docMap.put(fessConfig.getResponseFieldUrlLink(), viewHelper.getUrlLink(docMap));
docMap.put(fessConfig.getResponseFieldSitePath(), viewHelper.getSitePath(docMap));
}
return docMap;
}
protected void calculatePageInfo(final int start, final int size) {
pageSize = size;
allPageCount = (int) ((allRecordCount - 1) / pageSize) + 1;

6
src/main/resources/fess_config.properties
View file

@ -166,6 +166,8 @@ index.field.mimetype=mimetype
index.field.parent_id=parent_id
index.field.important_content=important_content
index.field.content=content
index.field.content_minhash=content_minhash
index.field.content_minhash_bits=content_minhash_bits
index.field.cache=cache
index.field.digest=digest
index.field.title=title
@ -221,6 +223,10 @@ query.additional.search.fields=
query.additional.facet.fields=
query.additional.sort.fields=
query.additional.not.analyzed.fields=
query.collapse.max.concurrent.group.results=4
query.collapse.inner.hits.name=similar_docs
query.collapse.inner.hits.size=0
query.collapse.inner.hits.sorts=
query.default.languages=
query.language.mapping=\
ar=ar\n\

21
src/main/resources/fess_indices/fess.json
View file

@ -406,6 +406,12 @@
"alphanum_word_filter" : {
"type" : "alphanum_word",
"max_token_length" : 20
},
"minhash_filter" : {
"type" : "minhash",
"seed" : 1,
"bit" : 2,
"size" : 64
}
},
"tokenizer": {
@ -756,6 +762,21 @@
"lowercase",
"stemmer_en_filter"
]
},
"minhash_analyzer": {
"type": "custom",
"char_filter": [
"mapping_ja_filter"
],
"tokenizer": "unigram_synonym_tokenizer",
"filter": [
"alphanum_word_filter",
"cjk_bigram",
"stopword_en_filter",
"lowercase",
"stemmer_en_filter",
"minhash_filter"
]
}
}
}

8
src/main/resources/fess_indices/fess/doc.json
View file

@ -456,6 +456,14 @@
"analyzer": "standard_analyzer",
"term_vector": "with_positions_offsets"
},
"content_minhash": {
"type": "minhash",
"minhash_analyzer": "minhash_analyzer",
"copy_bits_to": "content_minhash_bits"
},
"content_minhash_bits": {
"type": "keyword"
},
"content_length": {
"type": "long"
},

2
src/main/resources/fess_label.properties
View file

@ -217,6 +217,7 @@ labels.search_result_favorited=Liked
labels.search_click_count=Viewed ({0})
labels.search_result_more=more..
labels.search_result_cache=Cache
labels.search_result_similar=Similar Results ({0})
labels.facet_label_title=Label
labels.facet_timestamp_title=Date Range
labels.facet_timestamp_1day=Past 24 Hours
@ -366,6 +367,7 @@ labels.default_label_value=Default Label Value
labels.default_sort_value=Default Sort Value
labels.append_query_param_enabled=Append Params to URL
labels.login_required=Login Required
labels.result_collapsed=Similar Result Collapsed
labels.login_link=Login Link
labels.thumbnail=Thumbnail View
labels.ignore_failure_type=Excluded Failure Type

2
src/main/resources/fess_label_en.properties
View file

@ -217,6 +217,7 @@ labels.search_result_favorited=Liked
labels.search_click_count=Viewed ({0})
labels.search_result_more=more..
labels.search_result_cache=Cache
labels.search_result_similar=Similar Results ({0})
labels.facet_label_title=Label
labels.facet_timestamp_title=Date Range
labels.facet_timestamp_1day=Past 24 Hours
@ -366,6 +367,7 @@ labels.default_label_value=Default Label Value
labels.default_sort_value=Default Sort Value
labels.append_query_param_enabled=Append Params to URL
labels.login_required=Login Required
labels.result_collapsed=Similar Result Collapsed
labels.login_link=Login Link
labels.thumbnail=Thumbnail View
labels.ignore_failure_type=Excluded Failure Type

2
src/main/resources/fess_label_ja.properties
View file

@ -210,6 +210,7 @@ labels.search_result_favorited=Liked
labels.search_click_count=\u30af\u30ea\u30c3\u30af\u6570 ({0})
labels.search_result_more=\u8a73\u7d30..
labels.search_result_cache=\u30ad\u30e3\u30c3\u30b7\u30e5
labels.search_result_similar=\u985e\u4f3c\u7d50\u679c ({0})
labels.facet_label_title=\u30e9\u30d9\u30eb
labels.facet_timestamp_title=\u671f\u9593
labels.facet_timestamp_1day=24\u6642\u9593\u4ee5\u5185
@ -359,6 +360,7 @@ labels.default_label_value=\u30c7\u30d5\u30a9\u30eb\u30c8\u306e\u30e9\u30d9\u30e
labels.default_sort_value=\u30c7\u30d5\u30a9\u30eb\u30c8\u306e\u30bd\u30fc\u30c8\u5024
labels.append_query_param_enabled=\u691c\u7d22\u30d1\u30e9\u30e1\u30fc\u30bf\u306e\u8ffd\u52a0
labels.login_required=\u30ed\u30b0\u30a4\u30f3\u304c\u5fc5\u8981
labels.result_collapsed=\u91cd\u8907\u7d50\u679c\u306e\u6298\u308a\u7573\u307f
labels.login_link=\u30ed\u30b0\u30a4\u30f3\u30ea\u30f3\u30af\u8868\u793a
labels.thumbnail=\u30b5\u30e0\u30cd\u30a4\u30eb\u8868\u793a
labels.ignore_failure_type=\u9664\u5916\u3059\u308b\u30a8\u30e9\u30fc\u306e\u7a2e\u985e

12
src/main/webapp/WEB-INF/view/admin/general/admin_general.jsp
View file

@ -69,6 +69,18 @@
</div>
</div>
</div>
<div class="form-group">
<label for="resultCollapsed" class="col-sm-3 control-label"><la:message
key="labels.result_collapsed" /></label>
<div class="col-sm-9">
<la:errors property="resultCollapsed" />
<div class="checkbox">
<label> <la:checkbox property="resultCollapsed" /> <la:message
key="labels.enabled" />
</label>
</div>
</div>
</div>
<div class="form-group">
<label for="thumbnail" class="col-sm-3 control-label"><la:message
key="labels.thumbnail" /></label>

12
src/main/webapp/WEB-INF/view/searchResults.jsp
View file

@ -56,6 +56,14 @@
</la:link>
</small>
</c:if>
<c:if test="${doc.similar_docs_count!=null}">
<small class="hidden-md-down"> <la:link
href="/search?q=${f:u(q)}&ex_q=${f:u(queryEntry.value)}&sh=${f:u(doc.similar_docs_hash)}${fe:facetQuery()}${fe:geoQuery()}">
<la:message key="labels.search_result_similar"
arg0="${fe:formatNumber(doc.similar_docs_count)}" />
</la:link>
</small>
</c:if>
</div>
<div class="more hidden-md-up">
<a href="#result${s.index}"><la:message
@ -130,7 +138,7 @@
<c:if
test="${countEntry.value != 0 && fe:labelexists(countEntry.key)}">
<li class="list-group-item"><la:link
href="/search?q=${f:u(q)}&ex_q=label%3a${f:u(countEntry.key)}${fe:pagingQuery(null)}${fe:facetQuery()}${fe:geoQuery()}">
href="/search?q=${f:u(q)}&ex_q=label%3a${f:u(countEntry.key)}&sh=${f:u(sh)}${fe:pagingQuery(null)}${fe:facetQuery()}${fe:geoQuery()}">
${f:h(fe:label(countEntry.key))}
<span class="label label-default label-pill pull-right">${f:h(countEntry.value)}</span>
</la:link></li>
@ -147,7 +155,7 @@
<c:forEach var="queryEntry" items="${facetQueryView.queryMap}">
<c:if test="${facetResponse.queryCountMap[queryEntry.value] != 0}">
<li class="list-group-item p-l-md"><la:link
href="/search?q=${f:u(q)}&ex_q=${f:u(queryEntry.value)}${fe:pagingQuery(queryEntry.value)}${fe:facetQuery()}${fe:geoQuery()}">
href="/search?q=${f:u(q)}&ex_q=${f:u(queryEntry.value)}&sh=${f:u(sh)}${fe:pagingQuery(queryEntry.value)}${fe:facetQuery()}${fe:geoQuery()}">
<la:message key="${queryEntry.key}" />
<span class="label label-default label-pill pull-right">${f:h(facetResponse.queryCountMap[queryEntry.value])}</span>
</la:link></li>