diff --git a/src/main/java/org/codelibs/fess/Constants.java b/src/main/java/org/codelibs/fess/Constants.java index c953b8faf..06c35973b 100644 --- a/src/main/java/org/codelibs/fess/Constants.java +++ b/src/main/java/org/codelibs/fess/Constants.java @@ -330,4 +330,6 @@ public class Constants extends CoreLibConstants { public static final String ES_API_ACCESS_TOKEN = "esApiAccessToken"; public static final String ADMIN_PACKAGE = "org.codelibs.fess.app.web.admin"; + + public static final String DEFAULT_FIELD = "_default"; } diff --git a/src/main/java/org/codelibs/fess/app/service/KeyMatchService.java b/src/main/java/org/codelibs/fess/app/service/KeyMatchService.java index b8814a606..e87aa1d79 100644 --- a/src/main/java/org/codelibs/fess/app/service/KeyMatchService.java +++ b/src/main/java/org/codelibs/fess/app/service/KeyMatchService.java @@ -27,6 +27,7 @@ import org.codelibs.fess.app.pager.KeyMatchPager; import org.codelibs.fess.es.config.cbean.KeyMatchCB; import org.codelibs.fess.es.config.exbhv.KeyMatchBhv; import org.codelibs.fess.es.config.exentity.KeyMatch; +import org.codelibs.fess.mylasta.direction.FessConfig; import org.dbflute.cbean.result.PagingResultBean; import org.dbflute.optional.OptionalEntity; @@ -37,9 +38,8 @@ public class KeyMatchService implements Serializable { @Resource protected KeyMatchBhv keyMatchBhv; - public KeyMatchService() { - super(); - } + @Resource + protected FessConfig fessConfig; public List getKeyMatchList(final KeyMatchPager keyMatchPager) { @@ -113,6 +113,7 @@ public class KeyMatchService implements Serializable { public List getAvailableKeyMatchList() { return keyMatchBhv.selectList(cb -> { cb.query().matchAll(); + cb.fetchFirst(fessConfig.getPageKeymatchMaxFetchSizeAsInteger()); }); } diff --git a/src/main/java/org/codelibs/fess/entity/QueryContext.java b/src/main/java/org/codelibs/fess/entity/QueryContext.java index c7325f6f1..dd2f1a315 100644 --- a/src/main/java/org/codelibs/fess/entity/QueryContext.java +++ b/src/main/java/org/codelibs/fess/entity/QueryContext.java @@ -16,6 +16,7 @@ package org.codelibs.fess.entity; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -44,6 +45,8 @@ public class QueryContext { private Map> fieldLogMap = null; + private boolean disableRoleQuery = false; + @SuppressWarnings("unchecked") public QueryContext(final String queryString, final boolean isQuery) { this.queryString = queryString; @@ -114,6 +117,13 @@ public class QueryContext { list.add(text); } + public List getDefaultKeyword() { + if (fieldLogMap != null) { + return fieldLogMap.getOrDefault(Constants.DEFAULT_FIELD, Collections.emptyList()); + } + return Collections.emptyList(); + } + public void addHighlightedQuery(String text) { if (highlightedQuerySet != null) { highlightedQuerySet.add(text); @@ -124,4 +134,11 @@ public class QueryContext { return queryString; } + public boolean roleQueryEnabled() { + return !disableRoleQuery; + } + + public void skipRoleQuery() { + disableRoleQuery = true; + } } diff --git a/src/main/java/org/codelibs/fess/es/client/FessEsClient.java b/src/main/java/org/codelibs/fess/es/client/FessEsClient.java index d009cc563..8871c6410 100644 --- a/src/main/java/org/codelibs/fess/es/client/FessEsClient.java +++ b/src/main/java/org/codelibs/fess/es/client/FessEsClient.java @@ -760,6 +760,7 @@ public class FessEsClient implements Client { private int size = Constants.DEFAULT_PAGE_SIZE; private GeoInfo geoInfo; private FacetInfo facetInfo; + private boolean administrativeAccess = false; public static SearchConditionBuilder builder(final SearchRequestBuilder searchRequestBuilder) { return new SearchConditionBuilder(searchRequestBuilder); @@ -775,6 +776,7 @@ public class FessEsClient implements Client { } public SearchConditionBuilder administrativeAccess() { + administrativeAccess = true; return this; } @@ -815,6 +817,9 @@ public class FessEsClient implements Client { } final QueryContext queryContext = queryHelper.build(query, context -> { + if (administrativeAccess) { + context.skipRoleQuery(); + } // geo if (geoInfo != null && geoInfo.isAvailable()) { context.addQuery(boolQuery -> { diff --git a/src/main/java/org/codelibs/fess/helper/KeyMatchHelper.java b/src/main/java/org/codelibs/fess/helper/KeyMatchHelper.java index 4f564452d..21fcff2c1 100644 --- a/src/main/java/org/codelibs/fess/helper/KeyMatchHelper.java +++ b/src/main/java/org/codelibs/fess/helper/KeyMatchHelper.java @@ -15,27 +15,32 @@ */ package org.codelibs.fess.helper; -import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import javax.annotation.PostConstruct; -import org.codelibs.core.lang.StringUtil; +import org.codelibs.core.misc.Pair; import org.codelibs.fess.app.service.KeyMatchService; import org.codelibs.fess.es.client.FessEsClient; import org.codelibs.fess.es.client.FessEsClient.SearchConditionBuilder; import org.codelibs.fess.es.config.exentity.KeyMatch; import org.codelibs.fess.mylasta.direction.FessConfig; import org.codelibs.fess.util.ComponentUtil; +import org.codelibs.fess.util.DocumentUtil; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder; +import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilder; +import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders; import org.lastaflute.di.core.SingletonLaContainer; public class KeyMatchHelper { - protected volatile Map keyMatchQueryMap = Collections.emptyMap(); - - protected ThreadLocal> searchWordList = new ThreadLocal<>(); + protected volatile Map> keyMatchQueryMap = Collections.emptyMap(); protected long reloadInterval = 1000L; @@ -51,29 +56,32 @@ public class KeyMatchHelper { protected void reload(final long interval) { final FessConfig fessConfig = ComponentUtil.getFessConfig(); final KeyMatchService keyMatchService = SingletonLaContainer.getComponent(KeyMatchService.class); - final List list = keyMatchService.getAvailableKeyMatchList(); - final Map keyMatchQueryMap = new HashMap(list.size()); - for (final KeyMatch keyMatch : list) { - final List> documentList = getDocumentList(keyMatch); - final List docIdList = new ArrayList(); - for (final Map map : documentList) { - final String docId = (String) map.get(fessConfig.getIndexFieldDocId()); - if (StringUtil.isNotBlank(docId)) { - docIdList.add(fessConfig.getIndexFieldDocId() + ":" + docId + "^" + keyMatch.getBoost()); - } - } - if (!docIdList.isEmpty()) { - keyMatchQueryMap.put(keyMatch.getTerm(), docIdList.toArray(new String[docIdList.size()])); - } + final Map> keyMatchQueryMap = new HashMap<>(); + keyMatchService + .getAvailableKeyMatchList() + .stream() + .forEach( + keyMatch -> { + BoolQueryBuilder boolQuery = QueryBuilders.boolQuery(); + getDocumentList(keyMatch).stream().map(doc -> { + return DocumentUtil.getValue(doc, fessConfig.getIndexFieldDocId(), String.class); + }).forEach(docId -> { + boolQuery.should(QueryBuilders.termQuery(fessConfig.getIndexFieldDocId(), docId)); + }); - if (reloadInterval > 0) { - try { - Thread.sleep(reloadInterval); - } catch (final InterruptedException e) { - // ignore - } - } - } + if (boolQuery.hasClauses()) { + keyMatchQueryMap.put(toLowerCase(keyMatch.getTerm()), + new Pair<>(boolQuery, ScoreFunctionBuilders.weightFactorFunction(keyMatch.getBoost()))); + } + + if (reloadInterval > 0) { + try { + Thread.sleep(reloadInterval); + } catch (final InterruptedException e) { + // ignore + } + } + }); this.keyMatchQueryMap = keyMatchQueryMap; } @@ -89,28 +97,6 @@ public class KeyMatchHelper { return documentList; } - public void clear() { - searchWordList.remove(); - } - - public void addSearchWord(final String word) { - final String[] values = keyMatchQueryMap.get(word); - if (values != null) { - List list = searchWordList.get(); - if (list == null) { - list = new ArrayList<>(); - searchWordList.set(list); - } - for (final String value : values) { - list.add(value); - } - } - } - - public List getDocIdQueryList() { - return searchWordList.get(); - } - public long getReloadInterval() { return reloadInterval; } @@ -118,4 +104,18 @@ public class KeyMatchHelper { public void setReloadInterval(final long reloadInterval) { this.reloadInterval = reloadInterval; } + + public void buildQuery(List keywordList, FunctionScoreQueryBuilder functionScoreQuery) { + keywordList.stream().forEach(keyword -> { + Pair pair = keyMatchQueryMap.get(toLowerCase(keyword)); + if (pair != null) { + functionScoreQuery.add(pair.getFirst(), pair.getSecond()); + } + }); + } + + private String toLowerCase(String term) { + return term != null ? term.toLowerCase(Locale.ROOT) : term; + } + } diff --git a/src/main/java/org/codelibs/fess/helper/QueryHelper.java b/src/main/java/org/codelibs/fess/helper/QueryHelper.java index 37b2908ab..4ff97b360 100644 --- a/src/main/java/org/codelibs/fess/helper/QueryHelper.java +++ b/src/main/java/org/codelibs/fess/helper/QueryHelper.java @@ -73,8 +73,6 @@ public class QueryHelper implements Serializable { protected static final long serialVersionUID = 1L; - private static final String DEFAULT_FIELD = "_default"; - protected static final String SCORE_FIELD = "score"; protected static final String INURL_FIELD = "inurl"; @@ -224,22 +222,17 @@ public class QueryHelper implements Serializable { final QueryContext queryContext = new QueryContext(q, true); buildBaseQuery(queryContext, context); - buildBoostQuery(queryContext); + buildRoleQuery(queryContext); - if (keyMatchHelper != null) { - final List docIdQueryList = keyMatchHelper.getDocIdQueryList(); - if (docIdQueryList != null && !docIdQueryList.isEmpty()) { - queryContext.addQuery(boolQuery -> { - for (final String docIdQuery : docIdQueryList) { - // TODO id query? - boolQuery.should(QueryBuilders.queryStringQuery(docIdQuery)); - } - }); - } + if (!queryContext.hasSorts() && defaultSortBuilders != null) { + queryContext.addSorts(defaultSortBuilders); } + return queryContext; + } - if (roleQueryHelper != null) { + protected void buildRoleQuery(final QueryContext queryContext) { + if (roleQueryHelper != null && queryContext.roleQueryEnabled()) { final Set roleSet = roleQueryHelper.build(); if (!roleSet.isEmpty()) { queryContext.addQuery(boolQuery -> { @@ -251,16 +244,14 @@ public class QueryHelper implements Serializable { }); } } - - if (!queryContext.hasSorts() && defaultSortBuilders != null) { - queryContext.addSorts(defaultSortBuilders); - } - return queryContext; } - private void buildBoostQuery(final QueryContext queryContext) { + protected void buildBoostQuery(final QueryContext queryContext) { queryContext.addFunctionScore(functionScoreQuery -> { functionScoreQuery.add(ScoreFunctionBuilders.fieldValueFactorFunction(fessConfig.getIndexFieldBoost())); + if (keyMatchHelper != null) { + keyMatchHelper.buildQuery(queryContext.getDefaultKeyword(), functionScoreQuery); + } }); } @@ -282,7 +273,7 @@ public class QueryHelper implements Serializable { } protected QueryParser getQueryParser() { - return new ExtendableQueryParser(DEFAULT_FIELD, new WhitespaceAnalyzer()); + return new ExtendableQueryParser(Constants.DEFAULT_FIELD, new WhitespaceAnalyzer()); } protected QueryBuilder convertQuery(final QueryContext context, final Query query) { @@ -329,7 +320,7 @@ public class QueryHelper implements Serializable { protected QueryBuilder convertWildcardQuery(final QueryContext context, final WildcardQuery wildcardQuery) { final String field = wildcardQuery.getField(); - if (DEFAULT_FIELD.equals(field)) { + if (Constants.DEFAULT_FIELD.equals(field)) { context.addFieldLog(field, wildcardQuery.getTerm().text()); return buildDefaultQueryBuilder(f -> QueryBuilders.wildcardQuery(f, wildcardQuery.getTerm().text())); } else if (isSearchField(field)) { @@ -337,7 +328,7 @@ public class QueryHelper implements Serializable { return QueryBuilders.wildcardQuery(field, wildcardQuery.getTerm().text()).boost(wildcardQuery.getBoost()); } else { final String origQuery = wildcardQuery.getTerm().toString(); - context.addFieldLog(DEFAULT_FIELD, origQuery); + context.addFieldLog(Constants.DEFAULT_FIELD, origQuery); context.addHighlightedQuery(origQuery); return buildDefaultQueryBuilder(f -> QueryBuilders.wildcardQuery(f, origQuery)); } @@ -345,7 +336,7 @@ public class QueryHelper implements Serializable { protected QueryBuilder convertPrefixQuery(final QueryContext context, final PrefixQuery prefixQuery) { final String field = prefixQuery.getField(); - if (DEFAULT_FIELD.equals(field)) { + if (Constants.DEFAULT_FIELD.equals(field)) { context.addFieldLog(field, prefixQuery.getPrefix().text()); return buildDefaultQueryBuilder(f -> QueryBuilders.prefixQuery(f, prefixQuery.getPrefix().text())); } else if (isSearchField(field)) { @@ -353,7 +344,7 @@ public class QueryHelper implements Serializable { return QueryBuilders.prefixQuery(field, prefixQuery.getPrefix().text()).boost(prefixQuery.getBoost()); } else { final String origQuery = prefixQuery.getPrefix().toString(); - context.addFieldLog(DEFAULT_FIELD, origQuery); + context.addFieldLog(Constants.DEFAULT_FIELD, origQuery); context.addHighlightedQuery(origQuery); return buildDefaultQueryBuilder(f -> QueryBuilders.prefixQuery(f, origQuery)); } @@ -363,7 +354,7 @@ public class QueryHelper implements Serializable { final Term term = fuzzyQuery.getTerm(); final String field = term.field(); // TODO fuzzy value - if (DEFAULT_FIELD.equals(field)) { + if (Constants.DEFAULT_FIELD.equals(field)) { context.addFieldLog(field, term.text()); return buildDefaultQueryBuilder(f -> QueryBuilders.fuzzyQuery(f, term.text()).fuzziness( Fuzziness.fromEdits(fuzzyQuery.getMaxEdits()))); @@ -373,7 +364,7 @@ public class QueryHelper implements Serializable { .fuzziness(Fuzziness.fromEdits(fuzzyQuery.getMaxEdits())); } else { final String origQuery = fuzzyQuery.toString(); - context.addFieldLog(DEFAULT_FIELD, origQuery); + context.addFieldLog(Constants.DEFAULT_FIELD, origQuery); context.addHighlightedQuery(origQuery); return buildDefaultQueryBuilder(f -> QueryBuilders.fuzzyQuery(f, origQuery).fuzziness( Fuzziness.fromEdits(fuzzyQuery.getMaxEdits()))); @@ -405,7 +396,7 @@ public class QueryHelper implements Serializable { return rangeQuery; } else { final String origQuery = termRangeQuery.toString(); - context.addFieldLog(DEFAULT_FIELD, origQuery); + context.addFieldLog(Constants.DEFAULT_FIELD, origQuery); context.addHighlightedQuery(origQuery); return buildDefaultQueryBuilder(f -> QueryBuilders.matchPhraseQuery(f, origQuery)); } @@ -414,7 +405,7 @@ public class QueryHelper implements Serializable { protected QueryBuilder convertTermQuery(final QueryContext context, final TermQuery termQuery) { final String field = termQuery.getTerm().field(); final String text = termQuery.getTerm().text(); - if (DEFAULT_FIELD.equals(field)) { + if (Constants.DEFAULT_FIELD.equals(field)) { context.addFieldLog(field, text); context.addHighlightedQuery(text); return buildDefaultQueryBuilder(f -> QueryBuilders.matchPhraseQuery(f, text)); @@ -449,7 +440,7 @@ public class QueryHelper implements Serializable { return QueryBuilders.matchPhraseQuery(field, text).boost(termQuery.getBoost()); } else { final String origQuery = termQuery.toString(); - context.addFieldLog(DEFAULT_FIELD, origQuery); + context.addFieldLog(Constants.DEFAULT_FIELD, origQuery); context.addHighlightedQuery(origQuery); return buildDefaultQueryBuilder(f -> QueryBuilders.matchPhraseQuery(f, origQuery)); } diff --git a/src/main/java/org/codelibs/fess/job/SuggestJob.java b/src/main/java/org/codelibs/fess/job/SuggestJob.java index 3e34f7f73..709b250ad 100644 --- a/src/main/java/org/codelibs/fess/job/SuggestJob.java +++ b/src/main/java/org/codelibs/fess/job/SuggestJob.java @@ -82,7 +82,6 @@ public class SuggestJob { try { executeSuggestCreater(); - ComponentUtil.getKeyMatchHelper().update(); } catch (final FessSystemException e) { throw e; } catch (final Exception e) { diff --git a/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java b/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java index 31615d356..0472f058c 100644 --- a/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java +++ b/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java @@ -139,6 +139,9 @@ public interface FessConfig extends FessEnv { /** The key of the configuration. e.g. 1000 */ String PAGE_DOCBOOST_MAX_FETCH_SIZE = "page.docboost.max.fetch.size"; + /** The key of the configuration. e.g. 1000 */ + String PAGE_KEYMATCH_MAX_FETCH_SIZE = "page.keymatch.max.fetch.size"; + /** The key of the configuration. e.g. 1000 */ String PAGE_ROLE_MAX_FETCH_SIZE = "page.role.max.fetch.size"; @@ -608,6 +611,21 @@ public interface FessConfig extends FessEnv { */ Integer getPageDocboostMaxFetchSizeAsInteger(); + /** + * Get the value for the key 'page.keymatch.max.fetch.size'.
+ * The value is, e.g. 1000
+ * @return The value of found property. (NotNull: if not found, exception but basically no way) + */ + String getPageKeymatchMaxFetchSize(); + + /** + * Get the value for the key 'page.keymatch.max.fetch.size' as {@link Integer}.
+ * The value is, e.g. 1000
+ * @return The value of found property. (NotNull: if not found, exception but basically no way) + * @throws NumberFormatException When the property is not integer. + */ + Integer getPageKeymatchMaxFetchSizeAsInteger(); + /** * Get the value for the key 'page.role.max.fetch.size'.
* The value is, e.g. 1000
@@ -1071,6 +1089,14 @@ public interface FessConfig extends FessEnv { return getAsInteger(FessConfig.PAGE_DOCBOOST_MAX_FETCH_SIZE); } + public String getPageKeymatchMaxFetchSize() { + return get(FessConfig.PAGE_KEYMATCH_MAX_FETCH_SIZE); + } + + public Integer getPageKeymatchMaxFetchSizeAsInteger() { + return getAsInteger(FessConfig.PAGE_KEYMATCH_MAX_FETCH_SIZE); + } + public String getPageRoleMaxFetchSize() { return get(FessConfig.PAGE_ROLE_MAX_FETCH_SIZE); } diff --git a/src/main/resources/fess_config.properties b/src/main/resources/fess_config.properties index 68a3486ea..f6e6e51bb 100644 --- a/src/main/resources/fess_config.properties +++ b/src/main/resources/fess_config.properties @@ -85,6 +85,7 @@ paging.page.range.fill.limit = true # max page size page.docboost.max.fetch.size=1000 +page.keymatch.max.fetch.size=1000 page.role.max.fetch.size=1000 page.group.max.fetch.size=1000