diff --git a/src/main/java/org/codelibs/fess/app/service/BoostDocumentRuleService.java b/src/main/java/org/codelibs/fess/app/service/BoostDocumentRuleService.java index d09fdcd18..0ee137d8b 100644 --- a/src/main/java/org/codelibs/fess/app/service/BoostDocumentRuleService.java +++ b/src/main/java/org/codelibs/fess/app/service/BoostDocumentRuleService.java @@ -27,6 +27,7 @@ import org.codelibs.fess.app.pager.BoostDocumentRulePager; import org.codelibs.fess.es.config.cbean.BoostDocumentRuleCB; import org.codelibs.fess.es.config.exbhv.BoostDocumentRuleBhv; import org.codelibs.fess.es.config.exentity.BoostDocumentRule; +import org.codelibs.fess.mylasta.direction.FessConfig; import org.dbflute.cbean.result.PagingResultBean; import org.dbflute.optional.OptionalEntity; @@ -37,9 +38,8 @@ public class BoostDocumentRuleService implements Serializable { @Resource protected BoostDocumentRuleBhv boostDocumentRuleBhv; - public BoostDocumentRuleService() { - super(); - } + @Resource + protected FessConfig fessConfig; public List getBoostDocumentRuleList(final BoostDocumentRulePager boostDocumentRulePager) { @@ -112,7 +112,9 @@ public class BoostDocumentRuleService implements Serializable { public List getAvailableBoostDocumentRuleList() { return boostDocumentRuleBhv.selectList(cb -> { + cb.query().matchAll(); cb.query().addOrderBy_SortOrder_Asc(); + cb.fetchFirst(fessConfig.getPageDocboostMaxFetchSizeAsInteger()); }); } diff --git a/src/main/java/org/codelibs/fess/app/web/admin/log/AdminLogAction.java b/src/main/java/org/codelibs/fess/app/web/admin/log/AdminLogAction.java index f3718f382..e47959331 100644 --- a/src/main/java/org/codelibs/fess/app/web/admin/log/AdminLogAction.java +++ b/src/main/java/org/codelibs/fess/app/web/admin/log/AdminLogAction.java @@ -17,6 +17,7 @@ package org.codelibs.fess.app.web.admin.log; import java.io.IOException; import java.io.InputStream; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -40,8 +41,6 @@ import org.lastaflute.web.callback.ActionRuntime; import org.lastaflute.web.response.ActionResponse; import org.lastaflute.web.response.HtmlResponse; -import com.google.common.base.Charsets; - /** * @author codelibs * @author Keiichi Watanabe @@ -64,7 +63,7 @@ public class AdminLogAction extends FessAdminAction { @Execute public ActionResponse download(final String id) { - String filename = new String(Base64.getDecoder().decode(id), Charsets.UTF_8).replace("..", "").replaceAll("\\s", ""); + String filename = new String(Base64.getDecoder().decode(id), StandardCharsets.UTF_8).replace("..", "").replaceAll("\\s", ""); final String logFilePath = systemHelper.getLogFilePath(); if (StringUtil.isNotBlank(logFilePath)) { Path path = Paths.get(logFilePath, filename); @@ -89,7 +88,7 @@ public class AdminLogAction extends FessAdminAction { stream.filter(entry -> entry.getFileName().toString().endsWith(".log")).forEach(filePath -> { Map map = new HashMap<>(); String name = filePath.getFileName().toString(); - map.put("id", Base64.getEncoder().encodeToString(name.getBytes(Charsets.UTF_8))); + map.put("id", Base64.getEncoder().encodeToString(name.getBytes(StandardCharsets.UTF_8))); map.put("name", name); try { map.put("lastModified", new Date(Files.getLastModifiedTime(filePath).toMillis())); diff --git a/src/main/java/org/codelibs/fess/app/web/admin/user/AdminUserAction.java b/src/main/java/org/codelibs/fess/app/web/admin/user/AdminUserAction.java index 5375eaa22..2acf4d44d 100644 --- a/src/main/java/org/codelibs/fess/app/web/admin/user/AdminUserAction.java +++ b/src/main/java/org/codelibs/fess/app/web/admin/user/AdminUserAction.java @@ -121,8 +121,8 @@ public class AdminUserAction extends FessAdminAction { } private void registerForms(final RenderData data) { - data.register("roleItems", roleService.getAvailableRoleList(fessConfig.getFormRoleListSizeAsInteger())); - data.register("groupItems", groupService.getAvailableGroupList(fessConfig.getFormGroupListSizeAsInteger())); + data.register("roleItems", roleService.getAvailableRoleList(fessConfig.getPageRoleMaxFetchSizeAsInteger())); + data.register("groupItems", groupService.getAvailableGroupList(fessConfig.getPageGroupMaxFetchSizeAsInteger())); } // =================================================================================== diff --git a/src/main/java/org/codelibs/fess/entity/QueryContext.java b/src/main/java/org/codelibs/fess/entity/QueryContext.java index ab25e2f9a..c7325f6f1 100644 --- a/src/main/java/org/codelibs/fess/entity/QueryContext.java +++ b/src/main/java/org/codelibs/fess/entity/QueryContext.java @@ -29,6 +29,7 @@ import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder; import org.elasticsearch.search.sort.SortBuilder; import org.lastaflute.web.util.LaRequestUtil; @@ -59,6 +60,12 @@ public class QueryContext { } } + public void addFunctionScore(final Consumer functionScoreQuery) { + FunctionScoreQueryBuilder builder = QueryBuilders.functionScoreQuery(queryBuilder); + functionScoreQuery.accept(builder); + queryBuilder = builder; + } + public void addQuery(final Consumer boolQuery) { BoolQueryBuilder builder; if (queryBuilder instanceof BoolQueryBuilder) { diff --git a/src/main/java/org/codelibs/fess/es/client/FessEsClient.java b/src/main/java/org/codelibs/fess/es/client/FessEsClient.java index dfd5c235f..d009cc563 100644 --- a/src/main/java/org/codelibs/fess/es/client/FessEsClient.java +++ b/src/main/java/org/codelibs/fess/es/client/FessEsClient.java @@ -34,7 +34,6 @@ import java.util.stream.Collectors; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; -import org.apache.commons.codec.Charsets; import org.codelibs.core.beans.util.BeanUtil; import org.codelibs.core.io.FileUtil; import org.codelibs.core.io.ResourceUtil; @@ -50,8 +49,8 @@ import org.codelibs.fess.entity.PingResponse; import org.codelibs.fess.entity.QueryContext; import org.codelibs.fess.exception.FessSystemException; import org.codelibs.fess.exception.ResultOffsetExceededException; +import org.codelibs.fess.exception.SearchQueryException; import org.codelibs.fess.helper.QueryHelper; -import org.codelibs.fess.indexer.FessSearchQueryException; import org.codelibs.fess.mylasta.direction.FessConfig; import org.codelibs.fess.util.ComponentUtil; import org.codelibs.fess.util.StreamUtil; @@ -841,7 +840,7 @@ public class FessEsClient implements Client { if (facetInfo != null) { StreamUtil.of(facetInfo.field).forEach(f -> { if (queryHelper.isFacetField(f)) { - final String encodedField = BaseEncoding.base64().encode(f.getBytes(Charsets.UTF_8)); + final String encodedField = BaseEncoding.base64().encode(f.getBytes(StandardCharsets.UTF_8)); final TermsBuilder termsBuilder = AggregationBuilders.terms(Constants.FACET_FIELD_PREFIX + encodedField).field(f); // TODO order if (facetInfo.limit != null) { @@ -850,7 +849,7 @@ public class FessEsClient implements Client { } searchRequestBuilder.addAggregation(termsBuilder); } else { - throw new FessSearchQueryException("Invalid facet field: " + f); + throw new SearchQueryException("Invalid facet field: " + f); } }); StreamUtil.of(facetInfo.query).forEach( diff --git a/src/main/java/org/codelibs/fess/es/config/allcommon/EsAbstractBehavior.java b/src/main/java/org/codelibs/fess/es/config/allcommon/EsAbstractBehavior.java index 3167ce40c..15bebea3a 100644 --- a/src/main/java/org/codelibs/fess/es/config/allcommon/EsAbstractBehavior.java +++ b/src/main/java/org/codelibs/fess/es/config/allcommon/EsAbstractBehavior.java @@ -94,7 +94,7 @@ public abstract class EsAbstractBehavior docIdQueryList = keyMatchHelper.getDocIdQueryList(); if (docIdQueryList != null && !docIdQueryList.isEmpty()) { @@ -238,7 +242,6 @@ public class QueryHelper implements Serializable { if (roleQueryHelper != null) { final Set roleSet = roleQueryHelper.build(); if (!roleSet.isEmpty()) { - final FessConfig fessConfig = ComponentUtil.getFessConfig(); queryContext.addQuery(boolQuery -> { BoolQueryBuilder roleQuery = QueryBuilders.boolQuery(); roleSet.stream().forEach(name -> { @@ -255,6 +258,12 @@ public class QueryHelper implements Serializable { return queryContext; } + private void buildBoostQuery(final QueryContext queryContext) { + queryContext.addFunctionScore(functionScoreQuery -> { + functionScoreQuery.add(ScoreFunctionBuilders.fieldValueFactorFunction(fessConfig.getIndexFieldBoost())); + }); + } + public void buildBaseQuery(final QueryContext queryContext, final Consumer context) { final QueryParser queryParser = getQueryParser(); try { @@ -456,7 +465,6 @@ public class QueryHelper implements Serializable { } private QueryBuilder buildDefaultQueryBuilder(final Function builder) { - final FessConfig fessConfig = ComponentUtil.getFessConfig(); // TODO boost final BoolQueryBuilder boolQuery = QueryBuilders.boolQuery(); final QueryBuilder titleQuery = builder.apply(fessConfig.getIndexFieldTitle()); diff --git a/src/main/java/org/codelibs/fess/helper/WebFsIndexHelper.java b/src/main/java/org/codelibs/fess/helper/WebFsIndexHelper.java index 4be4a9928..4c0202a21 100644 --- a/src/main/java/org/codelibs/fess/helper/WebFsIndexHelper.java +++ b/src/main/java/org/codelibs/fess/helper/WebFsIndexHelper.java @@ -36,7 +36,6 @@ import org.codelibs.fess.crawler.interval.FessIntervalController; import org.codelibs.fess.crawler.service.DataService; import org.codelibs.fess.crawler.service.UrlFilterService; import org.codelibs.fess.crawler.service.UrlQueueService; -import org.codelibs.fess.es.config.exentity.BoostDocumentRule; import org.codelibs.fess.es.config.exentity.FileConfig; import org.codelibs.fess.es.config.exentity.WebConfig; import org.codelibs.fess.indexer.IndexUpdater; @@ -386,9 +385,9 @@ public class WebFsIndexHelper implements Serializable { indexUpdater.setSessionIdList(sessionIdList); indexUpdater.setDaemon(true); indexUpdater.setCrawlerList(crawlerList); - for (final BoostDocumentRule rule : boostDocumentRuleService.getAvailableBoostDocumentRuleList()) { + boostDocumentRuleService.getAvailableBoostDocumentRuleList().forEach(rule -> { indexUpdater.addDocBoostMatcher(new org.codelibs.fess.indexer.DocBoostMatcher(rule)); - } + }); indexUpdater.start(); int startedCrawlerNum = 0; diff --git a/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java b/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java index 6c85ad48a..31615d356 100644 --- a/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java +++ b/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java @@ -136,11 +136,14 @@ public interface FessConfig extends FessEnv { /** The key of the configuration. e.g. true */ String PAGING_PAGE_RANGE_FILL_LIMIT = "paging.page.range.fill.limit"; - /** The key of the configuration. e.g. 100 */ - String FORM_ROLE_LIST_SIZE = "form.role.list.size"; + /** The key of the configuration. e.g. 1000 */ + String PAGE_DOCBOOST_MAX_FETCH_SIZE = "page.docboost.max.fetch.size"; - /** The key of the configuration. e.g. 100 */ - String FORM_GROUP_LIST_SIZE = "form.group.list.size"; + /** The key of the configuration. e.g. 1000 */ + String PAGE_ROLE_MAX_FETCH_SIZE = "page.role.max.fetch.size"; + + /** The key of the configuration. e.g. 1000 */ + String PAGE_GROUP_MAX_FETCH_SIZE = "page.group.max.fetch.size"; /** The key of the configuration. e.g. Administrator */ String MAIL_FROM_NAME = "mail.from.name"; @@ -589,34 +592,51 @@ public interface FessConfig extends FessEnv { boolean isPagingPageRangeFillLimit(); /** - * Get the value for the key 'form.role.list.size'.
- * The value is, e.g. 100
+ * Get the value for the key 'page.docboost.max.fetch.size'.
+ * The value is, e.g. 1000
+ * comment: max page size * @return The value of found property. (NotNull: if not found, exception but basically no way) */ - String getFormRoleListSize(); + String getPageDocboostMaxFetchSize(); /** - * Get the value for the key 'form.role.list.size' as {@link Integer}.
- * The value is, e.g. 100
+ * Get the value for the key 'page.docboost.max.fetch.size' as {@link Integer}.
+ * The value is, e.g. 1000
+ * comment: max page size * @return The value of found property. (NotNull: if not found, exception but basically no way) * @throws NumberFormatException When the property is not integer. */ - Integer getFormRoleListSizeAsInteger(); + Integer getPageDocboostMaxFetchSizeAsInteger(); /** - * Get the value for the key 'form.group.list.size'.
- * The value is, e.g. 100
+ * Get the value for the key 'page.role.max.fetch.size'.
+ * The value is, e.g. 1000
* @return The value of found property. (NotNull: if not found, exception but basically no way) */ - String getFormGroupListSize(); + String getPageRoleMaxFetchSize(); /** - * Get the value for the key 'form.group.list.size' as {@link Integer}.
- * The value is, e.g. 100
+ * Get the value for the key 'page.role.max.fetch.size' as {@link Integer}.
+ * The value is, e.g. 1000
* @return The value of found property. (NotNull: if not found, exception but basically no way) * @throws NumberFormatException When the property is not integer. */ - Integer getFormGroupListSizeAsInteger(); + Integer getPageRoleMaxFetchSizeAsInteger(); + + /** + * Get the value for the key 'page.group.max.fetch.size'.
+ * The value is, e.g. 1000
+ * @return The value of found property. (NotNull: if not found, exception but basically no way) + */ + String getPageGroupMaxFetchSize(); + + /** + * Get the value for the key 'page.group.max.fetch.size' as {@link Integer}.
+ * The value is, e.g. 1000
+ * @return The value of found property. (NotNull: if not found, exception but basically no way) + * @throws NumberFormatException When the property is not integer. + */ + Integer getPageGroupMaxFetchSizeAsInteger(); /** * Get the value for the key 'mail.from.name'.
@@ -1043,20 +1063,28 @@ public interface FessConfig extends FessEnv { return is(FessConfig.PAGING_PAGE_RANGE_FILL_LIMIT); } - public String getFormRoleListSize() { - return get(FessConfig.FORM_ROLE_LIST_SIZE); + public String getPageDocboostMaxFetchSize() { + return get(FessConfig.PAGE_DOCBOOST_MAX_FETCH_SIZE); } - public Integer getFormRoleListSizeAsInteger() { - return getAsInteger(FessConfig.FORM_ROLE_LIST_SIZE); + public Integer getPageDocboostMaxFetchSizeAsInteger() { + return getAsInteger(FessConfig.PAGE_DOCBOOST_MAX_FETCH_SIZE); } - public String getFormGroupListSize() { - return get(FessConfig.FORM_GROUP_LIST_SIZE); + public String getPageRoleMaxFetchSize() { + return get(FessConfig.PAGE_ROLE_MAX_FETCH_SIZE); } - public Integer getFormGroupListSizeAsInteger() { - return getAsInteger(FessConfig.FORM_GROUP_LIST_SIZE); + public Integer getPageRoleMaxFetchSizeAsInteger() { + return getAsInteger(FessConfig.PAGE_ROLE_MAX_FETCH_SIZE); + } + + public String getPageGroupMaxFetchSize() { + return get(FessConfig.PAGE_GROUP_MAX_FETCH_SIZE); + } + + public Integer getPageGroupMaxFetchSizeAsInteger() { + return getAsInteger(FessConfig.PAGE_GROUP_MAX_FETCH_SIZE); } public String getMailFromName() { diff --git a/src/main/java/org/codelibs/fess/util/FacetResponse.java b/src/main/java/org/codelibs/fess/util/FacetResponse.java index 5ace0f252..fcfbc65f5 100644 --- a/src/main/java/org/codelibs/fess/util/FacetResponse.java +++ b/src/main/java/org/codelibs/fess/util/FacetResponse.java @@ -15,6 +15,7 @@ */ package org.codelibs.fess.util; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; @@ -25,7 +26,6 @@ import org.elasticsearch.search.aggregations.Aggregations; import org.elasticsearch.search.aggregations.bucket.filter.Filter; import org.elasticsearch.search.aggregations.bucket.terms.Terms; -import com.google.common.base.Charsets; import com.google.common.io.BaseEncoding; public class FacetResponse { @@ -34,17 +34,19 @@ public class FacetResponse { protected List fieldList = new ArrayList(); public FacetResponse(final Aggregations aggregations) { - aggregations.forEach(aggregation -> { - if (aggregation.getName().startsWith(Constants.FACET_FIELD_PREFIX)) { - final Terms termFacet = (Terms) aggregation; - fieldList.add(new Field(termFacet)); - } else if (aggregation.getName().startsWith(Constants.FACET_QUERY_PREFIX)) { - final Filter queryFacet = (Filter) aggregation; - final String encodedQuery = queryFacet.getName().substring(Constants.FACET_QUERY_PREFIX.length()); - queryCountMap.put(new String(BaseEncoding.base64().decode(encodedQuery), Charsets.UTF_8), queryFacet.getDocCount()); - } + aggregations + .forEach(aggregation -> { + if (aggregation.getName().startsWith(Constants.FACET_FIELD_PREFIX)) { + final Terms termFacet = (Terms) aggregation; + fieldList.add(new Field(termFacet)); + } else if (aggregation.getName().startsWith(Constants.FACET_QUERY_PREFIX)) { + final Filter queryFacet = (Filter) aggregation; + final String encodedQuery = queryFacet.getName().substring(Constants.FACET_QUERY_PREFIX.length()); + queryCountMap.put(new String(BaseEncoding.base64().decode(encodedQuery), StandardCharsets.UTF_8), + queryFacet.getDocCount()); + } - }); + }); } public boolean hasFacetResponse() { @@ -58,7 +60,7 @@ public class FacetResponse { public Field(final Terms termFacet) { final String encodedField = termFacet.getName().substring(Constants.FACET_FIELD_PREFIX.length()); - name = new String(BaseEncoding.base64().decode(encodedField), Charsets.UTF_8); + name = new String(BaseEncoding.base64().decode(encodedField), StandardCharsets.UTF_8); for (final Terms.Bucket tfEntry : termFacet.getBuckets()) { valueCountMap.put(tfEntry.getKeyAsString(), tfEntry.getDocCount()); } diff --git a/src/main/resources/fess_config.properties b/src/main/resources/fess_config.properties index f0c452ef9..68a3486ea 100644 --- a/src/main/resources/fess_config.properties +++ b/src/main/resources/fess_config.properties @@ -83,8 +83,10 @@ paging.page.range.size = 3 # The option 'fillLimit' of page range for paging paging.page.range.fill.limit = true -form.role.list.size = 100 -form.group.list.size = 100 +# max page size +page.docboost.max.fetch.size=1000 +page.role.max.fetch.size=1000 +page.group.max.fetch.size=1000 # ---------------------------------------------------------- # Mail diff --git a/src/main/webapp/WEB-INF/view/admin/boostdoc/admin_boostdoc_edit.jsp b/src/main/webapp/WEB-INF/view/admin/boostdoc/admin_boostdoc_edit.jsp index a9be33755..d2e633078 100644 --- a/src/main/webapp/WEB-INF/view/admin/boostdoc/admin_boostdoc_edit.jsp +++ b/src/main/webapp/WEB-INF/view/admin/boostdoc/admin_boostdoc_edit.jsp @@ -51,14 +51,14 @@
- +
- +