diff --git a/pom.xml b/pom.xml index 4c74411a4..ed385d2c8 100644 --- a/pom.xml +++ b/pom.xml @@ -62,7 +62,7 @@ 1.0.0 - 2.0.0 + 2.0.1-SNAPSHOT 2.1.0 diff --git a/src/main/java/org/codelibs/fess/Constants.java b/src/main/java/org/codelibs/fess/Constants.java index d7165c81a..dbd9cf041 100644 --- a/src/main/java/org/codelibs/fess/Constants.java +++ b/src/main/java/org/codelibs/fess/Constants.java @@ -136,6 +136,8 @@ public class Constants extends CoreLibConstants { public static final String SUGGEST_SEARCH_LOG_PROPERTY = "suggest.searchlog"; + public static final String SUGGEST_DOCUMENTS_PROPERTY = "suggest.document"; + public static final String PURGE_SUGGEST_SEARCH_LOG_DAY_PROPERTY = "purge.suggest.searchlog.day"; public static final String AUTH_CIPHER = "authenticationCipher"; diff --git a/src/main/java/org/codelibs/fess/api/suggest/SuggestApiManager.java b/src/main/java/org/codelibs/fess/api/suggest/SuggestApiManager.java index 493e74871..ffa9719b1 100644 --- a/src/main/java/org/codelibs/fess/api/suggest/SuggestApiManager.java +++ b/src/main/java/org/codelibs/fess/api/suggest/SuggestApiManager.java @@ -17,6 +17,7 @@ package org.codelibs.fess.api.suggest; import java.io.IOException; +import javax.annotation.Resource; import javax.servlet.FilterChain; import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; @@ -24,6 +25,8 @@ import javax.servlet.http.HttpServletResponse; import org.apache.commons.lang.StringUtils; import org.codelibs.core.lang.StringUtil; +import org.codelibs.core.misc.DynamicProperties; +import org.codelibs.fess.Constants; import org.codelibs.fess.api.BaseApiManager; import org.codelibs.fess.api.json.JsonApiManager; import org.codelibs.fess.helper.SuggestHelper; @@ -41,6 +44,9 @@ public class SuggestApiManager extends BaseApiManager { setPathPrefix("/suggest"); } + @Resource + protected DynamicProperties crawlerProperties; + @Override public boolean matches(final HttpServletRequest request) { final String servletPath = request.getServletPath(); @@ -68,6 +74,14 @@ public class SuggestApiManager extends BaseApiManager { final SuggestResponse suggestResponse = builder.execute().getResponse(); + builder.addKind(SuggestItem.Kind.USER.toString()); + if (Constants.TRUE.equals(crawlerProperties.getProperty(Constants.SUGGEST_SEARCH_LOG_PROPERTY, Constants.TRUE))) { + builder.addKind(SuggestItem.Kind.QUERY.toString()); + } + if (Constants.TRUE.equals(crawlerProperties.getProperty(Constants.SUGGEST_DOCUMENTS_PROPERTY, Constants.TRUE))) { + builder.addKind(SuggestItem.Kind.DOCUMENT.toString()); + } + buf.append("\"result\":{"); buf.append("\"took\":\"").append(suggestResponse.getTookMs()).append('\"'); @@ -147,7 +161,7 @@ public class SuggestApiManager extends BaseApiManager { } protected static RequestParameter parse(final HttpServletRequest request) { - final String query = request.getParameter("q"); + final String query = request.getParameter("query"); final String fieldsStr = request.getParameter("fields"); final String[] fields; if (StringUtils.isNotBlank(fieldsStr)) { diff --git a/src/main/java/org/codelibs/fess/app/web/admin/general/AdminGeneralAction.java b/src/main/java/org/codelibs/fess/app/web/admin/general/AdminGeneralAction.java index be4ecd0c4..36eb996f8 100644 --- a/src/main/java/org/codelibs/fess/app/web/admin/general/AdminGeneralAction.java +++ b/src/main/java/org/codelibs/fess/app/web/admin/general/AdminGeneralAction.java @@ -110,6 +110,8 @@ public class AdminGeneralAction extends FessAdminAction { updateProperty(Constants.NOTIFICATION_TO_PROPERTY, form.notificationTo); updateProperty(Constants.SUGGEST_SEARCH_LOG_PROPERTY, form.suggestSearchLog != null && Constants.ON.equalsIgnoreCase(form.suggestSearchLog) ? Constants.TRUE : Constants.FALSE); + updateProperty(Constants.SUGGEST_DOCUMENTS_PROPERTY, + form.suggestDocuments != null && Constants.ON.equalsIgnoreCase(form.suggestDocuments) ? Constants.TRUE : Constants.FALSE); updateProperty(Constants.PURGE_SUGGEST_SEARCH_LOG_DAY_PROPERTY, form.purgeSuggestSearchLogDay.toString()); crawlerProperties.store(); @@ -142,8 +144,9 @@ public class AdminGeneralAction extends FessAdminAction { form.purgeByBots = crawlerProperties.getProperty(Constants.PURGE_BY_BOTS_PROPERTY, Constants.DEFAULT_PURGE_BY_BOTS); form.notificationTo = crawlerProperties.getProperty(Constants.NOTIFICATION_TO_PROPERTY, StringUtil.EMPTY); form.suggestSearchLog = crawlerProperties.getProperty(Constants.SUGGEST_SEARCH_LOG_PROPERTY, Constants.TRUE); + form.suggestDocuments = crawlerProperties.getProperty(Constants.SUGGEST_DOCUMENTS_PROPERTY, Constants.TRUE); form.purgeSuggestSearchLogDay = - Integer.parseInt(crawlerProperties.getProperty(Constants.PURGE_SUGGEST_SEARCH_LOG_DAY_PROPERTY, "30")); + Integer.parseInt(crawlerProperties.getProperty(Constants.PURGE_SUGGEST_SEARCH_LOG_DAY_PROPERTY, "-1")); } private void updateProperty(final String key, final String value) { diff --git a/src/main/java/org/codelibs/fess/app/web/admin/general/EditForm.java b/src/main/java/org/codelibs/fess/app/web/admin/general/EditForm.java index c661eee6e..9d5dc1425 100644 --- a/src/main/java/org/codelibs/fess/app/web/admin/general/EditForm.java +++ b/src/main/java/org/codelibs/fess/app/web/admin/general/EditForm.java @@ -108,6 +108,9 @@ public class EditForm implements Serializable { @Size(max = 10) public String suggestSearchLog; + @Size(max = 10) + public String suggestDocuments; + @Min(0) @Max(100000) @ValidateTypeFailure diff --git a/src/main/java/org/codelibs/fess/exec/SuggestCreator.java b/src/main/java/org/codelibs/fess/exec/SuggestCreator.java index 61a7ece07..d489bedb3 100644 --- a/src/main/java/org/codelibs/fess/exec/SuggestCreator.java +++ b/src/main/java/org/codelibs/fess/exec/SuggestCreator.java @@ -23,6 +23,7 @@ import java.util.concurrent.atomic.AtomicInteger; import javax.annotation.Resource; import org.codelibs.core.lang.StringUtil; +import org.codelibs.core.misc.DynamicProperties; import org.codelibs.fess.Constants; import org.codelibs.fess.crawler.client.EsClient; import org.codelibs.fess.es.client.FessEsClient; @@ -123,6 +124,7 @@ public class SuggestCreator implements Serializable { final CountDownLatch latch = new CountDownLatch(1); final SuggestHelper suggestHelper = ComponentUtil.getSuggestHelper(); + suggestHelper.indexFromDocuments(ret -> { logger.info("Success index from documents."); result.set(0); @@ -141,8 +143,14 @@ public class SuggestCreator implements Serializable { private int purge(final LocalDateTime time) { final SuggestHelper suggestHelper = ComponentUtil.getSuggestHelper(); + final DynamicProperties crawlerProperties = ComponentUtil.getCrawlerProperties(); + try { - suggestHelper.purge(time); + suggestHelper.purgeDocumentSuggest(time); + final long cleanupDay = Long.parseLong(crawlerProperties.getProperty(Constants.PURGE_SUGGEST_SEARCH_LOG_DAY_PROPERTY, "-1")); + if (cleanupDay > 0) { + suggestHelper.purgeSearchlogSuggest(time.minusDays(cleanupDay)); + } return 0; } catch (final Exception e) { logger.info("Purge error.", e); diff --git a/src/main/java/org/codelibs/fess/helper/SuggestHelper.java b/src/main/java/org/codelibs/fess/helper/SuggestHelper.java index 6431843ff..c9a44f2be 100644 --- a/src/main/java/org/codelibs/fess/helper/SuggestHelper.java +++ b/src/main/java/org/codelibs/fess/helper/SuggestHelper.java @@ -142,7 +142,7 @@ public class SuggestHelper { new ESSourceReader(fessEsClient, suggester.settings(), fessConfig.getIndexDocumentIndex(), fessConfig.getIndexDocumentType()); - suggester.indexer().indexFromDocument(reader, 2, 100).done(response -> { + suggester.indexer().indexFromDocument(reader, 2, 100).then(response -> { suggester.refresh(); //TODO delete old doc @@ -151,13 +151,24 @@ public class SuggestHelper { }).error(t -> error.accept(t)); } - public void purge(final LocalDateTime time) { + public void purgeDocumentSuggest(final LocalDateTime time) { final BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); - boolQueryBuilder.must(QueryBuilders.rangeQuery(FieldNames.TIMESTAMP).lt(time.format(DateTimeFormatter.BASIC_ISO_DATE))); + boolQueryBuilder.must(QueryBuilders.rangeQuery(FieldNames.TIMESTAMP).lt(time.format(DateTimeFormatter.ISO_DATE))); boolQueryBuilder.must(QueryBuilders.termQuery(FieldNames.KINDS, SuggestItem.Kind.DOCUMENT.toString())); boolQueryBuilder.mustNot(QueryBuilders.termQuery(FieldNames.KINDS, SuggestItem.Kind.QUERY.toString())); - boolQueryBuilder.must(QueryBuilders.termQuery(FieldNames.KINDS, SuggestItem.Kind.USER.toString())); + boolQueryBuilder.mustNot(QueryBuilders.termQuery(FieldNames.KINDS, SuggestItem.Kind.USER.toString())); + + SuggestUtil.deleteByQuery(fessEsClient, suggester.getIndex(), suggester.getType(), boolQueryBuilder); + } + + public void purgeSearchlogSuggest(final LocalDateTime time) { + final BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); + boolQueryBuilder.must(QueryBuilders.rangeQuery(FieldNames.TIMESTAMP).lt(time.format(DateTimeFormatter.ISO_DATE))); + + boolQueryBuilder.mustNot(QueryBuilders.termQuery(FieldNames.KINDS, SuggestItem.Kind.DOCUMENT.toString())); + boolQueryBuilder.must(QueryBuilders.termQuery(FieldNames.KINDS, SuggestItem.Kind.QUERY.toString())); + boolQueryBuilder.mustNot(QueryBuilders.termQuery(FieldNames.KINDS, SuggestItem.Kind.USER.toString())); SuggestUtil.deleteByQuery(fessEsClient, suggester.getIndex(), suggester.getType(), boolQueryBuilder); } diff --git a/src/main/resources/fess_label.properties b/src/main/resources/fess_label.properties index 57618d12f..273e81732 100644 --- a/src/main/resources/fess_label.properties +++ b/src/main/resources/fess_label.properties @@ -325,6 +325,7 @@ labels.duplicate_host_configuration=Duplicate Host labels.duplicate_host_title_details=Duplicate Host labels.dashboard_title_configuration=System Configuration labels.suggest_search_log_enabled=Suggest by Search Words +labels.suggest_documents_enabled=Suggest by Documents labels.purge_suggest_search_log_day=Purge Suggest Documents Before labels.crawling_info_title=Crawling Information labels.crawling_info_title_confirm=Crawling Information diff --git a/src/main/resources/fess_label_en.properties b/src/main/resources/fess_label_en.properties index 5584037be..3d6909714 100644 --- a/src/main/resources/fess_label_en.properties +++ b/src/main/resources/fess_label_en.properties @@ -323,6 +323,7 @@ labels.duplicate_host_configuration=Duplicate Host labels.duplicate_host_title_details=Duplicate Host labels.dashboard_title_configuration=System Configuration labels.suggest_search_log_enabled=Suggest by Search Words +labels.suggest_documents_enabled=Suggest by Documents labels.purge_suggest_search_log_day=Purge Suggest Documents Before labels.crawling_info_title=Crawling Information labels.crawling_info_title_confirm=Crawling Information diff --git a/src/main/resources/fess_label_ja.properties b/src/main/resources/fess_label_ja.properties index 3145c24f4..dc0277c2d 100644 --- a/src/main/resources/fess_label_ja.properties +++ b/src/main/resources/fess_label_ja.properties @@ -321,7 +321,8 @@ labels.duplicate_name = \u91cd\u8907\u540d labels.duplicate_host_configuration = \u91cd\u8907\u30db\u30b9\u30c8 labels.duplicate_host_title_details = \u91cd\u8907\u30db\u30b9\u30c8 labels.dashboard_title_configuration = \u30b7\u30b9\u30c6\u30e0\u69cb\u6210 -labels.suggest_search_log_enabled = \u691c\u7d22\u8a9e\u3067\u30b5\u30b8\u30a7\u30b9\u30c8\u751f\u6210 +labels.suggest_search_log_enabled = \u691c\u7d22\u8a9e\u3067\u30b5\u30b8\u30a7\u30b9\u30c8 +labels.suggest_documents_enabled = \u30c9\u30ad\u30e5\u30e1\u30f3\u30c8\u3067\u30b5\u30b8\u30a7\u30b9\u30c8 labels.purge_suggest_search_log_day = \u4ee5\u524d\u306e\u30b5\u30b8\u30a7\u30b9\u30c8\u60c5\u5831\u3092\u524a\u9664 labels.crawling_info_title = \u30af\u30ed\u30fc\u30eb\u60c5\u5831 labels.crawling_info_title_confirm = \u30af\u30ed\u30fc\u30eb\u60c5\u5831 diff --git a/src/main/webapp/WEB-INF/view/admin/general/admin_general.jsp b/src/main/webapp/WEB-INF/view/admin/general/admin_general.jsp index bc21a02cc..9476bde26 100644 --- a/src/main/webapp/WEB-INF/view/admin/general/admin_general.jsp +++ b/src/main/webapp/WEB-INF/view/admin/general/admin_general.jsp @@ -253,6 +253,18 @@ +
+ +
+ +
+ +
+
+