0ct0pu5/fess
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

apply general settings to suggest

Browse source
This commit is contained in:
y_fujita 2015-12-10 15:11:40 +09:00
parent 9080bf7902
commit 7e43b8abb4
11 changed files with 65 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
pom.xml
View file

@ -62,7 +62,7 @@
<crawler.version>1.0.0</crawler.version>
<!-- Suggest -->
<suggest.version>2.0.0</suggest.version>
<suggest.version>2.0.1-SNAPSHOT</suggest.version>
<!-- Elasticsearch -->
<elasticsearch.version>2.1.0</elasticsearch.version>

2
src/main/java/org/codelibs/fess/Constants.java
View file

@ -136,6 +136,8 @@ public class Constants extends CoreLibConstants {
public static final String SUGGEST_SEARCH_LOG_PROPERTY = "suggest.searchlog";
public static final String SUGGEST_DOCUMENTS_PROPERTY = "suggest.document";
public static final String PURGE_SUGGEST_SEARCH_LOG_DAY_PROPERTY = "purge.suggest.searchlog.day";
public static final String AUTH_CIPHER = "authenticationCipher";

16
src/main/java/org/codelibs/fess/api/suggest/SuggestApiManager.java
View file

@ -17,6 +17,7 @@ package org.codelibs.fess.api.suggest;
import java.io.IOException;
import javax.annotation.Resource;
import javax.servlet.FilterChain;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
@ -24,6 +25,8 @@ import javax.servlet.http.HttpServletResponse;
import org.apache.commons.lang.StringUtils;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.core.misc.DynamicProperties;
import org.codelibs.fess.Constants;
import org.codelibs.fess.api.BaseApiManager;
import org.codelibs.fess.api.json.JsonApiManager;
import org.codelibs.fess.helper.SuggestHelper;
@ -41,6 +44,9 @@ public class SuggestApiManager extends BaseApiManager {
setPathPrefix("/suggest");
}
@Resource
protected DynamicProperties crawlerProperties;
@Override
public boolean matches(final HttpServletRequest request) {
final String servletPath = request.getServletPath();
@ -68,6 +74,14 @@ public class SuggestApiManager extends BaseApiManager {
final SuggestResponse suggestResponse = builder.execute().getResponse();
builder.addKind(SuggestItem.Kind.USER.toString());
if (Constants.TRUE.equals(crawlerProperties.getProperty(Constants.SUGGEST_SEARCH_LOG_PROPERTY, Constants.TRUE))) {
builder.addKind(SuggestItem.Kind.QUERY.toString());
}
if (Constants.TRUE.equals(crawlerProperties.getProperty(Constants.SUGGEST_DOCUMENTS_PROPERTY, Constants.TRUE))) {
builder.addKind(SuggestItem.Kind.DOCUMENT.toString());
}
buf.append("\"result\":{");
buf.append("\"took\":\"").append(suggestResponse.getTookMs()).append('\"');
@ -147,7 +161,7 @@ public class SuggestApiManager extends BaseApiManager {
}
protected static RequestParameter parse(final HttpServletRequest request) {
final String query = request.getParameter("q");
final String query = request.getParameter("query");
final String fieldsStr = request.getParameter("fields");
final String[] fields;
if (StringUtils.isNotBlank(fieldsStr)) {

5
src/main/java/org/codelibs/fess/app/web/admin/general/AdminGeneralAction.java
View file

@ -110,6 +110,8 @@ public class AdminGeneralAction extends FessAdminAction {
updateProperty(Constants.NOTIFICATION_TO_PROPERTY, form.notificationTo);
updateProperty(Constants.SUGGEST_SEARCH_LOG_PROPERTY,
form.suggestSearchLog != null && Constants.ON.equalsIgnoreCase(form.suggestSearchLog) ? Constants.TRUE : Constants.FALSE);
updateProperty(Constants.SUGGEST_DOCUMENTS_PROPERTY,
form.suggestDocuments != null && Constants.ON.equalsIgnoreCase(form.suggestDocuments) ? Constants.TRUE : Constants.FALSE);
updateProperty(Constants.PURGE_SUGGEST_SEARCH_LOG_DAY_PROPERTY, form.purgeSuggestSearchLogDay.toString());
crawlerProperties.store();
@ -142,8 +144,9 @@ public class AdminGeneralAction extends FessAdminAction {
form.purgeByBots = crawlerProperties.getProperty(Constants.PURGE_BY_BOTS_PROPERTY, Constants.DEFAULT_PURGE_BY_BOTS);
form.notificationTo = crawlerProperties.getProperty(Constants.NOTIFICATION_TO_PROPERTY, StringUtil.EMPTY);
form.suggestSearchLog = crawlerProperties.getProperty(Constants.SUGGEST_SEARCH_LOG_PROPERTY, Constants.TRUE);
form.suggestDocuments = crawlerProperties.getProperty(Constants.SUGGEST_DOCUMENTS_PROPERTY, Constants.TRUE);
form.purgeSuggestSearchLogDay =
Integer.parseInt(crawlerProperties.getProperty(Constants.PURGE_SUGGEST_SEARCH_LOG_DAY_PROPERTY, "30"));
Integer.parseInt(crawlerProperties.getProperty(Constants.PURGE_SUGGEST_SEARCH_LOG_DAY_PROPERTY, "-1"));
}
private void updateProperty(final String key, final String value) {

3
src/main/java/org/codelibs/fess/app/web/admin/general/EditForm.java
View file

@ -108,6 +108,9 @@ public class EditForm implements Serializable {
@Size(max = 10)
public String suggestSearchLog;
@Size(max = 10)
public String suggestDocuments;
@Min(0)
@Max(100000)
@ValidateTypeFailure

10
src/main/java/org/codelibs/fess/exec/SuggestCreator.java
View file

@ -23,6 +23,7 @@ import java.util.concurrent.atomic.AtomicInteger;
import javax.annotation.Resource;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.core.misc.DynamicProperties;
import org.codelibs.fess.Constants;
import org.codelibs.fess.crawler.client.EsClient;
import org.codelibs.fess.es.client.FessEsClient;
@ -123,6 +124,7 @@ public class SuggestCreator implements Serializable {
final CountDownLatch latch = new CountDownLatch(1);
final SuggestHelper suggestHelper = ComponentUtil.getSuggestHelper();
suggestHelper.indexFromDocuments(ret -> {
logger.info("Success index from documents.");
result.set(0);
@ -141,8 +143,14 @@ public class SuggestCreator implements Serializable {
private int purge(final LocalDateTime time) {
final SuggestHelper suggestHelper = ComponentUtil.getSuggestHelper();
final DynamicProperties crawlerProperties = ComponentUtil.getCrawlerProperties();
try {
suggestHelper.purge(time);
suggestHelper.purgeDocumentSuggest(time);
final long cleanupDay = Long.parseLong(crawlerProperties.getProperty(Constants.PURGE_SUGGEST_SEARCH_LOG_DAY_PROPERTY, "-1"));
if (cleanupDay > 0) {
suggestHelper.purgeSearchlogSuggest(time.minusDays(cleanupDay));
}
return 0;
} catch (final Exception e) {
logger.info("Purge error.", e);

19
src/main/java/org/codelibs/fess/helper/SuggestHelper.java
View file

@ -142,7 +142,7 @@ public class SuggestHelper {
new ESSourceReader(fessEsClient, suggester.settings(), fessConfig.getIndexDocumentIndex(),
fessConfig.getIndexDocumentType());
suggester.indexer().indexFromDocument(reader, 2, 100).done(response -> {
suggester.indexer().indexFromDocument(reader, 2, 100).then(response -> {
suggester.refresh();
//TODO delete old doc
@ -151,13 +151,24 @@ public class SuggestHelper {
}).error(t -> error.accept(t));
}
public void purge(final LocalDateTime time) {
public void purgeDocumentSuggest(final LocalDateTime time) {
final BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
boolQueryBuilder.must(QueryBuilders.rangeQuery(FieldNames.TIMESTAMP).lt(time.format(DateTimeFormatter.BASIC_ISO_DATE)));
boolQueryBuilder.must(QueryBuilders.rangeQuery(FieldNames.TIMESTAMP).lt(time.format(DateTimeFormatter.ISO_DATE)));
boolQueryBuilder.must(QueryBuilders.termQuery(FieldNames.KINDS, SuggestItem.Kind.DOCUMENT.toString()));
boolQueryBuilder.mustNot(QueryBuilders.termQuery(FieldNames.KINDS, SuggestItem.Kind.QUERY.toString()));
boolQueryBuilder.must(QueryBuilders.termQuery(FieldNames.KINDS, SuggestItem.Kind.USER.toString()));
boolQueryBuilder.mustNot(QueryBuilders.termQuery(FieldNames.KINDS, SuggestItem.Kind.USER.toString()));
SuggestUtil.deleteByQuery(fessEsClient, suggester.getIndex(), suggester.getType(), boolQueryBuilder);
}
public void purgeSearchlogSuggest(final LocalDateTime time) {
final BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
boolQueryBuilder.must(QueryBuilders.rangeQuery(FieldNames.TIMESTAMP).lt(time.format(DateTimeFormatter.ISO_DATE)));
boolQueryBuilder.mustNot(QueryBuilders.termQuery(FieldNames.KINDS, SuggestItem.Kind.DOCUMENT.toString()));
boolQueryBuilder.must(QueryBuilders.termQuery(FieldNames.KINDS, SuggestItem.Kind.QUERY.toString()));
boolQueryBuilder.mustNot(QueryBuilders.termQuery(FieldNames.KINDS, SuggestItem.Kind.USER.toString()));
SuggestUtil.deleteByQuery(fessEsClient, suggester.getIndex(), suggester.getType(), boolQueryBuilder);
}

1
src/main/resources/fess_label.properties
View file

@ -325,6 +325,7 @@ labels.duplicate_host_configuration=Duplicate Host
labels.duplicate_host_title_details=Duplicate Host
labels.dashboard_title_configuration=System Configuration
labels.suggest_search_log_enabled=Suggest by Search Words
labels.suggest_documents_enabled=Suggest by Documents
labels.purge_suggest_search_log_day=Purge Suggest Documents Before
labels.crawling_info_title=Crawling Information
labels.crawling_info_title_confirm=Crawling Information

1
src/main/resources/fess_label_en.properties
View file

@ -323,6 +323,7 @@ labels.duplicate_host_configuration=Duplicate Host
labels.duplicate_host_title_details=Duplicate Host
labels.dashboard_title_configuration=System Configuration
labels.suggest_search_log_enabled=Suggest by Search Words
labels.suggest_documents_enabled=Suggest by Documents
labels.purge_suggest_search_log_day=Purge Suggest Documents Before
labels.crawling_info_title=Crawling Information
labels.crawling_info_title_confirm=Crawling Information

3
src/main/resources/fess_label_ja.properties
View file

@ -321,7 +321,8 @@ labels.duplicate_name = \u91cd\u8907\u540d
labels.duplicate_host_configuration = \u91cd\u8907\u30db\u30b9\u30c8
labels.duplicate_host_title_details = \u91cd\u8907\u30db\u30b9\u30c8
labels.dashboard_title_configuration = \u30b7\u30b9\u30c6\u30e0\u69cb\u6210
labels.suggest_search_log_enabled = \u691c\u7d22\u8a9e\u3067\u30b5\u30b8\u30a7\u30b9\u30c8\u751f\u6210
labels.suggest_search_log_enabled = \u691c\u7d22\u8a9e\u3067\u30b5\u30b8\u30a7\u30b9\u30c8
labels.suggest_documents_enabled = \u30c9\u30ad\u30e5\u30e1\u30f3\u30c8\u3067\u30b5\u30b8\u30a7\u30b9\u30c8
labels.purge_suggest_search_log_day = \u4ee5\u524d\u306e\u30b5\u30b8\u30a7\u30b9\u30c8\u60c5\u5831\u3092\u524a\u9664
labels.crawling_info_title = \u30af\u30ed\u30fc\u30eb\u60c5\u5831
labels.crawling_info_title_confirm = \u30af\u30ed\u30fc\u30eb\u60c5\u5831

12
src/main/webapp/WEB-INF/view/admin/general/admin_general.jsp
View file

@ -253,6 +253,18 @@
</div>
</div>
</div>
<div class="form-group">
<label for="suggestSearchLog" class="col-sm-3 control-label"><la:message
key="labels.suggest_documents_enabled" /></label>
<div class="col-sm-9">
<la:errors property="suggestDocuments" />
<div class="checkbox">
<label> <la:checkbox property="suggestDocuments" />
<la:message key="labels.enabled" />
</label>
</div>
</div>
</div>
<div class="form-group">
<label for="purgeSuggestSearchLogDay"
class="col-sm-3 control-label"><la:message