0ct0pu5/fess
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

#502 improve search log processing for suggest and remove systemProperties field.

Browse source
This commit is contained in:
Shinsuke Sugaya 2016-06-05 17:59:02 +09:00
parent 0368c8ea39
commit 37f9ee43ff
37 changed files with 447 additions and 220 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
src/main/java/org/codelibs/fess/api/es/EsApiManager.java
View file

@ -21,7 +21,6 @@ import java.io.OutputStream;
import java.util.Locale;
import java.util.UUID;
import javax.annotation.Resource;
import javax.servlet.FilterChain;
import javax.servlet.ServletException;
import javax.servlet.ServletInputStream;
@ -32,7 +31,6 @@ import javax.servlet.http.HttpServletResponse;
import org.apache.catalina.connector.ClientAbortException;
import org.codelibs.core.io.CopyUtil;
import org.codelibs.core.io.InputStreamUtil;
import org.codelibs.core.misc.DynamicProperties;
import org.codelibs.elasticsearch.runner.net.Curl.Method;
import org.codelibs.elasticsearch.runner.net.CurlRequest;
import org.codelibs.fess.Constants;
@ -52,9 +50,6 @@ public class EsApiManager extends BaseApiManager {
private static final Logger logger = LoggerFactory.getLogger(EsApiManager.class);
@Resource
protected DynamicProperties systemProperties;
protected String[] acceptedRoles = new String[] { "admin" };
public EsApiManager() {

5
src/main/java/org/codelibs/fess/api/json/JsonApiManager.java
View file

@ -25,7 +25,6 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
import javax.annotation.Resource;
import javax.servlet.FilterChain;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
@ -33,7 +32,6 @@ import javax.servlet.http.HttpServletResponse;
import org.codelibs.core.CoreLibConstants;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.core.misc.DynamicProperties;
import org.codelibs.fess.Constants;
import org.codelibs.fess.api.BaseApiManager;
import org.codelibs.fess.app.service.FavoriteLogService;
@ -68,9 +66,6 @@ public class JsonApiManager extends BaseApiManager {
private static final Logger logger = LoggerFactory.getLogger(JsonApiManager.class);
@Resource
protected DynamicProperties systemProperties;
public JsonApiManager() {
setPathPrefix("/json");
}

5
src/main/java/org/codelibs/fess/api/suggest/SuggestApiManager.java
View file

@ -19,7 +19,6 @@ import static org.codelibs.core.stream.StreamUtil.stream;
import java.io.IOException;
import javax.annotation.Resource;
import javax.servlet.FilterChain;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
@ -27,7 +26,6 @@ import javax.servlet.http.HttpServletResponse;
import org.apache.commons.lang.StringUtils;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.core.misc.DynamicProperties;
import org.codelibs.fess.api.BaseApiManager;
import org.codelibs.fess.api.json.JsonApiManager;
import org.codelibs.fess.helper.RoleQueryHelper;
@ -42,9 +40,6 @@ import org.slf4j.LoggerFactory;
public class SuggestApiManager extends BaseApiManager {
private static final Logger logger = LoggerFactory.getLogger(SuggestApiManager.class);
@Resource
protected DynamicProperties systemProperties;
public SuggestApiManager() {
setPathPrefix("/suggest");
}

4
src/main/java/org/codelibs/fess/app/service/FailureUrlService.java
View file

@ -28,7 +28,6 @@ import org.apache.commons.io.output.StringBuilderWriter;
import org.apache.commons.lang3.StringUtils;
import org.codelibs.core.beans.util.BeanUtil;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.core.misc.DynamicProperties;
import org.codelibs.fess.Constants;
import org.codelibs.fess.app.pager.FailureUrlPager;
import org.codelibs.fess.es.config.cbean.FailureUrlCB;
@ -46,9 +45,6 @@ public class FailureUrlService implements Serializable {
private static final long serialVersionUID = 1L;
@Resource
protected DynamicProperties systemProperties;
@Resource
protected FailureUrlBhv failureUrlBhv;

5
src/main/java/org/codelibs/fess/app/service/SearchService.java
View file

@ -29,7 +29,6 @@ import javax.annotation.Resource;
import javax.servlet.http.HttpServletRequest;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.core.misc.DynamicProperties;
import org.codelibs.fess.Constants;
import org.codelibs.fess.entity.QueryContext;
import org.codelibs.fess.entity.SearchRenderData;
@ -64,8 +63,6 @@ public class SearchService {
// ===================================================================================
// Attribute
//
@Resource
protected DynamicProperties systemProperties;
@Resource
protected FessEsClient fessEsClient;
@ -196,7 +193,7 @@ public class SearchService {
langSet.remove(Constants.ALL_LANGUAGES);
}
return langSet.toArray(new String[langSet.size()]);
} else if (Constants.TRUE.equals(systemProperties.getProperty(Constants.USE_BROWSER_LOCALE_FOR_SEARCH_PROPERTY, Constants.FALSE))) {
} else if (fessConfig.isBrowserLocaleForSearchUsed()) {
final Set<String> langSet = new HashSet<>();
final Enumeration<Locale> locales = request.getLocales();
if (locales != null) {

5
src/main/java/org/codelibs/fess/app/service/UserService.java
View file

@ -32,6 +32,7 @@ import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.util.ComponentUtil;
import org.dbflute.cbean.result.PagingResultBean;
import org.dbflute.optional.OptionalEntity;
import org.jsoup.helper.StringUtil;
public class UserService implements Serializable {
@ -70,6 +71,10 @@ public class UserService implements Serializable {
}
public void store(final User user) {
if (StringUtil.isBlank(user.getSurname())) {
user.setSurname(user.getName());
}
ComponentUtil.getLdapManager().insert(user);
userBhv.insertOrUpdate(user, op -> {

3
src/main/java/org/codelibs/fess/app/web/admin/badword/AdminBadwordAction.java
View file

@ -27,7 +27,6 @@ import java.nio.file.Path;
import javax.annotation.Resource;
import org.codelibs.core.misc.DynamicProperties;
import org.codelibs.fess.Constants;
import org.codelibs.fess.app.pager.BadWordPager;
import org.codelibs.fess.app.service.BadWordService;
@ -58,8 +57,6 @@ public class AdminBadwordAction extends FessAdminAction {
@Resource
private BadWordPager badWordPager;
@Resource
protected DynamicProperties systemProperties;
@Resource
protected SuggestHelper suggestHelper;
// ===================================================================================

7
src/main/java/org/codelibs/fess/app/web/admin/design/AdminDesignAction.java
View file

@ -23,13 +23,10 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import javax.annotation.Resource;
import org.apache.commons.io.FileUtils;
import org.codelibs.core.io.FileUtil;
import org.codelibs.core.io.ResourceUtil;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.core.misc.DynamicProperties;
import org.codelibs.fess.Constants;
import org.codelibs.fess.app.web.base.FessAdminAction;
import org.codelibs.fess.exception.FessSystemException;
@ -54,8 +51,6 @@ public class AdminDesignAction extends FessAdminAction implements Serializable {
// ===================================================================================
// Attribute
// =========
@Resource
private DynamicProperties systemProperties;
// ===================================================================================
// Hook
@ -81,7 +76,7 @@ public class AdminDesignAction extends FessAdminAction implements Serializable {
}
private boolean editable() {
return Constants.TRUE.equals(systemProperties.getProperty(Constants.WEB_DESIGN_EDITOR_PROPERTY, Constants.TRUE));
return fessConfig.isWebDesignEditorEnabled();
}
private List<String> loadFileNameItems() {

3
src/main/java/org/codelibs/fess/app/web/admin/dict/kuromoji/AdminDictKuromojiAction.java
View file

@ -23,7 +23,6 @@ import javax.annotation.Resource;
import org.codelibs.core.beans.util.BeanUtil;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.core.misc.DynamicProperties;
import org.codelibs.fess.Constants;
import org.codelibs.fess.app.pager.KuromojiPager;
import org.codelibs.fess.app.service.KuromojiService;
@ -53,8 +52,6 @@ public class AdminDictKuromojiAction extends FessAdminAction {
private KuromojiService kuromojiService;
@Resource
private KuromojiPager kuromojiPager;
@Resource
protected DynamicProperties systemProperties;
// ===================================================================================
// Hook

3
src/main/java/org/codelibs/fess/app/web/admin/dict/seunjeon/AdminDictSeunjeonAction.java
View file

@ -25,7 +25,6 @@ import javax.annotation.Resource;
import org.codelibs.core.beans.util.BeanUtil;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.core.misc.DynamicProperties;
import org.codelibs.fess.Constants;
import org.codelibs.fess.app.pager.SeunjeonPager;
import org.codelibs.fess.app.service.SeunjeonService;
@ -56,8 +55,6 @@ public class AdminDictSeunjeonAction extends FessAdminAction {
private SeunjeonService seunjeonService;
@Resource
private SeunjeonPager seunjeonPager;
@Resource
protected DynamicProperties systemProperties;
// ===================================================================================
// Hook

3
src/main/java/org/codelibs/fess/app/web/admin/dict/synonym/AdminDictSynonymAction.java
View file

@ -25,7 +25,6 @@ import javax.annotation.Resource;
import org.codelibs.core.beans.util.BeanUtil;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.core.misc.DynamicProperties;
import org.codelibs.fess.Constants;
import org.codelibs.fess.app.pager.SynonymPager;
import org.codelibs.fess.app.service.SynonymService;
@ -56,8 +55,6 @@ public class AdminDictSynonymAction extends FessAdminAction {
private SynonymService synonymService;
@Resource
private SynonymPager synonymPager;
@Resource
protected DynamicProperties systemProperties;
// ===================================================================================
// Hook

3
src/main/java/org/codelibs/fess/app/web/admin/elevateword/AdminElevatewordAction.java
View file

@ -27,7 +27,6 @@ import java.nio.file.Path;
import javax.annotation.Resource;
import org.codelibs.core.misc.DynamicProperties;
import org.codelibs.fess.Constants;
import org.codelibs.fess.app.pager.ElevateWordPager;
import org.codelibs.fess.app.service.ElevateWordService;
@ -59,8 +58,6 @@ public class AdminElevatewordAction extends FessAdminAction {
@Resource
private ElevateWordPager elevateWordPager;
@Resource
protected DynamicProperties systemProperties;
@Resource
protected SuggestHelper suggestHelper;
@Resource
private LabelTypeService labelTypeService;

1
src/main/java/org/codelibs/fess/app/web/admin/user/CreateForm.java
View file

@ -44,7 +44,6 @@ public class CreateForm implements Serializable {
@Size(max = 100)
public String confirmPassword;
@Required
@Size(max = 1000)
public String surname;

4
src/main/java/org/codelibs/fess/app/web/admin/wizard/AdminWizardAction.java
View file

@ -173,7 +173,7 @@ public class AdminWizardAction extends FessAdminAction {
wConfig.setUpdatedTime(now);
wConfig.setUrls(configPath);
wConfig.setUserAgent(getDefaultString("default.config.web.userAgent", ComponentUtil.getUserAgentName()));
wConfig.setPermissions(ComponentUtil.getFessConfig().getSearchDefaultPermissionsAsArray());
wConfig.setPermissions(ComponentUtil.getFessConfig().getSearchDefaultDisplayEncodedPermissions());
webConfigService.store(wConfig);
@ -201,7 +201,7 @@ public class AdminWizardAction extends FessAdminAction {
fConfig.setUpdatedBy(username);
fConfig.setUpdatedTime(now);
fConfig.setPaths(configPath);
fConfig.setPermissions(ComponentUtil.getFessConfig().getSearchDefaultPermissionsAsArray());
fConfig.setPermissions(ComponentUtil.getFessConfig().getSearchDefaultDisplayEncodedPermissions());
fileConfigService.store(fConfig);
}

4
src/main/java/org/codelibs/fess/app/web/base/FessSearchAction.java
View file

@ -25,7 +25,6 @@ import javax.servlet.http.HttpServletRequest;
import org.apache.commons.lang3.StringEscapeUtils;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.core.misc.DynamicProperties;
import org.codelibs.core.net.URLUtil;
import org.codelibs.fess.Constants;
import org.codelibs.fess.app.web.login.LoginAction;
@ -79,9 +78,6 @@ public abstract class FessSearchAction extends FessBaseAction {
@Resource
protected PopularWordHelper popularWordHelper;
@Resource
protected DynamicProperties systemProperties;
@Resource
protected HttpServletRequest request;

2
src/main/java/org/codelibs/fess/app/web/go/GoAction.java
View file

@ -135,7 +135,7 @@ public class GoAction extends FessSearchAction {
final String targetUrl = pathMappingHelper.replaceUrl(url);
if (isFileSystemPath(targetUrl)) {
if (Constants.TRUE.equals(systemProperties.getProperty(Constants.SEARCH_FILE_PROXY_PROPERTY, Constants.TRUE))) {
if (fessConfig.isSearchFileProxyEnabled()) {
final ViewHelper viewHelper = ComponentUtil.getViewHelper();
try {
return viewHelper.asContentResponse(doc);

5
src/main/java/org/codelibs/fess/dict/DictionaryManager.java
View file

@ -25,10 +25,8 @@ import java.util.List;
import java.util.Map;
import javax.annotation.PostConstruct;
import javax.annotation.Resource;
import org.codelibs.core.io.FileUtil;
import org.codelibs.core.misc.DynamicProperties;
import org.codelibs.elasticsearch.runner.net.Curl;
import org.codelibs.elasticsearch.runner.net.CurlResponse;
import org.codelibs.fess.Constants;
@ -40,9 +38,6 @@ import org.slf4j.LoggerFactory;
public class DictionaryManager {
private static final Logger logger = LoggerFactory.getLogger(DictionaryManager.class);
@Resource
protected DynamicProperties systemProperties;
protected List<DictionaryCreator> creatorList = new ArrayList<>();
@PostConstruct

125
src/main/java/org/codelibs/fess/es/config/allcommon/EsAbstractBehavior.java
View file

@ -42,14 +42,12 @@ import org.dbflute.util.DfTypeUtil;
import org.elasticsearch.action.bulk.BulkItemResponse;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.count.CountRequestBuilder;
import org.elasticsearch.action.delete.DeleteRequestBuilder;
import org.elasticsearch.action.delete.DeleteResponse;
import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.action.update.UpdateRequestBuilder;
import org.elasticsearch.client.Client;
import org.elasticsearch.search.SearchHit;
@ -69,6 +67,11 @@ public abstract class EsAbstractBehavior<ENTITY extends Entity, CB extends Condi
protected String scrollForDelete = "1m";
protected int sizeForCursor = 100;
protected String scrollForCursor = "1m";
protected String searchTimeout = "3m";
protected String indexTimeout = "3m";
protected String scrollSearchTimeout = "3m";
protected String bulkTimeout = "3m";
protected String deleteTimeout = "3m";
protected abstract String asEsIndex();
@ -84,8 +87,12 @@ public abstract class EsAbstractBehavior<ENTITY extends Entity, CB extends Condi
@Override
protected int delegateSelectCountUniquely(final ConditionBean cb) {
// #pending check response and cast problem
final CountRequestBuilder builder = client.prepareCount(asEsIndex()).setTypes(asEsSearchType());
return (int) ((EsAbstractConditionBean) cb).build(builder).execute().actionGet().getCount();
final SearchRequestBuilder builder = client.prepareSearch(asEsIndex()).setTypes(asEsSearchType());
final EsAbstractConditionBean esCb = (EsAbstractConditionBean) cb;
if (esCb.getPreference() != null) {
builder.setPreference(esCb.getPreference());
}
return (int) esCb.build(builder).execute().actionGet(searchTimeout).getHits().getTotalHits();
}
@Override
@ -116,8 +123,12 @@ public abstract class EsAbstractBehavior<ENTITY extends Entity, CB extends Condi
}
builder.setFrom(from);
builder.setSize(size);
((EsAbstractConditionBean) cb).request().build(builder);
final SearchResponse response = ((EsAbstractConditionBean) cb).build(builder).execute().actionGet();
final EsAbstractConditionBean esCb = (EsAbstractConditionBean) cb;
if (esCb.getPreference() != null) {
builder.setPreference(esCb.getPreference());
}
esCb.request().build(builder);
final SearchResponse response = esCb.build(builder).execute().actionGet(searchTimeout);
final EsPagingResultBean<RESULT> list = new EsPagingResultBean<>();
final SearchHits searchHits = response.getHits();
@ -192,20 +203,24 @@ public abstract class EsAbstractBehavior<ENTITY extends Entity, CB extends Condi
}
protected void delegateBulkRequest(final ConditionBean cb, Function<SearchHits, Boolean> handler) {
final SearchRequestBuilder builder =
client.prepareSearch(asEsIndex()).setTypes(asEsIndexType()).setSearchType(SearchType.SCAN).setScroll(scrollForCursor)
.setSize(sizeForCursor);
((EsAbstractConditionBean) cb).request().build(builder);
final SearchResponse response = ((EsAbstractConditionBean) cb).build(builder).execute().actionGet();
String scrollId = response.getScrollId();
while (scrollId != null) {
final SearchResponse scrollResponse = client.prepareSearchScroll(scrollId).setScroll(scrollForDelete).execute().actionGet();
scrollId = scrollResponse.getScrollId();
final SearchHits searchHits = scrollResponse.getHits();
SearchResponse response = null;
while (true) {
if (response == null) {
final SearchRequestBuilder builder =
client.prepareSearch(asEsIndex()).setTypes(asEsIndexType()).setScroll(scrollForCursor).setSize(sizeForCursor);
final EsAbstractConditionBean esCb = (EsAbstractConditionBean) cb;
if (esCb.getPreference() != null) {
builder.setPreference(esCb.getPreference());
}
esCb.request().build(builder);
response = esCb.build(builder).execute().actionGet(scrollSearchTimeout);
} else {
final String scrollId = response.getScrollId();
response = client.prepareSearchScroll(scrollId).setScroll(scrollForDelete).execute().actionGet(scrollSearchTimeout);
}
final SearchHits searchHits = response.getHits();
final SearchHit[] hits = searchHits.getHits();
if (hits.length == 0) {
scrollId = null;
break;
}
@ -237,7 +252,7 @@ public abstract class EsAbstractBehavior<ENTITY extends Entity, CB extends Condi
final EsAbstractEntity esEntity = (EsAbstractEntity) entity;
IndexRequestBuilder builder = createInsertRequest(esEntity);
final IndexResponse response = builder.execute().actionGet();
final IndexResponse response = builder.execute().actionGet(indexTimeout);
esEntity.asDocMeta().id(response.getId());
return response.isCreated() ? 1 : 0;
}
@ -260,7 +275,7 @@ public abstract class EsAbstractBehavior<ENTITY extends Entity, CB extends Condi
final EsAbstractEntity esEntity = (EsAbstractEntity) entity;
final IndexRequestBuilder builder = createUpdateRequest(esEntity);
final IndexResponse response = builder.execute().actionGet();
final IndexResponse response = builder.execute().actionGet(indexTimeout);
long version = response.getVersion();
if (version != -1) {
esEntity.asDocMeta().version(version);
@ -287,7 +302,7 @@ public abstract class EsAbstractBehavior<ENTITY extends Entity, CB extends Condi
final EsAbstractEntity esEntity = (EsAbstractEntity) entity;
final DeleteRequestBuilder builder = createDeleteRequest(esEntity);
final DeleteResponse response = builder.execute().actionGet();
final DeleteResponse response = builder.execute().actionGet(deleteTimeout);
return response.isFound() ? 1 : 0;
}
@ -302,21 +317,25 @@ public abstract class EsAbstractBehavior<ENTITY extends Entity, CB extends Condi
@Override
protected int delegateQueryDelete(final ConditionBean cb, final DeleteOption<? extends ConditionBean> option) {
final SearchRequestBuilder builder =
client.prepareSearch(asEsIndex()).setTypes(asEsIndexType()).setSearchType(SearchType.SCAN).setScroll(scrollForDelete)
.setSize(sizeForDelete);
((EsAbstractConditionBean) cb).request().build(builder);
final SearchResponse response = ((EsAbstractConditionBean) cb).build(builder).execute().actionGet();
SearchResponse response = null;
int count = 0;
String scrollId = response.getScrollId();
while (scrollId != null) {
final SearchResponse scrollResponse = client.prepareSearchScroll(scrollId).setScroll(scrollForDelete).execute().actionGet();
scrollId = scrollResponse.getScrollId();
final SearchHits searchHits = scrollResponse.getHits();
while (true) {
if (response == null) {
final SearchRequestBuilder builder =
client.prepareSearch(asEsIndex()).setTypes(asEsIndexType()).setScroll(scrollForDelete).setSize(sizeForDelete);
final EsAbstractConditionBean esCb = (EsAbstractConditionBean) cb;
if (esCb.getPreference() != null) {
esCb.setPreference(esCb.getPreference());
}
esCb.request().build(builder);
response = esCb.build(builder).execute().actionGet(scrollSearchTimeout);
} else {
final String scrollId = response.getScrollId();
response = client.prepareSearchScroll(scrollId).setScroll(scrollForDelete).execute().actionGet(scrollSearchTimeout);
}
final SearchHits searchHits = response.getHits();
final SearchHit[] hits = searchHits.getHits();
if (hits.length == 0) {
scrollId = null;
break;
}
@ -325,7 +344,7 @@ public abstract class EsAbstractBehavior<ENTITY extends Entity, CB extends Condi
bulkRequest.add(client.prepareDelete(asEsIndex(), asEsIndexType(), hit.getId()));
}
count += hits.length;
final BulkResponse bulkResponse = bulkRequest.execute().actionGet();
final BulkResponse bulkResponse = bulkRequest.execute().actionGet(bulkTimeout);
if (bulkResponse.hasFailures()) {
throw new IllegalBehaviorStateException(bulkResponse.buildFailureMessage());
}
@ -390,7 +409,7 @@ public abstract class EsAbstractBehavior<ENTITY extends Entity, CB extends Condi
builderCall.callback(bulkBuilder);
}
final BulkResponse response = bulkBuilder.execute().actionGet();
final BulkResponse response = bulkBuilder.execute().actionGet(bulkTimeout);
final BulkItemResponse[] itemResponses = response.getItems();
if (itemResponses.length != entityList.size()) {
throw new IllegalStateException("Invalid response size: " + itemResponses.length + " != " + entityList.size());
@ -419,6 +438,42 @@ public abstract class EsAbstractBehavior<ENTITY extends Entity, CB extends Condi
return true;
}
public void setSizeForDelete(int sizeForDelete) {
this.sizeForDelete = sizeForDelete;
}
public void setScrollForDelete(String scrollForDelete) {
this.scrollForDelete = scrollForDelete;
}
public void setSizeForCursor(int sizeForCursor) {
this.sizeForCursor = sizeForCursor;
}
public void setScrollForCursor(String scrollForCursor) {
this.scrollForCursor = scrollForCursor;
}
public void setSearchTimeout(String searchTimeout) {
this.searchTimeout = searchTimeout;
}
public void setIndexTimeout(String indexTimeout) {
this.indexTimeout = indexTimeout;
}
public void setScrollSearchTimeout(String scrollSearchTimeout) {
this.scrollSearchTimeout = scrollSearchTimeout;
}
public void setBulkTimeout(String bulkTimeout) {
this.bulkTimeout = bulkTimeout;
}
public void setDeleteTimeout(String deleteTimeout) {
this.deleteTimeout = deleteTimeout;
}
// ===================================================================================
// Assist Logic
// ============

11
src/main/java/org/codelibs/fess/es/config/allcommon/EsAbstractConditionBean.java
View file

@ -51,7 +51,8 @@ public abstract class EsAbstractConditionBean implements ConditionBean {
// =========
protected final SqlClause _sqlClause = new EsSqlClause(asTableDbName());
protected int _safetyMaxResultSize;
private SearchRequestParams _searchRequestParams = new SearchRequestParams();
protected SearchRequestParams _searchRequestParams = new SearchRequestParams();
protected String _preference;
// ===================================================================================
// Builder
@ -599,6 +600,14 @@ public abstract class EsAbstractConditionBean implements ConditionBean {
return _searchRequestParams;
}
public void setPreference(final String preference) {
_preference = preference;
}
public String getPreference() {
return _preference;
}
// ===================================================================================
// General Helper
// ==============

2
src/main/java/org/codelibs/fess/es/config/bsentity/BsRoleType.java
View file

@ -15,7 +15,7 @@
*/
package org.codelibs.fess.es.config.bsentity;
import java.time.*;
import java.time.LocalDateTime;
import java.util.HashMap;
import java.util.Map;

125
src/main/java/org/codelibs/fess/es/log/allcommon/EsAbstractBehavior.java
View file

@ -42,14 +42,12 @@ import org.dbflute.util.DfTypeUtil;
import org.elasticsearch.action.bulk.BulkItemResponse;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.count.CountRequestBuilder;
import org.elasticsearch.action.delete.DeleteRequestBuilder;
import org.elasticsearch.action.delete.DeleteResponse;
import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.action.update.UpdateRequestBuilder;
import org.elasticsearch.client.Client;
import org.elasticsearch.search.SearchHit;
@ -69,6 +67,11 @@ public abstract class EsAbstractBehavior<ENTITY extends Entity, CB extends Condi
protected String scrollForDelete = "1m";
protected int sizeForCursor = 100;
protected String scrollForCursor = "1m";
protected String searchTimeout = "3m";
protected String indexTimeout = "3m";
protected String scrollSearchTimeout = "3m";
protected String bulkTimeout = "3m";
protected String deleteTimeout = "3m";
protected abstract String asEsIndex();
@ -84,8 +87,12 @@ public abstract class EsAbstractBehavior<ENTITY extends Entity, CB extends Condi
@Override
protected int delegateSelectCountUniquely(final ConditionBean cb) {
// #pending check response and cast problem
final CountRequestBuilder builder = client.prepareCount(asEsIndex()).setTypes(asEsSearchType());
return (int) ((EsAbstractConditionBean) cb).build(builder).execute().actionGet().getCount();
final SearchRequestBuilder builder = client.prepareSearch(asEsIndex()).setTypes(asEsSearchType());
final EsAbstractConditionBean esCb = (EsAbstractConditionBean) cb;
if (esCb.getPreference() != null) {
builder.setPreference(esCb.getPreference());
}
return (int) esCb.build(builder).execute().actionGet(searchTimeout).getHits().getTotalHits();
}
@Override
@ -116,8 +123,12 @@ public abstract class EsAbstractBehavior<ENTITY extends Entity, CB extends Condi
}
builder.setFrom(from);
builder.setSize(size);
((EsAbstractConditionBean) cb).request().build(builder);
final SearchResponse response = ((EsAbstractConditionBean) cb).build(builder).execute().actionGet();
final EsAbstractConditionBean esCb = (EsAbstractConditionBean) cb;
if (esCb.getPreference() != null) {
builder.setPreference(esCb.getPreference());
}
esCb.request().build(builder);
final SearchResponse response = esCb.build(builder).execute().actionGet(searchTimeout);
final EsPagingResultBean<RESULT> list = new EsPagingResultBean<>();
final SearchHits searchHits = response.getHits();
@ -192,20 +203,24 @@ public abstract class EsAbstractBehavior<ENTITY extends Entity, CB extends Condi
}
protected void delegateBulkRequest(final ConditionBean cb, Function<SearchHits, Boolean> handler) {
final SearchRequestBuilder builder =
client.prepareSearch(asEsIndex()).setTypes(asEsIndexType()).setSearchType(SearchType.SCAN).setScroll(scrollForCursor)
.setSize(sizeForCursor);
((EsAbstractConditionBean) cb).request().build(builder);
final SearchResponse response = ((EsAbstractConditionBean) cb).build(builder).execute().actionGet();
String scrollId = response.getScrollId();
while (scrollId != null) {
final SearchResponse scrollResponse = client.prepareSearchScroll(scrollId).setScroll(scrollForDelete).execute().actionGet();
scrollId = scrollResponse.getScrollId();
final SearchHits searchHits = scrollResponse.getHits();
SearchResponse response = null;
while (true) {
if (response == null) {
final SearchRequestBuilder builder =
client.prepareSearch(asEsIndex()).setTypes(asEsIndexType()).setScroll(scrollForCursor).setSize(sizeForCursor);
final EsAbstractConditionBean esCb = (EsAbstractConditionBean) cb;
if (esCb.getPreference() != null) {
builder.setPreference(esCb.getPreference());
}
esCb.request().build(builder);
response = esCb.build(builder).execute().actionGet(scrollSearchTimeout);
} else {
final String scrollId = response.getScrollId();
response = client.prepareSearchScroll(scrollId).setScroll(scrollForDelete).execute().actionGet(scrollSearchTimeout);
}
final SearchHits searchHits = response.getHits();
final SearchHit[] hits = searchHits.getHits();
if (hits.length == 0) {
scrollId = null;
break;
}
@ -237,7 +252,7 @@ public abstract class EsAbstractBehavior<ENTITY extends Entity, CB extends Condi
final EsAbstractEntity esEntity = (EsAbstractEntity) entity;
IndexRequestBuilder builder = createInsertRequest(esEntity);
final IndexResponse response = builder.execute().actionGet();
final IndexResponse response = builder.execute().actionGet(indexTimeout);
esEntity.asDocMeta().id(response.getId());
return response.isCreated() ? 1 : 0;
}
@ -260,7 +275,7 @@ public abstract class EsAbstractBehavior<ENTITY extends Entity, CB extends Condi
final EsAbstractEntity esEntity = (EsAbstractEntity) entity;
final IndexRequestBuilder builder = createUpdateRequest(esEntity);
final IndexResponse response = builder.execute().actionGet();
final IndexResponse response = builder.execute().actionGet(indexTimeout);
long version = response.getVersion();
if (version != -1) {
esEntity.asDocMeta().version(version);
@ -287,7 +302,7 @@ public abstract class EsAbstractBehavior<ENTITY extends Entity, CB extends Condi
final EsAbstractEntity esEntity = (EsAbstractEntity) entity;
final DeleteRequestBuilder builder = createDeleteRequest(esEntity);
final DeleteResponse response = builder.execute().actionGet();
final DeleteResponse response = builder.execute().actionGet(deleteTimeout);
return response.isFound() ? 1 : 0;
}
@ -302,21 +317,25 @@ public abstract class EsAbstractBehavior<ENTITY extends Entity, CB extends Condi
@Override
protected int delegateQueryDelete(final ConditionBean cb, final DeleteOption<? extends ConditionBean> option) {
final SearchRequestBuilder builder =
client.prepareSearch(asEsIndex()).setTypes(asEsIndexType()).setSearchType(SearchType.SCAN).setScroll(scrollForDelete)
.setSize(sizeForDelete);
((EsAbstractConditionBean) cb).request().build(builder);
final SearchResponse response = ((EsAbstractConditionBean) cb).build(builder).execute().actionGet();
SearchResponse response = null;
int count = 0;
String scrollId = response.getScrollId();
while (scrollId != null) {
final SearchResponse scrollResponse = client.prepareSearchScroll(scrollId).setScroll(scrollForDelete).execute().actionGet();
scrollId = scrollResponse.getScrollId();
final SearchHits searchHits = scrollResponse.getHits();
while (true) {
if (response == null) {
final SearchRequestBuilder builder =
client.prepareSearch(asEsIndex()).setTypes(asEsIndexType()).setScroll(scrollForDelete).setSize(sizeForDelete);
final EsAbstractConditionBean esCb = (EsAbstractConditionBean) cb;
if (esCb.getPreference() != null) {
esCb.setPreference(esCb.getPreference());
}
esCb.request().build(builder);
response = esCb.build(builder).execute().actionGet(scrollSearchTimeout);
} else {
final String scrollId = response.getScrollId();
response = client.prepareSearchScroll(scrollId).setScroll(scrollForDelete).execute().actionGet(scrollSearchTimeout);
}
final SearchHits searchHits = response.getHits();
final SearchHit[] hits = searchHits.getHits();
if (hits.length == 0) {
scrollId = null;
break;
}
@ -325,7 +344,7 @@ public abstract class EsAbstractBehavior<ENTITY extends Entity, CB extends Condi
bulkRequest.add(client.prepareDelete(asEsIndex(), asEsIndexType(), hit.getId()));
}
count += hits.length;
final BulkResponse bulkResponse = bulkRequest.execute().actionGet();
final BulkResponse bulkResponse = bulkRequest.execute().actionGet(bulkTimeout);
if (bulkResponse.hasFailures()) {
throw new IllegalBehaviorStateException(bulkResponse.buildFailureMessage());
}
@ -390,7 +409,7 @@ public abstract class EsAbstractBehavior<ENTITY extends Entity, CB extends Condi
builderCall.callback(bulkBuilder);
}
final BulkResponse response = bulkBuilder.execute().actionGet();
final BulkResponse response = bulkBuilder.execute().actionGet(bulkTimeout);
final BulkItemResponse[] itemResponses = response.getItems();
if (itemResponses.length != entityList.size()) {
throw new IllegalStateException("Invalid response size: " + itemResponses.length + " != " + entityList.size());
@ -419,6 +438,42 @@ public abstract class EsAbstractBehavior<ENTITY extends Entity, CB extends Condi
return true;
}
public void setSizeForDelete(int sizeForDelete) {
this.sizeForDelete = sizeForDelete;
}
public void setScrollForDelete(String scrollForDelete) {
this.scrollForDelete = scrollForDelete;
}
public void setSizeForCursor(int sizeForCursor) {
this.sizeForCursor = sizeForCursor;
}
public void setScrollForCursor(String scrollForCursor) {
this.scrollForCursor = scrollForCursor;
}
public void setSearchTimeout(String searchTimeout) {
this.searchTimeout = searchTimeout;
}
public void setIndexTimeout(String indexTimeout) {
this.indexTimeout = indexTimeout;
}
public void setScrollSearchTimeout(String scrollSearchTimeout) {
this.scrollSearchTimeout = scrollSearchTimeout;
}
public void setBulkTimeout(String bulkTimeout) {
this.bulkTimeout = bulkTimeout;
}
public void setDeleteTimeout(String deleteTimeout) {
this.deleteTimeout = deleteTimeout;
}
// ===================================================================================
// Assist Logic
// ============

11
src/main/java/org/codelibs/fess/es/log/allcommon/EsAbstractConditionBean.java
View file

@ -51,7 +51,8 @@ public abstract class EsAbstractConditionBean implements ConditionBean {
// =========
protected final SqlClause _sqlClause = new EsSqlClause(asTableDbName());
protected int _safetyMaxResultSize;
private SearchRequestParams _searchRequestParams = new SearchRequestParams();
protected SearchRequestParams _searchRequestParams = new SearchRequestParams();
protected String _preference;
// ===================================================================================
// Builder
@ -599,6 +600,14 @@ public abstract class EsAbstractConditionBean implements ConditionBean {
return _searchRequestParams;
}
public void setPreference(final String preference) {
_preference = preference;
}
public String getPreference() {
return _preference;
}
// ===================================================================================
// General Helper
// ==============

125
src/main/java/org/codelibs/fess/es/user/allcommon/EsAbstractBehavior.java
View file

@ -42,14 +42,12 @@ import org.dbflute.util.DfTypeUtil;
import org.elasticsearch.action.bulk.BulkItemResponse;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.count.CountRequestBuilder;
import org.elasticsearch.action.delete.DeleteRequestBuilder;
import org.elasticsearch.action.delete.DeleteResponse;
import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.action.update.UpdateRequestBuilder;
import org.elasticsearch.client.Client;
import org.elasticsearch.search.SearchHit;
@ -69,6 +67,11 @@ public abstract class EsAbstractBehavior<ENTITY extends Entity, CB extends Condi
protected String scrollForDelete = "1m";
protected int sizeForCursor = 100;
protected String scrollForCursor = "1m";
protected String searchTimeout = "3m";
protected String indexTimeout = "3m";
protected String scrollSearchTimeout = "3m";
protected String bulkTimeout = "3m";
protected String deleteTimeout = "3m";
protected abstract String asEsIndex();
@ -84,8 +87,12 @@ public abstract class EsAbstractBehavior<ENTITY extends Entity, CB extends Condi
@Override
protected int delegateSelectCountUniquely(final ConditionBean cb) {
// #pending check response and cast problem
final CountRequestBuilder builder = client.prepareCount(asEsIndex()).setTypes(asEsSearchType());
return (int) ((EsAbstractConditionBean) cb).build(builder).execute().actionGet().getCount();
final SearchRequestBuilder builder = client.prepareSearch(asEsIndex()).setTypes(asEsSearchType());
final EsAbstractConditionBean esCb = (EsAbstractConditionBean) cb;
if (esCb.getPreference() != null) {
builder.setPreference(esCb.getPreference());
}
return (int) esCb.build(builder).execute().actionGet(searchTimeout).getHits().getTotalHits();
}
@Override
@ -116,8 +123,12 @@ public abstract class EsAbstractBehavior<ENTITY extends Entity, CB extends Condi
}
builder.setFrom(from);
builder.setSize(size);
((EsAbstractConditionBean) cb).request().build(builder);
final SearchResponse response = ((EsAbstractConditionBean) cb).build(builder).execute().actionGet();
final EsAbstractConditionBean esCb = (EsAbstractConditionBean) cb;
if (esCb.getPreference() != null) {
builder.setPreference(esCb.getPreference());
}
esCb.request().build(builder);
final SearchResponse response = esCb.build(builder).execute().actionGet(searchTimeout);
final EsPagingResultBean<RESULT> list = new EsPagingResultBean<>();
final SearchHits searchHits = response.getHits();
@ -192,20 +203,24 @@ public abstract class EsAbstractBehavior<ENTITY extends Entity, CB extends Condi
}
protected void delegateBulkRequest(final ConditionBean cb, Function<SearchHits, Boolean> handler) {
final SearchRequestBuilder builder =
client.prepareSearch(asEsIndex()).setTypes(asEsIndexType()).setSearchType(SearchType.SCAN).setScroll(scrollForCursor)
.setSize(sizeForCursor);
((EsAbstractConditionBean) cb).request().build(builder);
final SearchResponse response = ((EsAbstractConditionBean) cb).build(builder).execute().actionGet();
String scrollId = response.getScrollId();
while (scrollId != null) {
final SearchResponse scrollResponse = client.prepareSearchScroll(scrollId).setScroll(scrollForDelete).execute().actionGet();
scrollId = scrollResponse.getScrollId();
final SearchHits searchHits = scrollResponse.getHits();
SearchResponse response = null;
while (true) {
if (response == null) {
final SearchRequestBuilder builder =
client.prepareSearch(asEsIndex()).setTypes(asEsIndexType()).setScroll(scrollForCursor).setSize(sizeForCursor);
final EsAbstractConditionBean esCb = (EsAbstractConditionBean) cb;
if (esCb.getPreference() != null) {
builder.setPreference(esCb.getPreference());
}
esCb.request().build(builder);
response = esCb.build(builder).execute().actionGet(scrollSearchTimeout);
} else {
final String scrollId = response.getScrollId();
response = client.prepareSearchScroll(scrollId).setScroll(scrollForDelete).execute().actionGet(scrollSearchTimeout);
}
final SearchHits searchHits = response.getHits();
final SearchHit[] hits = searchHits.getHits();
if (hits.length == 0) {
scrollId = null;
break;
}
@ -237,7 +252,7 @@ public abstract class EsAbstractBehavior<ENTITY extends Entity, CB extends Condi
final EsAbstractEntity esEntity = (EsAbstractEntity) entity;
IndexRequestBuilder builder = createInsertRequest(esEntity);
final IndexResponse response = builder.execute().actionGet();
final IndexResponse response = builder.execute().actionGet(indexTimeout);
esEntity.asDocMeta().id(response.getId());
return response.isCreated() ? 1 : 0;
}
@ -260,7 +275,7 @@ public abstract class EsAbstractBehavior<ENTITY extends Entity, CB extends Condi
final EsAbstractEntity esEntity = (EsAbstractEntity) entity;
final IndexRequestBuilder builder = createUpdateRequest(esEntity);
final IndexResponse response = builder.execute().actionGet();
final IndexResponse response = builder.execute().actionGet(indexTimeout);
long version = response.getVersion();
if (version != -1) {
esEntity.asDocMeta().version(version);
@ -287,7 +302,7 @@ public abstract class EsAbstractBehavior<ENTITY extends Entity, CB extends Condi
final EsAbstractEntity esEntity = (EsAbstractEntity) entity;
final DeleteRequestBuilder builder = createDeleteRequest(esEntity);
final DeleteResponse response = builder.execute().actionGet();
final DeleteResponse response = builder.execute().actionGet(deleteTimeout);
return response.isFound() ? 1 : 0;
}
@ -302,21 +317,25 @@ public abstract class EsAbstractBehavior<ENTITY extends Entity, CB extends Condi
@Override
protected int delegateQueryDelete(final ConditionBean cb, final DeleteOption<? extends ConditionBean> option) {
final SearchRequestBuilder builder =
client.prepareSearch(asEsIndex()).setTypes(asEsIndexType()).setSearchType(SearchType.SCAN).setScroll(scrollForDelete)
.setSize(sizeForDelete);
((EsAbstractConditionBean) cb).request().build(builder);
final SearchResponse response = ((EsAbstractConditionBean) cb).build(builder).execute().actionGet();
SearchResponse response = null;
int count = 0;
String scrollId = response.getScrollId();
while (scrollId != null) {
final SearchResponse scrollResponse = client.prepareSearchScroll(scrollId).setScroll(scrollForDelete).execute().actionGet();
scrollId = scrollResponse.getScrollId();
final SearchHits searchHits = scrollResponse.getHits();
while (true) {
if (response == null) {
final SearchRequestBuilder builder =
client.prepareSearch(asEsIndex()).setTypes(asEsIndexType()).setScroll(scrollForDelete).setSize(sizeForDelete);
final EsAbstractConditionBean esCb = (EsAbstractConditionBean) cb;
if (esCb.getPreference() != null) {
esCb.setPreference(esCb.getPreference());
}
esCb.request().build(builder);
response = esCb.build(builder).execute().actionGet(scrollSearchTimeout);
} else {
final String scrollId = response.getScrollId();
response = client.prepareSearchScroll(scrollId).setScroll(scrollForDelete).execute().actionGet(scrollSearchTimeout);
}
final SearchHits searchHits = response.getHits();
final SearchHit[] hits = searchHits.getHits();
if (hits.length == 0) {
scrollId = null;
break;
}
@ -325,7 +344,7 @@ public abstract class EsAbstractBehavior<ENTITY extends Entity, CB extends Condi
bulkRequest.add(client.prepareDelete(asEsIndex(), asEsIndexType(), hit.getId()));
}
count += hits.length;
final BulkResponse bulkResponse = bulkRequest.execute().actionGet();
final BulkResponse bulkResponse = bulkRequest.execute().actionGet(bulkTimeout);
if (bulkResponse.hasFailures()) {
throw new IllegalBehaviorStateException(bulkResponse.buildFailureMessage());
}
@ -390,7 +409,7 @@ public abstract class EsAbstractBehavior<ENTITY extends Entity, CB extends Condi
builderCall.callback(bulkBuilder);
}
final BulkResponse response = bulkBuilder.execute().actionGet();
final BulkResponse response = bulkBuilder.execute().actionGet(bulkTimeout);
final BulkItemResponse[] itemResponses = response.getItems();
if (itemResponses.length != entityList.size()) {
throw new IllegalStateException("Invalid response size: " + itemResponses.length + " != " + entityList.size());
@ -419,6 +438,42 @@ public abstract class EsAbstractBehavior<ENTITY extends Entity, CB extends Condi
return true;
}
public void setSizeForDelete(int sizeForDelete) {
this.sizeForDelete = sizeForDelete;
}
public void setScrollForDelete(String scrollForDelete) {
this.scrollForDelete = scrollForDelete;
}
public void setSizeForCursor(int sizeForCursor) {
this.sizeForCursor = sizeForCursor;
}
public void setScrollForCursor(String scrollForCursor) {
this.scrollForCursor = scrollForCursor;
}
public void setSearchTimeout(String searchTimeout) {
this.searchTimeout = searchTimeout;
}
public void setIndexTimeout(String indexTimeout) {
this.indexTimeout = indexTimeout;
}
public void setScrollSearchTimeout(String scrollSearchTimeout) {
this.scrollSearchTimeout = scrollSearchTimeout;
}
public void setBulkTimeout(String bulkTimeout) {
this.bulkTimeout = bulkTimeout;
}
public void setDeleteTimeout(String deleteTimeout) {
this.deleteTimeout = deleteTimeout;
}
// ===================================================================================
// Assist Logic
// ============

11
src/main/java/org/codelibs/fess/es/user/allcommon/EsAbstractConditionBean.java
View file

@ -51,7 +51,8 @@ public abstract class EsAbstractConditionBean implements ConditionBean {
// =========
protected final SqlClause _sqlClause = new EsSqlClause(asTableDbName());
protected int _safetyMaxResultSize;
private SearchRequestParams _searchRequestParams = new SearchRequestParams();
protected SearchRequestParams _searchRequestParams = new SearchRequestParams();
protected String _preference;
// ===================================================================================
// Builder
@ -599,6 +600,14 @@ public abstract class EsAbstractConditionBean implements ConditionBean {
return _searchRequestParams;
}
public void setPreference(final String preference) {
_preference = preference;
}
public String getPreference() {
return _preference;
}
// ===================================================================================
// General Helper
// ==============

7
src/main/java/org/codelibs/fess/exec/Crawler.java
View file

@ -80,9 +80,6 @@ public class Crawler implements Serializable {
@Resource
protected CrawlingInfoService crawlingInfoService;
@Resource
protected DynamicProperties systemProperties;
public static class Options {
@Option(name = "-s", aliases = "--sessionId", metaVar = "sessionId", usage = "Session ID")
@ -302,7 +299,8 @@ public class Crawler implements Serializable {
}
protected void sendMail(final Map<String, String> infoMap) {
final String toStrs = (String) systemProperties.get(Constants.NOTIFICATION_TO_PROPERTY);
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final String toStrs = fessConfig.getNotificationTo();
if (StringUtil.isNotBlank(toStrs)) {
final String[] toAddresses = toStrs.split(",");
final Map<String, String> dataMap = new HashMap<>();
@ -314,7 +312,6 @@ public class Crawler implements Serializable {
logger.debug("\ninfoMap: {}\ndataMap: {}", infoMap, dataMap);
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final Postbox postbox = ComponentUtil.getComponent(Postbox.class);
CrawlerPostcard.droppedInto(postbox, postcard -> {
postcard.setFrom(fessConfig.getMailFromAddress(), fessConfig.getMailFromName());

4
src/main/java/org/codelibs/fess/helper/DataIndexHelper.java
View file

@ -25,7 +25,6 @@ import java.util.Map;
import javax.annotation.Resource;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.core.misc.DynamicProperties;
import org.codelibs.fess.Constants;
import org.codelibs.fess.app.service.DataConfigService;
import org.codelibs.fess.app.service.FailureUrlService;
@ -48,9 +47,6 @@ public class DataIndexHelper implements Serializable {
private static final String DELETE_OLD_DOCS = "delete_old_docs";
@Resource
protected DynamicProperties systemProperties;
@Resource
public DataConfigService dataConfigService;

4
src/main/java/org/codelibs/fess/helper/QueryHelper.java
View file

@ -49,7 +49,6 @@ import org.apache.lucene.search.TermRangeQuery;
import org.apache.lucene.search.WildcardQuery;
import org.apache.lucene.util.BytesRef;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.core.misc.DynamicProperties;
import org.codelibs.fess.Constants;
import org.codelibs.fess.entity.FacetInfo;
import org.codelibs.fess.entity.GeoInfo;
@ -80,9 +79,6 @@ public class QueryHelper implements Serializable {
protected static final String INURL_FIELD = "inurl";
@Resource
protected DynamicProperties systemProperties;
@Resource
protected FessConfig fessConfig;

3
src/main/java/org/codelibs/fess/helper/RoleQueryHelper.java
View file

@ -31,7 +31,6 @@ import javax.servlet.http.HttpServletRequest;
import org.codelibs.core.crypto.CachedCipher;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.fess.Constants;
import org.codelibs.fess.mylasta.action.FessUserBean;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.util.ComponentUtil;
@ -109,7 +108,7 @@ public class RoleQueryHelper {
final RequestManager requestManager = ComponentUtil.getRequestManager();
requestManager.findUserBean(FessUserBean.class)
.ifPresent(fessUserBean -> stream(fessUserBean.getPermissions()).of(stream -> stream.forEach(roleList::add)))
.orElse(() -> roleList.add(fessConfig.getRoleSearchUserPrefix() + Constants.GUEST_USER));
.orElse(() -> roleList.addAll(fessConfig.getSearchGuestPermissionList()));
if (defaultRoleList != null) {
roleList.addAll(defaultRoleList);

14
src/main/java/org/codelibs/fess/helper/SearchLogHelper.java
View file

@ -26,13 +26,11 @@ import java.util.Queue;
import java.util.concurrent.ConcurrentLinkedQueue;
import javax.annotation.PostConstruct;
import javax.annotation.Resource;
import javax.servlet.http.HttpServletRequest;
import org.apache.commons.lang3.StringUtils;
import org.codelibs.core.collection.LruHashMap;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.core.misc.DynamicProperties;
import org.codelibs.fess.Constants;
import org.codelibs.fess.app.service.SearchService;
import org.codelibs.fess.es.log.exbhv.ClickLogBhv;
@ -59,9 +57,6 @@ import org.slf4j.LoggerFactory;
public class SearchLogHelper {
private static final Logger logger = LoggerFactory.getLogger(SearchLogHelper.class);
@Resource
protected DynamicProperties systemProperties;
public long userCheckInterval = 5 * 60 * 1000L;// 5 min
public int userInfoCacheSize = 1000;
@ -190,7 +185,8 @@ public class SearchLogHelper {
}
protected void processSearchLogQueue(final Queue<SearchLog> queue) {
final String value = ComponentUtil.getFessConfig().getPurgeByBots();
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final String value = fessConfig.getPurgeByBots();
String[] botNames;
if (StringUtil.isBlank(value)) {
botNames = StringUtil.EMPTY_STRINGS;
@ -245,8 +241,10 @@ public class SearchLogHelper {
if (!searchLogList.isEmpty()) {
storeSearchLogList(searchLogList);
final SuggestHelper suggestHelper = ComponentUtil.getSuggestHelper();
suggestHelper.indexFromSearchLog(searchLogList);
if (fessConfig.isSuggestSearchLog()) {
final SuggestHelper suggestHelper = ComponentUtil.getSuggestHelper();
suggestHelper.indexFromSearchLog(searchLogList);
}
}
}

8
src/main/java/org/codelibs/fess/helper/SuggestHelper.java
View file

@ -27,7 +27,6 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.function.Consumer;
import java.util.regex.Pattern;
import javax.annotation.PostConstruct;
import javax.annotation.Resource;
@ -76,8 +75,6 @@ public class SuggestHelper {
private List<String> contentFieldList;
private final List<Pattern> roleFilterList = new ArrayList<>();
@PostConstruct
public void init() {
fessConfig = ComponentUtil.getFessConfig();
@ -88,9 +85,6 @@ public class SuggestHelper {
split(fessConfig.getSuggestFieldRoles(), ",").of(
stream -> stream.filter(StringUtil::isNotBlank).forEach(f -> roleFieldNameSet.add(f)));
contentFieldList = Arrays.asList(stream(fessConfig.getSuggestFieldContents()).get(stream -> stream.toArray(n -> new String[n])));
split(fessConfig.getSuggestRoleFilters(), ",").of(stream -> stream.filter(StringUtil::isNotBlank).forEach(filter -> {
roleFilterList.add(Pattern.compile(filter));
}));
fessEsClient.admin().cluster().prepareHealth().setWaitForYellowStatus().execute().actionGet(fessConfig.getIndexHealthTimeout());
@ -136,7 +130,7 @@ public class SuggestHelper {
if (sb.length() > 0) {
stream(searchLog.getRoles()).of(stream -> stream.forEach(role -> roles.add(role)));
if (roles.stream().allMatch(v -> roleFilterList.stream().anyMatch(pattern -> pattern.matcher(v).matches()))) {
if (fessConfig.isValidSearchLogPermissions(roles.toArray(new String[roles.size()]))) {
suggester.indexer().indexFromSearchWord(sb.toString(), fields.toArray(new String[fields.size()]),
tags.toArray(new String[tags.size()]), roles.toArray(new String[roles.size()]), 1);
}

10
src/main/java/org/codelibs/fess/helper/ViewHelper.java
View file

@ -93,15 +93,15 @@ public class ViewHelper implements Serializable {
private static final Logger logger = LoggerFactory.getLogger(ViewHelper.class);
@Resource
protected DynamicProperties systemProperties;
@Resource
protected PathMappingHelper pathMappingHelper;
@Resource
protected UserAgentHelper userAgentHelper;
@Resource
protected DynamicProperties systemProperties;
public int descriptionLength = 200;
public int titleLength = 50;
@ -309,8 +309,8 @@ public class ViewHelper implements Serializable {
}
protected String appendQueryParameter(final Map<String, Object> document, final String url) {
if (Constants.TRUE.equals(systemProperties.get(Constants.APPEND_QUERY_PARAMETER_PROPERTY))) {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final FessConfig fessConfig = ComponentUtil.getFessConfig();
if (fessConfig.isAppendQueryParameter()) {
final String mimetype = DocumentUtil.getValue(document, fessConfig.getIndexFieldMimetype(), String.class);
if (StringUtil.isNotBlank(mimetype)) {
if ("application/pdf".equals(mimetype)) {

4
src/main/java/org/codelibs/fess/helper/WebFsIndexHelper.java
View file

@ -24,7 +24,6 @@ import java.util.Map;
import javax.annotation.Resource;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.core.misc.DynamicProperties;
import org.codelibs.fess.Constants;
import org.codelibs.fess.app.service.BoostDocumentRuleService;
import org.codelibs.fess.app.service.FailureUrlService;
@ -53,9 +52,6 @@ public class WebFsIndexHelper implements Serializable {
private static final Logger logger = LoggerFactory.getLogger(WebFsIndexHelper.class);
@Resource
protected DynamicProperties systemProperties;
@Resource
public WebConfigService webConfigService;

58
src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java
View file

@ -457,9 +457,15 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
/** The key of the configuration. e.g. admin */
String AUTHENTICATION_ADMIN_ROLES = "authentication.admin.roles";
/** The key of the configuration. e.g. {role}guest */
/** The key of the configuration. e.g. */
String ROLE_SEARCH_DEFAULT_PERMISSIONS = "role.search.default.permissions";
/** The key of the configuration. e.g. {role}guest */
String ROLE_SEARCH_DEFAULT_DISPLAY_PERMISSIONS = "role.search.default.display.permissions";
/** The key of the configuration. e.g. {role}guest */
String ROLE_SEARCH_GUEST_PERMISSIONS = "role.search.guest.permissions";
/** The key of the configuration. e.g. 1 */
String ROLE_SEARCH_USER_PREFIX = "role.search.user.prefix";
@ -739,8 +745,8 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
/** The key of the configuration. e.g. 60 */
String SUGGEST_POPULAR_WORD_CACHE_EXPIRE = "suggest.popular.word.cache.expire";
/** The key of the configuration. e.g. Rguest */
String SUGGEST_ROLE_FILTERS = "suggest.role.filters";
/** The key of the configuration. e.g. {user}guest,{role}guest */
String SUGGEST_SEARCH_LOG_PERMISSIONS = "suggest.search.log.permissions";
/** The key of the configuration. e.g. false */
String LDAP_ADMIN_ENABLED = "ldap.admin.enabled";
@ -2159,11 +2165,33 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
/**
* Get the value for the key 'role.search.default.permissions'. <br>
* The value is, e.g. {role}guest <br>
* The value is, e.g. <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getRoleSearchDefaultPermissions();
/**
* Get the value for the key 'role.search.default.permissions' as {@link Integer}. <br>
* The value is, e.g. <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
* @throws NumberFormatException When the property is not integer.
*/
Integer getRoleSearchDefaultPermissionsAsInteger();
/**
* Get the value for the key 'role.search.default.display.permissions'. <br>
* The value is, e.g. {role}guest <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getRoleSearchDefaultDisplayPermissions();
/**
* Get the value for the key 'role.search.guest.permissions'. <br>
* The value is, e.g. {role}guest <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getRoleSearchGuestPermissions();
/**
* Get the value for the key 'role.search.user.prefix'. <br>
* The value is, e.g. 1 <br>
@ -3214,11 +3242,11 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
Integer getSuggestPopularWordCacheExpireAsInteger();
/**
* Get the value for the key 'suggest.role.filters'. <br>
* The value is, e.g. Rguest <br>
* Get the value for the key 'suggest.search.log.permissions'. <br>
* The value is, e.g. {user}guest,{role}guest <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getSuggestRoleFilters();
String getSuggestSearchLogPermissions();
/**
* Get the value for the key 'ldap.admin.enabled'. <br>
@ -4306,6 +4334,18 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
return get(FessConfig.ROLE_SEARCH_DEFAULT_PERMISSIONS);
}
public Integer getRoleSearchDefaultPermissionsAsInteger() {
return getAsInteger(FessConfig.ROLE_SEARCH_DEFAULT_PERMISSIONS);
}
public String getRoleSearchDefaultDisplayPermissions() {
return get(FessConfig.ROLE_SEARCH_DEFAULT_DISPLAY_PERMISSIONS);
}
public String getRoleSearchGuestPermissions() {
return get(FessConfig.ROLE_SEARCH_GUEST_PERMISSIONS);
}
public String getRoleSearchUserPrefix() {
return get(FessConfig.ROLE_SEARCH_USER_PREFIX);
}
@ -4866,8 +4906,8 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
return getAsInteger(FessConfig.SUGGEST_POPULAR_WORD_CACHE_EXPIRE);
}
public String getSuggestRoleFilters() {
return get(FessConfig.SUGGEST_ROLE_FILTERS);
public String getSuggestSearchLogPermissions() {
return get(FessConfig.SUGGEST_SEARCH_LOG_PERMISSIONS);
}
public String getLdapAdminEnabled() {

66
src/main/java/org/codelibs/fess/mylasta/direction/FessProp.java
View file

@ -20,6 +20,7 @@ import static org.codelibs.core.stream.StreamUtil.stream;
import java.util.Collections;
import java.util.Enumeration;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
@ -47,6 +48,10 @@ import org.lastaflute.web.util.LaRequestUtil;
public interface FessProp {
public static final String SEARCH_GUEST_PERMISSION_LIST = "searchGuestPermissionList";
public static final String SUGGEST_SEARCH_LOG_PERMISSIONS = "suggestSearchLogPermissions";
public static final String GROUP_VALUE_PREFIX = "group:";
public static final String ROLE_VALUE_PREFIX = "role:";
@ -111,6 +116,18 @@ public interface FessProp {
setSystemProperty(key, Integer.toString(value));
}
public default boolean isWebDesignEditorEnabled() {
return getSystemPropertyAsBoolean(Constants.WEB_DESIGN_EDITOR_PROPERTY, true);
}
public default boolean isSearchFileProxyEnabled() {
return getSystemPropertyAsBoolean(Constants.SEARCH_FILE_PROXY_PROPERTY, true);
}
public default boolean isBrowserLocaleForSearchUsed() {
return getSystemPropertyAsBoolean(Constants.USE_BROWSER_LOCALE_FOR_SEARCH_PROPERTY, false);
}
public default String[] getDefaultSortValues(final OptionalThing<FessUserBean> userBean) {
@SuppressWarnings("unchecked")
Map<String, String> map = (Map<String, String>) propMap.get(DEFAULT_SORT_VALUES);
@ -898,8 +915,17 @@ public interface FessProp {
.toArray(n -> new String[n]));
}
String getRoleSearchDefaultDisplayPermissions();
public default String[] getSearchDefaultDisplayEncodedPermissions() {
final PermissionHelper permissionHelper = ComponentUtil.getPermissionHelper();
return stream(getRoleSearchDefaultDisplayPermissions().split(","))
.get(stream -> stream.map(p -> permissionHelper.encode(p)).filter(StringUtil::isNotBlank).distinct()
.toArray(n -> new String[n]));
}
public default String getSearchDefaultDisplayPermission() {
return stream(getRoleSearchDefaultPermissions().split(",")).get(
return stream(getRoleSearchDefaultDisplayPermissions().split(",")).get(
stream -> stream.filter(StringUtil::isNotBlank).distinct().collect(Collectors.joining("\n")));
}
@ -910,4 +936,42 @@ public interface FessProp {
stream -> stream.map(s -> s.trim()).filter(StringUtil::isNotBlank).toArray(n -> new String[n]));
}
String getSuggestSearchLogPermissions();
public default boolean isValidSearchLogPermissions(final String[] permissions) {
if (permissions == null) {
return false;
}
@SuppressWarnings("unchecked")
List<String> validPermissionList = (List<String>) propMap.get(SUGGEST_SEARCH_LOG_PERMISSIONS);
if (validPermissionList == null) {
final PermissionHelper permissionHelper = ComponentUtil.getPermissionHelper();
validPermissionList =
stream(getSuggestSearchLogPermissions().split(",")).get(
stream -> stream.map(s -> permissionHelper.encode(s)).filter(StringUtil::isNotBlank)
.collect(Collectors.toList()));
propMap.put(SUGGEST_SEARCH_LOG_PERMISSIONS, validPermissionList);
}
final List<String> list = validPermissionList;
return stream(permissions).get(stream -> stream.allMatch(v -> list.contains(v)));
}
String getRoleSearchUserPrefix();
String getRoleSearchGuestPermissions();
public default List<String> getSearchGuestPermissionList() {
@SuppressWarnings("unchecked")
List<String> list = (List<String>) propMap.get(SEARCH_GUEST_PERMISSION_LIST);
if (list == null) {
final PermissionHelper permissionHelper = ComponentUtil.getPermissionHelper();
list =
stream(getRoleSearchGuestPermissions().split(",")).get(
stream -> stream.map(s -> permissionHelper.encode(s)).filter(StringUtil::isNotBlank)
.collect(Collectors.toList()));
list.add(getRoleSearchUserPrefix() + Constants.GUEST_USER);
propMap.put(SEARCH_GUEST_PERMISSION_LIST, list);
}
return list;
}
}

8
src/main/resources/fess_config.properties
View file

@ -256,7 +256,9 @@ index.backup.targets=.fess_basic_config,.fess_config,.fess_user
authentication.admin.users=admin
authentication.admin.roles=admin
role.search.default.permissions={role}guest
role.search.default.permissions=
role.search.default.display.permissions={role}guest
role.search.guest.permissions={role}guest
role.search.user.prefix=1
role.search.group.prefix=2
@ -396,9 +398,7 @@ suggest.update.request.interval=1
suggest.source.reader.scroll.size=1
suggest.popular.word.cache.size=1000
suggest.popular.word.cache.expire=60
suggest.role.filters=\
Rguest
suggest.search.log.permissions={user}guest,{role}guest
# ----------------------------------------------------------
# LDAP

2
src/main/resources/fess_indices/.fess_user/role.bulk
View file

@ -1,2 +1,4 @@
{"index":{"_index":".fess_user","_type":"role","_id":"YWRtaW4="}}
{"name":"admin"}
{"index":{"_index":".fess_user","_type":"role","_id":"Z3Vlc3Q="}}
{"name":"guest"}

2
src/main/resources/fess_indices/.fess_user/user.bulk
View file

@ -1,2 +1,2 @@
{"index":{"_index":".fess_user","_type":"user","_id":"YWRtaW4="}}
{"password":"8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918","roles":["YWRtaW4="],"name":"admin"}
{"password":"8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918","surname":"admin","roles":["YWRtaW4=","Z3Vlc3Q="],"name":"admin"}