diff --git a/src/main/assemblies/files/fess.in.bat b/src/main/assemblies/files/fess.in.bat index 8ff37d914..59e99150e 100644 --- a/src/main/assemblies/files/fess.in.bat +++ b/src/main/assemblies/files/fess.in.bat @@ -84,7 +84,11 @@ set APP_NAME=fess set ES_HOME=%FESS_HOME%/es set FESS_CLASSPATH=%FESS_HOME%\lib\classes +if ""%FESS_JAVA_OPTS%""=="""" ( +set FESS_JAVA_OPTS=-Dfess +) else ( set FESS_JAVA_OPTS=-Dfess %FESS_JAVA_OPTS% +) set FESS_JAVA_OPTS=%FESS_JAVA_OPTS% -Des-foreground=yes set FESS_JAVA_OPTS=%FESS_JAVA_OPTS% -Dfess.home="%FESS_HOME%" set FESS_JAVA_OPTS=%FESS_JAVA_OPTS% -Dfess.es.dir="%ES_HOME%" diff --git a/src/main/config/es/fess_config.json b/src/main/config/es/fess_config.json index dd87101d5..f7339ec08 100644 --- a/src/main/config/es/fess_config.json +++ b/src/main/config/es/fess_config.json @@ -975,6 +975,10 @@ "createdTime" : { "type" : "long" }, + "target" : { + "type" : "string", + "index" : "not_analyzed" + }, "generator" : { "type" : "string", "index" : "not_analyzed" diff --git a/src/main/java/org/codelibs/fess/Constants.java b/src/main/java/org/codelibs/fess/Constants.java index 4bd6216b3..1c555cc91 100644 --- a/src/main/java/org/codelibs/fess/Constants.java +++ b/src/main/java/org/codelibs/fess/Constants.java @@ -404,4 +404,6 @@ public class Constants extends CoreLibConstants { public static final String PERMISSIONS = "permissions"; public static final String CIPHER_PREFIX = "{cipher}"; + + public static final String SYSTEM_USER = "system"; } diff --git a/src/main/java/org/codelibs/fess/app/service/BadWordService.java b/src/main/java/org/codelibs/fess/app/service/BadWordService.java index c6f891c25..847159b0a 100644 --- a/src/main/java/org/codelibs/fess/app/service/BadWordService.java +++ b/src/main/java/org/codelibs/fess/app/service/BadWordService.java @@ -123,11 +123,11 @@ public class BadWordService { } else if (badWord == null) { badWord = new BadWord(); badWord.setSuggestWord(targetWord); - badWord.setCreatedBy("system"); + badWord.setCreatedBy(Constants.SYSTEM_USER); badWord.setCreatedTime(now); badWordBhv.insert(badWord); } else { - badWord.setUpdatedBy("system"); + badWord.setUpdatedBy(Constants.SYSTEM_USER); badWord.setUpdatedTime(now); badWordBhv.update(badWord); } diff --git a/src/main/java/org/codelibs/fess/app/service/ElevateWordService.java b/src/main/java/org/codelibs/fess/app/service/ElevateWordService.java index db64be5f3..52c4a7111 100644 --- a/src/main/java/org/codelibs/fess/app/service/ElevateWordService.java +++ b/src/main/java/org/codelibs/fess/app/service/ElevateWordService.java @@ -221,7 +221,7 @@ public class ElevateWordService { elevateWord.setPermissions(permissions); elevateWord.setTargetLabel(label); elevateWord.setBoost(StringUtil.isBlank(boost) ? 1.0f : Float.parseFloat(boost)); - elevateWord.setCreatedBy("system"); + elevateWord.setCreatedBy(Constants.SYSTEM_USER); elevateWord.setCreatedTime(now); elevateWordBhv.insert(elevateWord); } else if (StringUtil.isBlank(reading) && StringUtil.isBlank(boost)) { @@ -230,7 +230,7 @@ public class ElevateWordService { elevateWord.setReading(reading); elevateWord.setPermissions(permissions); elevateWord.setBoost(StringUtil.isBlank(boost) ? 1.0f : Float.parseFloat(boost)); - elevateWord.setUpdatedBy("system"); + elevateWord.setUpdatedBy(Constants.SYSTEM_USER); elevateWord.setUpdatedTime(now); elevateWordBhv.update(elevateWord); } diff --git a/src/main/java/org/codelibs/fess/app/web/admin/backup/AdminBackupAction.java b/src/main/java/org/codelibs/fess/app/web/admin/backup/AdminBackupAction.java index 56c7352fd..796e49d6c 100644 --- a/src/main/java/org/codelibs/fess/app/web/admin/backup/AdminBackupAction.java +++ b/src/main/java/org/codelibs/fess/app/web/admin/backup/AdminBackupAction.java @@ -77,7 +77,7 @@ public class AdminBackupAction extends FessAdminAction { if (fileName.startsWith("system") && fileName.endsWith(".properties")) { try (final InputStream in = form.bulkFile.getInputStream()) { ComponentUtil.getSystemProperties().load(in); - } catch (IOException e) { + } catch (final IOException e) { logger.warn("Failed to process system.properties file: " + form.bulkFile.getFileName(), e); } } else { diff --git a/src/main/java/org/codelibs/fess/app/web/admin/upgrade/AdminUpgradeAction.java b/src/main/java/org/codelibs/fess/app/web/admin/upgrade/AdminUpgradeAction.java index a750a470e..23e7c026e 100644 --- a/src/main/java/org/codelibs/fess/app/web/admin/upgrade/AdminUpgradeAction.java +++ b/src/main/java/org/codelibs/fess/app/web/admin/upgrade/AdminUpgradeAction.java @@ -59,11 +59,14 @@ import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse; import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse; import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse; import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetaData; +import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse; import org.elasticsearch.action.admin.indices.open.OpenIndexRequest; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.IndicesAdminClient; +import org.elasticsearch.cluster.metadata.MappingMetaData; +import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; @@ -169,7 +172,7 @@ public class AdminUpgradeAction extends FessAdminAction { saveInfo(messages -> messages.addSuccessUpgradeFrom(GLOBAL)); - fessEsClient.refresh(); + systemHelper.reloadConfiguration(); } catch (final Exception e) { logger.warn("Failed to upgrade data.", e); saveError(messages -> messages.addErrorsFailedToUpgradeFrom(GLOBAL, VERSION_10_1, e.getLocalizedMessage())); @@ -182,7 +185,7 @@ public class AdminUpgradeAction extends FessAdminAction { saveInfo(messages -> messages.addSuccessUpgradeFrom(GLOBAL)); - fessEsClient.refresh(); + systemHelper.reloadConfiguration(); } catch (final Exception e) { logger.warn("Failed to upgrade data.", e); saveError(messages -> messages.addErrorsFailedToUpgradeFrom(GLOBAL, VERSION_10_1, e.getLocalizedMessage())); @@ -196,7 +199,7 @@ public class AdminUpgradeAction extends FessAdminAction { saveInfo(messages -> messages.addSuccessUpgradeFrom(GLOBAL)); - fessEsClient.refresh(); + systemHelper.reloadConfiguration(); } catch (final Exception e) { logger.warn("Failed to upgrade data.", e); saveError(messages -> messages.addErrorsFailedToUpgradeFrom(GLOBAL, VERSION_10_0, e.getLocalizedMessage())); @@ -224,11 +227,25 @@ public class AdminUpgradeAction extends FessAdminAction { final String searchIndex = fessConfig.getIndexDocumentSearchIndex(); // update mapping + addMapping(indicesClient, updateIndex, "access_token", "fess_indices/.fess_config"); + addMapping(indicesClient, updateIndex, "thumbnail_queue", "fess_indices/.fess_config"); + addFieldMapping(indicesClient, updateIndex, "doc", "filename", "{\"properties\":{\"filename\":{\"type\":\"string\",\"index\":\"not_analyzed\"}}}"); addFieldMapping(indicesClient, searchIndex, "doc", "filename", "{\"properties\":{\"filename\":{\"type\":\"string\",\"index\":\"not_analyzed\"}}}"); addFieldMapping(indicesClient, configIndex, "job_log", "lastUpdated", "{\"properties\":{\"lastUpdated\":{\"type\":\"long\"}}}"); + + // data migration + addData(configIndex, + "scheduled_job", + "thumbnail_generate", + "{\"name\":\"Thumbnail Generator\",\"target\":\"all\",\"cronExpression\":\"* * * * *\",\"scriptType\":\"groovy\",\"scriptData\":\"return container.getComponent(\\\"generateThumbnailJob\\\").execute();\",\"jobLogging\":false,\"crawler\":false,\"available\":true,\"sortOrder\":7,\"createdBy\":\"system\",\"createdTime\":0,\"updatedBy\":\"system\",\"updatedTime\":0}"); + addData(configIndex, + "scheduled_job", + "ping_es", + "{\"name\":\"Ping Elasticsearch\",\"target\":\"all\",\"cronExpression\":\"* * * * *\",\"scriptType\":\"groovy\",\"scriptData\":\"return container.getComponent(\\\"pingEsJob\\\").execute();\",\"jobLogging\":false,\"crawler\":false,\"available\":true,\"sortOrder\":8,\"createdBy\":\"system\",\"createdTime\":0,\"updatedBy\":\"system\",\"updatedTime\":0}"); + } private void upgradeFrom10_1() { @@ -1030,6 +1047,34 @@ public class AdminUpgradeAction extends FessAdminAction { } } + private void addMapping(final IndicesAdminClient indicesClient, final String index, final String type, final String indexResourcePath) { + final GetMappingsResponse getMappingsResponse = + indicesClient.prepareGetMappings(index).execute().actionGet(fessConfig.getIndexIndicesTimeout()); + final ImmutableOpenMap indexMappings = getMappingsResponse.mappings().get(index); + if (indexMappings == null || !indexMappings.containsKey(type)) { + String source = null; + final String mappingFile = indexResourcePath + "/" + type + ".json"; + try { + source = FileUtil.readUTF8(mappingFile); + } catch (final Exception e) { + logger.warn(mappingFile + " is not found.", e); + } + try { + final PutMappingResponse putMappingResponse = + indicesClient.preparePutMapping(index).setType(type).setSource(source).execute() + .actionGet(fessConfig.getIndexIndicesTimeout()); + if (putMappingResponse.isAcknowledged()) { + logger.info("Created " + index + "/" + type + " mapping."); + } else { + logger.warn("Failed to create " + index + "/" + type + " mapping."); + } + // TODO bulk + } catch (final Exception e) { + logger.warn("Failed to create " + index + "/" + type + " mapping.", e); + } + } + } + private void addFieldMapping(final IndicesAdminClient indicesClient, final String index, final String type, final String field, final String source) { final GetFieldMappingsResponse gfmResponse = diff --git a/src/main/java/org/codelibs/fess/ds/impl/GitBucketDataStoreImpl.java b/src/main/java/org/codelibs/fess/ds/impl/GitBucketDataStoreImpl.java index 283a90485..c007eb676 100644 --- a/src/main/java/org/codelibs/fess/ds/impl/GitBucketDataStoreImpl.java +++ b/src/main/java/org/codelibs/fess/ds/impl/GitBucketDataStoreImpl.java @@ -74,12 +74,12 @@ public class GitBucketDataStoreImpl extends AbstractDataStoreImpl { final CrawlingConfig crawlingConfig = new CrawlingConfigWrapper(dataConfig) { @Override - public Map initializeClientFactory(CrawlerClientFactory crawlerClientFactory) { + public Map initializeClientFactory(final CrawlerClientFactory crawlerClientFactory) { final Map paramMap = super.initializeClientFactory(crawlerClientFactory); - List headerList = new ArrayList<>(); - RequestHeader[] headers = (RequestHeader[]) paramMap.get(HcHttpClient.REQUERT_HEADERS_PROPERTY); + final List headerList = new ArrayList<>(); + final RequestHeader[] headers = (RequestHeader[]) paramMap.get(HcHttpClient.REQUERT_HEADERS_PROPERTY); if (headers != null) { - for (RequestHeader header : headers) { + for (final RequestHeader header : headers) { headerList.add(header); } } @@ -175,7 +175,7 @@ public class GitBucketDataStoreImpl extends AbstractDataStoreImpl { } private void storeFileContent(final String rootURL, final String authToken, final String owner, final String name, - List roleList, final String path, final CrawlingConfig crawlingConfig, final IndexUpdateCallback callback, + final List roleList, final String path, final CrawlingConfig crawlingConfig, final IndexUpdateCallback callback, final Map paramMap, final Map scriptMap, final Map defaultDataMap) { final String apiUrl = rootURL + "api/v3/repos/" + owner + "/" + name + "/contents/" + path; final String viewUrl = rootURL + owner + "/" + name + "/blob/master/" + path; @@ -199,7 +199,7 @@ public class GitBucketDataStoreImpl extends AbstractDataStoreImpl { } protected void collectFileNames(final String rootURL, final String authToken, final String owner, final String name, final String path, - final int depth, final long readInterval, Consumer consumer) { + final int depth, final long readInterval, final Consumer consumer) { if (MAX_DEPTH <= depth) { return; diff --git a/src/main/java/org/codelibs/fess/entity/PingResponse.java b/src/main/java/org/codelibs/fess/entity/PingResponse.java index 0c57252b3..ab39ff568 100644 --- a/src/main/java/org/codelibs/fess/entity/PingResponse.java +++ b/src/main/java/org/codelibs/fess/entity/PingResponse.java @@ -21,13 +21,13 @@ import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.cluster.health.ClusterHealthStatus; public class PingResponse { - private int status; + private final int status; - private List failures; + private final List failures; - private String clusterName; + private final String clusterName; - private String clusterStatus; + private final String clusterStatus; public PingResponse(final ClusterHealthResponse response) { status = response.getStatus() == ClusterHealthStatus.RED ? 1 : 0; diff --git a/src/main/java/org/codelibs/fess/es/config/bsbhv/BsThumbnailQueueBhv.java b/src/main/java/org/codelibs/fess/es/config/bsbhv/BsThumbnailQueueBhv.java index ff12bb127..db44bf458 100644 --- a/src/main/java/org/codelibs/fess/es/config/bsbhv/BsThumbnailQueueBhv.java +++ b/src/main/java/org/codelibs/fess/es/config/bsbhv/BsThumbnailQueueBhv.java @@ -75,6 +75,7 @@ public abstract class BsThumbnailQueueBhv extends EsAbstractBehavior ((ThumbnailQueue) et).setCreatedBy(DfTypeUtil.toString(vl)), "createdBy"); setupEpg(_epgMap, et -> ((ThumbnailQueue) et).getCreatedTime(), (et, vl) -> ((ThumbnailQueue) et).setCreatedTime(DfTypeUtil.toLong(vl)), "createdTime"); + setupEpg(_epgMap, et -> ((ThumbnailQueue) et).getTarget(), (et, vl) -> ((ThumbnailQueue) et).setTarget(DfTypeUtil.toString(vl)), + "target"); setupEpg(_epgMap, et -> ((ThumbnailQueue) et).getGenerator(), (et, vl) -> ((ThumbnailQueue) et).setGenerator(DfTypeUtil.toString(vl)), "generator"); setupEpg(_epgMap, et -> ((ThumbnailQueue) et).getPath(), (et, vl) -> ((ThumbnailQueue) et).setPath(DfTypeUtil.toString(vl)), "path"); @@ -127,6 +129,8 @@ public class ThumbnailQueueDbm extends AbstractDBMeta { false, "String", 0, 0, null, false, null, null, null, null, null, false); protected final ColumnInfo _columnCreatedTime = cci("createdTime", "createdTime", null, null, Long.class, "createdTime", null, false, false, false, "Long", 0, 0, null, false, null, null, null, null, null, false); + protected final ColumnInfo _columnTarget = cci("target", "target", null, null, String.class, "target", null, false, false, false, + "String", 0, 0, null, false, null, null, null, null, null, false); protected final ColumnInfo _columnGenerator = cci("generator", "generator", null, null, String.class, "generator", null, false, false, false, "String", 0, 0, null, false, null, null, null, null, null, false); protected final ColumnInfo _columnPath = cci("path", "path", null, null, String.class, "path", null, false, false, false, "String", 0, @@ -142,6 +146,10 @@ public class ThumbnailQueueDbm extends AbstractDBMeta { return _columnCreatedTime; } + public ColumnInfo columnTarget() { + return _columnTarget; + } + public ColumnInfo columnGenerator() { return _columnGenerator; } @@ -158,6 +166,7 @@ public class ThumbnailQueueDbm extends AbstractDBMeta { List ls = newArrayList(); ls.add(columnCreatedBy()); ls.add(columnCreatedTime()); + ls.add(columnTarget()); ls.add(columnGenerator()); ls.add(columnPath()); ls.add(columnUrl()); diff --git a/src/main/java/org/codelibs/fess/es/config/cbean/bs/BsThumbnailQueueCB.java b/src/main/java/org/codelibs/fess/es/config/cbean/bs/BsThumbnailQueueCB.java index 6eda159c9..30e51dd69 100644 --- a/src/main/java/org/codelibs/fess/es/config/cbean/bs/BsThumbnailQueueCB.java +++ b/src/main/java/org/codelibs/fess/es/config/cbean/bs/BsThumbnailQueueCB.java @@ -155,6 +155,10 @@ public class BsThumbnailQueueCB extends EsAbstractConditionBean { doColumn("createdTime"); } + public void columnTarget() { + doColumn("target"); + } + public void columnGenerator() { doColumn("generator"); } diff --git a/src/main/java/org/codelibs/fess/es/config/cbean/cq/bs/BsThumbnailQueueCQ.java b/src/main/java/org/codelibs/fess/es/config/cbean/cq/bs/BsThumbnailQueueCQ.java index f038381b5..16988e66c 100644 --- a/src/main/java/org/codelibs/fess/es/config/cbean/cq/bs/BsThumbnailQueueCQ.java +++ b/src/main/java/org/codelibs/fess/es/config/cbean/cq/bs/BsThumbnailQueueCQ.java @@ -543,6 +543,213 @@ public abstract class BsThumbnailQueueCQ extends EsAbstractConditionQuery { return this; } + public void setTarget_Equal(String target) { + setTarget_Term(target, null); + } + + public void setTarget_Equal(String target, ConditionOptionCall opLambda) { + setTarget_Term(target, opLambda); + } + + public void setTarget_Term(String target) { + setTarget_Term(target, null); + } + + public void setTarget_Term(String target, ConditionOptionCall opLambda) { + TermQueryBuilder builder = regTermQ("target", target); + if (opLambda != null) { + opLambda.callback(builder); + } + } + + public void setTarget_NotEqual(String target) { + setTarget_NotTerm(target, null); + } + + public void setTarget_NotTerm(String target) { + setTarget_NotTerm(target, null); + } + + public void setTarget_NotEqual(String target, ConditionOptionCall opLambda) { + setTarget_NotTerm(target, opLambda); + } + + public void setTarget_NotTerm(String target, ConditionOptionCall opLambda) { + not(not -> not.setTarget_Term(target), opLambda); + } + + public void setTarget_Terms(Collection targetList) { + setTarget_Terms(targetList, null); + } + + public void setTarget_Terms(Collection targetList, ConditionOptionCall opLambda) { + TermsQueryBuilder builder = regTermsQ("target", targetList); + if (opLambda != null) { + opLambda.callback(builder); + } + } + + public void setTarget_InScope(Collection targetList) { + setTarget_Terms(targetList, null); + } + + public void setTarget_InScope(Collection targetList, ConditionOptionCall opLambda) { + setTarget_Terms(targetList, opLambda); + } + + public void setTarget_Match(String target) { + setTarget_Match(target, null); + } + + public void setTarget_Match(String target, ConditionOptionCall opLambda) { + MatchQueryBuilder builder = regMatchQ("target", target); + if (opLambda != null) { + opLambda.callback(builder); + } + } + + public void setTarget_MatchPhrase(String target) { + setTarget_MatchPhrase(target, null); + } + + public void setTarget_MatchPhrase(String target, ConditionOptionCall opLambda) { + MatchQueryBuilder builder = regMatchPhraseQ("target", target); + if (opLambda != null) { + opLambda.callback(builder); + } + } + + public void setTarget_MatchPhrasePrefix(String target) { + setTarget_MatchPhrasePrefix(target, null); + } + + public void setTarget_MatchPhrasePrefix(String target, ConditionOptionCall opLambda) { + MatchQueryBuilder builder = regMatchPhrasePrefixQ("target", target); + if (opLambda != null) { + opLambda.callback(builder); + } + } + + public void setTarget_Fuzzy(String target) { + setTarget_Fuzzy(target, null); + } + + public void setTarget_Fuzzy(String target, ConditionOptionCall opLambda) { + FuzzyQueryBuilder builder = regFuzzyQ("target", target); + if (opLambda != null) { + opLambda.callback(builder); + } + } + + public void setTarget_Prefix(String target) { + setTarget_Prefix(target, null); + } + + public void setTarget_Prefix(String target, ConditionOptionCall opLambda) { + PrefixQueryBuilder builder = regPrefixQ("target", target); + if (opLambda != null) { + opLambda.callback(builder); + } + } + + public void setTarget_Wildcard(String target) { + setTarget_Wildcard(target, null); + } + + public void setTarget_Wildcard(String target, ConditionOptionCall opLambda) { + WildcardQueryBuilder builder = regWildcardQ("target", target); + if (opLambda != null) { + opLambda.callback(builder); + } + } + + public void setTarget_Regexp(String target) { + setTarget_Regexp(target, null); + } + + public void setTarget_Regexp(String target, ConditionOptionCall opLambda) { + RegexpQueryBuilder builder = regRegexpQ("target", target); + if (opLambda != null) { + opLambda.callback(builder); + } + } + + public void setTarget_GreaterThan(String target) { + setTarget_GreaterThan(target, null); + } + + public void setTarget_GreaterThan(String target, ConditionOptionCall opLambda) { + RangeQueryBuilder builder = regRangeQ("target", ConditionKey.CK_GREATER_THAN, target); + if (opLambda != null) { + opLambda.callback(builder); + } + } + + public void setTarget_LessThan(String target) { + setTarget_LessThan(target, null); + } + + public void setTarget_LessThan(String target, ConditionOptionCall opLambda) { + RangeQueryBuilder builder = regRangeQ("target", ConditionKey.CK_LESS_THAN, target); + if (opLambda != null) { + opLambda.callback(builder); + } + } + + public void setTarget_GreaterEqual(String target) { + setTarget_GreaterEqual(target, null); + } + + public void setTarget_GreaterEqual(String target, ConditionOptionCall opLambda) { + RangeQueryBuilder builder = regRangeQ("target", ConditionKey.CK_GREATER_EQUAL, target); + if (opLambda != null) { + opLambda.callback(builder); + } + } + + public void setTarget_LessEqual(String target) { + setTarget_LessEqual(target, null); + } + + public void setTarget_LessEqual(String target, ConditionOptionCall opLambda) { + RangeQueryBuilder builder = regRangeQ("target", ConditionKey.CK_LESS_EQUAL, target); + if (opLambda != null) { + opLambda.callback(builder); + } + } + + public void setTarget_Exists() { + setTarget_Exists(null); + } + + public void setTarget_Exists(ConditionOptionCall opLambda) { + ExistsQueryBuilder builder = regExistsQ("target"); + if (opLambda != null) { + opLambda.callback(builder); + } + } + + public void setTarget_CommonTerms(String target) { + setTarget_CommonTerms(target, null); + } + + public void setTarget_CommonTerms(String target, ConditionOptionCall opLambda) { + CommonTermsQueryBuilder builder = regCommonTermsQ("target", target); + if (opLambda != null) { + opLambda.callback(builder); + } + } + + public BsThumbnailQueueCQ addOrderBy_Target_Asc() { + regOBA("target"); + return this; + } + + public BsThumbnailQueueCQ addOrderBy_Target_Desc() { + regOBD("target"); + return this; + } + public void setGenerator_Equal(String generator) { setGenerator_Term(generator, null); } diff --git a/src/main/java/org/codelibs/fess/es/config/exentity/AccessToken.java b/src/main/java/org/codelibs/fess/es/config/exentity/AccessToken.java index 42a2acdc5..8b2c39bc7 100644 --- a/src/main/java/org/codelibs/fess/es/config/exentity/AccessToken.java +++ b/src/main/java/org/codelibs/fess/es/config/exentity/AccessToken.java @@ -50,7 +50,7 @@ public class AccessToken extends BsAccessToken { return new Date(getExpiredTime().longValue()); } - public void setExpires(Date date) { + public void setExpires(final Date date) { setExpiredTime(date != null ? date.getTime() : null); } diff --git a/src/main/java/org/codelibs/fess/es/config/exentity/CrawlingConfigWrapper.java b/src/main/java/org/codelibs/fess/es/config/exentity/CrawlingConfigWrapper.java index b2be78c86..17a0b8ef9 100644 --- a/src/main/java/org/codelibs/fess/es/config/exentity/CrawlingConfigWrapper.java +++ b/src/main/java/org/codelibs/fess/es/config/exentity/CrawlingConfigWrapper.java @@ -21,49 +21,59 @@ import org.codelibs.fess.crawler.client.CrawlerClientFactory; public class CrawlingConfigWrapper implements CrawlingConfig { - private CrawlingConfig crawlingConfig; + private final CrawlingConfig crawlingConfig; public CrawlingConfigWrapper(final CrawlingConfig crawlingConfig) { this.crawlingConfig = crawlingConfig; } + @Override public String getId() { return crawlingConfig.getId(); } + @Override public String getName() { return crawlingConfig.getName(); } + @Override public String[] getPermissions() { return crawlingConfig.getPermissions(); } + @Override public String[] getLabelTypeValues() { return crawlingConfig.getLabelTypeValues(); } + @Override public String getDocumentBoost() { return crawlingConfig.getDocumentBoost(); } - public String getIndexingTarget(String input) { + @Override + public String getIndexingTarget(final String input) { return crawlingConfig.getIndexingTarget(input); } + @Override public String getConfigId() { return crawlingConfig.getConfigId(); } + @Override public Integer getTimeToLive() { return crawlingConfig.getTimeToLive(); } - public Map initializeClientFactory(CrawlerClientFactory crawlerClientFactory) { + @Override + public Map initializeClientFactory(final CrawlerClientFactory crawlerClientFactory) { return crawlingConfig.initializeClientFactory(crawlerClientFactory); } - public Map getConfigParameterMap(ConfigName name) { + @Override + public Map getConfigParameterMap(final ConfigName name) { return crawlingConfig.getConfigParameterMap(name); } } diff --git a/src/main/java/org/codelibs/fess/helper/SystemHelper.java b/src/main/java/org/codelibs/fess/helper/SystemHelper.java index b459ca683..655483640 100644 --- a/src/main/java/org/codelibs/fess/helper/SystemHelper.java +++ b/src/main/java/org/codelibs/fess/helper/SystemHelper.java @@ -299,6 +299,7 @@ public class SystemHelper { } public void reloadConfiguration() { + ComponentUtil.getFessEsClient().refresh(); ComponentUtil.getLabelTypeHelper().init(); ComponentUtil.getPathMappingHelper().init(); ComponentUtil.getSuggestHelper().init(); @@ -316,7 +317,7 @@ public class SystemHelper { this.random = random; } - public boolean isChangedClusterState(int status) { + public boolean isChangedClusterState(final int status) { return previousClusterState.getAndSet(status) != status; } diff --git a/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java b/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java index 142d3fc9c..d4e94f1f0 100644 --- a/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java +++ b/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java @@ -652,6 +652,9 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction /** The key of the configuration. e.g. true */ String THUMBNAIL_HTML_PHANTOMJS_ENABLED = "thumbnail.html.phantomjs.enabled"; + /** The key of the configuration. e.g. all */ + String THUMBNAIL_GENERATOR_TARGETS = "thumbnail.generator.targets"; + /** The key of the configuration. e.g. Administrator */ String MAIL_FROM_NAME = "mail.from.name"; @@ -3163,6 +3166,13 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction */ boolean isThumbnailHtmlPhantomjsEnabled(); + /** + * Get the value for the key 'thumbnail.generator.targets'.
+ * The value is, e.g. all
+ * @return The value of found property. (NotNull: if not found, exception but basically no way) + */ + String getThumbnailGeneratorTargets(); + /** * Get the value for the key 'mail.from.name'.
* The value is, e.g. Administrator
@@ -5398,6 +5408,10 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction return is(FessConfig.THUMBNAIL_HTML_PHANTOMJS_ENABLED); } + public String getThumbnailGeneratorTargets() { + return get(FessConfig.THUMBNAIL_GENERATOR_TARGETS); + } + public String getMailFromName() { return get(FessConfig.MAIL_FROM_NAME); } diff --git a/src/main/java/org/codelibs/fess/mylasta/direction/FessProp.java b/src/main/java/org/codelibs/fess/mylasta/direction/FessProp.java index 5dcdb6fe1..7da3e4177 100644 --- a/src/main/java/org/codelibs/fess/mylasta/direction/FessProp.java +++ b/src/main/java/org/codelibs/fess/mylasta/direction/FessProp.java @@ -1438,4 +1438,12 @@ public interface FessProp { stream -> stream.filter(StringUtil::isNotBlank).map(s -> s.trim()).forEach(list::add)); return list.toArray(new String[list.size()]); } + + String getThumbnailGeneratorTargets(); + + public default String[] getThumbnailGeneratorTargetsAsArray() { + return getThumbnailGeneratorTargets().split(","); + + } + } diff --git a/src/main/java/org/codelibs/fess/thumbnail/ThumbnailManager.java b/src/main/java/org/codelibs/fess/thumbnail/ThumbnailManager.java index 3aaa19a57..4f8ebfd3d 100644 --- a/src/main/java/org/codelibs/fess/thumbnail/ThumbnailManager.java +++ b/src/main/java/org/codelibs/fess/thumbnail/ThumbnailManager.java @@ -29,7 +29,6 @@ import java.util.Map; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -45,6 +44,7 @@ import org.codelibs.fess.es.config.exbhv.ThumbnailQueueBhv; import org.codelibs.fess.es.config.exentity.ThumbnailQueue; import org.codelibs.fess.exception.FessSystemException; import org.codelibs.fess.exception.JobProcessingException; +import org.codelibs.fess.helper.SystemHelper; import org.codelibs.fess.mylasta.direction.FessConfig; import org.codelibs.fess.util.ComponentUtil; import org.codelibs.fess.util.DocumentUtil; @@ -52,6 +52,8 @@ import org.lastaflute.web.util.LaRequestUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.collect.Lists; + public class ThumbnailManager { private static final String DEFAULT_SCREENSHOT_DIR = "/WEB-INF/thumbnails"; @@ -116,12 +118,12 @@ public class ThumbnailManager { final List> taskList = new ArrayList<>(); while (generating) { try { - Tuple3 task = thumbnailTaskQueue.poll(thumbnailTaskQueueTimeout, TimeUnit.MILLISECONDS); + final Tuple3 task = thumbnailTaskQueue.poll(thumbnailTaskQueueTimeout, TimeUnit.MILLISECONDS); if (task == null) { if (!taskList.isEmpty()) { storeQueue(taskList); } - } else { + } else if (!taskList.contains(task)) { taskList.add(task); if (taskList.size() > thumbnailTaskBulkSize) { storeQueue(taskList); @@ -158,13 +160,22 @@ public class ThumbnailManager { } protected void storeQueue(final List> taskList) { - List list = taskList.stream().filter(entity -> entity != null).map(task -> { - ThumbnailQueue entity = new ThumbnailQueue(); - entity.setGenerator(task.getValue1()); - entity.setUrl(task.getValue2()); - entity.setPath(task.getValue3()); - return entity; - }).collect(Collectors.toList()); + final FessConfig fessConfig = ComponentUtil.getFessConfig(); + final SystemHelper systemHelper = ComponentUtil.getSystemHelper(); + final String[] targets = fessConfig.getThumbnailGeneratorTargetsAsArray(); + final List list = new ArrayList<>(); + taskList.stream().filter(entity -> entity != null).forEach(task -> { + for (final String target : targets) { + final ThumbnailQueue entity = new ThumbnailQueue(); + entity.setGenerator(task.getValue1()); + entity.setUrl(task.getValue2()); + entity.setPath(task.getValue3()); + entity.setTarget(target); + entity.setCreatedBy(Constants.SYSTEM_USER); + entity.setCreatedTime(systemHelper.getCurrentTimeAsLong()); + list.add(entity); + } + }); taskList.clear(); final ThumbnailQueueBhv thumbnailQueueBhv = ComponentUtil.getComponent(ThumbnailQueueBhv.class); thumbnailQueueBhv.batchInsert(list); @@ -175,6 +186,11 @@ public class ThumbnailManager { final List idList = new ArrayList<>(); final ThumbnailQueueBhv thumbnailQueueBhv = ComponentUtil.getComponent(ThumbnailQueueBhv.class); thumbnailQueueBhv.selectList(cb -> { + if (StringUtil.isBlank(fessConfig.getSchedulerTargetName())) { + cb.query().setTarget_Equal(Constants.DEFAULT_JOB_TARGET); + } else { + cb.query().setTarget_InScope(Lists.newArrayList(Constants.DEFAULT_JOB_TARGET, fessConfig.getSchedulerTargetName())); + } cb.query().addOrderBy_CreatedTime_Asc(); cb.fetchFirst(fessConfig.getPageThumbnailQueueMaxFetchSizeAsInteger()); }).forEach(entity -> { @@ -213,7 +229,7 @@ public class ThumbnailManager { if (generator.isTarget(docMap)) { final String url = DocumentUtil.getValue(docMap, fessConfig.getIndexFieldUrl(), String.class); final String path = getImageFilename(docMap); - Tuple3 task = new Tuple3(generator.getName(), url, path); + final Tuple3 task = new Tuple3<>(generator.getName(), url, path); thumbnailTaskQueue.offer(task); break; } diff --git a/src/main/java/org/codelibs/fess/thumbnail/impl/BaseThumbnailGenerator.java b/src/main/java/org/codelibs/fess/thumbnail/impl/BaseThumbnailGenerator.java index 2a660ee5e..2f05e8b51 100644 --- a/src/main/java/org/codelibs/fess/thumbnail/impl/BaseThumbnailGenerator.java +++ b/src/main/java/org/codelibs/fess/thumbnail/impl/BaseThumbnailGenerator.java @@ -105,11 +105,12 @@ public abstract class BaseThumbnailGenerator implements ThumbnailGenerator { this.generatorList = generatorList; } + @Override public String getName() { return name; } - public void setName(String name) { + public void setName(final String name) { this.name = name; } diff --git a/src/main/java/org/codelibs/fess/thumbnail/impl/WebDriverGenerator.java b/src/main/java/org/codelibs/fess/thumbnail/impl/WebDriverGenerator.java index 2fac91744..8fadff087 100644 --- a/src/main/java/org/codelibs/fess/thumbnail/impl/WebDriverGenerator.java +++ b/src/main/java/org/codelibs/fess/thumbnail/impl/WebDriverGenerator.java @@ -139,7 +139,7 @@ public class WebDriverGenerator extends BaseThumbnailGenerator { final File thumbnail = ((TakesScreenshot) webDriver).getScreenshotAs(OutputType.FILE); convert(thumbnail, outputFile); return true; - } catch (UnreachableBrowserException e) { + } catch (final UnreachableBrowserException e) { if (logger.isDebugEnabled()) { logger.debug("WebDriver is not available.", e); } @@ -265,7 +265,7 @@ public class WebDriverGenerator extends BaseThumbnailGenerator { this.thumbnailHeight = thumbnailHeight; } - public void setUnreachableCheckInterval(long unreachableCheckInterval) { + public void setUnreachableCheckInterval(final long unreachableCheckInterval) { this.unreachableCheckInterval = unreachableCheckInterval; } } diff --git a/src/main/resources/fess_config.properties b/src/main/resources/fess_config.properties index 1f465e316..a2fbe499e 100644 --- a/src/main/resources/fess_config.properties +++ b/src/main/resources/fess_config.properties @@ -349,6 +349,7 @@ paging.search.page.size=20 paging.search.page.max.size=100 thumbnail.html.phantomjs.enabled=true +thumbnail.generator.targets=all # ---------------------------------------------------------- # Mail diff --git a/src/main/resources/fess_indices/.fess_config/thumbnail_queue.json b/src/main/resources/fess_indices/.fess_config/thumbnail_queue.json index 39534599e..4457e2b41 100644 --- a/src/main/resources/fess_indices/.fess_config/thumbnail_queue.json +++ b/src/main/resources/fess_indices/.fess_config/thumbnail_queue.json @@ -19,6 +19,10 @@ "type": "string", "index": "not_analyzed" }, + "target": { + "type": "string", + "index": "not_analyzed" + }, "createdTime": { "type": "long" }, diff --git a/src/main/webapp/WEB-INF/view/admin/accesstoken/admin_accesstoken.jsp b/src/main/webapp/WEB-INF/view/admin/accesstoken/admin_accesstoken.jsp index d647cf29a..ae204c8cf 100644 --- a/src/main/webapp/WEB-INF/view/admin/accesstoken/admin_accesstoken.jsp +++ b/src/main/webapp/WEB-INF/view/admin/accesstoken/admin_accesstoken.jsp @@ -10,7 +10,7 @@
- +
diff --git a/src/main/webapp/WEB-INF/view/admin/accesstoken/admin_accesstoken_details.jsp b/src/main/webapp/WEB-INF/view/admin/accesstoken/admin_accesstoken_details.jsp index ecf2b4877..f6bd0bad5 100644 --- a/src/main/webapp/WEB-INF/view/admin/accesstoken/admin_accesstoken_details.jsp +++ b/src/main/webapp/WEB-INF/view/admin/accesstoken/admin_accesstoken_details.jsp @@ -10,7 +10,7 @@
- +
diff --git a/src/main/webapp/WEB-INF/view/admin/accesstoken/admin_accesstoken_edit.jsp b/src/main/webapp/WEB-INF/view/admin/accesstoken/admin_accesstoken_edit.jsp index 4e3102ec5..82b5e8f33 100644 --- a/src/main/webapp/WEB-INF/view/admin/accesstoken/admin_accesstoken_edit.jsp +++ b/src/main/webapp/WEB-INF/view/admin/accesstoken/admin_accesstoken_edit.jsp @@ -10,7 +10,7 @@
- +