0ct0pu5/fess
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch '10.3.x'

Browse source
This commit is contained in:
Shinsuke Sugaya 2016-11-05 07:34:33 +09:00
commit 5676f46667
27 changed files with 389 additions and 41 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
src/main/assemblies/files/fess.in.bat
View file

@ -84,7 +84,11 @@ set APP_NAME=fess
set ES_HOME=%FESS_HOME%/es
set FESS_CLASSPATH=%FESS_HOME%\lib\classes
if ""%FESS_JAVA_OPTS%""=="""" (
set FESS_JAVA_OPTS=-Dfess
) else (
set FESS_JAVA_OPTS=-Dfess %FESS_JAVA_OPTS%
)
set FESS_JAVA_OPTS=%FESS_JAVA_OPTS% -Des-foreground=yes
set FESS_JAVA_OPTS=%FESS_JAVA_OPTS% -Dfess.home="%FESS_HOME%"
set FESS_JAVA_OPTS=%FESS_JAVA_OPTS% -Dfess.es.dir="%ES_HOME%"

4
src/main/config/es/fess_config.json
View file

@ -975,6 +975,10 @@
"createdTime" : {
"type" : "long"
},
"target" : {
"type" : "string",
"index" : "not_analyzed"
},
"generator" : {
"type" : "string",
"index" : "not_analyzed"

2
src/main/java/org/codelibs/fess/Constants.java
View file

@ -404,4 +404,6 @@ public class Constants extends CoreLibConstants {
public static final String PERMISSIONS = "permissions";
public static final String CIPHER_PREFIX = "{cipher}";
public static final String SYSTEM_USER = "system";
}

4
src/main/java/org/codelibs/fess/app/service/BadWordService.java
View file

@ -123,11 +123,11 @@ public class BadWordService {
} else if (badWord == null) {
badWord = new BadWord();
badWord.setSuggestWord(targetWord);
badWord.setCreatedBy("system");
badWord.setCreatedBy(Constants.SYSTEM_USER);
badWord.setCreatedTime(now);
badWordBhv.insert(badWord);
} else {
badWord.setUpdatedBy("system");
badWord.setUpdatedBy(Constants.SYSTEM_USER);
badWord.setUpdatedTime(now);
badWordBhv.update(badWord);
}

4
src/main/java/org/codelibs/fess/app/service/ElevateWordService.java
View file

@ -221,7 +221,7 @@ public class ElevateWordService {
elevateWord.setPermissions(permissions);
elevateWord.setTargetLabel(label);
elevateWord.setBoost(StringUtil.isBlank(boost) ? 1.0f : Float.parseFloat(boost));
elevateWord.setCreatedBy("system");
elevateWord.setCreatedBy(Constants.SYSTEM_USER);
elevateWord.setCreatedTime(now);
elevateWordBhv.insert(elevateWord);
} else if (StringUtil.isBlank(reading) && StringUtil.isBlank(boost)) {
@ -230,7 +230,7 @@ public class ElevateWordService {
elevateWord.setReading(reading);
elevateWord.setPermissions(permissions);
elevateWord.setBoost(StringUtil.isBlank(boost) ? 1.0f : Float.parseFloat(boost));
elevateWord.setUpdatedBy("system");
elevateWord.setUpdatedBy(Constants.SYSTEM_USER);
elevateWord.setUpdatedTime(now);
elevateWordBhv.update(elevateWord);
}

2
src/main/java/org/codelibs/fess/app/web/admin/backup/AdminBackupAction.java
View file

@ -77,7 +77,7 @@ public class AdminBackupAction extends FessAdminAction {
if (fileName.startsWith("system") && fileName.endsWith(".properties")) {
try (final InputStream in = form.bulkFile.getInputStream()) {
ComponentUtil.getSystemProperties().load(in);
} catch (IOException e) {
} catch (final IOException e) {
logger.warn("Failed to process system.properties file: " + form.bulkFile.getFileName(), e);
}
} else {

51
src/main/java/org/codelibs/fess/app/web/admin/upgrade/AdminUpgradeAction.java
View file

@ -59,11 +59,14 @@ import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse;
import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetaData;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
import org.elasticsearch.action.admin.indices.open.OpenIndexRequest;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.support.IndicesOptions;
import org.elasticsearch.client.IndicesAdminClient;
import org.elasticsearch.cluster.metadata.MappingMetaData;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentParser;
@ -169,7 +172,7 @@ public class AdminUpgradeAction extends FessAdminAction {
saveInfo(messages -> messages.addSuccessUpgradeFrom(GLOBAL));
fessEsClient.refresh();
systemHelper.reloadConfiguration();
} catch (final Exception e) {
logger.warn("Failed to upgrade data.", e);
saveError(messages -> messages.addErrorsFailedToUpgradeFrom(GLOBAL, VERSION_10_1, e.getLocalizedMessage()));
@ -182,7 +185,7 @@ public class AdminUpgradeAction extends FessAdminAction {
saveInfo(messages -> messages.addSuccessUpgradeFrom(GLOBAL));
fessEsClient.refresh();
systemHelper.reloadConfiguration();
} catch (final Exception e) {
logger.warn("Failed to upgrade data.", e);
saveError(messages -> messages.addErrorsFailedToUpgradeFrom(GLOBAL, VERSION_10_1, e.getLocalizedMessage()));
@ -196,7 +199,7 @@ public class AdminUpgradeAction extends FessAdminAction {
saveInfo(messages -> messages.addSuccessUpgradeFrom(GLOBAL));
fessEsClient.refresh();
systemHelper.reloadConfiguration();
} catch (final Exception e) {
logger.warn("Failed to upgrade data.", e);
saveError(messages -> messages.addErrorsFailedToUpgradeFrom(GLOBAL, VERSION_10_0, e.getLocalizedMessage()));
@ -224,11 +227,25 @@ public class AdminUpgradeAction extends FessAdminAction {
final String searchIndex = fessConfig.getIndexDocumentSearchIndex();
// update mapping
addMapping(indicesClient, updateIndex, "access_token", "fess_indices/.fess_config");
addMapping(indicesClient, updateIndex, "thumbnail_queue", "fess_indices/.fess_config");
addFieldMapping(indicesClient, updateIndex, "doc", "filename",
"{\"properties\":{\"filename\":{\"type\":\"string\",\"index\":\"not_analyzed\"}}}");
addFieldMapping(indicesClient, searchIndex, "doc", "filename",
"{\"properties\":{\"filename\":{\"type\":\"string\",\"index\":\"not_analyzed\"}}}");
addFieldMapping(indicesClient, configIndex, "job_log", "lastUpdated", "{\"properties\":{\"lastUpdated\":{\"type\":\"long\"}}}");
// data migration
addData(configIndex,
"scheduled_job",
"thumbnail_generate",
"{\"name\":\"Thumbnail Generator\",\"target\":\"all\",\"cronExpression\":\"* * * * *\",\"scriptType\":\"groovy\",\"scriptData\":\"return container.getComponent(\\\"generateThumbnailJob\\\").execute();\",\"jobLogging\":false,\"crawler\":false,\"available\":true,\"sortOrder\":7,\"createdBy\":\"system\",\"createdTime\":0,\"updatedBy\":\"system\",\"updatedTime\":0}");
addData(configIndex,
"scheduled_job",
"ping_es",
"{\"name\":\"Ping Elasticsearch\",\"target\":\"all\",\"cronExpression\":\"* * * * *\",\"scriptType\":\"groovy\",\"scriptData\":\"return container.getComponent(\\\"pingEsJob\\\").execute();\",\"jobLogging\":false,\"crawler\":false,\"available\":true,\"sortOrder\":8,\"createdBy\":\"system\",\"createdTime\":0,\"updatedBy\":\"system\",\"updatedTime\":0}");
}
private void upgradeFrom10_1() {
@ -1030,6 +1047,34 @@ public class AdminUpgradeAction extends FessAdminAction {
}
}
private void addMapping(final IndicesAdminClient indicesClient, final String index, final String type, final String indexResourcePath) {
final GetMappingsResponse getMappingsResponse =
indicesClient.prepareGetMappings(index).execute().actionGet(fessConfig.getIndexIndicesTimeout());
final ImmutableOpenMap<String, MappingMetaData> indexMappings = getMappingsResponse.mappings().get(index);
if (indexMappings == null || !indexMappings.containsKey(type)) {
String source = null;
final String mappingFile = indexResourcePath + "/" + type + ".json";
try {
source = FileUtil.readUTF8(mappingFile);
} catch (final Exception e) {
logger.warn(mappingFile + " is not found.", e);
}
try {
final PutMappingResponse putMappingResponse =
indicesClient.preparePutMapping(index).setType(type).setSource(source).execute()
.actionGet(fessConfig.getIndexIndicesTimeout());
if (putMappingResponse.isAcknowledged()) {
logger.info("Created " + index + "/" + type + " mapping.");
} else {
logger.warn("Failed to create " + index + "/" + type + " mapping.");
}
// TODO bulk
} catch (final Exception e) {
logger.warn("Failed to create " + index + "/" + type + " mapping.", e);
}
}
}
private void addFieldMapping(final IndicesAdminClient indicesClient, final String index, final String type, final String field,
final String source) {
final GetFieldMappingsResponse gfmResponse =

12
src/main/java/org/codelibs/fess/ds/impl/GitBucketDataStoreImpl.java
View file

@ -74,12 +74,12 @@ public class GitBucketDataStoreImpl extends AbstractDataStoreImpl {
final CrawlingConfig crawlingConfig = new CrawlingConfigWrapper(dataConfig) {
@Override
public Map<String, Object> initializeClientFactory(CrawlerClientFactory crawlerClientFactory) {
public Map<String, Object> initializeClientFactory(final CrawlerClientFactory crawlerClientFactory) {
final Map<String, Object> paramMap = super.initializeClientFactory(crawlerClientFactory);
List<RequestHeader> headerList = new ArrayList<>();
RequestHeader[] headers = (RequestHeader[]) paramMap.get(HcHttpClient.REQUERT_HEADERS_PROPERTY);
final List<RequestHeader> headerList = new ArrayList<>();
final RequestHeader[] headers = (RequestHeader[]) paramMap.get(HcHttpClient.REQUERT_HEADERS_PROPERTY);
if (headers != null) {
for (RequestHeader header : headers) {
for (final RequestHeader header : headers) {
headerList.add(header);
}
}
@ -175,7 +175,7 @@ public class GitBucketDataStoreImpl extends AbstractDataStoreImpl {
}
private void storeFileContent(final String rootURL, final String authToken, final String owner, final String name,
List<String> roleList, final String path, final CrawlingConfig crawlingConfig, final IndexUpdateCallback callback,
final List<String> roleList, final String path, final CrawlingConfig crawlingConfig, final IndexUpdateCallback callback,
final Map<String, String> paramMap, final Map<String, String> scriptMap, final Map<String, Object> defaultDataMap) {
final String apiUrl = rootURL + "api/v3/repos/" + owner + "/" + name + "/contents/" + path;
final String viewUrl = rootURL + owner + "/" + name + "/blob/master/" + path;
@ -199,7 +199,7 @@ public class GitBucketDataStoreImpl extends AbstractDataStoreImpl {
}
protected void collectFileNames(final String rootURL, final String authToken, final String owner, final String name, final String path,
final int depth, final long readInterval, Consumer<String> consumer) {
final int depth, final long readInterval, final Consumer<String> consumer) {
if (MAX_DEPTH <= depth) {
return;

8
src/main/java/org/codelibs/fess/entity/PingResponse.java
View file

@ -21,13 +21,13 @@ import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.cluster.health.ClusterHealthStatus;
public class PingResponse {
private int status;
private final int status;
private List<String> failures;
private final List<String> failures;
private String clusterName;
private final String clusterName;
private String clusterStatus;
private final String clusterStatus;
public PingResponse(final ClusterHealthResponse response) {
status = response.getStatus() == ClusterHealthStatus.RED ? 1 : 0;

1
src/main/java/org/codelibs/fess/es/config/bsbhv/BsThumbnailQueueBhv.java
View file

@ -75,6 +75,7 @@ public abstract class BsThumbnailQueueBhv extends EsAbstractBehavior<ThumbnailQu
final RESULT result = entityType.newInstance();
result.setCreatedBy(DfTypeUtil.toString(source.get("createdBy")));
result.setCreatedTime(DfTypeUtil.toLong(source.get("createdTime")));
result.setTarget(DfTypeUtil.toString(source.get("target")));
result.setGenerator(DfTypeUtil.toString(source.get("generator")));
result.setPath(DfTypeUtil.toString(source.get("path")));
result.setUrl(DfTypeUtil.toString(source.get("url")));

17
src/main/java/org/codelibs/fess/es/config/bsentity/BsThumbnailQueue.java
View file

@ -43,6 +43,9 @@ public class BsThumbnailQueue extends EsAbstractEntity {
/** createdTime */
protected Long createdTime;
/** target */
protected String target;
/** generator */
protected String generator;
@ -79,6 +82,9 @@ public class BsThumbnailQueue extends EsAbstractEntity {
if (createdTime != null) {
sourceMap.put("createdTime", createdTime);
}
if (target != null) {
sourceMap.put("target", target);
}
if (generator != null) {
sourceMap.put("generator", generator);
}
@ -99,6 +105,7 @@ public class BsThumbnailQueue extends EsAbstractEntity {
StringBuilder sb = new StringBuilder();
sb.append(dm).append(createdBy);
sb.append(dm).append(createdTime);
sb.append(dm).append(target);
sb.append(dm).append(generator);
sb.append(dm).append(path);
sb.append(dm).append(url);
@ -132,6 +139,16 @@ public class BsThumbnailQueue extends EsAbstractEntity {
this.createdTime = value;
}
public String getTarget() {
checkSpecifiedProperty("target");
return convertEmptyToNull(target);
}
public void setTarget(String value) {
registerModifiedProperty("target");
this.target = value;
}
public String getGenerator() {
checkSpecifiedProperty("generator");
return convertEmptyToNull(generator);

9
src/main/java/org/codelibs/fess/es/config/bsentity/dbmeta/ThumbnailQueueDbm.java
View file

@ -83,6 +83,8 @@ public class ThumbnailQueueDbm extends AbstractDBMeta {
(et, vl) -> ((ThumbnailQueue) et).setCreatedBy(DfTypeUtil.toString(vl)), "createdBy");
setupEpg(_epgMap, et -> ((ThumbnailQueue) et).getCreatedTime(),
(et, vl) -> ((ThumbnailQueue) et).setCreatedTime(DfTypeUtil.toLong(vl)), "createdTime");
setupEpg(_epgMap, et -> ((ThumbnailQueue) et).getTarget(), (et, vl) -> ((ThumbnailQueue) et).setTarget(DfTypeUtil.toString(vl)),
"target");
setupEpg(_epgMap, et -> ((ThumbnailQueue) et).getGenerator(),
(et, vl) -> ((ThumbnailQueue) et).setGenerator(DfTypeUtil.toString(vl)), "generator");
setupEpg(_epgMap, et -> ((ThumbnailQueue) et).getPath(), (et, vl) -> ((ThumbnailQueue) et).setPath(DfTypeUtil.toString(vl)), "path");
@ -127,6 +129,8 @@ public class ThumbnailQueueDbm extends AbstractDBMeta {
false, "String", 0, 0, null, false, null, null, null, null, null, false);
protected final ColumnInfo _columnCreatedTime = cci("createdTime", "createdTime", null, null, Long.class, "createdTime", null, false,
false, false, "Long", 0, 0, null, false, null, null, null, null, null, false);
protected final ColumnInfo _columnTarget = cci("target", "target", null, null, String.class, "target", null, false, false, false,
"String", 0, 0, null, false, null, null, null, null, null, false);
protected final ColumnInfo _columnGenerator = cci("generator", "generator", null, null, String.class, "generator", null, false, false,
false, "String", 0, 0, null, false, null, null, null, null, null, false);
protected final ColumnInfo _columnPath = cci("path", "path", null, null, String.class, "path", null, false, false, false, "String", 0,
@ -142,6 +146,10 @@ public class ThumbnailQueueDbm extends AbstractDBMeta {
return _columnCreatedTime;
}
public ColumnInfo columnTarget() {
return _columnTarget;
}
public ColumnInfo columnGenerator() {
return _columnGenerator;
}
@ -158,6 +166,7 @@ public class ThumbnailQueueDbm extends AbstractDBMeta {
List<ColumnInfo> ls = newArrayList();
ls.add(columnCreatedBy());
ls.add(columnCreatedTime());
ls.add(columnTarget());
ls.add(columnGenerator());
ls.add(columnPath());
ls.add(columnUrl());

4
src/main/java/org/codelibs/fess/es/config/cbean/bs/BsThumbnailQueueCB.java
View file

@ -155,6 +155,10 @@ public class BsThumbnailQueueCB extends EsAbstractConditionBean {
doColumn("createdTime");
}
public void columnTarget() {
doColumn("target");
}
public void columnGenerator() {
doColumn("generator");
}

207
src/main/java/org/codelibs/fess/es/config/cbean/cq/bs/BsThumbnailQueueCQ.java
View file

@ -543,6 +543,213 @@ public abstract class BsThumbnailQueueCQ extends EsAbstractConditionQuery {
return this;
}
public void setTarget_Equal(String target) {
setTarget_Term(target, null);
}
public void setTarget_Equal(String target, ConditionOptionCall<TermQueryBuilder> opLambda) {
setTarget_Term(target, opLambda);
}
public void setTarget_Term(String target) {
setTarget_Term(target, null);
}
public void setTarget_Term(String target, ConditionOptionCall<TermQueryBuilder> opLambda) {
TermQueryBuilder builder = regTermQ("target", target);
if (opLambda != null) {
opLambda.callback(builder);
}
}
public void setTarget_NotEqual(String target) {
setTarget_NotTerm(target, null);
}
public void setTarget_NotTerm(String target) {
setTarget_NotTerm(target, null);
}
public void setTarget_NotEqual(String target, ConditionOptionCall<BoolQueryBuilder> opLambda) {
setTarget_NotTerm(target, opLambda);
}
public void setTarget_NotTerm(String target, ConditionOptionCall<BoolQueryBuilder> opLambda) {
not(not -> not.setTarget_Term(target), opLambda);
}
public void setTarget_Terms(Collection<String> targetList) {
setTarget_Terms(targetList, null);
}
public void setTarget_Terms(Collection<String> targetList, ConditionOptionCall<TermsQueryBuilder> opLambda) {
TermsQueryBuilder builder = regTermsQ("target", targetList);
if (opLambda != null) {
opLambda.callback(builder);
}
}
public void setTarget_InScope(Collection<String> targetList) {
setTarget_Terms(targetList, null);
}
public void setTarget_InScope(Collection<String> targetList, ConditionOptionCall<TermsQueryBuilder> opLambda) {
setTarget_Terms(targetList, opLambda);
}
public void setTarget_Match(String target) {
setTarget_Match(target, null);
}
public void setTarget_Match(String target, ConditionOptionCall<MatchQueryBuilder> opLambda) {
MatchQueryBuilder builder = regMatchQ("target", target);
if (opLambda != null) {
opLambda.callback(builder);
}
}
public void setTarget_MatchPhrase(String target) {
setTarget_MatchPhrase(target, null);
}
public void setTarget_MatchPhrase(String target, ConditionOptionCall<MatchQueryBuilder> opLambda) {
MatchQueryBuilder builder = regMatchPhraseQ("target", target);
if (opLambda != null) {
opLambda.callback(builder);
}
}
public void setTarget_MatchPhrasePrefix(String target) {
setTarget_MatchPhrasePrefix(target, null);
}
public void setTarget_MatchPhrasePrefix(String target, ConditionOptionCall<MatchQueryBuilder> opLambda) {
MatchQueryBuilder builder = regMatchPhrasePrefixQ("target", target);
if (opLambda != null) {
opLambda.callback(builder);
}
}
public void setTarget_Fuzzy(String target) {
setTarget_Fuzzy(target, null);
}
public void setTarget_Fuzzy(String target, ConditionOptionCall<FuzzyQueryBuilder> opLambda) {
FuzzyQueryBuilder builder = regFuzzyQ("target", target);
if (opLambda != null) {
opLambda.callback(builder);
}
}
public void setTarget_Prefix(String target) {
setTarget_Prefix(target, null);
}
public void setTarget_Prefix(String target, ConditionOptionCall<PrefixQueryBuilder> opLambda) {
PrefixQueryBuilder builder = regPrefixQ("target", target);
if (opLambda != null) {
opLambda.callback(builder);
}
}
public void setTarget_Wildcard(String target) {
setTarget_Wildcard(target, null);
}
public void setTarget_Wildcard(String target, ConditionOptionCall<WildcardQueryBuilder> opLambda) {
WildcardQueryBuilder builder = regWildcardQ("target", target);
if (opLambda != null) {
opLambda.callback(builder);
}
}
public void setTarget_Regexp(String target) {
setTarget_Regexp(target, null);
}
public void setTarget_Regexp(String target, ConditionOptionCall<RegexpQueryBuilder> opLambda) {
RegexpQueryBuilder builder = regRegexpQ("target", target);
if (opLambda != null) {
opLambda.callback(builder);
}
}
public void setTarget_GreaterThan(String target) {
setTarget_GreaterThan(target, null);
}
public void setTarget_GreaterThan(String target, ConditionOptionCall<RangeQueryBuilder> opLambda) {
RangeQueryBuilder builder = regRangeQ("target", ConditionKey.CK_GREATER_THAN, target);
if (opLambda != null) {
opLambda.callback(builder);
}
}
public void setTarget_LessThan(String target) {
setTarget_LessThan(target, null);
}
public void setTarget_LessThan(String target, ConditionOptionCall<RangeQueryBuilder> opLambda) {
RangeQueryBuilder builder = regRangeQ("target", ConditionKey.CK_LESS_THAN, target);
if (opLambda != null) {
opLambda.callback(builder);
}
}
public void setTarget_GreaterEqual(String target) {
setTarget_GreaterEqual(target, null);
}
public void setTarget_GreaterEqual(String target, ConditionOptionCall<RangeQueryBuilder> opLambda) {
RangeQueryBuilder builder = regRangeQ("target", ConditionKey.CK_GREATER_EQUAL, target);
if (opLambda != null) {
opLambda.callback(builder);
}
}
public void setTarget_LessEqual(String target) {
setTarget_LessEqual(target, null);
}
public void setTarget_LessEqual(String target, ConditionOptionCall<RangeQueryBuilder> opLambda) {
RangeQueryBuilder builder = regRangeQ("target", ConditionKey.CK_LESS_EQUAL, target);
if (opLambda != null) {
opLambda.callback(builder);
}
}
public void setTarget_Exists() {
setTarget_Exists(null);
}
public void setTarget_Exists(ConditionOptionCall<ExistsQueryBuilder> opLambda) {
ExistsQueryBuilder builder = regExistsQ("target");
if (opLambda != null) {
opLambda.callback(builder);
}
}
public void setTarget_CommonTerms(String target) {
setTarget_CommonTerms(target, null);
}
public void setTarget_CommonTerms(String target, ConditionOptionCall<CommonTermsQueryBuilder> opLambda) {
CommonTermsQueryBuilder builder = regCommonTermsQ("target", target);
if (opLambda != null) {
opLambda.callback(builder);
}
}
public BsThumbnailQueueCQ addOrderBy_Target_Asc() {
regOBA("target");
return this;
}
public BsThumbnailQueueCQ addOrderBy_Target_Desc() {
regOBD("target");
return this;
}
public void setGenerator_Equal(String generator) {
setGenerator_Term(generator, null);
}

2
src/main/java/org/codelibs/fess/es/config/exentity/AccessToken.java
View file

@ -50,7 +50,7 @@ public class AccessToken extends BsAccessToken {
return new Date(getExpiredTime().longValue());
}
public void setExpires(Date date) {
public void setExpires(final Date date) {
setExpiredTime(date != null ? date.getTime() : null);
}

18
src/main/java/org/codelibs/fess/es/config/exentity/CrawlingConfigWrapper.java
View file

@ -21,49 +21,59 @@ import org.codelibs.fess.crawler.client.CrawlerClientFactory;
public class CrawlingConfigWrapper implements CrawlingConfig {
private CrawlingConfig crawlingConfig;
private final CrawlingConfig crawlingConfig;
public CrawlingConfigWrapper(final CrawlingConfig crawlingConfig) {
this.crawlingConfig = crawlingConfig;
}
@Override
public String getId() {
return crawlingConfig.getId();
}
@Override
public String getName() {
return crawlingConfig.getName();
}
@Override
public String[] getPermissions() {
return crawlingConfig.getPermissions();
}
@Override
public String[] getLabelTypeValues() {
return crawlingConfig.getLabelTypeValues();
}
@Override
public String getDocumentBoost() {
return crawlingConfig.getDocumentBoost();
}
public String getIndexingTarget(String input) {
@Override
public String getIndexingTarget(final String input) {
return crawlingConfig.getIndexingTarget(input);
}
@Override
public String getConfigId() {
return crawlingConfig.getConfigId();
}
@Override
public Integer getTimeToLive() {
return crawlingConfig.getTimeToLive();
}
public Map<String, Object> initializeClientFactory(CrawlerClientFactory crawlerClientFactory) {
@Override
public Map<String, Object> initializeClientFactory(final CrawlerClientFactory crawlerClientFactory) {
return crawlingConfig.initializeClientFactory(crawlerClientFactory);
}
public Map<String, String> getConfigParameterMap(ConfigName name) {
@Override
public Map<String, String> getConfigParameterMap(final ConfigName name) {
return crawlingConfig.getConfigParameterMap(name);
}
}

3
src/main/java/org/codelibs/fess/helper/SystemHelper.java
View file

@ -299,6 +299,7 @@ public class SystemHelper {
}
public void reloadConfiguration() {
ComponentUtil.getFessEsClient().refresh();
ComponentUtil.getLabelTypeHelper().init();
ComponentUtil.getPathMappingHelper().init();
ComponentUtil.getSuggestHelper().init();
@ -316,7 +317,7 @@ public class SystemHelper {
this.random = random;
}
public boolean isChangedClusterState(int status) {
public boolean isChangedClusterState(final int status) {
return previousClusterState.getAndSet(status) != status;
}

14
src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java
View file

@ -652,6 +652,9 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
/** The key of the configuration. e.g. true */
String THUMBNAIL_HTML_PHANTOMJS_ENABLED = "thumbnail.html.phantomjs.enabled";
/** The key of the configuration. e.g. all */
String THUMBNAIL_GENERATOR_TARGETS = "thumbnail.generator.targets";
/** The key of the configuration. e.g. Administrator */
String MAIL_FROM_NAME = "mail.from.name";
@ -3163,6 +3166,13 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
*/
boolean isThumbnailHtmlPhantomjsEnabled();
/**
* Get the value for the key 'thumbnail.generator.targets'. <br>
* The value is, e.g. all <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getThumbnailGeneratorTargets();
/**
* Get the value for the key 'mail.from.name'. <br>
* The value is, e.g. Administrator <br>
@ -5398,6 +5408,10 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
return is(FessConfig.THUMBNAIL_HTML_PHANTOMJS_ENABLED);
}
public String getThumbnailGeneratorTargets() {
return get(FessConfig.THUMBNAIL_GENERATOR_TARGETS);
}
public String getMailFromName() {
return get(FessConfig.MAIL_FROM_NAME);
}

8
src/main/java/org/codelibs/fess/mylasta/direction/FessProp.java
View file

@ -1438,4 +1438,12 @@ public interface FessProp {
stream -> stream.filter(StringUtil::isNotBlank).map(s -> s.trim()).forEach(list::add));
return list.toArray(new String[list.size()]);
}
String getThumbnailGeneratorTargets();
public default String[] getThumbnailGeneratorTargetsAsArray() {
return getThumbnailGeneratorTargets().split(",");
}
}

38
src/main/java/org/codelibs/fess/thumbnail/ThumbnailManager.java
View file

@ -29,7 +29,6 @@ import java.util.Map;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
@ -45,6 +44,7 @@ import org.codelibs.fess.es.config.exbhv.ThumbnailQueueBhv;
import org.codelibs.fess.es.config.exentity.ThumbnailQueue;
import org.codelibs.fess.exception.FessSystemException;
import org.codelibs.fess.exception.JobProcessingException;
import org.codelibs.fess.helper.SystemHelper;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.util.ComponentUtil;
import org.codelibs.fess.util.DocumentUtil;
@ -52,6 +52,8 @@ import org.lastaflute.web.util.LaRequestUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.collect.Lists;
public class ThumbnailManager {
private static final String DEFAULT_SCREENSHOT_DIR = "/WEB-INF/thumbnails";
@ -116,12 +118,12 @@ public class ThumbnailManager {
final List<Tuple3<String, String, String>> taskList = new ArrayList<>();
while (generating) {
try {
Tuple3<String, String, String> task = thumbnailTaskQueue.poll(thumbnailTaskQueueTimeout, TimeUnit.MILLISECONDS);
final Tuple3<String, String, String> task = thumbnailTaskQueue.poll(thumbnailTaskQueueTimeout, TimeUnit.MILLISECONDS);
if (task == null) {
if (!taskList.isEmpty()) {
storeQueue(taskList);
}
} else {
} else if (!taskList.contains(task)) {
taskList.add(task);
if (taskList.size() > thumbnailTaskBulkSize) {
storeQueue(taskList);
@ -158,13 +160,22 @@ public class ThumbnailManager {
}
protected void storeQueue(final List<Tuple3<String, String, String>> taskList) {
List<ThumbnailQueue> list = taskList.stream().filter(entity -> entity != null).map(task -> {
ThumbnailQueue entity = new ThumbnailQueue();
entity.setGenerator(task.getValue1());
entity.setUrl(task.getValue2());
entity.setPath(task.getValue3());
return entity;
}).collect(Collectors.toList());
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
final String[] targets = fessConfig.getThumbnailGeneratorTargetsAsArray();
final List<ThumbnailQueue> list = new ArrayList<>();
taskList.stream().filter(entity -> entity != null).forEach(task -> {
for (final String target : targets) {
final ThumbnailQueue entity = new ThumbnailQueue();
entity.setGenerator(task.getValue1());
entity.setUrl(task.getValue2());
entity.setPath(task.getValue3());
entity.setTarget(target);
entity.setCreatedBy(Constants.SYSTEM_USER);
entity.setCreatedTime(systemHelper.getCurrentTimeAsLong());
list.add(entity);
}
});
taskList.clear();
final ThumbnailQueueBhv thumbnailQueueBhv = ComponentUtil.getComponent(ThumbnailQueueBhv.class);
thumbnailQueueBhv.batchInsert(list);
@ -175,6 +186,11 @@ public class ThumbnailManager {
final List<String> idList = new ArrayList<>();
final ThumbnailQueueBhv thumbnailQueueBhv = ComponentUtil.getComponent(ThumbnailQueueBhv.class);
thumbnailQueueBhv.selectList(cb -> {
if (StringUtil.isBlank(fessConfig.getSchedulerTargetName())) {
cb.query().setTarget_Equal(Constants.DEFAULT_JOB_TARGET);
} else {
cb.query().setTarget_InScope(Lists.newArrayList(Constants.DEFAULT_JOB_TARGET, fessConfig.getSchedulerTargetName()));
}
cb.query().addOrderBy_CreatedTime_Asc();
cb.fetchFirst(fessConfig.getPageThumbnailQueueMaxFetchSizeAsInteger());
}).forEach(entity -> {
@ -213,7 +229,7 @@ public class ThumbnailManager {
if (generator.isTarget(docMap)) {
final String url = DocumentUtil.getValue(docMap, fessConfig.getIndexFieldUrl(), String.class);
final String path = getImageFilename(docMap);
Tuple3<String, String, String> task = new Tuple3<String, String, String>(generator.getName(), url, path);
final Tuple3<String, String, String> task = new Tuple3<>(generator.getName(), url, path);
thumbnailTaskQueue.offer(task);
break;
}

3
src/main/java/org/codelibs/fess/thumbnail/impl/BaseThumbnailGenerator.java
View file

@ -105,11 +105,12 @@ public abstract class BaseThumbnailGenerator implements ThumbnailGenerator {
this.generatorList = generatorList;
}
@Override
public String getName() {
return name;
}
public void setName(String name) {
public void setName(final String name) {
this.name = name;
}

4
src/main/java/org/codelibs/fess/thumbnail/impl/WebDriverGenerator.java
View file

@ -139,7 +139,7 @@ public class WebDriverGenerator extends BaseThumbnailGenerator {
final File thumbnail = ((TakesScreenshot) webDriver).getScreenshotAs(OutputType.FILE);
convert(thumbnail, outputFile);
return true;
} catch (UnreachableBrowserException e) {
} catch (final UnreachableBrowserException e) {
if (logger.isDebugEnabled()) {
logger.debug("WebDriver is not available.", e);
}
@ -265,7 +265,7 @@ public class WebDriverGenerator extends BaseThumbnailGenerator {
this.thumbnailHeight = thumbnailHeight;
}
public void setUnreachableCheckInterval(long unreachableCheckInterval) {
public void setUnreachableCheckInterval(final long unreachableCheckInterval) {
this.unreachableCheckInterval = unreachableCheckInterval;
}
}

1
src/main/resources/fess_config.properties
View file

@ -349,6 +349,7 @@ paging.search.page.size=20
paging.search.page.max.size=100
thumbnail.html.phantomjs.enabled=true
thumbnail.generator.targets=all
# ----------------------------------------------------------
# Mail

4
src/main/resources/fess_indices/.fess_config/thumbnail_queue.json
View file

@ -19,6 +19,10 @@
"type": "string",
"index": "not_analyzed"
},
"target": {
"type": "string",
"index": "not_analyzed"
},
"createdTime": {
"type": "long"
},

2
src/main/webapp/WEB-INF/view/admin/accesstoken/admin_accesstoken.jsp
View file

@ -10,7 +10,7 @@
<div class="wrapper">
<jsp:include page="/WEB-INF/view/common/admin/header.jsp"></jsp:include>
<jsp:include page="/WEB-INF/view/common/admin/sidebar.jsp">
<jsp:param name="menuCategoryType" value="crawl" />
<jsp:param name="menuCategoryType" value="system" />
<jsp:param name="menuType" value="accessToken" />
</jsp:include>
<div class="content-wrapper">

2
src/main/webapp/WEB-INF/view/admin/accesstoken/admin_accesstoken_details.jsp
View file

@ -10,7 +10,7 @@
<div class="wrapper">
<jsp:include page="/WEB-INF/view/common/admin/header.jsp"></jsp:include>
<jsp:include page="/WEB-INF/view/common/admin/sidebar.jsp">
<jsp:param name="menuCategoryType" value="crawl" />
<jsp:param name="menuCategoryType" value="system" />
<jsp:param name="menuType" value="accessToken" />
</jsp:include>
<div class="content-wrapper">

2
src/main/webapp/WEB-INF/view/admin/accesstoken/admin_accesstoken_edit.jsp
View file

@ -10,7 +10,7 @@
<div class="wrapper">
<jsp:include page="/WEB-INF/view/common/admin/header.jsp"></jsp:include>
<jsp:include page="/WEB-INF/view/common/admin/sidebar.jsp">
<jsp:param name="menuCategoryType" value="crawl" />
<jsp:param name="menuCategoryType" value="system" />
<jsp:param name="menuType" value="accessToken" />
</jsp:include>
<div class="content-wrapper">