0ct0pu5/fess
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix #899 thumbnail generator access thumbnail content via CrawlerClient

Browse source
This commit is contained in:
Shinsuke Sugaya 2017-07-15 19:45:40 +09:00
parent 6f31f6dbb8
commit f9d3dfe4f5
24 changed files with 400 additions and 643 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
src/main/config/es/fess_config.json
View file

@ -870,9 +870,6 @@
},
"path" : {
"type": "keyword"
},
"url" : {
"type": "keyword"
}
}
},

2
src/main/java/org/codelibs/fess/crawler/transformer/AbstractFessFileTransformer.java
View file

@ -313,6 +313,8 @@ public abstract class AbstractFessFileTransformer extends AbstractTransformer im
putResultDataBody(dataMap, fessConfig.getIndexFieldParentId(), crawlingInfoHelper.generateId(dataMap));
putResultDataBody(dataMap, fessConfig.getIndexFieldUrl(), url); // set again
}
// thumbnail
putResultDataBody(dataMap, fessConfig.getIndexFieldThumbnail(), responseData.getUrl());
// from config
final Map<String, String> scriptConfigMap = crawlingConfig.getConfigParameterMap(ConfigName.SCRIPT);

1
src/main/java/org/codelibs/fess/es/config/bsbhv/BsThumbnailQueueBhv.java
View file

@ -79,7 +79,6 @@ public abstract class BsThumbnailQueueBhv extends EsAbstractBehavior<ThumbnailQu
result.setGenerator(DfTypeUtil.toString(source.get("generator")));
result.setThumbnailId(DfTypeUtil.toString(source.get("thumbnail_id")));
result.setPath(DfTypeUtil.toString(source.get("path")));
result.setUrl(DfTypeUtil.toString(source.get("url")));
return updateEntity(source, result);
} catch (InstantiationException | IllegalAccessException e) {
final String msg = "Cannot create a new instance: " + entityType.getName();

17
src/main/java/org/codelibs/fess/es/config/bsentity/BsThumbnailQueue.java
View file

@ -55,9 +55,6 @@ public class BsThumbnailQueue extends EsAbstractEntity {
/** path */
protected String path;
/** url */
protected String url;
// [Referrers] *comment only
// ===================================================================================
@ -97,9 +94,6 @@ public class BsThumbnailQueue extends EsAbstractEntity {
if (path != null) {
addFieldToSource(sourceMap, "path", path);
}
if (url != null) {
addFieldToSource(sourceMap, "url", url);
}
return sourceMap;
}
@ -119,7 +113,6 @@ public class BsThumbnailQueue extends EsAbstractEntity {
sb.append(dm).append(generator);
sb.append(dm).append(thumbnailId);
sb.append(dm).append(path);
sb.append(dm).append(url);
if (sb.length() > dm.length()) {
sb.delete(0, dm.length());
}
@ -189,14 +182,4 @@ public class BsThumbnailQueue extends EsAbstractEntity {
registerModifiedProperty("path");
this.path = value;
}
public String getUrl() {
checkSpecifiedProperty("url");
return convertEmptyToNull(url);
}
public void setUrl(String value) {
registerModifiedProperty("url");
this.url = value;
}
}

8
src/main/java/org/codelibs/fess/es/config/bsentity/dbmeta/ThumbnailQueueDbm.java
View file

@ -90,7 +90,6 @@ public class ThumbnailQueueDbm extends AbstractDBMeta {
setupEpg(_epgMap, et -> ((ThumbnailQueue) et).getThumbnailId(),
(et, vl) -> ((ThumbnailQueue) et).setThumbnailId(DfTypeUtil.toString(vl)), "thumbnailId");
setupEpg(_epgMap, et -> ((ThumbnailQueue) et).getPath(), (et, vl) -> ((ThumbnailQueue) et).setPath(DfTypeUtil.toString(vl)), "path");
setupEpg(_epgMap, et -> ((ThumbnailQueue) et).getUrl(), (et, vl) -> ((ThumbnailQueue) et).setUrl(DfTypeUtil.toString(vl)), "url");
}
@Override
@ -139,8 +138,6 @@ public class ThumbnailQueueDbm extends AbstractDBMeta {
false, false, false, "keyword", 0, 0, null, false, null, null, null, null, null, false);
protected final ColumnInfo _columnPath = cci("path", "path", null, null, String.class, "path", null, false, false, false, "keyword", 0,
0, null, false, null, null, null, null, null, false);
protected final ColumnInfo _columnUrl = cci("url", "url", null, null, String.class, "url", null, false, false, false, "keyword", 0, 0,
null, false, null, null, null, null, null, false);
public ColumnInfo columnCreatedBy() {
return _columnCreatedBy;
@ -166,10 +163,6 @@ public class ThumbnailQueueDbm extends AbstractDBMeta {
return _columnPath;
}
public ColumnInfo columnUrl() {
return _columnUrl;
}
protected List<ColumnInfo> ccil() {
List<ColumnInfo> ls = newArrayList();
ls.add(columnCreatedBy());
@ -178,7 +171,6 @@ public class ThumbnailQueueDbm extends AbstractDBMeta {
ls.add(columnGenerator());
ls.add(columnThumbnailId());
ls.add(columnPath());
ls.add(columnUrl());
return ls;
}

4
src/main/java/org/codelibs/fess/es/config/cbean/bs/BsThumbnailQueueCB.java
View file

@ -199,9 +199,5 @@ public class BsThumbnailQueueCB extends EsAbstractConditionBean {
public void columnPath() {
doColumn("path");
}
public void columnUrl() {
doColumn("url");
}
}
}

130
src/main/java/org/codelibs/fess/es/config/cbean/ca/bs/BsThumbnailQueueCA.java
View file

@ -987,134 +987,4 @@ public abstract class BsThumbnailQueueCA extends EsAbstractConditionAggregation
}
}
public void setUrl_Terms() {
setUrl_Terms(null);
}
public void setUrl_Terms(ConditionOptionCall<TermsAggregationBuilder> opLambda) {
setUrl_Terms("url", opLambda, null);
}
public void setUrl_Terms(ConditionOptionCall<TermsAggregationBuilder> opLambda, OperatorCall<BsThumbnailQueueCA> aggsLambda) {
setUrl_Terms("url", opLambda, aggsLambda);
}
public void setUrl_Terms(String name, ConditionOptionCall<TermsAggregationBuilder> opLambda, OperatorCall<BsThumbnailQueueCA> aggsLambda) {
TermsAggregationBuilder builder = regTermsA(name, "url");
if (opLambda != null) {
opLambda.callback(builder);
}
if (aggsLambda != null) {
ThumbnailQueueCA ca = new ThumbnailQueueCA();
aggsLambda.callback(ca);
ca.getAggregationBuilderList().forEach(builder::subAggregation);
}
}
public void setUrl_SignificantTerms() {
setUrl_SignificantTerms(null);
}
public void setUrl_SignificantTerms(ConditionOptionCall<SignificantTermsAggregationBuilder> opLambda) {
setUrl_SignificantTerms("url", opLambda, null);
}
public void setUrl_SignificantTerms(ConditionOptionCall<SignificantTermsAggregationBuilder> opLambda,
OperatorCall<BsThumbnailQueueCA> aggsLambda) {
setUrl_SignificantTerms("url", opLambda, aggsLambda);
}
public void setUrl_SignificantTerms(String name, ConditionOptionCall<SignificantTermsAggregationBuilder> opLambda,
OperatorCall<BsThumbnailQueueCA> aggsLambda) {
SignificantTermsAggregationBuilder builder = regSignificantTermsA(name, "url");
if (opLambda != null) {
opLambda.callback(builder);
}
if (aggsLambda != null) {
ThumbnailQueueCA ca = new ThumbnailQueueCA();
aggsLambda.callback(ca);
ca.getAggregationBuilderList().forEach(builder::subAggregation);
}
}
public void setUrl_IpRange() {
setUrl_IpRange(null);
}
public void setUrl_IpRange(ConditionOptionCall<IpRangeAggregationBuilder> opLambda) {
setUrl_IpRange("url", opLambda, null);
}
public void setUrl_IpRange(ConditionOptionCall<IpRangeAggregationBuilder> opLambda, OperatorCall<BsThumbnailQueueCA> aggsLambda) {
setUrl_IpRange("url", opLambda, aggsLambda);
}
public void setUrl_IpRange(String name, ConditionOptionCall<IpRangeAggregationBuilder> opLambda,
OperatorCall<BsThumbnailQueueCA> aggsLambda) {
IpRangeAggregationBuilder builder = regIpRangeA(name, "url");
if (opLambda != null) {
opLambda.callback(builder);
}
if (aggsLambda != null) {
ThumbnailQueueCA ca = new ThumbnailQueueCA();
aggsLambda.callback(ca);
ca.getAggregationBuilderList().forEach(builder::subAggregation);
}
}
public void setUrl_Count() {
setUrl_Count(null);
}
public void setUrl_Count(ConditionOptionCall<ValueCountAggregationBuilder> opLambda) {
setUrl_Count("url", opLambda);
}
public void setUrl_Count(String name, ConditionOptionCall<ValueCountAggregationBuilder> opLambda) {
ValueCountAggregationBuilder builder = regCountA(name, "url");
if (opLambda != null) {
opLambda.callback(builder);
}
}
public void setUrl_Cardinality() {
setUrl_Cardinality(null);
}
public void setUrl_Cardinality(ConditionOptionCall<CardinalityAggregationBuilder> opLambda) {
setUrl_Cardinality("url", opLambda);
}
public void setUrl_Cardinality(String name, ConditionOptionCall<CardinalityAggregationBuilder> opLambda) {
CardinalityAggregationBuilder builder = regCardinalityA(name, "url");
if (opLambda != null) {
opLambda.callback(builder);
}
}
public void setUrl_Missing() {
setUrl_Missing(null);
}
public void setUrl_Missing(ConditionOptionCall<MissingAggregationBuilder> opLambda) {
setUrl_Missing("url", opLambda, null);
}
public void setUrl_Missing(ConditionOptionCall<MissingAggregationBuilder> opLambda, OperatorCall<BsThumbnailQueueCA> aggsLambda) {
setUrl_Missing("url", opLambda, aggsLambda);
}
public void setUrl_Missing(String name, ConditionOptionCall<MissingAggregationBuilder> opLambda,
OperatorCall<BsThumbnailQueueCA> aggsLambda) {
MissingAggregationBuilder builder = regMissingA(name, "url");
if (opLambda != null) {
opLambda.callback(builder);
}
if (aggsLambda != null) {
ThumbnailQueueCA ca = new ThumbnailQueueCA();
aggsLambda.callback(ca);
ca.getAggregationBuilderList().forEach(builder::subAggregation);
}
}
}

222
src/main/java/org/codelibs/fess/es/config/cbean/cq/bs/BsThumbnailQueueCQ.java
View file

@ -1474,226 +1474,4 @@ public abstract class BsThumbnailQueueCQ extends EsAbstractConditionQuery {
return this;
}
public void setUrl_Equal(String url) {
setUrl_Term(url, null);
}
public void setUrl_Equal(String url, ConditionOptionCall<TermQueryBuilder> opLambda) {
setUrl_Term(url, opLambda);
}
public void setUrl_Term(String url) {
setUrl_Term(url, null);
}
public void setUrl_Term(String url, ConditionOptionCall<TermQueryBuilder> opLambda) {
TermQueryBuilder builder = regTermQ("url", url);
if (opLambda != null) {
opLambda.callback(builder);
}
}
public void setUrl_NotEqual(String url) {
setUrl_NotTerm(url, null);
}
public void setUrl_NotTerm(String url) {
setUrl_NotTerm(url, null);
}
public void setUrl_NotEqual(String url, ConditionOptionCall<BoolQueryBuilder> opLambda) {
setUrl_NotTerm(url, opLambda);
}
public void setUrl_NotTerm(String url, ConditionOptionCall<BoolQueryBuilder> opLambda) {
not(not -> not.setUrl_Term(url), opLambda);
}
public void setUrl_Terms(Collection<String> urlList) {
setUrl_Terms(urlList, null);
}
public void setUrl_Terms(Collection<String> urlList, ConditionOptionCall<TermsQueryBuilder> opLambda) {
TermsQueryBuilder builder = regTermsQ("url", urlList);
if (opLambda != null) {
opLambda.callback(builder);
}
}
public void setUrl_InScope(Collection<String> urlList) {
setUrl_Terms(urlList, null);
}
public void setUrl_InScope(Collection<String> urlList, ConditionOptionCall<TermsQueryBuilder> opLambda) {
setUrl_Terms(urlList, opLambda);
}
public void setUrl_Match(String url) {
setUrl_Match(url, null);
}
public void setUrl_Match(String url, ConditionOptionCall<MatchQueryBuilder> opLambda) {
MatchQueryBuilder builder = regMatchQ("url", url);
if (opLambda != null) {
opLambda.callback(builder);
}
}
public void setUrl_MatchPhrase(String url) {
setUrl_MatchPhrase(url, null);
}
public void setUrl_MatchPhrase(String url, ConditionOptionCall<MatchPhraseQueryBuilder> opLambda) {
MatchPhraseQueryBuilder builder = regMatchPhraseQ("url", url);
if (opLambda != null) {
opLambda.callback(builder);
}
}
public void setUrl_MatchPhrasePrefix(String url) {
setUrl_MatchPhrasePrefix(url, null);
}
public void setUrl_MatchPhrasePrefix(String url, ConditionOptionCall<MatchPhrasePrefixQueryBuilder> opLambda) {
MatchPhrasePrefixQueryBuilder builder = regMatchPhrasePrefixQ("url", url);
if (opLambda != null) {
opLambda.callback(builder);
}
}
public void setUrl_Fuzzy(String url) {
setUrl_Fuzzy(url, null);
}
public void setUrl_Fuzzy(String url, ConditionOptionCall<MatchQueryBuilder> opLambda) {
MatchQueryBuilder builder = regFuzzyQ("url", url);
if (opLambda != null) {
opLambda.callback(builder);
}
}
public void setUrl_Prefix(String url) {
setUrl_Prefix(url, null);
}
public void setUrl_Prefix(String url, ConditionOptionCall<PrefixQueryBuilder> opLambda) {
PrefixQueryBuilder builder = regPrefixQ("url", url);
if (opLambda != null) {
opLambda.callback(builder);
}
}
public void setUrl_Wildcard(String url) {
setUrl_Wildcard(url, null);
}
public void setUrl_Wildcard(String url, ConditionOptionCall<WildcardQueryBuilder> opLambda) {
WildcardQueryBuilder builder = regWildcardQ("url", url);
if (opLambda != null) {
opLambda.callback(builder);
}
}
public void setUrl_Regexp(String url) {
setUrl_Regexp(url, null);
}
public void setUrl_Regexp(String url, ConditionOptionCall<RegexpQueryBuilder> opLambda) {
RegexpQueryBuilder builder = regRegexpQ("url", url);
if (opLambda != null) {
opLambda.callback(builder);
}
}
public void setUrl_SpanTerm(String url) {
setUrl_SpanTerm("url", null);
}
public void setUrl_SpanTerm(String url, ConditionOptionCall<SpanTermQueryBuilder> opLambda) {
SpanTermQueryBuilder builder = regSpanTermQ("url", url);
if (opLambda != null) {
opLambda.callback(builder);
}
}
public void setUrl_GreaterThan(String url) {
setUrl_GreaterThan(url, null);
}
public void setUrl_GreaterThan(String url, ConditionOptionCall<RangeQueryBuilder> opLambda) {
final Object _value = url;
RangeQueryBuilder builder = regRangeQ("url", ConditionKey.CK_GREATER_THAN, _value);
if (opLambda != null) {
opLambda.callback(builder);
}
}
public void setUrl_LessThan(String url) {
setUrl_LessThan(url, null);
}
public void setUrl_LessThan(String url, ConditionOptionCall<RangeQueryBuilder> opLambda) {
final Object _value = url;
RangeQueryBuilder builder = regRangeQ("url", ConditionKey.CK_LESS_THAN, _value);
if (opLambda != null) {
opLambda.callback(builder);
}
}
public void setUrl_GreaterEqual(String url) {
setUrl_GreaterEqual(url, null);
}
public void setUrl_GreaterEqual(String url, ConditionOptionCall<RangeQueryBuilder> opLambda) {
final Object _value = url;
RangeQueryBuilder builder = regRangeQ("url", ConditionKey.CK_GREATER_EQUAL, _value);
if (opLambda != null) {
opLambda.callback(builder);
}
}
public void setUrl_LessEqual(String url) {
setUrl_LessEqual(url, null);
}
public void setUrl_LessEqual(String url, ConditionOptionCall<RangeQueryBuilder> opLambda) {
final Object _value = url;
RangeQueryBuilder builder = regRangeQ("url", ConditionKey.CK_LESS_EQUAL, _value);
if (opLambda != null) {
opLambda.callback(builder);
}
}
public void setUrl_Exists() {
setUrl_Exists(null);
}
public void setUrl_Exists(ConditionOptionCall<ExistsQueryBuilder> opLambda) {
ExistsQueryBuilder builder = regExistsQ("url");
if (opLambda != null) {
opLambda.callback(builder);
}
}
public void setUrl_CommonTerms(String url) {
setUrl_CommonTerms(url, null);
}
public void setUrl_CommonTerms(String url, ConditionOptionCall<CommonTermsQueryBuilder> opLambda) {
CommonTermsQueryBuilder builder = regCommonTermsQ("url", url);
if (opLambda != null) {
opLambda.callback(builder);
}
}
public BsThumbnailQueueCQ addOrderBy_Url_Asc() {
regOBA("url");
return this;
}
public BsThumbnailQueueCQ addOrderBy_Url_Desc() {
regOBD("url");
return this;
}
}

2
src/main/java/org/codelibs/fess/es/config/exentity/ThumbnailQueue.java
View file

@ -43,6 +43,6 @@ public class ThumbnailQueue extends BsThumbnailQueue {
@Override
public String toString() {
return "ThumbnailQueue [createdBy=" + createdBy + ", createdTime=" + createdTime + ", generator=" + generator + ", path=" + path
+ ", url=" + url + "]";
+ "]";
}
}

29
src/main/java/org/codelibs/fess/exception/ThumbnailGenerationException.java Normal file
View file

@ -0,0 +1,29 @@
/*
* Copyright 2012-2017 CodeLibs Project and the Others.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
* either express or implied. See the License for the specific language
* governing permissions and limitations under the License.
*/
package org.codelibs.fess.exception;
public class ThumbnailGenerationException extends FessSystemException {
private static final long serialVersionUID = 1L;
public ThumbnailGenerationException(String message, Exception cause) {
super(message, cause);
}
public ThumbnailGenerationException(String message) {
super(message, false, false);
}
}

63
src/main/java/org/codelibs/fess/helper/CrawlingConfigHelper.java
View file

@ -17,6 +17,10 @@ package org.codelibs.fess.helper;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import javax.annotation.PostConstruct;
import org.codelibs.fess.app.service.DataConfigService;
import org.codelibs.fess.app.service.FileConfigService;
@ -27,6 +31,9 @@ import org.codelibs.fess.util.ComponentUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
public class CrawlingConfigHelper {
private static final Logger logger = LoggerFactory.getLogger(CrawlingConfigHelper.class);
@ -35,6 +42,13 @@ public class CrawlingConfigHelper {
protected int count = 1;
protected Cache<String, CrawlingConfig> crawlingConfigCache;
@PostConstruct
public void init() {
crawlingConfigCache = CacheBuilder.newBuilder().maximumSize(100).expireAfterWrite(10, TimeUnit.MINUTES).build();
}
public ConfigType getConfigType(final String configId) {
if (configId == null || configId.length() < 2) {
return null;
@ -58,29 +72,40 @@ public class CrawlingConfigHelper {
}
public CrawlingConfig getCrawlingConfig(final String configId) {
final ConfigType configType = getConfigType(configId);
if (configType == null) {
return null;
}
final String id = getId(configId);
if (id == null) {
return null;
}
switch (configType) {
case WEB:
final WebConfigService webConfigService = ComponentUtil.getComponent(WebConfigService.class);
return webConfigService.getWebConfig(id).get();
case FILE:
final FileConfigService fileConfigService = ComponentUtil.getComponent(FileConfigService.class);
return fileConfigService.getFileConfig(id).get();
case DATA:
final DataConfigService dataConfigService = ComponentUtil.getComponent(DataConfigService.class);
return dataConfigService.getDataConfig(id).get();
default:
try {
return crawlingConfigCache.get(configId, () -> {
final ConfigType configType = getConfigType(configId);
if (configType == null) {
return null;
}
final String id = getId(configId);
if (id == null) {
return null;
}
switch (configType) {
case WEB:
final WebConfigService webConfigService = ComponentUtil.getComponent(WebConfigService.class);
return webConfigService.getWebConfig(id).get();
case FILE:
final FileConfigService fileConfigService = ComponentUtil.getComponent(FileConfigService.class);
return fileConfigService.getFileConfig(id).get();
case DATA:
final DataConfigService dataConfigService = ComponentUtil.getComponent(DataConfigService.class);
return dataConfigService.getDataConfig(id).get();
default:
return null;
}
});
} catch (ExecutionException e) {
logger.warn("Failed to access a crawling config cache: " + configId, e);
return null;
}
}
public void refresh() {
crawlingConfigCache.invalidateAll();
}
public synchronized String store(final String sessionId, final CrawlingConfig crawlingConfig) {
final String sessionCountId = sessionId + "-" + count;
crawlingConfigMap.put(sessionCountId, crawlingConfig);

11
src/main/java/org/codelibs/fess/helper/IndexingHelper.java
View file

@ -57,7 +57,16 @@ public class IndexingHelper {
try {
if (fessConfig.isThumbnailCrawlerEnabled()) {
final ThumbnailManager thumbnailManager = ComponentUtil.getThumbnailManager();
docList.stream().forEach(doc -> thumbnailManager.offer(doc));
docList.stream().forEach(
doc -> {
if (!thumbnailManager.offer(doc)) {
if (logger.isDebugEnabled()) {
logger.debug("Removing " + doc.get(fessConfig.getIndexFieldThumbnail()) + " from "
+ doc.get(fessConfig.getIndexFieldUrl()));
}
doc.remove(fessConfig.getIndexFieldThumbnail());
}
});
}
synchronized (fessEsClient) {
deleteOldDocuments(fessEsClient, docList);

34
src/main/java/org/codelibs/fess/helper/ViewHelper.java
View file

@ -39,6 +39,7 @@ import javax.annotation.Resource;
import javax.servlet.http.HttpServletRequest;
import org.apache.catalina.connector.ClientAbortException;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringEscapeUtils;
import org.apache.commons.lang3.StringUtils;
import org.codelibs.core.CoreLibConstants;
@ -46,9 +47,6 @@ import org.codelibs.core.lang.StringUtil;
import org.codelibs.core.misc.Base64Util;
import org.codelibs.core.misc.DynamicProperties;
import org.codelibs.fess.Constants;
import org.codelibs.fess.app.service.DataConfigService;
import org.codelibs.fess.app.service.FileConfigService;
import org.codelibs.fess.app.service.WebConfigService;
import org.codelibs.fess.crawler.builder.RequestDataBuilder;
import org.codelibs.fess.crawler.client.CrawlerClient;
import org.codelibs.fess.crawler.client.CrawlerClientFactory;
@ -56,7 +54,6 @@ import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.util.CharUtil;
import org.codelibs.fess.entity.FacetQueryView;
import org.codelibs.fess.es.config.exentity.CrawlingConfig;
import org.codelibs.fess.es.config.exentity.CrawlingConfig.ConfigType;
import org.codelibs.fess.exception.FessSystemException;
import org.codelibs.fess.helper.UserAgentHelper.UserAgentType;
import org.codelibs.fess.mylasta.direction.FessConfig;
@ -491,21 +488,7 @@ public class ViewHelper {
if (configId.length() < 2) {
throw new FessSystemException("Invalid configId: " + configId);
}
final ConfigType configType = crawlingConfigHelper.getConfigType(configId);
CrawlingConfig config = null;
if (logger.isDebugEnabled()) {
logger.debug("configType: " + configType + ", configId: " + configId);
}
if (ConfigType.WEB == configType) {
final WebConfigService webConfigService = ComponentUtil.getComponent(WebConfigService.class);
config = webConfigService.getWebConfig(crawlingConfigHelper.getId(configId)).get();
} else if (ConfigType.FILE == configType) {
final FileConfigService fileConfigService = ComponentUtil.getComponent(FileConfigService.class);
config = fileConfigService.getFileConfig(crawlingConfigHelper.getId(configId)).get();
} else if (ConfigType.DATA == configType) {
final DataConfigService dataConfigService = ComponentUtil.getComponent(DataConfigService.class);
config = dataConfigService.getDataConfig(crawlingConfigHelper.getId(configId)).get();
}
final CrawlingConfig config = crawlingConfigHelper.getCrawlingConfig(configId);
if (config == null) {
throw new FessSystemException("No crawlingConfig: " + configId);
}
@ -520,15 +503,16 @@ public class ViewHelper {
}
protected StreamResponse writeContent(final String configId, final String url, final CrawlerClient client) {
final ResponseData responseData = client.execute(RequestDataBuilder.newRequestData().get().url(url).build());
final StreamResponse response = new StreamResponse(StringUtil.EMPTY);
final ResponseData responseData = client.execute(RequestDataBuilder.newRequestData().get().url(url).build());
if (responseData.getHttpStatusCode() == 404) {
response.httpStatus(responseData.getHttpStatusCode());
IOUtils.closeQuietly(responseData);
return response;
}
writeFileName(response, responseData);
writeContentType(response, responseData);
writeNoCache(response, responseData);
if (responseData.getHttpStatusCode() == 404) {
response.httpStatus(responseData.getHttpStatusCode());
return response;
}
response.stream(out -> {
try (final InputStream is = new BufferedInputStream(responseData.getResponseBody())) {
out.write(is);
@ -537,7 +521,7 @@ public class ViewHelper {
throw new FessSystemException("Failed to write a content. configId: " + configId + ", url: " + url, e);
}
} finally {
responseData.close();
IOUtils.closeQuietly(responseData);
}
if (logger.isDebugEnabled()) {
logger.debug("Finished to write " + url);

48
src/main/java/org/codelibs/fess/mylasta/action/FessLabels.java
View file

@ -1499,54 +1499,6 @@ public class FessLabels extends UserMessages {
/** The key of the message: Use Default */
public static final String LABELS_design_use_default_button = "{labels.design_use_default_button}";
/** The key of the message: Top Page */
public static final String LABELS_design_file_index = "{labels.design_file_index}";
/** The key of the message: Footer */
public static final String LABELS_design_file_footer = "{labels.design_file_footer}";
/** The key of the message: Results Page (Frame) */
public static final String LABELS_design_file_search = "{labels.design_file_search}";
/** The key of the message: Results Page (Content) */
public static final String LABELS_design_file_searchResults = "{labels.design_file_searchResults}";
/** The key of the message: Results Page (No Result) */
public static final String LABELS_design_file_searchNoResult = "{labels.design_file_searchNoResult}";
/** The key of the message: Search Options Page */
public static final String LABELS_design_file_searchOptions = "{labels.design_file_searchOptions}";
/** The key of the message: Help Page (Content) */
public static final String LABELS_design_file_help = "{labels.design_file_help}";
/** The key of the message: Header */
public static final String LABELS_design_file_header = "{labels.design_file_header}";
/** The key of the message: Search Error Page */
public static final String LABELS_design_file_error = "{labels.design_file_error}";
/** The key of the message: Cache Page */
public static final String LABELS_design_file_cache = "{labels.design_file_cache}";
/** The key of the message: Error Page (Not Found) */
public static final String LABELS_design_file_errorNotFound = "{labels.design_file_errorNotFound}";
/** The key of the message: Error Page (System Error) */
public static final String LABELS_design_file_errorSystem = "{labels.design_file_errorSystem}";
/** The key of the message: Error Page (Redirect) */
public static final String LABELS_design_file_errorRedirect = "{labels.design_file_errorRedirect}";
/** The key of the message: Error Page (BadRequest) */
public static final String LABELS_design_file_errorBadRequest = "{labels.design_file_errorBadRequest}";
/** The key of the message: Login Page */
public static final String LABELS_design_file_login = "{labels.design_file_login}";
/** The key of the message: Profile Page */
public static final String LABELS_design_file_profile = "{labels.design_file_profile}";
/** The key of the message: Edit JSP File */
public static final String LABELS_design_title_edit_content = "{labels.design_title_edit_content}";

6
src/main/java/org/codelibs/fess/thumbnail/ThumbnailGenerator.java
View file

@ -18,13 +18,13 @@ package org.codelibs.fess.thumbnail;
import java.io.File;
import java.util.Map;
import org.codelibs.core.misc.Tuple4;
import org.codelibs.core.misc.Tuple3;
public interface ThumbnailGenerator {
String getName();
boolean generate(String thumbnailId, String url, File outputFile);
boolean generate(String thumbnailId, File outputFile);
boolean isTarget(Map<String, Object> docMap);
@ -32,5 +32,5 @@ public interface ThumbnailGenerator {
void destroy();
Tuple4<String, String, String, String> createTask(String path, Map<String, Object> docMap);
Tuple3<String, String, String> createTask(String path, Map<String, Object> docMap);
}

73
src/main/java/org/codelibs/fess/thumbnail/ThumbnailManager.java
View file

@ -36,7 +36,7 @@ import javax.servlet.http.HttpSession;
import org.codelibs.core.collection.LruHashMap;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.core.misc.Tuple4;
import org.codelibs.core.misc.Tuple3;
import org.codelibs.fess.Constants;
import org.codelibs.fess.es.client.FessEsClient;
import org.codelibs.fess.es.config.exbhv.ThumbnailQueueBhv;
@ -70,7 +70,7 @@ public class ThumbnailManager {
private final List<ThumbnailGenerator> generatorList = new ArrayList<>();
private BlockingQueue<Tuple4<String, String, String, String>> thumbnailTaskQueue;
private BlockingQueue<Tuple3<String, String, String>> thumbnailTaskQueue;
private volatile boolean generating;
@ -116,37 +116,35 @@ public class ThumbnailManager {
thumbnailTaskQueue = new LinkedBlockingQueue<>(thumbnailTaskQueueSize);
generating = true;
thumbnailQueueThread =
new Thread((Runnable) () -> {
final List<Tuple4<String, String, String, String>> taskList = new ArrayList<>();
while (generating) {
try {
final Tuple4<String, String, String, String> task =
thumbnailTaskQueue.poll(thumbnailTaskQueueTimeout, TimeUnit.MILLISECONDS);
if (task == null) {
if (!taskList.isEmpty()) {
storeQueue(taskList);
}
} else if (!taskList.contains(task)) {
taskList.add(task);
if (taskList.size() > thumbnailTaskBulkSize) {
storeQueue(taskList);
}
}
} catch (final InterruptedException e) {
if (logger.isDebugEnabled()) {
logger.debug("Interupted task.", e);
}
} catch (final Exception e) {
if (generating) {
logger.warn("Failed to generate thumbnail.", e);
}
thumbnailQueueThread = new Thread((Runnable) () -> {
final List<Tuple3<String, String, String>> taskList = new ArrayList<>();
while (generating) {
try {
final Tuple3<String, String, String> task = thumbnailTaskQueue.poll(thumbnailTaskQueueTimeout, TimeUnit.MILLISECONDS);
if (task == null) {
if (!taskList.isEmpty()) {
storeQueue(taskList);
}
} else if (!taskList.contains(task)) {
taskList.add(task);
if (taskList.size() > thumbnailTaskBulkSize) {
storeQueue(taskList);
}
}
if (!taskList.isEmpty()) {
storeQueue(taskList);
} catch (final InterruptedException e) {
if (logger.isDebugEnabled()) {
logger.debug("Interupted task.", e);
}
}, "ThumbnailGenerator");
} catch (final Exception e) {
if (generating) {
logger.warn("Failed to generate thumbnail.", e);
}
}
}
if (!taskList.isEmpty()) {
storeQueue(taskList);
}
}, "ThumbnailGenerator");
thumbnailQueueThread.start();
}
@ -172,7 +170,7 @@ public class ThumbnailManager {
return "-D" + FESS_THUMBNAIL_PATH + "=" + baseDir.getAbsolutePath();
}
protected void storeQueue(final List<Tuple4<String, String, String, String>> taskList) {
protected void storeQueue(final List<Tuple3<String, String, String>> taskList) {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
final String[] targets = fessConfig.getThumbnailGeneratorTargetsAsArray();
@ -182,8 +180,7 @@ public class ThumbnailManager {
final ThumbnailQueue entity = new ThumbnailQueue();
entity.setGenerator(task.getValue1());
entity.setThumbnailId(task.getValue2());
entity.setUrl(task.getValue3());
entity.setPath(task.getValue4());
entity.setPath(task.getValue3());
entity.setTarget(target);
entity.setCreatedBy(Constants.SYSTEM_USER);
entity.setCreatedTime(systemHelper.getCurrentTimeAsLong());
@ -224,7 +221,7 @@ public class ThumbnailManager {
if (noImageFile.isFile() && !noImageFile.delete()) {
logger.warn("Failed to delete " + noImageFile.getAbsolutePath());
}
if (!generator.generate(entity.getThumbnailId(), entity.getUrl(), outputFile)) {
if (!generator.generate(entity.getThumbnailId(), outputFile)) {
new File(outputFile.getAbsolutePath() + NOIMAGE_FILE_SUFFIX).setLastModified(System.currentTimeMillis());
} else {
final long interval = fessConfig.getThumbnailGeneratorIntervalAsInteger().longValue();
@ -248,11 +245,11 @@ public class ThumbnailManager {
return idList.size();
}
public void offer(final Map<String, Object> docMap) {
public boolean offer(final Map<String, Object> docMap) {
for (final ThumbnailGenerator generator : generatorList) {
if (generator.isTarget(docMap)) {
final String path = getImageFilename(docMap);
final Tuple4<String, String, String, String> task = generator.createTask(path, docMap);
final Tuple3<String, String, String> task = generator.createTask(path, docMap);
if (task != null) {
if (logger.isDebugEnabled()) {
logger.debug("Add thumbnail task: " + task);
@ -260,13 +257,15 @@ public class ThumbnailManager {
if (!thumbnailTaskQueue.offer(task)) {
logger.warn("Failed to add thumbnail task: " + task);
}
return true;
}
return;
return false;
}
}
if (logger.isDebugEnabled()) {
logger.debug("Thumbnail generator is not found: " + (docMap != null ? docMap.get("url") : docMap));
}
return false;
}
protected String getImageFilename(final Map<String, Object> docMap) {

81
src/main/java/org/codelibs/fess/thumbnail/impl/BaseThumbnailGenerator.java
View file

@ -22,8 +22,20 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.BiPredicate;
import java.util.function.Predicate;
import org.codelibs.core.misc.Tuple4;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.core.misc.Tuple3;
import org.codelibs.fess.crawler.builder.RequestDataBuilder;
import org.codelibs.fess.crawler.client.CrawlerClient;
import org.codelibs.fess.crawler.client.CrawlerClientFactory;
import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.es.client.FessEsClient;
import org.codelibs.fess.es.config.exentity.CrawlingConfig;
import org.codelibs.fess.exception.ThumbnailGenerationException;
import org.codelibs.fess.helper.CrawlingConfigHelper;
import org.codelibs.fess.helper.IndexingHelper;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.thumbnail.ThumbnailGenerator;
import org.codelibs.fess.util.ComponentUtil;
@ -50,6 +62,9 @@ public abstract class BaseThumbnailGenerator implements ThumbnailGenerator {
@Override
public boolean isTarget(final Map<String, Object> docMap) {
if (!docMap.containsKey(ComponentUtil.getFessConfig().getIndexFieldThumbnail())) {
return false;
}
for (final Map.Entry<String, String> entry : conditionMap.entrySet()) {
final Object value = docMap.get(entry.getKey());
if (value instanceof String && !((String) value).matches(entry.getValue())) {
@ -92,11 +107,10 @@ public abstract class BaseThumbnailGenerator implements ThumbnailGenerator {
}
@Override
public Tuple4<String, String, String, String> createTask(final String path, final Map<String, Object> docMap) {
public Tuple3<String, String, String> createTask(final String path, final Map<String, Object> docMap) {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final String thumbnailId = DocumentUtil.getValue(docMap, fessConfig.getIndexFieldId(), String.class);
final String url = DocumentUtil.getValue(docMap, fessConfig.getIndexFieldUrl(), String.class);
final Tuple4<String, String, String, String> task = new Tuple4<>(getName(), thumbnailId, url, path);
final Tuple3<String, String, String> task = new Tuple3<>(getName(), thumbnailId, path);
if (logger.isDebugEnabled()) {
logger.debug("Create thumbnail task: " + task);
}
@ -114,17 +128,70 @@ public abstract class BaseThumbnailGenerator implements ThumbnailGenerator {
return value;
}
protected void updateThumbnailField(final String thumbnailId, final String url, final String value) {
protected void updateThumbnailField(final String thumbnailId, final String value) {
// TODO bulk
final FessConfig fessConfig = ComponentUtil.getFessConfig();
try {
ComponentUtil.getIndexingHelper().updateDocument(ComponentUtil.getFessEsClient(), thumbnailId,
fessConfig.getIndexFieldThumbnail(), null);
fessConfig.getIndexFieldThumbnail(), value);
} catch (final Exception e) {
logger.warn("Failed to update thumbnail field at " + url, e);
logger.warn("Failed to update thumbnail field at " + thumbnailId, e);
}
}
protected boolean process(final String id, final BiPredicate<String, String> consumer) {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final FessEsClient fessEsClient = ComponentUtil.getFessEsClient();
final IndexingHelper indexingHelper = ComponentUtil.getIndexingHelper();
try {
final Map<String, Object> doc =
indexingHelper.getDocument(fessEsClient, id,
new String[] { fessConfig.getIndexFieldThumbnail(), fessConfig.getIndexFieldConfigId() });
if (doc == null) {
throw new ThumbnailGenerationException("Document is not found: " + id);
}
final String url = DocumentUtil.getValue(doc, fessConfig.getIndexFieldThumbnail(), String.class);
if (StringUtil.isBlank(url)) {
throw new ThumbnailGenerationException("Invalid thumbnail: " + url);
}
final String configId = DocumentUtil.getValue(doc, fessConfig.getIndexFieldConfigId(), String.class);
if (configId == null || configId.length() < 2) {
throw new ThumbnailGenerationException("Invalid configId: " + configId);
}
return consumer.test(configId, url);
} catch (final ThumbnailGenerationException e) {
if (e.getCause() == null) {
logger.debug(e.getMessage());
} else {
logger.warn("Failed to process " + id, e);
}
} catch (final Exception e) {
logger.warn("Failed to process " + id, e);
}
return false;
}
protected boolean process(final String id, final Predicate<ResponseData> consumer) {
return process(id, (configId, url) -> {
final CrawlingConfigHelper crawlingConfigHelper = ComponentUtil.getCrawlingConfigHelper();
CrawlingConfig config = crawlingConfigHelper.getCrawlingConfig(configId);
if (config == null) {
throw new ThumbnailGenerationException("No CrawlingConfig: " + configId);
}
final CrawlerClientFactory crawlerClientFactory = ComponentUtil.getComponent(CrawlerClientFactory.class);
config.initializeClientFactory(crawlerClientFactory);
final CrawlerClient client = crawlerClientFactory.getClient(url);
if (client == null) {
throw new ThumbnailGenerationException("No CrawlerClient: " + configId + ", url: " + url);
}
try (final ResponseData responseData = client.execute(RequestDataBuilder.newRequestData().get().url(url).build())) {
return consumer.test(responseData);
} catch (final Exception e) {
throw new ThumbnailGenerationException("Failed to process a thumbnail content: " + url, e);
}
});
}
public void setGeneratorList(final List<String> generatorList) {
this.generatorList = generatorList;
}

67
src/main/java/org/codelibs/fess/thumbnail/impl/CommandGenerator.java
View file

@ -29,6 +29,8 @@ import javax.annotation.Resource;
import javax.servlet.ServletContext;
import org.apache.commons.io.IOUtils;
import org.codelibs.core.io.CopyUtil;
import org.codelibs.core.lang.StringUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -61,9 +63,9 @@ public class CommandGenerator extends BaseThumbnailGenerator {
}
@Override
public boolean generate(final String thumbnailId, final String url, final File outputFile) {
public boolean generate(final String thumbnailId, final File outputFile) {
if (logger.isDebugEnabled()) {
logger.debug("Generate Thumbnail: " + url);
logger.debug("Generate Thumbnail: " + thumbnailId);
}
if (outputFile.exists()) {
@ -82,12 +84,47 @@ public class CommandGenerator extends BaseThumbnailGenerator {
return false;
}
final String outputPath = outputFile.getAbsolutePath();
final List<String> cmdList = new ArrayList<>();
for (final String value : commandList) {
cmdList.add(expandPath(value.replace("${url}", url).replace("${outputFile}", outputPath)));
}
return process(thumbnailId, responseData -> {
File tempFile = null;
try {
tempFile = File.createTempFile("thumbnail_", "");
CopyUtil.copy(responseData.getResponseBody(), tempFile);
final String tempPath = tempFile.getAbsolutePath();
final String outputPath = outputFile.getAbsolutePath();
final List<String> cmdList = new ArrayList<>();
for (final String value : commandList) {
cmdList.add(expandPath(value.replace("${url}", tempPath).replace("${outputFile}", outputPath)));
}
executeCommand(thumbnailId, cmdList);
if (outputFile.isFile() && outputFile.length() == 0) {
logger.warn("Thumbnail File is empty. ID is " + thumbnailId);
if (outputFile.delete()) {
logger.info("Deleted: " + outputFile.getAbsolutePath());
}
updateThumbnailField(thumbnailId, StringUtil.EMPTY);
return false;
}
if (logger.isDebugEnabled()) {
logger.debug("Thumbnail File: " + outputPath);
}
return true;
} catch (final Exception e) {
logger.warn("Failed to process ", e);
if (tempFile != null && !tempFile.delete()) {
logger.debug("Failed to delete " + tempFile.getAbsolutePath());
}
updateThumbnailField(thumbnailId, StringUtil.EMPTY);
return false;
}
});
}
protected void executeCommand(final String thumbnailId, final List<String> cmdList) {
ProcessBuilder pb = null;
Process p = null;
@ -125,26 +162,12 @@ public class CommandGenerator extends BaseThumbnailGenerator {
p.destroy();
}
} catch (final Exception e) {
logger.warn("Failed to generate a thumbnail of " + url, e);
logger.warn("Failed to generate a thumbnail of " + thumbnailId, e);
}
if (task != null) {
task.cancel();
task = null;
}
if (outputFile.isFile() && outputFile.length() == 0) {
logger.warn("Thumbnail File is empty. URL is " + url);
if (outputFile.delete()) {
logger.info("Deleted: " + outputFile.getAbsolutePath());
}
return false;
}
if (logger.isDebugEnabled()) {
logger.debug("Thumbnail File: " + outputPath);
}
updateThumbnailField(thumbnailId, url, url);
return true;
}
protected static class ProcessDestroyer extends TimerTask {

2
src/main/java/org/codelibs/fess/thumbnail/impl/EmptyGenerator.java
View file

@ -20,7 +20,7 @@ import java.io.File;
public class EmptyGenerator extends BaseThumbnailGenerator {
@Override
public boolean generate(final String thumbnailId, final String url, final File outputFile) {
public boolean generate(final String thumbnailId, final File outputFile) {
return false;
}

113
src/main/java/org/codelibs/fess/thumbnail/impl/HtmlTagBasedGenerator.java
View file

@ -29,8 +29,7 @@ import javax.imageio.ImageReader;
import javax.imageio.stream.ImageInputStream;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.core.misc.Tuple4;
import org.codelibs.elasticsearch.runner.net.Curl;
import org.codelibs.core.misc.Tuple3;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.util.ComponentUtil;
import org.codelibs.fess.util.DocumentUtil;
@ -46,17 +45,10 @@ public class HtmlTagBasedGenerator extends BaseThumbnailGenerator {
}
@Override
public Tuple4<String, String, String, String> createTask(final String path, final Map<String, Object> docMap) {
public Tuple3<String, String, String> createTask(final String path, final Map<String, Object> docMap) {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final String thumbnailId = DocumentUtil.getValue(docMap, fessConfig.getIndexFieldId(), String.class);
final String url = DocumentUtil.getValue(docMap, fessConfig.getIndexFieldThumbnail(), String.class);
if (StringUtil.isBlank(url)) {
if (logger.isDebugEnabled()) {
logger.debug("Thumbnail task is not found: " + path + ", " + thumbnailId + ", " + url);
}
return null;
}
final Tuple4<String, String, String, String> task = new Tuple4<>(getName(), thumbnailId, url, path);
final Tuple3<String, String, String> task = new Tuple3<>(getName(), thumbnailId, path);
if (logger.isDebugEnabled()) {
logger.debug("Create thumbnail task: " + task);
}
@ -64,9 +56,9 @@ public class HtmlTagBasedGenerator extends BaseThumbnailGenerator {
}
@Override
public boolean generate(final String thumbnailId, final String url, final File outputFile) {
public boolean generate(final String thumbnailId, final File outputFile) {
if (logger.isDebugEnabled()) {
logger.debug("Generate Thumbnail: " + url);
logger.debug("Generate Thumbnail: " + thumbnailId);
}
if (outputFile.exists()) {
@ -85,47 +77,64 @@ public class HtmlTagBasedGenerator extends BaseThumbnailGenerator {
return false;
}
final FessConfig fessConfig = ComponentUtil.getFessConfig();
Curl.get(url)
.proxy(fessConfig.getHttpProxy())
.execute(
con -> {
boolean created = false;
try (ImageInputStream input = ImageIO.createImageInputStream(con.getInputStream())) {
switch (saveImage(input, outputFile)) {
case OK:
created = true;
break;
case FAILED:
logger.warn("Failed to create thumbnail: " + thumbnailId + " -> " + url);
break;
case INVALID_SIZE:
if (logger.isDebugEnabled()) {
logger.debug("Invalid thumbnail size: " + thumbnailId + " -> " + url);
}
break;
default:
logger.error("Unknown thumbnail result: " + thumbnailId + " -> " + url);
break;
}
} catch (final Throwable t) {
if (logger.isDebugEnabled()) {
logger.warn("Failed to create thumbnail: " + thumbnailId + " -> " + url, t);
} else {
logger.warn("Failed to create thumbnail: " + thumbnailId + " -> " + url + " ("
+ t.getClass().getCanonicalName() + ": " + t.getMessage() + ")");
}
} finally {
if (!created) {
updateThumbnailField(thumbnailId, url, StringUtil.EMPTY);
if (outputFile.exists() && !outputFile.delete()) {
logger.warn("Failed to delete " + outputFile.getAbsolutePath());
}
}
return process(
thumbnailId,
responseData -> {
if (!isImageMimeType(responseData.getMimeType())) {
if (logger.isDebugEnabled()) {
logger.debug("Thumbnail is not image: " + thumbnailId + " : " + responseData.getUrl());
}
updateThumbnailField(thumbnailId, StringUtil.EMPTY);
return false;
}
boolean created = false;
try (ImageInputStream input = ImageIO.createImageInputStream(responseData.getResponseBody())) {
switch (saveImage(input, outputFile)) {
case OK:
created = true;
break;
case FAILED:
logger.warn("Failed to create thumbnail: " + thumbnailId);
break;
case INVALID_SIZE:
if (logger.isDebugEnabled()) {
logger.debug("Invalid thumbnail size: " + thumbnailId);
}
});
break;
default:
logger.error("Unknown thumbnail result: " + thumbnailId);
break;
}
} catch (final Throwable t) {
if (logger.isDebugEnabled()) {
logger.warn("Failed to create thumbnail: " + thumbnailId, t);
} else {
logger.warn("Failed to create thumbnail: " + thumbnailId + " (" + t.getClass().getCanonicalName() + ": "
+ t.getMessage() + ")");
}
} finally {
if (!created) {
updateThumbnailField(thumbnailId, StringUtil.EMPTY);
if (outputFile.exists() && !outputFile.delete()) {
logger.warn("Failed to delete " + outputFile.getAbsolutePath());
}
}
}
return outputFile.exists();
});
return outputFile.exists();
}
protected boolean isImageMimeType(final String mimeType) {
switch (mimeType) {
case "image/png":
case "image/gif":
case "image/jpeg":
case "image/bmp":
return true;
default:
return false;
}
}
protected Result saveImage(final ImageInputStream input, final File outputFile) throws IOException {

11
src/main/java/org/codelibs/fess/thumbnail/impl/WebDriverGenerator.java
View file

@ -50,6 +50,7 @@ import org.openqa.selenium.remote.UnreachableBrowserException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@Deprecated
public class WebDriverGenerator extends BaseThumbnailGenerator {
private static final Logger logger = LoggerFactory.getLogger(WebDriverGenerator.class);
@ -117,7 +118,13 @@ public class WebDriverGenerator extends BaseThumbnailGenerator {
}
@Override
public boolean generate(final String thumbnailId, final String url, final File outputFile) {
public boolean generate(final String thumbnailId, final File outputFile) {
return process(thumbnailId, (configId, url) -> {
return generate(thumbnailId, url, outputFile);
});
}
protected boolean generate(final String thumbnailId, final String url, final File outputFile) {
if (logger.isDebugEnabled()) {
logger.debug("Generate Thumbnail: " + url);
}
@ -154,7 +161,7 @@ public class WebDriverGenerator extends BaseThumbnailGenerator {
}
final File thumbnail = ((TakesScreenshot) webDriver).getScreenshotAs(OutputType.FILE);
convert(thumbnail, outputFile);
updateThumbnailField(thumbnailId, url, url);
updateThumbnailField(thumbnailId, url);
return true;
} catch (final UnreachableBrowserException | SessionNotFoundException e) {
if (logger.isDebugEnabled()) {

3
src/main/resources/fess_indices/.fess_config/thumbnail_queue.json
View file

@ -10,9 +10,6 @@
"thumbnail_id": {
"type": "keyword"
},
"url": {
"type": "keyword"
},
"path": {
"type": "keyword"
},

36
src/main/resources/fess_thumbnail.xml
View file

@ -20,42 +20,6 @@
<arg>"text/html"</arg>
</postConstruct>
</component>
<!--
<component name="htmlThumbnailGenerator" class="org.codelibs.fess.thumbnail.impl.WebDriverGenerator">
<property name="name">"htmlThumbnailGenerator"</property>
<property name="generatorList">
["${path}/phantomjs"]
</property>
<property name="webDriverCapabilities">
<component class="org.openqa.selenium.remote.DesiredCapabilities">
<postConstruct name="setCapability">
<arg>"phantomjs.binary.path"</arg>
<arg>"${path}/phantomjs"</arg>
</postConstruct>
</component>
</property>
<postConstruct name="addCondition">
<arg>"mimetype"</arg>
<arg>"text/html"</arg>
</postConstruct>
</component>
<component name="htmlThumbnailGenerator" class="org.codelibs.fess.thumbnail.impl.CommandGenerator">
<property name="name">"htmlThumbnailGenerator"</property>
<property name="commandList">
["${path}/generate-thumbnail",
"html",
"${url}",
"${outputFile}"]
</property>
<property name="generatorList">
["${path}/generate-thumbnail"]
</property>
<postConstruct name="addCondition">
<arg>"mimetype"</arg>
<arg>"text/html"</arg>
</postConstruct>
</component>
-->
<component name="msofficeThumbnailGenerator" class="org.codelibs.fess.thumbnail.impl.CommandGenerator">
<property name="name">"msofficeThumbnailGenerator"</property>
<property name="commandList">

77
src/main/webapp/WEB-INF/crawler/resources/crawler_thumbnail.xml
View file

@ -6,6 +6,12 @@
<postConstruct name="add">
<arg>htmlThumbnailGenerator</arg>
</postConstruct>
<postConstruct name="add">
<arg>msofficeThumbnailGenerator</arg>
</postConstruct>
<postConstruct name="add">
<arg>pdfThumbnailGenerator</arg>
</postConstruct>
</component>
<component name="htmlThumbnailGenerator" class="org.codelibs.fess.thumbnail.impl.HtmlTagBasedGenerator">
<property name="name">"htmlThumbnailGenerator"</property>
@ -14,5 +20,74 @@
<arg>"text/html"</arg>
</postConstruct>
</component>
<!-- TODO remove ServletContext from CommandGenerator -->
<component name="msofficeThumbnailGenerator" class="org.codelibs.fess.thumbnail.impl.EmptyGenerator">
<property name="name">"msofficeThumbnailGenerator"</property>
<property name="generatorList">
["${path}/generate-thumbnail"]
</property>
<postConstruct name="addCondition">
<arg>"mimetype"</arg>
<arg>"text/html"</arg>
</postConstruct>
<postConstruct name="addCondition">
<arg>"mimetype"</arg>
<arg>"application/vnd.openxmlformats-officedocument.wordprocessingml.document"</arg>
</postConstruct>
<postConstruct name="addCondition">
<arg>"mimetype"</arg>
<arg>"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"</arg>
</postConstruct>
<postConstruct name="addCondition">
<arg>"mimetype"</arg>
<arg>"application/vnd.openxmlformats-officedocument.presentationml.presentation"</arg>
</postConstruct>
<postConstruct name="addCondition">
<arg>"mimetype"</arg>
<arg>"application/msword"</arg>
</postConstruct>
<postConstruct name="addCondition">
<arg>"mimetype"</arg>
<arg>"application/vnd.ms-excel"</arg>
</postConstruct>
<postConstruct name="addCondition">
<arg>"mimetype"</arg>
<arg>"application/vnd.ms-excel.sheet.2"</arg>
</postConstruct>
<postConstruct name="addCondition">
<arg>"mimetype"</arg>
<arg>"application/vnd.ms-excel.sheet.3"</arg>
</postConstruct>
<postConstruct name="addCondition">
<arg>"mimetype"</arg>
<arg>"application/vnd.ms-excel.sheet.4"</arg>
</postConstruct>
<postConstruct name="addCondition">
<arg>"mimetype"</arg>
<arg>"application/vnd.ms-excel.workspace.3"</arg>
</postConstruct>
<postConstruct name="addCondition">
<arg>"mimetype"</arg>
<arg>"application/vnd.ms-excel.workspace.4"</arg>
</postConstruct>
<postConstruct name="addCondition">
<arg>"mimetype"</arg>
<arg>"application/vnd.ms-powerpoint"
</arg>
</postConstruct>
<postConstruct name="addCondition">
<arg>"mimetype"</arg>
<arg>"application/rtf"</arg>
</postConstruct>
</component>
<component name="pdfThumbnailGenerator" class="org.codelibs.fess.thumbnail.impl.EmptyGenerator">
<property name="name">"pdfThumbnailGenerator"</property>
<property name="generatorList">
["${path}/generate-thumbnail"]
</property>
<postConstruct name="addCondition">
<arg>"mimetype"</arg>
<arg>"application/pdf"
</arg>
</postConstruct>
</component>
</components>