fix #899 thumbnail generator access thumbnail content via CrawlerClient

src/main/config/es/fess_config.json

@@ -858,9 +858,6 @@
           },
           "path" : {
             "type": "keyword"
-          },
-          "url" : {
-            "type": "keyword"
           }
         }
       }

src/main/java/org/codelibs/fess/crawler/transformer/AbstractFessFileTransformer.java

@@ -310,6 +310,8 @@ public abstract class AbstractFessFileTransformer extends AbstractTransformer im
             putResultDataBody(dataMap, fessConfig.getIndexFieldParentId(), crawlingInfoHelper.generateId(dataMap));
             putResultDataBody(dataMap, fessConfig.getIndexFieldUrl(), url); // set again
         }
+        // thumbnail
+        putResultDataBody(dataMap, fessConfig.getIndexFieldThumbnail(), responseData.getUrl());
 
         // from config
         final Map<String, String> scriptConfigMap = crawlingConfig.getConfigParameterMap(ConfigName.SCRIPT);

src/main/java/org/codelibs/fess/es/config/bsbhv/BsThumbnailQueueBhv.java

@@ -79,7 +79,6 @@ public abstract class BsThumbnailQueueBhv extends EsAbstractBehavior<ThumbnailQu
             result.setGenerator(DfTypeUtil.toString(source.get("generator")));
             result.setThumbnailId(DfTypeUtil.toString(source.get("thumbnail_id")));
             result.setPath(DfTypeUtil.toString(source.get("path")));
-            result.setUrl(DfTypeUtil.toString(source.get("url")));
             return updateEntity(source, result);
         } catch (InstantiationException | IllegalAccessException e) {
             final String msg = "Cannot create a new instance: " + entityType.getName();

src/main/java/org/codelibs/fess/es/config/bsentity/BsThumbnailQueue.java

@@ -55,9 +55,6 @@ public class BsThumbnailQueue extends EsAbstractEntity {
     /** path */
     protected String path;
 
-    /** url */
-    protected String url;
-
     // [Referrers] *comment only
 
     // ===================================================================================
@@ -97,9 +94,6 @@ public class BsThumbnailQueue extends EsAbstractEntity {
         if (path != null) {
             addFieldToSource(sourceMap, "path", path);
         }
-        if (url != null) {
-            addFieldToSource(sourceMap, "url", url);
-        }
         return sourceMap;
     }
 
@@ -119,7 +113,6 @@ public class BsThumbnailQueue extends EsAbstractEntity {
         sb.append(dm).append(generator);
         sb.append(dm).append(thumbnailId);
         sb.append(dm).append(path);
-        sb.append(dm).append(url);
         if (sb.length() > dm.length()) {
             sb.delete(0, dm.length());
         }
@@ -189,14 +182,4 @@ public class BsThumbnailQueue extends EsAbstractEntity {
         registerModifiedProperty("path");
         this.path = value;
     }
-
-    public String getUrl() {
-        checkSpecifiedProperty("url");
-        return convertEmptyToNull(url);
-    }
-
-    public void setUrl(String value) {
-        registerModifiedProperty("url");
-        this.url = value;
-    }
 }

src/main/java/org/codelibs/fess/es/config/bsentity/dbmeta/ThumbnailQueueDbm.java

@@ -90,7 +90,6 @@ public class ThumbnailQueueDbm extends AbstractDBMeta {
         setupEpg(_epgMap, et -> ((ThumbnailQueue) et).getThumbnailId(),
                 (et, vl) -> ((ThumbnailQueue) et).setThumbnailId(DfTypeUtil.toString(vl)), "thumbnailId");
         setupEpg(_epgMap, et -> ((ThumbnailQueue) et).getPath(), (et, vl) -> ((ThumbnailQueue) et).setPath(DfTypeUtil.toString(vl)), "path");
-        setupEpg(_epgMap, et -> ((ThumbnailQueue) et).getUrl(), (et, vl) -> ((ThumbnailQueue) et).setUrl(DfTypeUtil.toString(vl)), "url");
     }
 
     @Override
@@ -139,8 +138,6 @@ public class ThumbnailQueueDbm extends AbstractDBMeta {
             false, false, false, "keyword", 0, 0, null, false, null, null, null, null, null, false);
     protected final ColumnInfo _columnPath = cci("path", "path", null, null, String.class, "path", null, false, false, false, "keyword", 0,
             0, null, false, null, null, null, null, null, false);
-    protected final ColumnInfo _columnUrl = cci("url", "url", null, null, String.class, "url", null, false, false, false, "keyword", 0, 0,
-            null, false, null, null, null, null, null, false);
 
     public ColumnInfo columnCreatedBy() {
         return _columnCreatedBy;
@@ -166,10 +163,6 @@ public class ThumbnailQueueDbm extends AbstractDBMeta {
         return _columnPath;
     }
 
-    public ColumnInfo columnUrl() {
-        return _columnUrl;
-    }
-
     protected List<ColumnInfo> ccil() {
         List<ColumnInfo> ls = newArrayList();
         ls.add(columnCreatedBy());
@@ -178,7 +171,6 @@ public class ThumbnailQueueDbm extends AbstractDBMeta {
         ls.add(columnGenerator());
         ls.add(columnThumbnailId());
         ls.add(columnPath());
-        ls.add(columnUrl());
         return ls;
     }
 

src/main/java/org/codelibs/fess/es/config/cbean/bs/BsThumbnailQueueCB.java

@@ -199,9 +199,5 @@ public class BsThumbnailQueueCB extends EsAbstractConditionBean {
         public void columnPath() {
             doColumn("path");
         }
-
-        public void columnUrl() {
-            doColumn("url");
-        }
     }
 }

src/main/java/org/codelibs/fess/es/config/cbean/ca/bs/BsThumbnailQueueCA.java

@@ -987,134 +987,4 @@ public abstract class BsThumbnailQueueCA extends EsAbstractConditionAggregation
         }
     }
 
-    public void setUrl_Terms() {
-        setUrl_Terms(null);
-    }
-
-    public void setUrl_Terms(ConditionOptionCall<TermsAggregationBuilder> opLambda) {
-        setUrl_Terms("url", opLambda, null);
-    }
-
-    public void setUrl_Terms(ConditionOptionCall<TermsAggregationBuilder> opLambda, OperatorCall<BsThumbnailQueueCA> aggsLambda) {
-        setUrl_Terms("url", opLambda, aggsLambda);
-    }
-
-    public void setUrl_Terms(String name, ConditionOptionCall<TermsAggregationBuilder> opLambda, OperatorCall<BsThumbnailQueueCA> aggsLambda) {
-        TermsAggregationBuilder builder = regTermsA(name, "url");
-        if (opLambda != null) {
-            opLambda.callback(builder);
-        }
-        if (aggsLambda != null) {
-            ThumbnailQueueCA ca = new ThumbnailQueueCA();
-            aggsLambda.callback(ca);
-            ca.getAggregationBuilderList().forEach(builder::subAggregation);
-        }
-    }
-
-    public void setUrl_SignificantTerms() {
-        setUrl_SignificantTerms(null);
-    }
-
-    public void setUrl_SignificantTerms(ConditionOptionCall<SignificantTermsAggregationBuilder> opLambda) {
-        setUrl_SignificantTerms("url", opLambda, null);
-    }
-
-    public void setUrl_SignificantTerms(ConditionOptionCall<SignificantTermsAggregationBuilder> opLambda,
-            OperatorCall<BsThumbnailQueueCA> aggsLambda) {
-        setUrl_SignificantTerms("url", opLambda, aggsLambda);
-    }
-
-    public void setUrl_SignificantTerms(String name, ConditionOptionCall<SignificantTermsAggregationBuilder> opLambda,
-            OperatorCall<BsThumbnailQueueCA> aggsLambda) {
-        SignificantTermsAggregationBuilder builder = regSignificantTermsA(name, "url");
-        if (opLambda != null) {
-            opLambda.callback(builder);
-        }
-        if (aggsLambda != null) {
-            ThumbnailQueueCA ca = new ThumbnailQueueCA();
-            aggsLambda.callback(ca);
-            ca.getAggregationBuilderList().forEach(builder::subAggregation);
-        }
-    }
-
-    public void setUrl_IpRange() {
-        setUrl_IpRange(null);
-    }
-
-    public void setUrl_IpRange(ConditionOptionCall<IpRangeAggregationBuilder> opLambda) {
-        setUrl_IpRange("url", opLambda, null);
-    }
-
-    public void setUrl_IpRange(ConditionOptionCall<IpRangeAggregationBuilder> opLambda, OperatorCall<BsThumbnailQueueCA> aggsLambda) {
-        setUrl_IpRange("url", opLambda, aggsLambda);
-    }
-
-    public void setUrl_IpRange(String name, ConditionOptionCall<IpRangeAggregationBuilder> opLambda,
-            OperatorCall<BsThumbnailQueueCA> aggsLambda) {
-        IpRangeAggregationBuilder builder = regIpRangeA(name, "url");
-        if (opLambda != null) {
-            opLambda.callback(builder);
-        }
-        if (aggsLambda != null) {
-            ThumbnailQueueCA ca = new ThumbnailQueueCA();
-            aggsLambda.callback(ca);
-            ca.getAggregationBuilderList().forEach(builder::subAggregation);
-        }
-    }
-
-    public void setUrl_Count() {
-        setUrl_Count(null);
-    }
-
-    public void setUrl_Count(ConditionOptionCall<ValueCountAggregationBuilder> opLambda) {
-        setUrl_Count("url", opLambda);
-    }
-
-    public void setUrl_Count(String name, ConditionOptionCall<ValueCountAggregationBuilder> opLambda) {
-        ValueCountAggregationBuilder builder = regCountA(name, "url");
-        if (opLambda != null) {
-            opLambda.callback(builder);
-        }
-    }
-
-    public void setUrl_Cardinality() {
-        setUrl_Cardinality(null);
-    }
-
-    public void setUrl_Cardinality(ConditionOptionCall<CardinalityAggregationBuilder> opLambda) {
-        setUrl_Cardinality("url", opLambda);
-    }
-
-    public void setUrl_Cardinality(String name, ConditionOptionCall<CardinalityAggregationBuilder> opLambda) {
-        CardinalityAggregationBuilder builder = regCardinalityA(name, "url");
-        if (opLambda != null) {
-            opLambda.callback(builder);
-        }
-    }
-
-    public void setUrl_Missing() {
-        setUrl_Missing(null);
-    }
-
-    public void setUrl_Missing(ConditionOptionCall<MissingAggregationBuilder> opLambda) {
-        setUrl_Missing("url", opLambda, null);
-    }
-
-    public void setUrl_Missing(ConditionOptionCall<MissingAggregationBuilder> opLambda, OperatorCall<BsThumbnailQueueCA> aggsLambda) {
-        setUrl_Missing("url", opLambda, aggsLambda);
-    }
-
-    public void setUrl_Missing(String name, ConditionOptionCall<MissingAggregationBuilder> opLambda,
-            OperatorCall<BsThumbnailQueueCA> aggsLambda) {
-        MissingAggregationBuilder builder = regMissingA(name, "url");
-        if (opLambda != null) {
-            opLambda.callback(builder);
-        }
-        if (aggsLambda != null) {
-            ThumbnailQueueCA ca = new ThumbnailQueueCA();
-            aggsLambda.callback(ca);
-            ca.getAggregationBuilderList().forEach(builder::subAggregation);
-        }
-    }
-
 }

src/main/java/org/codelibs/fess/es/config/cbean/cq/bs/BsThumbnailQueueCQ.java

@@ -1474,226 +1474,4 @@ public abstract class BsThumbnailQueueCQ extends EsAbstractConditionQuery {
         return this;
     }
 
-    public void setUrl_Equal(String url) {
-        setUrl_Term(url, null);
-    }
-
-    public void setUrl_Equal(String url, ConditionOptionCall<TermQueryBuilder> opLambda) {
-        setUrl_Term(url, opLambda);
-    }
-
-    public void setUrl_Term(String url) {
-        setUrl_Term(url, null);
-    }
-
-    public void setUrl_Term(String url, ConditionOptionCall<TermQueryBuilder> opLambda) {
-        TermQueryBuilder builder = regTermQ("url", url);
-        if (opLambda != null) {
-            opLambda.callback(builder);
-        }
-    }
-
-    public void setUrl_NotEqual(String url) {
-        setUrl_NotTerm(url, null);
-    }
-
-    public void setUrl_NotTerm(String url) {
-        setUrl_NotTerm(url, null);
-    }
-
-    public void setUrl_NotEqual(String url, ConditionOptionCall<BoolQueryBuilder> opLambda) {
-        setUrl_NotTerm(url, opLambda);
-    }
-
-    public void setUrl_NotTerm(String url, ConditionOptionCall<BoolQueryBuilder> opLambda) {
-        not(not -> not.setUrl_Term(url), opLambda);
-    }
-
-    public void setUrl_Terms(Collection<String> urlList) {
-        setUrl_Terms(urlList, null);
-    }
-
-    public void setUrl_Terms(Collection<String> urlList, ConditionOptionCall<TermsQueryBuilder> opLambda) {
-        TermsQueryBuilder builder = regTermsQ("url", urlList);
-        if (opLambda != null) {
-            opLambda.callback(builder);
-        }
-    }
-
-    public void setUrl_InScope(Collection<String> urlList) {
-        setUrl_Terms(urlList, null);
-    }
-
-    public void setUrl_InScope(Collection<String> urlList, ConditionOptionCall<TermsQueryBuilder> opLambda) {
-        setUrl_Terms(urlList, opLambda);
-    }
-
-    public void setUrl_Match(String url) {
-        setUrl_Match(url, null);
-    }
-
-    public void setUrl_Match(String url, ConditionOptionCall<MatchQueryBuilder> opLambda) {
-        MatchQueryBuilder builder = regMatchQ("url", url);
-        if (opLambda != null) {
-            opLambda.callback(builder);
-        }
-    }
-
-    public void setUrl_MatchPhrase(String url) {
-        setUrl_MatchPhrase(url, null);
-    }
-
-    public void setUrl_MatchPhrase(String url, ConditionOptionCall<MatchPhraseQueryBuilder> opLambda) {
-        MatchPhraseQueryBuilder builder = regMatchPhraseQ("url", url);
-        if (opLambda != null) {
-            opLambda.callback(builder);
-        }
-    }
-
-    public void setUrl_MatchPhrasePrefix(String url) {
-        setUrl_MatchPhrasePrefix(url, null);
-    }
-
-    public void setUrl_MatchPhrasePrefix(String url, ConditionOptionCall<MatchPhrasePrefixQueryBuilder> opLambda) {
-        MatchPhrasePrefixQueryBuilder builder = regMatchPhrasePrefixQ("url", url);
-        if (opLambda != null) {
-            opLambda.callback(builder);
-        }
-    }
-
-    public void setUrl_Fuzzy(String url) {
-        setUrl_Fuzzy(url, null);
-    }
-
-    public void setUrl_Fuzzy(String url, ConditionOptionCall<MatchQueryBuilder> opLambda) {
-        MatchQueryBuilder builder = regFuzzyQ("url", url);
-        if (opLambda != null) {
-            opLambda.callback(builder);
-        }
-    }
-
-    public void setUrl_Prefix(String url) {
-        setUrl_Prefix(url, null);
-    }
-
-    public void setUrl_Prefix(String url, ConditionOptionCall<PrefixQueryBuilder> opLambda) {
-        PrefixQueryBuilder builder = regPrefixQ("url", url);
-        if (opLambda != null) {
-            opLambda.callback(builder);
-        }
-    }
-
-    public void setUrl_Wildcard(String url) {
-        setUrl_Wildcard(url, null);
-    }
-
-    public void setUrl_Wildcard(String url, ConditionOptionCall<WildcardQueryBuilder> opLambda) {
-        WildcardQueryBuilder builder = regWildcardQ("url", url);
-        if (opLambda != null) {
-            opLambda.callback(builder);
-        }
-    }
-
-    public void setUrl_Regexp(String url) {
-        setUrl_Regexp(url, null);
-    }
-
-    public void setUrl_Regexp(String url, ConditionOptionCall<RegexpQueryBuilder> opLambda) {
-        RegexpQueryBuilder builder = regRegexpQ("url", url);
-        if (opLambda != null) {
-            opLambda.callback(builder);
-        }
-    }
-
-    public void setUrl_SpanTerm(String url) {
-        setUrl_SpanTerm("url", null);
-    }
-
-    public void setUrl_SpanTerm(String url, ConditionOptionCall<SpanTermQueryBuilder> opLambda) {
-        SpanTermQueryBuilder builder = regSpanTermQ("url", url);
-        if (opLambda != null) {
-            opLambda.callback(builder);
-        }
-    }
-
-    public void setUrl_GreaterThan(String url) {
-        setUrl_GreaterThan(url, null);
-    }
-
-    public void setUrl_GreaterThan(String url, ConditionOptionCall<RangeQueryBuilder> opLambda) {
-        final Object _value = url;
-        RangeQueryBuilder builder = regRangeQ("url", ConditionKey.CK_GREATER_THAN, _value);
-        if (opLambda != null) {
-            opLambda.callback(builder);
-        }
-    }
-
-    public void setUrl_LessThan(String url) {
-        setUrl_LessThan(url, null);
-    }
-
-    public void setUrl_LessThan(String url, ConditionOptionCall<RangeQueryBuilder> opLambda) {
-        final Object _value = url;
-        RangeQueryBuilder builder = regRangeQ("url", ConditionKey.CK_LESS_THAN, _value);
-        if (opLambda != null) {
-            opLambda.callback(builder);
-        }
-    }
-
-    public void setUrl_GreaterEqual(String url) {
-        setUrl_GreaterEqual(url, null);
-    }
-
-    public void setUrl_GreaterEqual(String url, ConditionOptionCall<RangeQueryBuilder> opLambda) {
-        final Object _value = url;
-        RangeQueryBuilder builder = regRangeQ("url", ConditionKey.CK_GREATER_EQUAL, _value);
-        if (opLambda != null) {
-            opLambda.callback(builder);
-        }
-    }
-
-    public void setUrl_LessEqual(String url) {
-        setUrl_LessEqual(url, null);
-    }
-
-    public void setUrl_LessEqual(String url, ConditionOptionCall<RangeQueryBuilder> opLambda) {
-        final Object _value = url;
-        RangeQueryBuilder builder = regRangeQ("url", ConditionKey.CK_LESS_EQUAL, _value);
-        if (opLambda != null) {
-            opLambda.callback(builder);
-        }
-    }
-
-    public void setUrl_Exists() {
-        setUrl_Exists(null);
-    }
-
-    public void setUrl_Exists(ConditionOptionCall<ExistsQueryBuilder> opLambda) {
-        ExistsQueryBuilder builder = regExistsQ("url");
-        if (opLambda != null) {
-            opLambda.callback(builder);
-        }
-    }
-
-    public void setUrl_CommonTerms(String url) {
-        setUrl_CommonTerms(url, null);
-    }
-
-    public void setUrl_CommonTerms(String url, ConditionOptionCall<CommonTermsQueryBuilder> opLambda) {
-        CommonTermsQueryBuilder builder = regCommonTermsQ("url", url);
-        if (opLambda != null) {
-            opLambda.callback(builder);
-        }
-    }
-
-    public BsThumbnailQueueCQ addOrderBy_Url_Asc() {
-        regOBA("url");
-        return this;
-    }
-
-    public BsThumbnailQueueCQ addOrderBy_Url_Desc() {
-        regOBD("url");
-        return this;
-    }
-
 }

src/main/java/org/codelibs/fess/es/config/exentity/ThumbnailQueue.java

@@ -43,6 +43,6 @@ public class ThumbnailQueue extends BsThumbnailQueue {
     @Override
     public String toString() {
         return "ThumbnailQueue [createdBy=" + createdBy + ", createdTime=" + createdTime + ", generator=" + generator + ", path=" + path
-                + ", url=" + url + "]";
+                + "]";
     }
 }

src/main/java/org/codelibs/fess/exception/ThumbnailGenerationException.java

@@ -0,0 +1,29 @@
+/*
+ * Copyright 2012-2017 CodeLibs Project and the Others.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+ * either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+package org.codelibs.fess.exception;
+
+public class ThumbnailGenerationException extends FessSystemException {
+
+    private static final long serialVersionUID = 1L;
+
+    public ThumbnailGenerationException(String message, Exception cause) {
+        super(message, cause);
+    }
+
+    public ThumbnailGenerationException(String message) {
+        super(message, false, false);
+    }
+}

src/main/java/org/codelibs/fess/helper/CrawlingConfigHelper.java

@@ -17,6 +17,10 @@ package org.codelibs.fess.helper;
 
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+
+import javax.annotation.PostConstruct;
 
 import org.codelibs.fess.app.service.DataConfigService;
 import org.codelibs.fess.app.service.FileConfigService;
@@ -27,6 +31,9 @@ import org.codelibs.fess.util.ComponentUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+
 public class CrawlingConfigHelper {
 
     private static final Logger logger = LoggerFactory.getLogger(CrawlingConfigHelper.class);
@@ -35,6 +42,13 @@ public class CrawlingConfigHelper {
 
     protected int count = 1;
 
+    protected Cache<String, CrawlingConfig> crawlingConfigCache;
+
+    @PostConstruct
+    public void init() {
+        crawlingConfigCache = CacheBuilder.newBuilder().maximumSize(100).expireAfterWrite(10, TimeUnit.MINUTES).build();
+    }
+
     public ConfigType getConfigType(final String configId) {
         if (configId == null || configId.length() < 2) {
             return null;
@@ -58,29 +72,40 @@ public class CrawlingConfigHelper {
     }
 
     public CrawlingConfig getCrawlingConfig(final String configId) {
-        final ConfigType configType = getConfigType(configId);
-        if (configType == null) {
-            return null;
-        }
-        final String id = getId(configId);
-        if (id == null) {
-            return null;
-        }
-        switch (configType) {
-        case WEB:
-            final WebConfigService webConfigService = ComponentUtil.getComponent(WebConfigService.class);
-            return webConfigService.getWebConfig(id).get();
-        case FILE:
-            final FileConfigService fileConfigService = ComponentUtil.getComponent(FileConfigService.class);
-            return fileConfigService.getFileConfig(id).get();
-        case DATA:
-            final DataConfigService dataConfigService = ComponentUtil.getComponent(DataConfigService.class);
-            return dataConfigService.getDataConfig(id).get();
-        default:
+        try {
+            return crawlingConfigCache.get(configId, () -> {
+                final ConfigType configType = getConfigType(configId);
+                if (configType == null) {
+                    return null;
+                }
+                final String id = getId(configId);
+                if (id == null) {
+                    return null;
+                }
+                switch (configType) {
+                case WEB:
+                    final WebConfigService webConfigService = ComponentUtil.getComponent(WebConfigService.class);
+                    return webConfigService.getWebConfig(id).get();
+                case FILE:
+                    final FileConfigService fileConfigService = ComponentUtil.getComponent(FileConfigService.class);
+                    return fileConfigService.getFileConfig(id).get();
+                case DATA:
+                    final DataConfigService dataConfigService = ComponentUtil.getComponent(DataConfigService.class);
+                    return dataConfigService.getDataConfig(id).get();
+                default:
+                    return null;
+                }
+            });
+        } catch (ExecutionException e) {
+            logger.warn("Failed to access a crawling config cache: " + configId, e);
             return null;
         }
     }
 
+    public void refresh() {
+        crawlingConfigCache.invalidateAll();
+    }
+
     public synchronized String store(final String sessionId, final CrawlingConfig crawlingConfig) {
         final String sessionCountId = sessionId + "-" + count;
         crawlingConfigMap.put(sessionCountId, crawlingConfig);

src/main/java/org/codelibs/fess/helper/IndexingHelper.java

@@ -57,7 +57,16 @@ public class IndexingHelper {
         try {
             if (fessConfig.isThumbnailCrawlerEnabled()) {
                 final ThumbnailManager thumbnailManager = ComponentUtil.getThumbnailManager();
-                docList.stream().forEach(doc -> thumbnailManager.offer(doc));
+                docList.stream().forEach(
+                        doc -> {
+                            if (!thumbnailManager.offer(doc)) {
+                                if (logger.isDebugEnabled()) {
+                                    logger.debug("Removing " + doc.get(fessConfig.getIndexFieldThumbnail()) + " from "
+                                            + doc.get(fessConfig.getIndexFieldUrl()));
+                                }
+                                doc.remove(fessConfig.getIndexFieldThumbnail());
+                            }
+                        });
             }
             synchronized (fessEsClient) {
                 deleteOldDocuments(fessEsClient, docList);

src/main/java/org/codelibs/fess/helper/ViewHelper.java

@@ -39,6 +39,7 @@ import javax.annotation.Resource;
 import javax.servlet.http.HttpServletRequest;
 
 import org.apache.catalina.connector.ClientAbortException;
+import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.StringEscapeUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.codelibs.core.CoreLibConstants;
@@ -46,9 +47,6 @@ import org.codelibs.core.lang.StringUtil;
 import org.codelibs.core.misc.Base64Util;
 import org.codelibs.core.misc.DynamicProperties;
 import org.codelibs.fess.Constants;
-import org.codelibs.fess.app.service.DataConfigService;
-import org.codelibs.fess.app.service.FileConfigService;
-import org.codelibs.fess.app.service.WebConfigService;
 import org.codelibs.fess.crawler.builder.RequestDataBuilder;
 import org.codelibs.fess.crawler.client.CrawlerClient;
 import org.codelibs.fess.crawler.client.CrawlerClientFactory;
@@ -56,7 +54,6 @@ import org.codelibs.fess.crawler.entity.ResponseData;
 import org.codelibs.fess.crawler.util.CharUtil;
 import org.codelibs.fess.entity.FacetQueryView;
 import org.codelibs.fess.es.config.exentity.CrawlingConfig;
-import org.codelibs.fess.es.config.exentity.CrawlingConfig.ConfigType;
 import org.codelibs.fess.exception.FessSystemException;
 import org.codelibs.fess.helper.UserAgentHelper.UserAgentType;
 import org.codelibs.fess.mylasta.direction.FessConfig;
@@ -491,21 +488,7 @@ public class ViewHelper {
         if (configId.length() < 2) {
             throw new FessSystemException("Invalid configId: " + configId);
         }
-        final ConfigType configType = crawlingConfigHelper.getConfigType(configId);
-        CrawlingConfig config = null;
-        if (logger.isDebugEnabled()) {
-            logger.debug("configType: " + configType + ", configId: " + configId);
-        }
-        if (ConfigType.WEB == configType) {
-            final WebConfigService webConfigService = ComponentUtil.getComponent(WebConfigService.class);
-            config = webConfigService.getWebConfig(crawlingConfigHelper.getId(configId)).get();
-        } else if (ConfigType.FILE == configType) {
-            final FileConfigService fileConfigService = ComponentUtil.getComponent(FileConfigService.class);
-            config = fileConfigService.getFileConfig(crawlingConfigHelper.getId(configId)).get();
-        } else if (ConfigType.DATA == configType) {
-            final DataConfigService dataConfigService = ComponentUtil.getComponent(DataConfigService.class);
-            config = dataConfigService.getDataConfig(crawlingConfigHelper.getId(configId)).get();
-        }
+        final CrawlingConfig config = crawlingConfigHelper.getCrawlingConfig(configId);
         if (config == null) {
             throw new FessSystemException("No crawlingConfig: " + configId);
         }
@@ -520,15 +503,16 @@ public class ViewHelper {
     }
 
     protected StreamResponse writeContent(final String configId, final String url, final CrawlerClient client) {
-        final ResponseData responseData = client.execute(RequestDataBuilder.newRequestData().get().url(url).build());
         final StreamResponse response = new StreamResponse(StringUtil.EMPTY);
-        writeFileName(response, responseData);
-        writeContentType(response, responseData);
-        writeNoCache(response, responseData);
+        final ResponseData responseData = client.execute(RequestDataBuilder.newRequestData().get().url(url).build());
         if (responseData.getHttpStatusCode() == 404) {
             response.httpStatus(responseData.getHttpStatusCode());
+            IOUtils.closeQuietly(responseData);
             return response;
         }
+        writeFileName(response, responseData);
+        writeContentType(response, responseData);
+        writeNoCache(response, responseData);
         response.stream(out -> {
             try (final InputStream is = new BufferedInputStream(responseData.getResponseBody())) {
                 out.write(is);
@@ -537,7 +521,7 @@ public class ViewHelper {
                     throw new FessSystemException("Failed to write a content. configId: " + configId + ", url: " + url, e);
                 }
             } finally {
-                responseData.close();
+                IOUtils.closeQuietly(responseData);
             }
             if (logger.isDebugEnabled()) {
                 logger.debug("Finished to write " + url);

src/main/java/org/codelibs/fess/mylasta/action/FessLabels.java

@@ -1481,54 +1481,6 @@ public class FessLabels extends UserMessages {
     /** The key of the message: Use Default */
     public static final String LABELS_design_use_default_button = "{labels.design_use_default_button}";
 
-    /** The key of the message: Top Page */
-    public static final String LABELS_design_file_index = "{labels.design_file_index}";
-
-    /** The key of the message: Footer */
-    public static final String LABELS_design_file_footer = "{labels.design_file_footer}";
-
-    /** The key of the message: Results Page (Frame) */
-    public static final String LABELS_design_file_search = "{labels.design_file_search}";
-
-    /** The key of the message: Results Page (Content) */
-    public static final String LABELS_design_file_searchResults = "{labels.design_file_searchResults}";
-
-    /** The key of the message: Results Page (No Result) */
-    public static final String LABELS_design_file_searchNoResult = "{labels.design_file_searchNoResult}";
-
-    /** The key of the message: Search Options Page */
-    public static final String LABELS_design_file_searchOptions = "{labels.design_file_searchOptions}";
-
-    /** The key of the message: Help Page (Content) */
-    public static final String LABELS_design_file_help = "{labels.design_file_help}";
-
-    /** The key of the message: Header */
-    public static final String LABELS_design_file_header = "{labels.design_file_header}";
-
-    /** The key of the message: Search Error Page */
-    public static final String LABELS_design_file_error = "{labels.design_file_error}";
-
-    /** The key of the message: Cache Page */
-    public static final String LABELS_design_file_cache = "{labels.design_file_cache}";
-
-    /** The key of the message: Error Page (Not Found) */
-    public static final String LABELS_design_file_errorNotFound = "{labels.design_file_errorNotFound}";
-
-    /** The key of the message: Error Page (System Error) */
-    public static final String LABELS_design_file_errorSystem = "{labels.design_file_errorSystem}";
-
-    /** The key of the message: Error Page (Redirect) */
-    public static final String LABELS_design_file_errorRedirect = "{labels.design_file_errorRedirect}";
-
-    /** The key of the message: Error Page (BadRequest) */
-    public static final String LABELS_design_file_errorBadRequest = "{labels.design_file_errorBadRequest}";
-
-    /** The key of the message: Login Page */
-    public static final String LABELS_design_file_login = "{labels.design_file_login}";
-
-    /** The key of the message: Profile Page */
-    public static final String LABELS_design_file_profile = "{labels.design_file_profile}";
-
     /** The key of the message: Edit JSP File */
     public static final String LABELS_design_title_edit_content = "{labels.design_title_edit_content}";
 

src/main/java/org/codelibs/fess/thumbnail/ThumbnailGenerator.java

@@ -18,13 +18,13 @@ package org.codelibs.fess.thumbnail;
 import java.io.File;
 import java.util.Map;
 
-import org.codelibs.core.misc.Tuple4;
+import org.codelibs.core.misc.Tuple3;
 
 public interface ThumbnailGenerator {
 
     String getName();
 
-    boolean generate(String thumbnailId, String url, File outputFile);
+    boolean generate(String thumbnailId, File outputFile);
 
     boolean isTarget(Map<String, Object> docMap);
 
@@ -32,5 +32,5 @@ public interface ThumbnailGenerator {
 
     void destroy();
 
-    Tuple4<String, String, String, String> createTask(String path, Map<String, Object> docMap);
+    Tuple3<String, String, String> createTask(String path, Map<String, Object> docMap);
 }

src/main/java/org/codelibs/fess/thumbnail/ThumbnailManager.java

@@ -36,7 +36,7 @@ import javax.servlet.http.HttpSession;
 
 import org.codelibs.core.collection.LruHashMap;
 import org.codelibs.core.lang.StringUtil;
-import org.codelibs.core.misc.Tuple4;
+import org.codelibs.core.misc.Tuple3;
 import org.codelibs.fess.Constants;
 import org.codelibs.fess.es.client.FessEsClient;
 import org.codelibs.fess.es.config.exbhv.ThumbnailQueueBhv;
@@ -70,7 +70,7 @@ public class ThumbnailManager {
 
     private final List<ThumbnailGenerator> generatorList = new ArrayList<>();
 
-    private BlockingQueue<Tuple4<String, String, String, String>> thumbnailTaskQueue;
+    private BlockingQueue<Tuple3<String, String, String>> thumbnailTaskQueue;
 
     private volatile boolean generating;
 
@@ -116,37 +116,35 @@ public class ThumbnailManager {
 
         thumbnailTaskQueue = new LinkedBlockingQueue<>(thumbnailTaskQueueSize);
         generating = true;
-        thumbnailQueueThread =
-                new Thread((Runnable) () -> {
-                    final List<Tuple4<String, String, String, String>> taskList = new ArrayList<>();
-                    while (generating) {
-                        try {
-                            final Tuple4<String, String, String, String> task =
-                                    thumbnailTaskQueue.poll(thumbnailTaskQueueTimeout, TimeUnit.MILLISECONDS);
-                            if (task == null) {
-                                if (!taskList.isEmpty()) {
-                                    storeQueue(taskList);
-                                }
-                            } else if (!taskList.contains(task)) {
-                                taskList.add(task);
-                                if (taskList.size() > thumbnailTaskBulkSize) {
-                                    storeQueue(taskList);
-                                }
-                            }
-                        } catch (final InterruptedException e) {
-                            if (logger.isDebugEnabled()) {
-                                logger.debug("Interupted task.", e);
-                            }
-                        } catch (final Exception e) {
-                            if (generating) {
-                                logger.warn("Failed to generate thumbnail.", e);
-                            }
+        thumbnailQueueThread = new Thread((Runnable) () -> {
+            final List<Tuple3<String, String, String>> taskList = new ArrayList<>();
+            while (generating) {
+                try {
+                    final Tuple3<String, String, String> task = thumbnailTaskQueue.poll(thumbnailTaskQueueTimeout, TimeUnit.MILLISECONDS);
+                    if (task == null) {
+                        if (!taskList.isEmpty()) {
+                            storeQueue(taskList);
                         }
+                    } else if (!taskList.contains(task)) {
+                        taskList.add(task);
+                        if (taskList.size() > thumbnailTaskBulkSize) {
+                            storeQueue(taskList);
+                        }
+                    }
+                } catch (final InterruptedException e) {
+                    if (logger.isDebugEnabled()) {
+                        logger.debug("Interupted task.", e);
                     }
-                    if (!taskList.isEmpty()) {
-                        storeQueue(taskList);
+                } catch (final Exception e) {
+                    if (generating) {
+                        logger.warn("Failed to generate thumbnail.", e);
                     }
-                }, "ThumbnailGenerator");
+                }
+            }
+            if (!taskList.isEmpty()) {
+                storeQueue(taskList);
+            }
+        }, "ThumbnailGenerator");
         thumbnailQueueThread.start();
     }
 
@@ -172,7 +170,7 @@ public class ThumbnailManager {
         return "-D" + FESS_THUMBNAIL_PATH + "=" + baseDir.getAbsolutePath();
     }
 
-    protected void storeQueue(final List<Tuple4<String, String, String, String>> taskList) {
+    protected void storeQueue(final List<Tuple3<String, String, String>> taskList) {
         final FessConfig fessConfig = ComponentUtil.getFessConfig();
         final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
         final String[] targets = fessConfig.getThumbnailGeneratorTargetsAsArray();
@@ -182,8 +180,7 @@ public class ThumbnailManager {
                 final ThumbnailQueue entity = new ThumbnailQueue();
                 entity.setGenerator(task.getValue1());
                 entity.setThumbnailId(task.getValue2());
-                entity.setUrl(task.getValue3());
-                entity.setPath(task.getValue4());
+                entity.setPath(task.getValue3());
                 entity.setTarget(target);
                 entity.setCreatedBy(Constants.SYSTEM_USER);
                 entity.setCreatedTime(systemHelper.getCurrentTimeAsLong());
@@ -224,7 +221,7 @@ public class ThumbnailManager {
                     if (noImageFile.isFile() && !noImageFile.delete()) {
                         logger.warn("Failed to delete " + noImageFile.getAbsolutePath());
                     }
-                    if (!generator.generate(entity.getThumbnailId(), entity.getUrl(), outputFile)) {
+                    if (!generator.generate(entity.getThumbnailId(), outputFile)) {
                         new File(outputFile.getAbsolutePath() + NOIMAGE_FILE_SUFFIX).setLastModified(System.currentTimeMillis());
                     } else {
                         final long interval = fessConfig.getThumbnailGeneratorIntervalAsInteger().longValue();
@@ -248,11 +245,11 @@ public class ThumbnailManager {
         return idList.size();
     }
 
-    public void offer(final Map<String, Object> docMap) {
+    public boolean offer(final Map<String, Object> docMap) {
         for (final ThumbnailGenerator generator : generatorList) {
             if (generator.isTarget(docMap)) {
                 final String path = getImageFilename(docMap);
-                final Tuple4<String, String, String, String> task = generator.createTask(path, docMap);
+                final Tuple3<String, String, String> task = generator.createTask(path, docMap);
                 if (task != null) {
                     if (logger.isDebugEnabled()) {
                         logger.debug("Add thumbnail task: " + task);
@@ -260,13 +257,15 @@ public class ThumbnailManager {
                     if (!thumbnailTaskQueue.offer(task)) {
                         logger.warn("Failed to add thumbnail task: " + task);
                     }
+                    return true;
                 }
-                return;
+                return false;
             }
         }
         if (logger.isDebugEnabled()) {
             logger.debug("Thumbnail generator is not found: " + (docMap != null ? docMap.get("url") : docMap));
         }
+        return false;
     }
 
     protected String getImageFilename(final Map<String, Object> docMap) {

src/main/java/org/codelibs/fess/thumbnail/impl/BaseThumbnailGenerator.java

@@ -22,8 +22,20 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-
-import org.codelibs.core.misc.Tuple4;
+import java.util.function.BiPredicate;
+import java.util.function.Predicate;
+
+import org.codelibs.core.lang.StringUtil;
+import org.codelibs.core.misc.Tuple3;
+import org.codelibs.fess.crawler.builder.RequestDataBuilder;
+import org.codelibs.fess.crawler.client.CrawlerClient;
+import org.codelibs.fess.crawler.client.CrawlerClientFactory;
+import org.codelibs.fess.crawler.entity.ResponseData;
+import org.codelibs.fess.es.client.FessEsClient;
+import org.codelibs.fess.es.config.exentity.CrawlingConfig;
+import org.codelibs.fess.exception.ThumbnailGenerationException;
+import org.codelibs.fess.helper.CrawlingConfigHelper;
+import org.codelibs.fess.helper.IndexingHelper;
 import org.codelibs.fess.mylasta.direction.FessConfig;
 import org.codelibs.fess.thumbnail.ThumbnailGenerator;
 import org.codelibs.fess.util.ComponentUtil;
@@ -50,6 +62,9 @@ public abstract class BaseThumbnailGenerator implements ThumbnailGenerator {
 
     @Override
     public boolean isTarget(final Map<String, Object> docMap) {
+        if (!docMap.containsKey(ComponentUtil.getFessConfig().getIndexFieldThumbnail())) {
+            return false;
+        }
         for (final Map.Entry<String, String> entry : conditionMap.entrySet()) {
             final Object value = docMap.get(entry.getKey());
             if (value instanceof String && !((String) value).matches(entry.getValue())) {
@@ -92,11 +107,10 @@ public abstract class BaseThumbnailGenerator implements ThumbnailGenerator {
     }
 
     @Override
-    public Tuple4<String, String, String, String> createTask(final String path, final Map<String, Object> docMap) {
+    public Tuple3<String, String, String> createTask(final String path, final Map<String, Object> docMap) {
         final FessConfig fessConfig = ComponentUtil.getFessConfig();
         final String thumbnailId = DocumentUtil.getValue(docMap, fessConfig.getIndexFieldId(), String.class);
-        final String url = DocumentUtil.getValue(docMap, fessConfig.getIndexFieldUrl(), String.class);
-        final Tuple4<String, String, String, String> task = new Tuple4<>(getName(), thumbnailId, url, path);
+        final Tuple3<String, String, String> task = new Tuple3<>(getName(), thumbnailId, path);
         if (logger.isDebugEnabled()) {
             logger.debug("Create thumbnail task: " + task);
         }
@@ -114,17 +128,70 @@ public abstract class BaseThumbnailGenerator implements ThumbnailGenerator {
         return value;
     }
 
-    protected void updateThumbnailField(final String thumbnailId, final String url, final String value) {
+    protected void updateThumbnailField(final String thumbnailId, final String value) {
         // TODO bulk
         final FessConfig fessConfig = ComponentUtil.getFessConfig();
         try {
             ComponentUtil.getIndexingHelper().updateDocument(ComponentUtil.getFessEsClient(), thumbnailId,
-                    fessConfig.getIndexFieldThumbnail(), null);
+                    fessConfig.getIndexFieldThumbnail(), value);
         } catch (final Exception e) {
-            logger.warn("Failed to update thumbnail field at " + url, e);
+            logger.warn("Failed to update thumbnail field at " + thumbnailId, e);
         }
     }
 
+    protected boolean process(final String id, final BiPredicate<String, String> consumer) {
+        final FessConfig fessConfig = ComponentUtil.getFessConfig();
+        final FessEsClient fessEsClient = ComponentUtil.getFessEsClient();
+        final IndexingHelper indexingHelper = ComponentUtil.getIndexingHelper();
+        try {
+            final Map<String, Object> doc =
+                    indexingHelper.getDocument(fessEsClient, id,
+                            new String[] { fessConfig.getIndexFieldThumbnail(), fessConfig.getIndexFieldConfigId() });
+            if (doc == null) {
+                throw new ThumbnailGenerationException("Document is not found: " + id);
+            }
+            final String url = DocumentUtil.getValue(doc, fessConfig.getIndexFieldThumbnail(), String.class);
+            if (StringUtil.isBlank(url)) {
+                throw new ThumbnailGenerationException("Invalid thumbnail: " + url);
+            }
+            final String configId = DocumentUtil.getValue(doc, fessConfig.getIndexFieldConfigId(), String.class);
+            if (configId == null || configId.length() < 2) {
+                throw new ThumbnailGenerationException("Invalid configId: " + configId);
+            }
+            return consumer.test(configId, url);
+        } catch (final ThumbnailGenerationException e) {
+            if (e.getCause() == null) {
+                logger.debug(e.getMessage());
+            } else {
+                logger.warn("Failed to process " + id, e);
+            }
+        } catch (final Exception e) {
+            logger.warn("Failed to process " + id, e);
+        }
+        return false;
+    }
+
+    protected boolean process(final String id, final Predicate<ResponseData> consumer) {
+        return process(id, (configId, url) -> {
+            final CrawlingConfigHelper crawlingConfigHelper = ComponentUtil.getCrawlingConfigHelper();
+            CrawlingConfig config = crawlingConfigHelper.getCrawlingConfig(configId);
+            if (config == null) {
+                throw new ThumbnailGenerationException("No CrawlingConfig: " + configId);
+            }
+            final CrawlerClientFactory crawlerClientFactory = ComponentUtil.getComponent(CrawlerClientFactory.class);
+            config.initializeClientFactory(crawlerClientFactory);
+            final CrawlerClient client = crawlerClientFactory.getClient(url);
+            if (client == null) {
+                throw new ThumbnailGenerationException("No CrawlerClient: " + configId + ", url: " + url);
+            }
+            try (final ResponseData responseData = client.execute(RequestDataBuilder.newRequestData().get().url(url).build())) {
+                return consumer.test(responseData);
+            } catch (final Exception e) {
+                throw new ThumbnailGenerationException("Failed to process a thumbnail content: " + url, e);
+            }
+        });
+    }
+
     public void setGeneratorList(final List<String> generatorList) {
         this.generatorList = generatorList;
     }

src/main/java/org/codelibs/fess/thumbnail/impl/CommandGenerator.java

@@ -29,6 +29,8 @@ import javax.annotation.Resource;
 import javax.servlet.ServletContext;
 
 import org.apache.commons.io.IOUtils;
+import org.codelibs.core.io.CopyUtil;
+import org.codelibs.core.lang.StringUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -61,9 +63,9 @@ public class CommandGenerator extends BaseThumbnailGenerator {
     }
 
     @Override
-    public boolean generate(final String thumbnailId, final String url, final File outputFile) {
+    public boolean generate(final String thumbnailId, final File outputFile) {
         if (logger.isDebugEnabled()) {
-            logger.debug("Generate Thumbnail: " + url);
+            logger.debug("Generate Thumbnail: " + thumbnailId);
         }
 
         if (outputFile.exists()) {
@@ -82,12 +84,47 @@ public class CommandGenerator extends BaseThumbnailGenerator {
             return false;
         }
 
-        final String outputPath = outputFile.getAbsolutePath();
-        final List<String> cmdList = new ArrayList<>();
-        for (final String value : commandList) {
-            cmdList.add(expandPath(value.replace("${url}", url).replace("${outputFile}", outputPath)));
-        }
+        return process(thumbnailId, responseData -> {
+            File tempFile = null;
+            try {
+                tempFile = File.createTempFile("thumbnail_", "");
+                CopyUtil.copy(responseData.getResponseBody(), tempFile);
+
+                final String tempPath = tempFile.getAbsolutePath();
+                final String outputPath = outputFile.getAbsolutePath();
+                final List<String> cmdList = new ArrayList<>();
+                for (final String value : commandList) {
+                    cmdList.add(expandPath(value.replace("${url}", tempPath).replace("${outputFile}", outputPath)));
+                }
 
+                executeCommand(thumbnailId, cmdList);
+
+                if (outputFile.isFile() && outputFile.length() == 0) {
+                    logger.warn("Thumbnail File is empty. ID is " + thumbnailId);
+                    if (outputFile.delete()) {
+                        logger.info("Deleted: " + outputFile.getAbsolutePath());
+                    }
+                    updateThumbnailField(thumbnailId, StringUtil.EMPTY);
+                    return false;
+                }
+
+                if (logger.isDebugEnabled()) {
+                    logger.debug("Thumbnail File: " + outputPath);
+                }
+                return true;
+            } catch (final Exception e) {
+                logger.warn("Failed to process ", e);
+                if (tempFile != null && !tempFile.delete()) {
+                    logger.debug("Failed to delete " + tempFile.getAbsolutePath());
+                }
+                updateThumbnailField(thumbnailId, StringUtil.EMPTY);
+                return false;
+            }
+        });
+
+    }
+
+    protected void executeCommand(final String thumbnailId, final List<String> cmdList) {
         ProcessBuilder pb = null;
         Process p = null;
 
@@ -125,26 +162,12 @@ public class CommandGenerator extends BaseThumbnailGenerator {
                 p.destroy();
             }
         } catch (final Exception e) {
-            logger.warn("Failed to generate a thumbnail of " + url, e);
+            logger.warn("Failed to generate a thumbnail of " + thumbnailId, e);
         }
         if (task != null) {
             task.cancel();
             task = null;
         }
-
-        if (outputFile.isFile() && outputFile.length() == 0) {
-            logger.warn("Thumbnail File is empty. URL is " + url);
-            if (outputFile.delete()) {
-                logger.info("Deleted: " + outputFile.getAbsolutePath());
-            }
-            return false;
-        }
-
-        if (logger.isDebugEnabled()) {
-            logger.debug("Thumbnail File: " + outputPath);
-        }
-        updateThumbnailField(thumbnailId, url, url);
-        return true;
     }
 
     protected static class ProcessDestroyer extends TimerTask {

src/main/java/org/codelibs/fess/thumbnail/impl/EmptyGenerator.java

@@ -20,7 +20,7 @@ import java.io.File;
 public class EmptyGenerator extends BaseThumbnailGenerator {
 
     @Override
-    public boolean generate(final String thumbnailId, final String url, final File outputFile) {
+    public boolean generate(final String thumbnailId, final File outputFile) {
         return false;
     }
 

src/main/java/org/codelibs/fess/thumbnail/impl/HtmlTagBasedGenerator.java

@@ -29,8 +29,7 @@ import javax.imageio.ImageReader;
 import javax.imageio.stream.ImageInputStream;
 
 import org.codelibs.core.lang.StringUtil;
-import org.codelibs.core.misc.Tuple4;
-import org.codelibs.elasticsearch.runner.net.Curl;
+import org.codelibs.core.misc.Tuple3;
 import org.codelibs.fess.mylasta.direction.FessConfig;
 import org.codelibs.fess.util.ComponentUtil;
 import org.codelibs.fess.util.DocumentUtil;
@@ -46,17 +45,10 @@ public class HtmlTagBasedGenerator extends BaseThumbnailGenerator {
     }
 
     @Override
-    public Tuple4<String, String, String, String> createTask(final String path, final Map<String, Object> docMap) {
+    public Tuple3<String, String, String> createTask(final String path, final Map<String, Object> docMap) {
         final FessConfig fessConfig = ComponentUtil.getFessConfig();
         final String thumbnailId = DocumentUtil.getValue(docMap, fessConfig.getIndexFieldId(), String.class);
-        final String url = DocumentUtil.getValue(docMap, fessConfig.getIndexFieldThumbnail(), String.class);
-        if (StringUtil.isBlank(url)) {
-            if (logger.isDebugEnabled()) {
-                logger.debug("Thumbnail task is not found: " + path + ", " + thumbnailId + ", " + url);
-            }
-            return null;
-        }
-        final Tuple4<String, String, String, String> task = new Tuple4<>(getName(), thumbnailId, url, path);
+        final Tuple3<String, String, String> task = new Tuple3<>(getName(), thumbnailId, path);
         if (logger.isDebugEnabled()) {
             logger.debug("Create thumbnail task: " + task);
         }
@@ -64,9 +56,9 @@ public class HtmlTagBasedGenerator extends BaseThumbnailGenerator {
     }
 
     @Override
-    public boolean generate(final String thumbnailId, final String url, final File outputFile) {
+    public boolean generate(final String thumbnailId, final File outputFile) {
         if (logger.isDebugEnabled()) {
-            logger.debug("Generate Thumbnail: " + url);
+            logger.debug("Generate Thumbnail: " + thumbnailId);
         }
 
         if (outputFile.exists()) {
@@ -85,47 +77,64 @@ public class HtmlTagBasedGenerator extends BaseThumbnailGenerator {
             return false;
         }
 
-        final FessConfig fessConfig = ComponentUtil.getFessConfig();
-        Curl.get(url)
-                .proxy(fessConfig.getHttpProxy())
-                .execute(
-                        con -> {
-                            boolean created = false;
-                            try (ImageInputStream input = ImageIO.createImageInputStream(con.getInputStream())) {
-                                switch (saveImage(input, outputFile)) {
-                                case OK:
-                                    created = true;
-                                    break;
-                                case FAILED:
-                                    logger.warn("Failed to create thumbnail: " + thumbnailId + " -> " + url);
-                                    break;
-                                case INVALID_SIZE:
-                                    if (logger.isDebugEnabled()) {
-                                        logger.debug("Invalid thumbnail size: " + thumbnailId + " -> " + url);
-                                    }
-                                    break;
-                                default:
-                                    logger.error("Unknown thumbnail result: " + thumbnailId + " -> " + url);
-                                    break;
-                                }
-                            } catch (final Throwable t) {
-                                if (logger.isDebugEnabled()) {
-                                    logger.warn("Failed to create thumbnail: " + thumbnailId + " -> " + url, t);
-                                } else {
-                                    logger.warn("Failed to create thumbnail: " + thumbnailId + " -> " + url + " ("
-                                            + t.getClass().getCanonicalName() + ": " + t.getMessage() + ")");
-                                }
-                            } finally {
-                                if (!created) {
-                                    updateThumbnailField(thumbnailId, url, StringUtil.EMPTY);
-                                    if (outputFile.exists() && !outputFile.delete()) {
-                                        logger.warn("Failed to delete " + outputFile.getAbsolutePath());
-                                    }
-                                }
+        return process(
+                thumbnailId,
+                responseData -> {
+                    if (!isImageMimeType(responseData.getMimeType())) {
+                        if (logger.isDebugEnabled()) {
+                            logger.debug("Thumbnail is not image: " + thumbnailId + " : " + responseData.getUrl());
+                        }
+                        updateThumbnailField(thumbnailId, StringUtil.EMPTY);
+                        return false;
+                    }
+                    boolean created = false;
+                    try (ImageInputStream input = ImageIO.createImageInputStream(responseData.getResponseBody())) {
+                        switch (saveImage(input, outputFile)) {
+                        case OK:
+                            created = true;
+                            break;
+                        case FAILED:
+                            logger.warn("Failed to create thumbnail: " + thumbnailId);
+                            break;
+                        case INVALID_SIZE:
+                            if (logger.isDebugEnabled()) {
+                                logger.debug("Invalid thumbnail size: " + thumbnailId);
+                            }
+                            break;
+                        default:
+                            logger.error("Unknown thumbnail result: " + thumbnailId);
+                            break;
+                        }
+                    } catch (final Throwable t) {
+                        if (logger.isDebugEnabled()) {
+                            logger.warn("Failed to create thumbnail: " + thumbnailId, t);
+                        } else {
+                            logger.warn("Failed to create thumbnail: " + thumbnailId + " (" + t.getClass().getCanonicalName() + ": "
+                                    + t.getMessage() + ")");
+                        }
+                    } finally {
+                        if (!created) {
+                            updateThumbnailField(thumbnailId, StringUtil.EMPTY);
+                            if (outputFile.exists() && !outputFile.delete()) {
+                                logger.warn("Failed to delete " + outputFile.getAbsolutePath());
                             }
-                        });
+                        }
+                    }
+                    return outputFile.exists();
+                });
 
-        return outputFile.exists();
+    }
+
+    protected boolean isImageMimeType(final String mimeType) {
+        switch (mimeType) {
+        case "image/png":
+        case "image/gif":
+        case "image/jpeg":
+        case "image/bmp":
+            return true;
+        default:
+            return false;
+        }
     }
 
     protected Result saveImage(final ImageInputStream input, final File outputFile) throws IOException {

src/main/java/org/codelibs/fess/thumbnail/impl/WebDriverGenerator.java

@@ -50,6 +50,7 @@ import org.openqa.selenium.remote.UnreachableBrowserException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+@Deprecated
 public class WebDriverGenerator extends BaseThumbnailGenerator {
 
     private static final Logger logger = LoggerFactory.getLogger(WebDriverGenerator.class);
@@ -117,7 +118,13 @@ public class WebDriverGenerator extends BaseThumbnailGenerator {
     }
 
     @Override
-    public boolean generate(final String thumbnailId, final String url, final File outputFile) {
+    public boolean generate(final String thumbnailId, final File outputFile) {
+        return process(thumbnailId, (configId, url) -> {
+            return generate(thumbnailId, url, outputFile);
+        });
+    }
+
+    protected boolean generate(final String thumbnailId, final String url, final File outputFile) {
         if (logger.isDebugEnabled()) {
             logger.debug("Generate Thumbnail: " + url);
         }
@@ -154,7 +161,7 @@ public class WebDriverGenerator extends BaseThumbnailGenerator {
                     }
                     final File thumbnail = ((TakesScreenshot) webDriver).getScreenshotAs(OutputType.FILE);
                     convert(thumbnail, outputFile);
-                    updateThumbnailField(thumbnailId, url, url);
+                    updateThumbnailField(thumbnailId, url);
                     return true;
                 } catch (final UnreachableBrowserException | SessionNotFoundException e) {
                     if (logger.isDebugEnabled()) {

src/main/resources/fess_indices/.fess_config/thumbnail_queue.json

@@ -10,9 +10,6 @@
       "thumbnail_id": {
         "type": "keyword"
       },
-      "url": {
-        "type": "keyword"
-      },
       "path": {
         "type": "keyword"
       },

src/main/resources/fess_thumbnail.xml

@@ -20,42 +20,6 @@
 			<arg>"text/html"</arg>
 		</postConstruct>
 	</component>
-<!--
-	<component name="htmlThumbnailGenerator" class="org.codelibs.fess.thumbnail.impl.WebDriverGenerator">
-		<property name="name">"htmlThumbnailGenerator"</property>
-		<property name="generatorList">
-			["${path}/phantomjs"]
-		</property>
-		<property name="webDriverCapabilities">
-			<component class="org.openqa.selenium.remote.DesiredCapabilities">
-				<postConstruct name="setCapability">
-					<arg>"phantomjs.binary.path"</arg>
-					<arg>"${path}/phantomjs"</arg>
-				</postConstruct>
-			</component>
-		</property>
-		<postConstruct name="addCondition">
-			<arg>"mimetype"</arg>
-			<arg>"text/html"</arg>
-		</postConstruct>
-	</component>
-	<component name="htmlThumbnailGenerator" class="org.codelibs.fess.thumbnail.impl.CommandGenerator">
-		<property name="name">"htmlThumbnailGenerator"</property>
-		<property name="commandList">
-			["${path}/generate-thumbnail",
-			"html",
-			"${url}",
-			"${outputFile}"]
-		</property>
-		<property name="generatorList">
-			["${path}/generate-thumbnail"]
-		</property>
-		<postConstruct name="addCondition">
-			<arg>"mimetype"</arg>
-			<arg>"text/html"</arg>
-		</postConstruct>
-	</component>
--->
 	<component name="msofficeThumbnailGenerator" class="org.codelibs.fess.thumbnail.impl.CommandGenerator">
 		<property name="name">"msofficeThumbnailGenerator"</property>
 		<property name="commandList">

src/main/webapp/WEB-INF/crawler/resources/crawler_thumbnail.xml

@@ -6,6 +6,12 @@
 		<postConstruct name="add">
 			<arg>htmlThumbnailGenerator</arg>
 		</postConstruct>
+		<postConstruct name="add">
+			<arg>msofficeThumbnailGenerator</arg>
+		</postConstruct>
+		<postConstruct name="add">
+			<arg>pdfThumbnailGenerator</arg>
+		</postConstruct>
 	</component>
 	<component name="htmlThumbnailGenerator" class="org.codelibs.fess.thumbnail.impl.HtmlTagBasedGenerator">
 		<property name="name">"htmlThumbnailGenerator"</property>
@@ -14,5 +20,74 @@
 			<arg>"text/html"</arg>
 		</postConstruct>
 	</component>
-	<!-- TODO remove ServletContext from CommandGenerator -->
+	<component name="msofficeThumbnailGenerator" class="org.codelibs.fess.thumbnail.impl.EmptyGenerator">
+		<property name="name">"msofficeThumbnailGenerator"</property>
+		<property name="generatorList">
+			["${path}/generate-thumbnail"]
+		</property>
+		<postConstruct name="addCondition">
+			<arg>"mimetype"</arg>
+			<arg>"text/html"</arg>
+		</postConstruct>
+		<postConstruct name="addCondition">
+			<arg>"mimetype"</arg>
+			<arg>"application/vnd.openxmlformats-officedocument.wordprocessingml.document"</arg>
+		</postConstruct>
+		<postConstruct name="addCondition">
+			<arg>"mimetype"</arg>
+			<arg>"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"</arg>
+		</postConstruct>
+		<postConstruct name="addCondition">
+			<arg>"mimetype"</arg>
+			<arg>"application/vnd.openxmlformats-officedocument.presentationml.presentation"</arg>
+		</postConstruct>
+		<postConstruct name="addCondition">
+			<arg>"mimetype"</arg>
+			<arg>"application/msword"</arg>
+		</postConstruct>
+		<postConstruct name="addCondition">
+			<arg>"mimetype"</arg>
+			<arg>"application/vnd.ms-excel"</arg>
+		</postConstruct>
+		<postConstruct name="addCondition">
+			<arg>"mimetype"</arg>
+			<arg>"application/vnd.ms-excel.sheet.2"</arg>
+		</postConstruct>
+		<postConstruct name="addCondition">
+			<arg>"mimetype"</arg>
+			<arg>"application/vnd.ms-excel.sheet.3"</arg>
+		</postConstruct>
+		<postConstruct name="addCondition">
+			<arg>"mimetype"</arg>
+			<arg>"application/vnd.ms-excel.sheet.4"</arg>
+		</postConstruct>
+		<postConstruct name="addCondition">
+			<arg>"mimetype"</arg>
+			<arg>"application/vnd.ms-excel.workspace.3"</arg>
+		</postConstruct>
+		<postConstruct name="addCondition">
+			<arg>"mimetype"</arg>
+			<arg>"application/vnd.ms-excel.workspace.4"</arg>
+		</postConstruct>
+		<postConstruct name="addCondition">
+			<arg>"mimetype"</arg>
+			<arg>"application/vnd.ms-powerpoint"
+			</arg>
+		</postConstruct>
+		<postConstruct name="addCondition">
+			<arg>"mimetype"</arg>
+			<arg>"application/rtf"</arg>
+		</postConstruct>
+	</component>
+	<component name="pdfThumbnailGenerator" class="org.codelibs.fess.thumbnail.impl.EmptyGenerator">
+		<property name="name">"pdfThumbnailGenerator"</property>
+		<property name="generatorList">
+			["${path}/generate-thumbnail"]
+		</property>
+		<postConstruct name="addCondition">
+			<arg>"mimetype"</arg>
+			<arg>"application/pdf"
+			</arg>
+		</postConstruct>
+	</component>
 </components>