fix #773 generate thumbnailes on crawler

src/main/java/org/codelibs/fess/helper/IndexingHelper.java

@@ -21,6 +21,7 @@ import java.util.Map;
 
 import org.codelibs.fess.es.client.FessEsClient;
 import org.codelibs.fess.mylasta.direction.FessConfig;
+import org.codelibs.fess.thumbnail.ThumbnailManager;
 import org.codelibs.fess.util.ComponentUtil;
 import org.codelibs.fess.util.DocList;
 import org.codelibs.fess.util.MemoryUtil;
@@ -43,6 +44,7 @@ public class IndexingHelper {
         if (docList.isEmpty()) {
             return;
         }
+        final FessConfig fessConfig = ComponentUtil.getFessConfig();
         final long execTime = System.currentTimeMillis();
         if (logger.isDebugEnabled()) {
             logger.debug("Sending " + docList.size() + " documents to a server.");
@@ -50,9 +52,12 @@ public class IndexingHelper {
         try {
             synchronized (fessEsClient) {
                 deleteOldDocuments(fessEsClient, docList);
-                final FessConfig fessConfig = ComponentUtil.getFessConfig();
                 fessEsClient.addAll(fessConfig.getIndexDocumentUpdateIndex(), fessConfig.getIndexDocumentType(), docList);
             }
+            if (fessConfig.isThumbnailCrawlerEnabled()) {
+                final ThumbnailManager thumbnailManager = ComponentUtil.getThumbnailManager();
+                docList.stream().forEach(doc -> thumbnailManager.offer(doc));
+            }
             if (logger.isInfoEnabled()) {
                 if (docList.getContentSize() > 0) {
                     logger.info("Sent " + docList.size() + " docs (Doc:{process " + docList.getProcessingTime() + "ms, send "

src/main/java/org/codelibs/fess/job/CrawlJob.java

@@ -331,6 +331,8 @@ public class CrawlJob {
             }
         }
 
+        cmdList.add(ComponentUtil.getThumbnailManager().getThumbnailPathOption());
+
         if (StringUtil.isNotBlank(jvmOptions)) {
             split(jvmOptions, " ").of(stream -> stream.filter(StringUtil::isNotBlank).forEach(s -> cmdList.add(s)));
         }

src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java

@@ -658,6 +658,9 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
     /** The key of the configuration. e.g. all */
     String THUMBNAIL_GENERATOR_TARGETS = "thumbnail.generator.targets";
 
+    /** The key of the configuration. e.g. false */
+    String THUMBNAIL_CRAWLER_ENABLED = "thumbnail.crawler.enabled";
+
     /** The key of the configuration. e.g. Administrator */
     String MAIL_FROM_NAME = "mail.from.name";
 
@@ -3190,6 +3193,20 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
      */
     String getThumbnailGeneratorTargets();
 
+    /**
+     * Get the value for the key 'thumbnail.crawler.enabled'. <br>
+     * The value is, e.g. false <br>
+     * @return The value of found property. (NotNull: if not found, exception but basically no way)
+     */
+    String getThumbnailCrawlerEnabled();
+
+    /**
+     * Is the property for the key 'thumbnail.crawler.enabled' true? <br>
+     * The value is, e.g. false <br>
+     * @return The determination, true or false. (if not found, exception but basically no way)
+     */
+    boolean isThumbnailCrawlerEnabled();
+
     /**
      * Get the value for the key 'mail.from.name'. <br>
      * The value is, e.g. Administrator <br>
@@ -5437,6 +5454,14 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
             return get(FessConfig.THUMBNAIL_GENERATOR_TARGETS);
         }
 
+        public String getThumbnailCrawlerEnabled() {
+            return get(FessConfig.THUMBNAIL_CRAWLER_ENABLED);
+        }
+
+        public boolean isThumbnailCrawlerEnabled() {
+            return is(FessConfig.THUMBNAIL_CRAWLER_ENABLED);
+        }
+
         public String getMailFromName() {
             return get(FessConfig.MAIL_FROM_NAME);
         }

src/main/java/org/codelibs/fess/thumbnail/ThumbnailManager.java

@@ -32,8 +32,6 @@ import java.util.concurrent.TimeUnit;
 
 import javax.annotation.PostConstruct;
 import javax.annotation.PreDestroy;
-import javax.annotation.Resource;
-import javax.servlet.ServletContext;
 import javax.servlet.http.HttpSession;
 
 import org.codelibs.core.collection.LruHashMap;
@@ -48,6 +46,7 @@ import org.codelibs.fess.helper.SystemHelper;
 import org.codelibs.fess.mylasta.direction.FessConfig;
 import org.codelibs.fess.util.ComponentUtil;
 import org.codelibs.fess.util.DocumentUtil;
+import org.codelibs.fess.util.ResourceUtil;
 import org.lastaflute.web.util.LaRequestUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -55,15 +54,14 @@ import org.slf4j.LoggerFactory;
 import com.google.common.collect.Lists;
 
 public class ThumbnailManager {
-    private static final String DEFAULT_SCREENSHOT_DIR = "/WEB-INF/thumbnails";
+    private static final String FESS_THUMBNAIL_PATH = "fess.thumbnail.path";
+
+    private static final String FESS_VAR_PATH = "fess.var.path";
 
     private static final String NOIMAGE_FILE_SUFFIX = ".txt";
 
     private static final Logger logger = LoggerFactory.getLogger(ThumbnailManager.class);
 
-    @Resource
-    protected ServletContext application;
-
     protected File baseDir;
 
     private final List<ThumbnailGenerator> generatorList = new ArrayList<>();
@@ -72,7 +70,7 @@ public class ThumbnailManager {
 
     private volatile boolean generating;
 
-    private Thread thumbnailGeneratorThread;
+    private Thread thumbnailQueueThread;
 
     protected int thumbnailPathCacheSize = 10;
 
@@ -90,15 +88,15 @@ public class ThumbnailManager {
 
     @PostConstruct
     public void init() {
-        final String varPath = System.getProperty("fess.var.path");
-        if (varPath != null) {
-            baseDir = new File(varPath, "thumbnails");
+        final String thumbnailPath = System.getProperty(FESS_THUMBNAIL_PATH);
+        if (thumbnailPath != null) {
+            baseDir = new File(thumbnailPath);
         } else {
-            final String path = application.getRealPath(DEFAULT_SCREENSHOT_DIR);
-            if (StringUtil.isNotBlank(path)) {
-                baseDir = new File(path);
+            final String varPath = System.getProperty(FESS_VAR_PATH);
+            if (varPath != null) {
+                baseDir = new File(varPath, "thumbnails");
             } else {
-                baseDir = new File("." + DEFAULT_SCREENSHOT_DIR);
+                baseDir = ResourceUtil.getThumbnailPath().toFile();
             }
         }
         if (baseDir.mkdirs()) {
@@ -114,7 +112,7 @@ public class ThumbnailManager {
 
         thumbnailTaskQueue = new LinkedBlockingQueue<>(thumbnailTaskQueueSize);
         generating = true;
-        thumbnailGeneratorThread = new Thread((Runnable) () -> {
+        thumbnailQueueThread = new Thread((Runnable) () -> {
             final List<Tuple3<String, String, String>> taskList = new ArrayList<>();
             while (generating) {
                 try {
@@ -138,15 +136,15 @@ public class ThumbnailManager {
                 }
             }
         }, "ThumbnailGenerator");
-        thumbnailGeneratorThread.start();
+        thumbnailQueueThread.start();
     }
 
     @PreDestroy
     public void destroy() {
         generating = false;
-        thumbnailGeneratorThread.interrupt();
+        thumbnailQueueThread.interrupt();
         try {
-            thumbnailGeneratorThread.join(10000);
+            thumbnailQueueThread.join(10000);
         } catch (final InterruptedException e) {
             logger.warn("Thumbnail thread is timeouted.", e);
         }
@@ -159,6 +157,10 @@ public class ThumbnailManager {
         });
     }
 
+    public String getThumbnailPathOption() {
+        return "-D" + FESS_THUMBNAIL_PATH + "=" + baseDir.getAbsolutePath();
+    }
+
     protected void storeQueue(final List<Tuple3<String, String, String>> taskList) {
         final FessConfig fessConfig = ComponentUtil.getFessConfig();
         final SystemHelper systemHelper = ComponentUtil.getSystemHelper();

src/main/java/org/codelibs/fess/thumbnail/impl/BaseThumbnailGenerator.java

@@ -23,16 +23,10 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
-import javax.annotation.Resource;
-import javax.servlet.ServletContext;
-
 import org.codelibs.fess.thumbnail.ThumbnailGenerator;
 
 public abstract class BaseThumbnailGenerator implements ThumbnailGenerator {
 
-    @Resource
-    protected ServletContext application;
-
     protected final Map<String, String> conditionMap = new HashMap<>();
 
     protected int directoryNameLength = 5;

src/main/java/org/codelibs/fess/thumbnail/impl/CommandGenerator.java

@@ -25,6 +25,8 @@ import java.util.Timer;
 import java.util.TimerTask;
 
 import javax.annotation.PostConstruct;
+import javax.annotation.Resource;
+import javax.servlet.ServletContext;
 
 import org.apache.commons.io.IOUtils;
 import org.slf4j.Logger;
@@ -33,6 +35,9 @@ import org.slf4j.LoggerFactory;
 public class CommandGenerator extends BaseThumbnailGenerator {
     private static final Logger logger = LoggerFactory.getLogger(CommandGenerator.class);
 
+    @Resource
+    protected ServletContext application;
+
     public List<String> commandList;
 
     public long commandTimeout = 10 * 1000L;// 10sec

src/main/java/org/codelibs/fess/thumbnail/impl/EmptyGenerator.java

@@ -0,0 +1,36 @@
+/*
+ * Copyright 2012-2016 CodeLibs Project and the Others.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+ * either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+package org.codelibs.fess.thumbnail.impl;
+
+import java.io.File;
+
+public class EmptyGenerator extends BaseThumbnailGenerator {
+
+    @Override
+    public boolean generate(String url, File outputFile) {
+        return false;
+    }
+
+    @Override
+    public void destroy() {
+        // nothing
+    }
+
+    @Override
+    public boolean isAvailable() {
+        return true;
+    }
+}

src/main/java/org/codelibs/fess/util/ResourceUtil.java

@@ -72,6 +72,10 @@ public class ResourceUtil {
         return getPath("dict", names);
     }
 
+    public static Path getThumbnailPath(final String... names) {
+        return getPath("thumbnails", names);
+    }
+
     protected static Path getPath(final String base, final String... names) {
 
         try {

src/main/resources/fess_config.properties

@@ -351,6 +351,7 @@ paging.search.page.max.size=100
 
 thumbnail.html.phantomjs.enabled=true
 thumbnail.generator.targets=all
+thumbnail.crawler.enabled=false
 
 # ----------------------------------------------------------
 #                                                       Mail

src/main/webapp/WEB-INF/crawler/resources/app.xml

@@ -7,6 +7,7 @@
 	<include path="fess.xml" />
 
 	<include path="crawler_es.xml" />
+	<include path="crawler_thumbnail.xml" />
 
 	<component name="indexingHelper" class="org.codelibs.fess.helper.IndexingHelper">
 	</component>

src/main/webapp/WEB-INF/crawler/resources/crawler_thumbnail.xml

@@ -0,0 +1,62 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE components PUBLIC "-//DBFLUTE//DTD LastaDi 1.0//EN"
+	"http://dbflute.org/meta/lastadi10.dtd">
+<components>
+	<component name="thumbnailManager" class="org.codelibs.fess.thumbnail.ThumbnailManager">
+		<postConstruct name="add">
+			<arg>htmlThumbnailGenerator</arg>
+		</postConstruct>
+		<postConstruct name="add">
+			<arg>msofficeThumbnailGenerator</arg>
+		</postConstruct>
+	</component>
+	<component name="htmlThumbnailGenerator" class="org.codelibs.fess.thumbnail.impl.EmptyGenerator">
+		<property name="name">"htmlThumbnailGenerator"</property>
+		<postConstruct name="addCondition">
+			<arg>"mimetype"</arg>
+			<arg>"text/html"</arg>
+		</postConstruct>
+	</component>
+	<component name="msofficeThumbnailGenerator" class="org.codelibs.fess.thumbnail.impl.EmptyGenerator">
+		<property name="name">"msofficeThumbnailGenerator"</property>
+		<postConstruct name="addCondition">
+			<arg>"mimetype"</arg>
+			<arg>"text/html"</arg>
+		</postConstruct>
+		<postConstruct name="addCondition">
+			<arg>"mimetype"</arg>
+			<arg>"application/vnd.openxmlformats-officedocument.wordprocessingml.document"
+			</arg>
+		</postConstruct>
+		<postConstruct name="addCondition">
+			<arg>"mimetype"</arg>
+			<arg>"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
+			</arg>
+		</postConstruct>
+		<postConstruct name="addCondition">
+			<arg>"mimetype"</arg>
+			<arg>"application/vnd.openxmlformats-officedocument.presentationml.presentation"
+			</arg>
+		</postConstruct>
+		<postConstruct name="addCondition">
+			<arg>"mimetype"</arg>
+			<arg>"application/msword"
+			</arg>
+		</postConstruct>
+		<postConstruct name="addCondition">
+			<arg>"mimetype"</arg>
+			<arg>"application/vnd.ms-excel"
+			</arg>
+		</postConstruct>
+		<postConstruct name="addCondition">
+			<arg>"mimetype"</arg>
+			<arg>"application/vnd.ms-powerpoint"
+			</arg>
+		</postConstruct>
+		<postConstruct name="addCondition">
+			<arg>"mimetype"</arg>
+			<arg>"application/rtf"
+			</arg>
+		</postConstruct>
+	</component>
+</components>