fix #1498 add thumbnail generator as a process

.gitignore

@@ -6,8 +6,9 @@
 /src/main/webapp/WEB-INF/classes/
 /src/main/webapp/WEB-INF/lib/
 /src/main/webapp/WEB-INF/site/
-/src/main/webapp/WEB-INF/crawler/lib/
-/src/main/webapp/WEB-INF/suggest/lib/
+/src/main/webapp/WEB-INF/env/crawler/lib/
+/src/main/webapp/WEB-INF/env/suggest/lib/
+/src/main/webapp/WEB-INF/env/thumbnail/lib/
 /src/main/webapp/WEB-INF/thumbnails/
 /src/main/webapp/jar/
 /dbflute_fess/log/*.log

pom.xml

@@ -524,6 +524,7 @@
 								<path>${project.build.directory}/fess/WEB-INF/classes/fess_config.properties</path>
 								<path>${project.build.directory}/fess/WEB-INF/classes/fess_env_crawler.properties</path>
 								<path>${project.build.directory}/fess/WEB-INF/classes/fess_env_suggest.properties</path>
+								<path>${project.build.directory}/fess/WEB-INF/classes/fess_env_thumbnail.properties</path>
 								<path>${project.build.directory}/fess/WEB-INF/classes/fess_env_web.properties</path>
 							</paths>
 							<dst>${packaging.fess.conf.dir}</dst>

src/main/java/org/codelibs/fess/exec/SuggestCreator.java

@@ -65,7 +65,7 @@ public class SuggestCreator {
         protected String propertiesPath;
 
         protected Options() {
-            // noghing
+            // nothing
         }
 
         @Override
@@ -89,7 +89,7 @@ public class SuggestCreator {
             parser.parseArgument(args);
         } catch (final CmdLineException e) {
             System.err.println(e.getMessage());
-            System.err.println("java " + Crawler.class.getCanonicalName() + " [options...] arguments...");
+            System.err.println("java " + SuggestCreator.class.getCanonicalName() + " [options...] arguments...");
             parser.printUsage(System.err);
             return;
         }
@@ -140,9 +140,9 @@ public class SuggestCreator {
             exitCode = process(options);
         } catch (final ContainerNotAvailableException e) {
             if (logger.isDebugEnabled()) {
-                logger.debug("Crawler is stopped.", e);
+                logger.debug("SuggestCreator is stopped.", e);
             } else if (logger.isInfoEnabled()) {
-                logger.info("Crawler is stopped.");
+                logger.info("SuggestCreator is stopped.");
             }
             exitCode = Constants.EXIT_FAIL;
         } catch (final Throwable t) {
@@ -155,7 +155,7 @@ public class SuggestCreator {
             destroyContainer();
         }
 
-        logger.info("Finished suggestCreator.");
+        logger.info("Finished SuggestCreator.");
         System.exit(exitCode);
     }
 

src/main/java/org/codelibs/fess/exec/ThumbnailGenerator.java

@@ -0,0 +1,191 @@
+/*
+ * Copyright 2012-2018 CodeLibs Project and the Others.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+ * either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+package org.codelibs.fess.exec;
+
+import java.io.File;
+import java.io.IOException;
+import java.lang.management.ManagementFactory;
+
+import javax.annotation.Resource;
+
+import org.codelibs.core.lang.StringUtil;
+import org.codelibs.core.misc.DynamicProperties;
+import org.codelibs.core.timer.TimeoutManager;
+import org.codelibs.core.timer.TimeoutTask;
+import org.codelibs.fess.Constants;
+import org.codelibs.fess.crawler.client.EsClient;
+import org.codelibs.fess.es.client.FessEsClient;
+import org.codelibs.fess.exception.ContainerNotAvailableException;
+import org.codelibs.fess.timer.SystemMonitorTarget;
+import org.codelibs.fess.util.ComponentUtil;
+import org.elasticsearch.monitor.jvm.JvmInfo;
+import org.elasticsearch.monitor.os.OsProbe;
+import org.elasticsearch.monitor.process.ProcessProbe;
+import org.kohsuke.args4j.CmdLineException;
+import org.kohsuke.args4j.CmdLineParser;
+import org.kohsuke.args4j.Option;
+import org.lastaflute.di.core.external.GenericExternalContext;
+import org.lastaflute.di.core.external.GenericExternalContextComponentDefRegister;
+import org.lastaflute.di.core.factory.SingletonLaContainerFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ThumbnailGenerator {
+
+    private static final Logger logger = LoggerFactory.getLogger(ThumbnailGenerator.class);
+
+    @Resource
+    public FessEsClient fessEsClient;
+
+    protected static class Options {
+        @Option(name = "-s", aliases = "--sessionId", metaVar = "sessionId", usage = "Session ID")
+        protected String sessionId;
+
+        @Option(name = "-n", aliases = "--name", metaVar = "name", usage = "Name")
+        protected String name;
+
+        @Option(name = "-p", aliases = "--properties", metaVar = "properties", usage = "Properties File")
+        protected String propertiesPath;
+
+        protected Options() {
+            // nothing
+        }
+
+        @Override
+        public String toString() {
+            return "Options [sessionId=" + sessionId + ", name=" + name + ", propertiesPath=" + propertiesPath + "]";
+        }
+    }
+
+    static void initializeProbes() {
+        // Force probes to be loaded
+        ProcessProbe.getInstance();
+        OsProbe.getInstance();
+        JvmInfo.jvmInfo();
+    }
+
+    public static void main(final String[] args) {
+        final Options options = new Options();
+
+        final CmdLineParser parser = new CmdLineParser(options);
+        try {
+            parser.parseArgument(args);
+        } catch (final CmdLineException e) {
+            System.err.println(e.getMessage());
+            System.err.println("java " + ThumbnailGenerator.class.getCanonicalName() + " [options...] arguments...");
+            parser.printUsage(System.err);
+            return;
+        }
+
+        if (logger.isDebugEnabled()) {
+            try {
+                ManagementFactory.getRuntimeMXBean().getInputArguments().stream().forEach(s -> logger.debug("Parameter: " + s));
+                System.getProperties().entrySet().stream().forEach(e -> logger.debug("Property: " + e.getKey() + "=" + e.getValue()));
+                System.getenv().entrySet().forEach(e -> logger.debug("Env: " + e.getKey() + "=" + e.getValue()));
+                logger.debug("Option: " + options);
+            } catch (final Exception e) {
+                // ignore
+            }
+        }
+
+        final String transportAddresses = System.getProperty(Constants.FESS_ES_TRANSPORT_ADDRESSES);
+        if (StringUtil.isNotBlank(transportAddresses)) {
+            System.setProperty(EsClient.TRANSPORT_ADDRESSES, transportAddresses);
+        }
+        final String clusterName = System.getProperty(Constants.FESS_ES_CLUSTER_NAME);
+        if (StringUtil.isNotBlank(clusterName)) {
+            System.setProperty(EsClient.CLUSTER_NAME, clusterName);
+        }
+
+        TimeoutTask systemMonitorTask = null;
+        int exitCode;
+        try {
+            SingletonLaContainerFactory.setConfigPath("app.xml");
+            SingletonLaContainerFactory.setExternalContext(new GenericExternalContext());
+            SingletonLaContainerFactory.setExternalContextComponentDefRegister(new GenericExternalContextComponentDefRegister());
+            SingletonLaContainerFactory.init();
+
+            final Thread shutdownCallback = new Thread("ShutdownHook") {
+                @Override
+                public void run() {
+                    if (logger.isDebugEnabled()) {
+                        logger.debug("Destroying LaContainer..");
+                    }
+                    destroyContainer();
+                }
+            };
+            Runtime.getRuntime().addShutdownHook(shutdownCallback);
+
+            systemMonitorTask =
+                    TimeoutManager.getInstance().addTimeoutTarget(new SystemMonitorTarget(),
+                            ComponentUtil.getFessConfig().getSuggestSystemMonitorIntervalAsInteger(), true);
+
+            exitCode = process(options);
+        } catch (final ContainerNotAvailableException e) {
+            if (logger.isDebugEnabled()) {
+                logger.debug("ThumbnailGenerator is stopped.", e);
+            } else if (logger.isInfoEnabled()) {
+                logger.info("ThumbnailGenerator is stopped.");
+            }
+            exitCode = Constants.EXIT_FAIL;
+        } catch (final Throwable t) {
+            logger.error("ThumbnailGenerator does not work correctly.", t);
+            exitCode = Constants.EXIT_FAIL;
+        } finally {
+            if (systemMonitorTask != null) {
+                systemMonitorTask.cancel();
+            }
+            destroyContainer();
+        }
+
+        logger.info("Finished ThumbnailGenerator.");
+        System.exit(exitCode);
+    }
+
+    private static void destroyContainer() {
+        synchronized (SingletonLaContainerFactory.class) {
+            SingletonLaContainerFactory.destroy();
+        }
+    }
+
+    private static int process(final Options options) {
+        final DynamicProperties systemProperties = ComponentUtil.getSystemProperties();
+
+        if (StringUtil.isNotBlank(options.propertiesPath)) {
+            systemProperties.reload(options.propertiesPath);
+        } else {
+            try {
+                final File propFile = File.createTempFile("thumbnail_", ".properties");
+                if (propFile.delete() && logger.isDebugEnabled()) {
+                    logger.debug("Deleted a temp file: " + propFile.getAbsolutePath());
+                }
+                systemProperties.reload(propFile.getAbsolutePath());
+                propFile.deleteOnExit();
+            } catch (final IOException e) {
+                logger.warn("Failed to create system properties file.", e);
+            }
+        }
+
+        int totalCount = 0;
+        int count = 1;
+        while (count != 0) {
+            count = ComponentUtil.getThumbnailManager().generate();
+            totalCount += count;
+        }
+        logger.info("Created " + totalCount + " thumbnail files.");
+        return 0;
+    }
+}

src/main/java/org/codelibs/fess/job/CrawlJob.java

@@ -227,9 +227,11 @@ public class CrawlJob {
             buf.append(confPath);
             buf.append(cpSeparator);
         }
-        // WEB-INF/crawler/resources
+        // WEB-INF/env/crawler/resources
         buf.append("WEB-INF");
         buf.append(File.separator);
+        buf.append("env");
+        buf.append(File.separator);
         buf.append("crawler");
         buf.append(File.separator);
         buf.append("resources");
@@ -247,10 +249,11 @@ public class CrawlJob {
             buf.append(targetClassesDir.getAbsolutePath());
         }
         // WEB-INF/lib
-        appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/lib")), "WEB-INF/lib" + File.separator);
-        // WEB-INF/crawler/lib
-        appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/crawler/lib")), "WEB-INF/crawler" + File.separator
-                + "lib" + File.separator);
+        appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/lib")), "WEB-INF" + File.separator + "lib"
+                + File.separator);
+        // WEB-INF/env/crawler/lib
+        appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/env/crawler/lib")), "WEB-INF" + File.separator
+                + "env" + File.separator + "crawler" + File.separator + "lib" + File.separator);
         final File targetLibDir = new File(targetDir, "fess" + File.separator + "WEB-INF" + File.separator + "lib");
         if (targetLibDir.isDirectory()) {
             appendJarFile(cpSeparator, buf, targetLibDir, targetLibDir.getAbsolutePath() + File.separator);

src/main/java/org/codelibs/fess/job/GenerateThumbnailJob.java

@@ -15,26 +15,317 @@
  */
 package org.codelibs.fess.job;
 
+import static org.codelibs.core.stream.StreamUtil.split;
+import static org.codelibs.core.stream.StreamUtil.stream;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.FilenameFilter;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.Properties;
+
+import javax.servlet.ServletContext;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.SystemUtils;
+import org.codelibs.core.lang.StringUtil;
+import org.codelibs.fess.Constants;
+import org.codelibs.fess.exception.FessSystemException;
+import org.codelibs.fess.exec.ThumbnailGenerator;
+import org.codelibs.fess.helper.ProcessHelper;
+import org.codelibs.fess.mylasta.direction.FessConfig;
 import org.codelibs.fess.util.ComponentUtil;
+import org.codelibs.fess.util.InputStreamThread;
+import org.codelibs.fess.util.JobProcess;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 public class GenerateThumbnailJob {
+    private static final String REMOTE_DEBUG_OPTIONS = "-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=localhost:8000";
+
     private static final Logger logger = LoggerFactory.getLogger(GenerateThumbnailJob.class);
 
+    protected JobExecutor jobExecutor;
+
+    protected String sessionId;
+
+    protected boolean useLocaleElasticsearch = true;
+
+    protected String logFilePath;
+
+    protected String logLevel;
+
+    protected String jvmOptions;
+
+    protected String lastaEnv;
+
+    public GenerateThumbnailJob jobExecutor(final JobExecutor jobExecutor) {
+        this.jobExecutor = jobExecutor;
+        return this;
+    }
+
+    public GenerateThumbnailJob sessionId(final String sessionId) {
+        this.sessionId = sessionId;
+        return this;
+    }
+
+    public GenerateThumbnailJob logFilePath(final String logFilePath) {
+        this.logFilePath = logFilePath;
+        return this;
+    }
+
+    public GenerateThumbnailJob logLevel(final String logLevel) {
+        this.logLevel = logLevel;
+        return this;
+    }
+
+    public GenerateThumbnailJob useLocaleElasticsearch(final boolean useLocaleElasticsearch) {
+        this.useLocaleElasticsearch = useLocaleElasticsearch;
+        return this;
+    }
+
+    public GenerateThumbnailJob remoteDebug() {
+        return jvmOptions(REMOTE_DEBUG_OPTIONS);
+    }
+
+    public GenerateThumbnailJob jvmOptions(final String option) {
+        this.jvmOptions = option;
+        return this;
+    }
+
+    public GenerateThumbnailJob lastaEnv(final String env) {
+        this.lastaEnv = env;
+        return this;
+    }
+
+    public String execute(final JobExecutor jobExecutor) {
+        jobExecutor(jobExecutor);
+        return execute();
+    }
+
     public String execute() {
-        int totalCount = 0;
-        int count = 1;
+        final StringBuilder resultBuf = new StringBuilder();
+
+        if (sessionId == null) { // create session id
+            final SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHmmss");
+            sessionId = sdf.format(new Date());
+        }
+        resultBuf.append("Session Id: ").append(sessionId).append("\n");
+        if (jobExecutor != null) {
+            jobExecutor.addShutdownListener(() -> ComponentUtil.getProcessHelper().destroyProcess(sessionId));
+        }
+
         try {
-            while (count != 0) {
-                count = ComponentUtil.getThumbnailManager().generate();
-                totalCount += count;
-            }
-            return "Created " + totalCount + " thumbnail files.";
+            executeThumbnailGenerator();
         } catch (final Exception e) {
             logger.error("Failed to purge user info.", e);
-            return e.getMessage();
+            resultBuf.append(e.getMessage()).append("\n");
+        }
+
+        return resultBuf.toString();
+
+    }
+
+    protected void executeThumbnailGenerator() {
+        final List<String> cmdList = new ArrayList<>();
+        final String cpSeparator = SystemUtils.IS_OS_WINDOWS ? ";" : ":";
+        final ServletContext servletContext = ComponentUtil.getComponent(ServletContext.class);
+        final ProcessHelper processHelper = ComponentUtil.getProcessHelper();
+        final FessConfig fessConfig = ComponentUtil.getFessConfig();
+
+        cmdList.add(fessConfig.getJavaCommandPath());
+
+        // -cp
+        cmdList.add("-cp");
+        final StringBuilder buf = new StringBuilder(100);
+        final String confPath = System.getProperty(Constants.FESS_CONF_PATH);
+        if (StringUtil.isNotBlank(confPath)) {
+            buf.append(confPath);
+            buf.append(cpSeparator);
+        }
+        // WEB-INF/env/thumbnail/resources
+        buf.append("WEB-INF");
+        buf.append(File.separator);
+        buf.append("env");
+        buf.append(File.separator);
+        buf.append("thumbnail");
+        buf.append(File.separator);
+        buf.append("resources");
+        buf.append(cpSeparator);
+        // WEB-INF/classes
+        buf.append("WEB-INF");
+        buf.append(File.separator);
+        buf.append("classes");
+        // target/classes
+        final String userDir = System.getProperty("user.dir");
+        final File targetDir = new File(userDir, "target");
+        final File targetClassesDir = new File(targetDir, "classes");
+        if (targetClassesDir.isDirectory()) {
+            buf.append(cpSeparator);
+            buf.append(targetClassesDir.getAbsolutePath());
+        }
+        // WEB-INF/lib
+        appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/lib")), "WEB-INF" + File.separator + "lib"
+                + File.separator);
+        // WEB-INF/crawler/lib
+        appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/env/thumbnail/lib")), "WEB-INF" + File.separator
+                + "env" + File.separator + "thumbnail" + File.separator + "lib" + File.separator);
+        final File targetLibDir = new File(targetDir, "fess" + File.separator + "WEB-INF" + File.separator + "lib");
+        if (targetLibDir.isDirectory()) {
+            appendJarFile(cpSeparator, buf, targetLibDir, targetLibDir.getAbsolutePath() + File.separator);
+        }
+        cmdList.add(buf.toString());
+
+        if (useLocaleElasticsearch) {
+            final String transportAddresses = System.getProperty(Constants.FESS_ES_TRANSPORT_ADDRESSES);
+            if (StringUtil.isNotBlank(transportAddresses)) {
+                cmdList.add("-D" + Constants.FESS_ES_TRANSPORT_ADDRESSES + "=" + transportAddresses);
+            }
+        }
+
+        final String clusterName = System.getProperty(Constants.FESS_ES_CLUSTER_NAME);
+        if (StringUtil.isNotBlank(clusterName)) {
+            cmdList.add("-D" + Constants.FESS_ES_CLUSTER_NAME + "=" + clusterName);
+        } else {
+            cmdList.add("-D" + Constants.FESS_ES_CLUSTER_NAME + "=" + fessConfig.getElasticsearchClusterName());
+        }
+
+        final String systemLastaEnv = System.getProperty("lasta.env");
+        if (StringUtil.isNotBlank(systemLastaEnv)) {
+            if (systemLastaEnv.equals("web")) {
+                cmdList.add("-Dlasta.env=thumbnail");
+            } else {
+                cmdList.add("-Dlasta.env=" + systemLastaEnv);
+            }
+        } else if (StringUtil.isNotBlank(lastaEnv)) {
+            cmdList.add("-Dlasta.env=" + lastaEnv);
+        }
+
+        cmdList.add("-Dfess.thumbnail.process=true");
+        if (logFilePath == null) {
+            final String value = System.getProperty("fess.log.path");
+            logFilePath = value != null ? value : new File(targetDir, "logs").getAbsolutePath();
+        }
+        cmdList.add("-Dfess.log.path=" + logFilePath);
+        addSystemProperty(cmdList, "fess.log.name", "fess-thumbnail", "-thumbnail");
+        if (logLevel == null) {
+            addSystemProperty(cmdList, "fess.log.level", null, null);
+        } else {
+            cmdList.add("-Dfess.log.level=" + logLevel);
+        }
+        stream(fessConfig.getJvmSuggestOptionsAsArray()).of(
+                stream -> stream.filter(StringUtil::isNotBlank).forEach(value -> cmdList.add(value)));
+
+        File ownTmpDir = null;
+        final String tmpDir = System.getProperty("java.io.tmpdir");
+        if (fessConfig.isUseOwnTmpDir() && StringUtil.isNotBlank(tmpDir)) {
+            ownTmpDir = new File(tmpDir, "fessTmpDir_" + sessionId);
+            if (ownTmpDir.mkdirs()) {
+                cmdList.add("-Djava.io.tmpdir=" + ownTmpDir.getAbsolutePath());
+            } else {
+                ownTmpDir = null;
+            }
+        }
+
+        if (StringUtil.isNotBlank(jvmOptions)) {
+            split(jvmOptions, " ").of(stream -> stream.filter(StringUtil::isNotBlank).forEach(s -> cmdList.add(s)));
+        }
+
+        cmdList.add(ThumbnailGenerator.class.getCanonicalName());
+
+        cmdList.add("--sessionId");
+        cmdList.add(sessionId);
+
+        File propFile = null;
+        try {
+            cmdList.add("-p");
+            propFile = File.createTempFile("thumbnail_", ".properties");
+            cmdList.add(propFile.getAbsolutePath());
+            try (FileOutputStream out = new FileOutputStream(propFile)) {
+                final Properties prop = new Properties();
+                prop.putAll(ComponentUtil.getSystemProperties());
+                prop.store(out, cmdList.toString());
+            }
+
+            final File baseDir = new File(servletContext.getRealPath("/WEB-INF")).getParentFile();
+
+            if (logger.isInfoEnabled()) {
+                logger.info("ThumbnailGenerator: \nDirectory=" + baseDir + "\nOptions=" + cmdList);
+            }
+
+            final JobProcess jobProcess = processHelper.startProcess(sessionId, cmdList, pb -> {
+                pb.directory(baseDir);
+                pb.redirectErrorStream(true);
+            });
+
+            final InputStreamThread it = jobProcess.getInputStreamThread();
+            it.start();
+
+            final Process currentProcess = jobProcess.getProcess();
+            currentProcess.waitFor();
+            it.join(5000);
+
+            final int exitValue = currentProcess.exitValue();
+
+            if (logger.isInfoEnabled()) {
+                logger.info("ThumbnailGenerator: Exit Code=" + exitValue + " - ThumbnailGenerator Process Output:\n" + it.getOutput());
+            }
+            if (exitValue != 0) {
+                throw new FessSystemException("Exit Code: " + exitValue + "\nOutput:\n" + it.getOutput());
+            }
+            ComponentUtil.getPopularWordHelper().clearCache();
+        } catch (final FessSystemException e) {
+            throw e;
+        } catch (final InterruptedException e) {
+            logger.warn("ThumbnailGenerator Process interrupted.");
+        } catch (final Exception e) {
+            throw new FessSystemException("ThumbnailGenerator Process terminated.", e);
+        } finally {
+            try {
+                processHelper.destroyProcess(sessionId);
+            } finally {
+                if (propFile != null && !propFile.delete()) {
+                    logger.warn("Failed to delete {}.", propFile.getAbsolutePath());
+                }
+                deleteTempDir(ownTmpDir);
+            }
         }
     }
 
+    private void addSystemProperty(final List<String> crawlerCmdList, final String name, final String defaultValue, final String appendValue) {
+        final String value = System.getProperty(name);
+        if (value != null) {
+            final StringBuilder buf = new StringBuilder();
+            buf.append("-D").append(name).append("=").append(value);
+            if (appendValue != null) {
+                buf.append(appendValue);
+            }
+            crawlerCmdList.add(buf.toString());
+        } else if (defaultValue != null) {
+            crawlerCmdList.add("-D" + name + "=" + defaultValue);
+        }
+    }
+
+    protected void deleteTempDir(final File ownTmpDir) {
+        if (ownTmpDir == null) {
+            return;
+        }
+        if (!FileUtils.deleteQuietly(ownTmpDir)) {
+            logger.warn("Could not delete a temp dir: " + ownTmpDir.getAbsolutePath());
+        }
+    }
+
+    protected void appendJarFile(final String cpSeparator, final StringBuilder buf, final File libDir, final String basePath) {
+        final File[] jarFiles = libDir.listFiles((FilenameFilter) (dir, name) -> name.toLowerCase().endsWith(".jar"));
+        if (jarFiles != null) {
+            for (final File file : jarFiles) {
+                buf.append(cpSeparator);
+                buf.append(basePath);
+                buf.append(file.getName());
+            }
+        }
+    }
 }

src/main/java/org/codelibs/fess/job/SuggestJob.java

@@ -146,9 +146,11 @@ public class SuggestJob {
             buf.append(confPath);
             buf.append(cpSeparator);
         }
-        // WEB-INF/suggest/resources
+        // WEB-INF/env/suggest/resources
         buf.append("WEB-INF");
         buf.append(File.separator);
+        buf.append("env");
+        buf.append(File.separator);
         buf.append("suggest");
         buf.append(File.separator);
         buf.append("resources");
@@ -166,10 +168,11 @@ public class SuggestJob {
             buf.append(targetClassesDir.getAbsolutePath());
         }
         // WEB-INF/lib
-        appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/lib")), "WEB-INF/lib" + File.separator);
+        appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/lib")), "WEB-INF" + File.separator + "lib"
+                + File.separator);
         // WEB-INF/crawler/lib
-        appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/suggest/lib")), "WEB-INF/suggest" + File.separator
-                + "lib" + File.separator);
+        appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/env/suggest/lib")), "WEB-INF" + File.separator
+                + "env" + File.separator + "suggest" + File.separator + "lib" + File.separator);
         final File targetLibDir = new File(targetDir, "fess" + File.separator + "WEB-INF" + File.separator + "lib");
         if (targetLibDir.isDirectory()) {
             appendJarFile(cpSeparator, buf, targetLibDir, targetLibDir.getAbsolutePath() + File.separator);
@@ -239,7 +242,7 @@ public class SuggestJob {
         File propFile = null;
         try {
             cmdList.add("-p");
-            propFile = File.createTempFile("crawler_", ".properties");
+            propFile = File.createTempFile("suggest_", ".properties");
             cmdList.add(propFile.getAbsolutePath());
             try (FileOutputStream out = new FileOutputStream(propFile)) {
                 final Properties prop = new Properties();

src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java

@@ -103,6 +103,35 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
     */
     String JVM_SUGGEST_OPTIONS = "jvm.suggest.options";
 
+    /** The key of the configuration. e.g. -Djava.awt.headless=true
+    -Dfile.encoding=UTF-8
+    -Djna.nosys=true
+    -Djdk.io.permissionsUseCanonicalPath=true
+    -server
+    -Xmx128m
+    -XX:MaxMetaspaceSize=128m
+    -XX:CompressedClassSpaceSize=32m
+    -XX:-UseGCOverheadLimit
+    -XX:+UseConcMarkSweepGC
+    -XX:CMSInitiatingOccupancyFraction=75
+    -XX:+UseCMSInitiatingOccupancyOnly
+    -XX:+UseTLAB
+    -XX:+DisableExplicitGC
+    -XX:+HeapDumpOnOutOfMemoryError
+    -XX:-OmitStackTraceInFastThrow
+    -Djcifs.smb.client.connTimeout=60000
+    -Djcifs.smb.client.soTimeout=35000
+    -Djcifs.smb.client.responseTimeout=30000
+    -Dgroovy.use.classvalue=true
+    -Dio.netty.noUnsafe=true
+    -Dio.netty.noKeySetOptimization=true
+    -Dio.netty.recycler.maxCapacityPerThread=0
+    -Dlog4j.shutdownHookEnabled=false
+    -Dlog4j2.disable.jmx=true
+    -Dlog4j.skipJansi=true
+    */
+    String JVM_THUMBNAIL_OPTIONS = "jvm.thumbnail.options";
+
     /** The key of the configuration. e.g. default_crawler */
     String JOB_SYSTEM_JOB_IDS = "job.system.job.ids";
 
@@ -1471,6 +1500,39 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
      */
     String getJvmSuggestOptions();
 
+    /**
+     * Get the value for the key 'jvm.thumbnail.options'. <br>
+     * The value is, e.g. -Djava.awt.headless=true
+    -Dfile.encoding=UTF-8
+    -Djna.nosys=true
+    -Djdk.io.permissionsUseCanonicalPath=true
+    -server
+    -Xmx128m
+    -XX:MaxMetaspaceSize=128m
+    -XX:CompressedClassSpaceSize=32m
+    -XX:-UseGCOverheadLimit
+    -XX:+UseConcMarkSweepGC
+    -XX:CMSInitiatingOccupancyFraction=75
+    -XX:+UseCMSInitiatingOccupancyOnly
+    -XX:+UseTLAB
+    -XX:+DisableExplicitGC
+    -XX:+HeapDumpOnOutOfMemoryError
+    -XX:-OmitStackTraceInFastThrow
+    -Djcifs.smb.client.connTimeout=60000
+    -Djcifs.smb.client.soTimeout=35000
+    -Djcifs.smb.client.responseTimeout=30000
+    -Dgroovy.use.classvalue=true
+    -Dio.netty.noUnsafe=true
+    -Dio.netty.noKeySetOptimization=true
+    -Dio.netty.recycler.maxCapacityPerThread=0
+    -Dlog4j.shutdownHookEnabled=false
+    -Dlog4j2.disable.jmx=true
+    -Dlog4j.skipJansi=true
+    <br>
+     * @return The value of found property. (NotNull: if not found, exception but basically no way)
+     */
+    String getJvmThumbnailOptions();
+
     /**
      * Get the value for the key 'job.system.job.ids'. <br>
      * The value is, e.g. default_crawler <br>
@@ -5605,6 +5667,10 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
             return get(FessConfig.JVM_SUGGEST_OPTIONS);
         }
 
+        public String getJvmThumbnailOptions() {
+            return get(FessConfig.JVM_THUMBNAIL_OPTIONS);
+        }
+
         public String getJobSystemJobIds() {
             return get(FessConfig.JOB_SYSTEM_JOB_IDS);
         }
@@ -7815,6 +7881,9 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
             defaultMap
                     .put(FessConfig.JVM_SUGGEST_OPTIONS,
                             "-Djava.awt.headless=true\n-Dfile.encoding=UTF-8\n-Djna.nosys=true\n-Djdk.io.permissionsUseCanonicalPath=true\n-server\n-Xmx256m\n-XX:MaxMetaspaceSize=128m\n-XX:CompressedClassSpaceSize=32m\n-XX:-UseGCOverheadLimit\n-XX:+UseConcMarkSweepGC\n-XX:CMSInitiatingOccupancyFraction=75\n-XX:+UseCMSInitiatingOccupancyOnly\n-XX:+UseTLAB\n-XX:+DisableExplicitGC\n-XX:+HeapDumpOnOutOfMemoryError\n-Dgroovy.use.classvalue=true\n-Dio.netty.noUnsafe=true\n-Dio.netty.noKeySetOptimization=true\n-Dio.netty.recycler.maxCapacityPerThread=0\n-Dlog4j.shutdownHookEnabled=false\n-Dlog4j2.disable.jmx=true\n-Dlog4j.skipJansi=true\n");
+            defaultMap
+                    .put(FessConfig.JVM_THUMBNAIL_OPTIONS,
+                            "-Djava.awt.headless=true\n-Dfile.encoding=UTF-8\n-Djna.nosys=true\n-Djdk.io.permissionsUseCanonicalPath=true\n-server\n-Xmx128m\n-XX:MaxMetaspaceSize=128m\n-XX:CompressedClassSpaceSize=32m\n-XX:-UseGCOverheadLimit\n-XX:+UseConcMarkSweepGC\n-XX:CMSInitiatingOccupancyFraction=75\n-XX:+UseCMSInitiatingOccupancyOnly\n-XX:+UseTLAB\n-XX:+DisableExplicitGC\n-XX:+HeapDumpOnOutOfMemoryError\n-XX:-OmitStackTraceInFastThrow\n-Djcifs.smb.client.connTimeout=60000\n-Djcifs.smb.client.soTimeout=35000\n-Djcifs.smb.client.responseTimeout=30000\n-Dgroovy.use.classvalue=true\n-Dio.netty.noUnsafe=true\n-Dio.netty.noKeySetOptimization=true\n-Dio.netty.recycler.maxCapacityPerThread=0\n-Dlog4j.shutdownHookEnabled=false\n-Dlog4j2.disable.jmx=true\n-Dlog4j.skipJansi=true\n");
             defaultMap.put(FessConfig.JOB_SYSTEM_JOB_IDS, "default_crawler");
             defaultMap.put(FessConfig.JOB_TEMPLATE_TITLE_WEB, "Web Crawler - {0}");
             defaultMap.put(FessConfig.JOB_TEMPLATE_TITLE_FILE, "File Crawler - {0}");

src/main/java/org/codelibs/fess/thumbnail/ThumbnailManager.java

@@ -112,7 +112,7 @@ public class ThumbnailManager {
         }
 
         thumbnailTaskQueue = new LinkedBlockingQueue<>(thumbnailTaskQueueSize);
-        generating = true;
+        generating = !Constants.TRUE.equalsIgnoreCase(System.getProperty("fess.thumbnail.process"));
         thumbnailQueueThread = new Thread((Runnable) () -> {
             final List<Tuple3<String, String, String>> taskList = new ArrayList<>();
             while (generating) {

src/main/java/org/codelibs/fess/thumbnail/impl/BaseThumbnailGenerator.java

@@ -194,6 +194,11 @@ public abstract class BaseThumbnailGenerator implements ThumbnailGenerator {
                     if (config == null) {
                         throw new ThumbnailGenerationException("No CrawlingConfig: " + configId);
                     }
+
+                    if (logger.isInfoEnabled()) {
+                        logger.info("Generating Thumbnail: " + url);
+                    }
+
                     final CrawlerClientFactory crawlerClientFactory = ComponentUtil.getComponent(CrawlerClientFactory.class);
                     config.initializeClientFactory(crawlerClientFactory);
                     final CrawlerClient client = crawlerClientFactory.getClient(url);

src/main/java/org/codelibs/fess/thumbnail/impl/CommandGenerator.java

@@ -26,8 +26,6 @@ import java.util.TimerTask;
 import java.util.concurrent.TimeUnit;
 
 import javax.annotation.PostConstruct;
-import javax.annotation.Resource;
-import javax.servlet.ServletContext;
 
 import org.codelibs.core.io.CloseableUtil;
 import org.codelibs.core.io.CopyUtil;
@@ -38,9 +36,6 @@ import org.slf4j.LoggerFactory;
 public class CommandGenerator extends BaseThumbnailGenerator {
     private static final Logger logger = LoggerFactory.getLogger(CommandGenerator.class);
 
-    @Resource
-    protected ServletContext application;
-
     protected List<String> commandList;
 
     protected long commandTimeout = 30 * 1000L;// 30sec
@@ -54,7 +49,7 @@ public class CommandGenerator extends BaseThumbnailGenerator {
     @PostConstruct
     public void init() {
         if (baseDir == null) {
-            baseDir = new File(application.getRealPath("/"));
+            baseDir = new File(System.getProperty("java.io.tmpdir"));
         }
         destoryTimer = new Timer("CommandGeneratorDestoryTimer-" + System.currentTimeMillis(), true);
     }

src/main/resources/fess_config.properties

@@ -75,6 +75,35 @@ jvm.suggest.options=\
 -Dlog4j.skipJansi=true\n\
 
 
+jvm.thumbnail.options=\
+-Djava.awt.headless=true\n\
+-Dfile.encoding=UTF-8\n\
+-Djna.nosys=true\n\
+-Djdk.io.permissionsUseCanonicalPath=true\n\
+-server\n\
+-Xmx128m\n\
+-XX:MaxMetaspaceSize=128m\n\
+-XX:CompressedClassSpaceSize=32m\n\
+-XX:-UseGCOverheadLimit\n\
+-XX:+UseConcMarkSweepGC\n\
+-XX:CMSInitiatingOccupancyFraction=75\n\
+-XX:+UseCMSInitiatingOccupancyOnly\n\
+-XX:+UseTLAB\n\
+-XX:+DisableExplicitGC\n\
+-XX:+HeapDumpOnOutOfMemoryError\n\
+-XX:-OmitStackTraceInFastThrow\n\
+-Djcifs.smb.client.connTimeout=60000\n\
+-Djcifs.smb.client.soTimeout=35000\n\
+-Djcifs.smb.client.responseTimeout=30000\n\
+-Dgroovy.use.classvalue=true\n\
+-Dio.netty.noUnsafe=true\n\
+-Dio.netty.noKeySetOptimization=true\n\
+-Dio.netty.recycler.maxCapacityPerThread=0\n\
+-Dlog4j.shutdownHookEnabled=false\n\
+-Dlog4j2.disable.jmx=true\n\
+-Dlog4j.skipJansi=true\n\
+
+
 #-Xdebug\n\
 #-Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=127.0.0.1:8000\n\
 

src/main/resources/fess_env_thumbnail.properties

@@ -0,0 +1,47 @@
+# _/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/
+# Fess environment configuration for Local Development
+# _/_/_/_/_/_/_/_/_/_/
+# ========================================================================================
+#                                                                                    Core
+#                                                                                   ======
+# The mode of Lasta Di's smart-deploy, should be cool in production (e.g. hot, cool, warm)
+lasta_di.smart.deploy.mode = warm
+
+# Is development environment here? (used for various purpose, you should set false if unknown)
+development.here = false
+
+# The title of environment (e.g. local or integration or production)
+environment.title = Production
+
+# Does it enable the Framework internal debug? (true only when emergency)
+framework.debug = false
+
+# one day: 86400000, three days: 259200000, five days: 432000000, one week: 604800000, one year: 31556926000
+# special script :: absolute mode: $(2014/07/10), relative mode: addDay(3).addMonth(4)
+# The milliseconds for (relative or absolute) adjust time (set only when test) @LongType *dynamic in development
+time.adjust.time.millis = 0
+
+# ----------------------------------------------------------
+#                                                      Mail
+#                                                     ------
+# Does it send mock mail? (true: no send actually, logging only)
+mail.send.mock = false
+
+# SMTP server settings for main: host:port
+mail.smtp.server.main.host.and.port = localhost:25
+
+# The prefix of subject to show test environment or not
+mail.subject.test.prefix = 
+
+# The common return path of all mail
+mail.return.path = root@localhost
+
+
+# ========================================================================================
+#                                                                                      DB
+#                                                                                     ====
+
+
+# ========================================================================================
+#                                                                                     Web
+#                                                                                    =====

src/main/webapp/WEB-INF/crawler/resources/crawler_thumbnail.xml

@@ -1,167 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!DOCTYPE components PUBLIC "-//DBFLUTE//DTD LastaDi 1.0//EN"
-	"http://dbflute.org/meta/lastadi10.dtd">
-<components>
-	<component name="thumbnailManager" class="org.codelibs.fess.thumbnail.ThumbnailManager">
-		<postConstruct name="add">
-			<arg>htmlThumbnailGenerator</arg>
-		</postConstruct>
-		<postConstruct name="add">
-			<arg>msofficeThumbnailGenerator</arg>
-		</postConstruct>
-		<postConstruct name="add">
-			<arg>pdfThumbnailGenerator</arg>
-		</postConstruct>
-		<postConstruct name="add">
-			<arg>psThumbnailGenerator</arg>
-		</postConstruct>
-		<postConstruct name="add">
-			<arg>imageThumbnailGenerator</arg>
-		</postConstruct>
-	</component>
-	<component name="htmlThumbnailGenerator" class="org.codelibs.fess.thumbnail.impl.HtmlTagBasedGenerator">
-		<property name="name">"htmlThumbnailGenerator"</property>
-		<postConstruct name="addCondition">
-			<arg>"mimetype"</arg>
-			<arg>"text/html"</arg>
-		</postConstruct>
-	</component>
-	<component name="msofficeThumbnailGenerator" class="org.codelibs.fess.thumbnail.impl.EmptyGenerator">
-		<property name="name">"msofficeThumbnailGenerator"</property>
-		<property name="generatorList">
-			["${path}/generate-thumbnail"]
-		</property>
-		<postConstruct name="addCondition">
-			<arg>"mimetype"</arg>
-			<arg>"application/vnd.openxmlformats-officedocument.wordprocessingml.document"</arg>
-		</postConstruct>
-		<postConstruct name="addCondition">
-			<arg>"mimetype"</arg>
-			<arg>"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"</arg>
-		</postConstruct>
-		<postConstruct name="addCondition">
-			<arg>"mimetype"</arg>
-			<arg>"application/vnd.openxmlformats-officedocument.presentationml.presentation"</arg>
-		</postConstruct>
-		<postConstruct name="addCondition">
-			<arg>"mimetype"</arg>
-			<arg>"application/msword"</arg>
-		</postConstruct>
-		<postConstruct name="addCondition">
-			<arg>"mimetype"</arg>
-			<arg>"application/vnd.ms-excel"</arg>
-		</postConstruct>
-		<postConstruct name="addCondition">
-			<arg>"mimetype"</arg>
-			<arg>"application/vnd.ms-excel.sheet.2"</arg>
-		</postConstruct>
-		<postConstruct name="addCondition">
-			<arg>"mimetype"</arg>
-			<arg>"application/vnd.ms-excel.sheet.3"</arg>
-		</postConstruct>
-		<postConstruct name="addCondition">
-			<arg>"mimetype"</arg>
-			<arg>"application/vnd.ms-excel.sheet.4"</arg>
-		</postConstruct>
-		<postConstruct name="addCondition">
-			<arg>"mimetype"</arg>
-			<arg>"application/vnd.ms-excel.workspace.3"</arg>
-		</postConstruct>
-		<postConstruct name="addCondition">
-			<arg>"mimetype"</arg>
-			<arg>"application/vnd.ms-excel.workspace.4"</arg>
-		</postConstruct>
-		<postConstruct name="addCondition">
-			<arg>"mimetype"</arg>
-			<arg>"application/vnd.ms-powerpoint"
-			</arg>
-		</postConstruct>
-		<postConstruct name="addCondition">
-			<arg>"mimetype"</arg>
-			<arg>"application/rtf"</arg>
-		</postConstruct>
-	</component>
-	<component name="pdfThumbnailGenerator" class="org.codelibs.fess.thumbnail.impl.EmptyGenerator">
-		<property name="name">"pdfThumbnailGenerator"</property>
-		<property name="generatorList">
-			["${path}/generate-thumbnail"]
-		</property>
-		<postConstruct name="addCondition">
-			<arg>"mimetype"</arg>
-			<arg>"application/pdf"
-			</arg>
-		</postConstruct>
-	</component>
-	<component name="psThumbnailGenerator" class="org.codelibs.fess.thumbnail.impl.EmptyGenerator">
-		<property name="name">"psThumbnailGenerator"</property>
-		<property name="generatorList">
-			["${path}/generate-thumbnail"]
-		</property>
-		<postConstruct name="addCondition">
-			<arg>"mimetype"</arg>
-			<arg>"application/postscript"
-			</arg>
-		</postConstruct>
-	</component>
-	<component name="imageThumbnailGenerator" class="org.codelibs.fess.thumbnail.impl.EmptyGenerator">
-		<property name="name">"imageThumbnailGenerator"</property>
-		<property name="generatorList">
-			["${path}/generate-thumbnail"]
-		</property>
-		<postConstruct name="addCondition">
-			<arg>"mimetype"</arg>
-			<arg>"image/jpeg"
-			</arg>
-		</postConstruct>
-		<postConstruct name="addCondition">
-			<arg>"mimetype"</arg>
-			<arg>"image/tiff"
-			</arg>
-		</postConstruct>
-		<postConstruct name="addCondition">
-			<arg>"mimetype"</arg>
-			<arg>"image/bmp"
-			</arg>
-		</postConstruct>
-		<postConstruct name="addCondition">
-			<arg>"mimetype"</arg>
-			<arg>"image/x-windows-bmp"
-			</arg>
-		</postConstruct>
-		<postConstruct name="addCondition">
-			<arg>"mimetype"</arg>
-			<arg>"image/x-ms-bmp"
-			</arg>
-		</postConstruct>
-		<postConstruct name="addCondition">
-			<arg>"mimetype"</arg>
-			<arg>"image/gif"
-			</arg>
-		</postConstruct>
-		<postConstruct name="addCondition">
-			<arg>"mimetype"</arg>
-			<arg>"image/png"
-			</arg>
-		</postConstruct>
-		<postConstruct name="addCondition">
-			<arg>"mimetype"</arg>
-			<arg>"image/vnd.adobe.photoshop"
-			</arg>
-		</postConstruct>
-		<postConstruct name="addCondition">
-			<arg>"mimetype"</arg>
-			<arg>"image/photoshop"
-			</arg>
-		</postConstruct>
-		<postConstruct name="addCondition">
-			<arg>"mimetype"</arg>
-			<arg>"application/x-photoshop"
-			</arg>
-		</postConstruct>
-		<postConstruct name="addCondition">
-			<arg>"mimetype"</arg>
-			<arg>"application/photoshop"
-			</arg>
-		</postConstruct>
-	</component>
-</components>

src/main/webapp/WEB-INF/crawler/resources/app.xml → src/main/webapp/WEB-INF/env/crawler/resources/app.xml

@@ -7,7 +7,7 @@
 	<include path="fess.xml" />
 
 	<include path="crawler_es.xml" />
-	<include path="crawler_thumbnail.xml" />
+	<include path="fess_thumbnail.xml" />
 
 	<component name="labelTypeHelper" class="org.codelibs.fess.helper.LabelTypeHelper">
 	</component>
@@ -18,7 +18,7 @@
 	<component name="duplicateHostHelper" class="org.codelibs.fess.helper.DuplicateHostHelper">
 	</component>
 	<component name="intervalControlHelper" class="org.codelibs.fess.helper.IntervalControlHelper">
-		<!-- 
+		<!--
 		<postConstruct name="addIntervalRule">
 			<arg>"5:00"</arg>
 			<arg>"10:00"</arg>
@@ -29,7 +29,7 @@
 	</component>
 	<component name="indexUpdater" class="org.codelibs.fess.indexer.IndexUpdater"
 		instance="prototype">
-		<!-- 
+		<!--
 		<property name="maxDocumentCacheSize">5</property>
 		<property name="unprocessedDocumentSize">100</property>
 		<property name="threadDump">false</property>
@@ -103,7 +103,7 @@
 		<postConstruct name="add">
 			<arg>"application/vnd.oasis.opendocument.presentation"</arg>
 			<arg>"odp"</arg>
-		</postConstruct>		
+		</postConstruct>
 		<postConstruct name="add">
 			<arg>"application/pdf"</arg>
 			<arg>"pdf"</arg>

src/main/webapp/WEB-INF/env/thumbnail/resources/app.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE components PUBLIC "-//DBFLUTE//DTD LastaDi 1.0//EN"
+	"http://dbflute.org/meta/lastadi10.dtd">
+<components>
+	<include path="convention.xml" />
+	<include path="lastaflute_core.xml"/>
+	<include path="fess.xml" />
+
+	<include path="crawler_es.xml" />
+	<include path="fess_thumbnail.xml"/>
+
+</components>

src/main/webapp/WEB-INF/env/thumbnail/resources/my_creator.xml

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE components PUBLIC "-//DBFLUTE//DTD LastaDi 1.0//EN"
+	"http://dbflute.org/meta/lastadi10.dtd">
+<components>
+</components>