0ct0pu5/fess
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix #1498 add thumbnail generator as a process

Browse source
This commit is contained in:
Shinsuke Sugaya 2018-02-12 22:38:57 +09:00
parent 8a68d25563
commit 268d4e77ce
20 changed files with 688 additions and 203 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
.gitignore vendored
View file

@ -6,8 +6,9 @@
/src/main/webapp/WEB-INF/classes/
/src/main/webapp/WEB-INF/lib/
/src/main/webapp/WEB-INF/site/
/src/main/webapp/WEB-INF/crawler/lib/
/src/main/webapp/WEB-INF/suggest/lib/
/src/main/webapp/WEB-INF/env/crawler/lib/
/src/main/webapp/WEB-INF/env/suggest/lib/
/src/main/webapp/WEB-INF/env/thumbnail/lib/
/src/main/webapp/WEB-INF/thumbnails/
/src/main/webapp/jar/
/dbflute_fess/log/*.log

1
pom.xml
View file

@ -524,6 +524,7 @@
<path>${project.build.directory}/fess/WEB-INF/classes/fess_config.properties</path>
<path>${project.build.directory}/fess/WEB-INF/classes/fess_env_crawler.properties</path>
<path>${project.build.directory}/fess/WEB-INF/classes/fess_env_suggest.properties</path>
<path>${project.build.directory}/fess/WEB-INF/classes/fess_env_thumbnail.properties</path>
<path>${project.build.directory}/fess/WEB-INF/classes/fess_env_web.properties</path>
</paths>
<dst>${packaging.fess.conf.dir}</dst>

10
src/main/java/org/codelibs/fess/exec/SuggestCreator.java
View file

@ -65,7 +65,7 @@ public class SuggestCreator {
protected String propertiesPath;
protected Options() {
// noghing
// nothing
}
@Override
@ -89,7 +89,7 @@ public class SuggestCreator {
parser.parseArgument(args);
} catch (final CmdLineException e) {
System.err.println(e.getMessage());
System.err.println("java " + Crawler.class.getCanonicalName() + " [options...] arguments...");
System.err.println("java " + SuggestCreator.class.getCanonicalName() + " [options...] arguments...");
parser.printUsage(System.err);
return;
}
@ -140,9 +140,9 @@ public class SuggestCreator {
exitCode = process(options);
} catch (final ContainerNotAvailableException e) {
if (logger.isDebugEnabled()) {
logger.debug("Crawler is stopped.", e);
logger.debug("SuggestCreator is stopped.", e);
} else if (logger.isInfoEnabled()) {
logger.info("Crawler is stopped.");
logger.info("SuggestCreator is stopped.");
}
exitCode = Constants.EXIT_FAIL;
} catch (final Throwable t) {
@ -155,7 +155,7 @@ public class SuggestCreator {
destroyContainer();
}
logger.info("Finished suggestCreator.");
logger.info("Finished SuggestCreator.");
System.exit(exitCode);
}

191
src/main/java/org/codelibs/fess/exec/ThumbnailGenerator.java Normal file
View file

@ -0,0 +1,191 @@
/*
* Copyright 2012-2018 CodeLibs Project and the Others.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
* either express or implied. See the License for the specific language
* governing permissions and limitations under the License.
*/
package org.codelibs.fess.exec;
import java.io.File;
import java.io.IOException;
import java.lang.management.ManagementFactory;
import javax.annotation.Resource;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.core.misc.DynamicProperties;
import org.codelibs.core.timer.TimeoutManager;
import org.codelibs.core.timer.TimeoutTask;
import org.codelibs.fess.Constants;
import org.codelibs.fess.crawler.client.EsClient;
import org.codelibs.fess.es.client.FessEsClient;
import org.codelibs.fess.exception.ContainerNotAvailableException;
import org.codelibs.fess.timer.SystemMonitorTarget;
import org.codelibs.fess.util.ComponentUtil;
import org.elasticsearch.monitor.jvm.JvmInfo;
import org.elasticsearch.monitor.os.OsProbe;
import org.elasticsearch.monitor.process.ProcessProbe;
import org.kohsuke.args4j.CmdLineException;
import org.kohsuke.args4j.CmdLineParser;
import org.kohsuke.args4j.Option;
import org.lastaflute.di.core.external.GenericExternalContext;
import org.lastaflute.di.core.external.GenericExternalContextComponentDefRegister;
import org.lastaflute.di.core.factory.SingletonLaContainerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class ThumbnailGenerator {
private static final Logger logger = LoggerFactory.getLogger(ThumbnailGenerator.class);
@Resource
public FessEsClient fessEsClient;
protected static class Options {
@Option(name = "-s", aliases = "--sessionId", metaVar = "sessionId", usage = "Session ID")
protected String sessionId;
@Option(name = "-n", aliases = "--name", metaVar = "name", usage = "Name")
protected String name;
@Option(name = "-p", aliases = "--properties", metaVar = "properties", usage = "Properties File")
protected String propertiesPath;
protected Options() {
// nothing
}
@Override
public String toString() {
return "Options [sessionId=" + sessionId + ", name=" + name + ", propertiesPath=" + propertiesPath + "]";
}
}
static void initializeProbes() {
// Force probes to be loaded
ProcessProbe.getInstance();
OsProbe.getInstance();
JvmInfo.jvmInfo();
}
public static void main(final String[] args) {
final Options options = new Options();
final CmdLineParser parser = new CmdLineParser(options);
try {
parser.parseArgument(args);
} catch (final CmdLineException e) {
System.err.println(e.getMessage());
System.err.println("java " + ThumbnailGenerator.class.getCanonicalName() + " [options...] arguments...");
parser.printUsage(System.err);
return;
}
if (logger.isDebugEnabled()) {
try {
ManagementFactory.getRuntimeMXBean().getInputArguments().stream().forEach(s -> logger.debug("Parameter: " + s));
System.getProperties().entrySet().stream().forEach(e -> logger.debug("Property: " + e.getKey() + "=" + e.getValue()));
System.getenv().entrySet().forEach(e -> logger.debug("Env: " + e.getKey() + "=" + e.getValue()));
logger.debug("Option: " + options);
} catch (final Exception e) {
// ignore
}
}
final String transportAddresses = System.getProperty(Constants.FESS_ES_TRANSPORT_ADDRESSES);
if (StringUtil.isNotBlank(transportAddresses)) {
System.setProperty(EsClient.TRANSPORT_ADDRESSES, transportAddresses);
}
final String clusterName = System.getProperty(Constants.FESS_ES_CLUSTER_NAME);
if (StringUtil.isNotBlank(clusterName)) {
System.setProperty(EsClient.CLUSTER_NAME, clusterName);
}
TimeoutTask systemMonitorTask = null;
int exitCode;
try {
SingletonLaContainerFactory.setConfigPath("app.xml");
SingletonLaContainerFactory.setExternalContext(new GenericExternalContext());
SingletonLaContainerFactory.setExternalContextComponentDefRegister(new GenericExternalContextComponentDefRegister());
SingletonLaContainerFactory.init();
final Thread shutdownCallback = new Thread("ShutdownHook") {
@Override
public void run() {
if (logger.isDebugEnabled()) {
logger.debug("Destroying LaContainer..");
}
destroyContainer();
}
};
Runtime.getRuntime().addShutdownHook(shutdownCallback);
systemMonitorTask =
TimeoutManager.getInstance().addTimeoutTarget(new SystemMonitorTarget(),
ComponentUtil.getFessConfig().getSuggestSystemMonitorIntervalAsInteger(), true);
exitCode = process(options);
} catch (final ContainerNotAvailableException e) {
if (logger.isDebugEnabled()) {
logger.debug("ThumbnailGenerator is stopped.", e);
} else if (logger.isInfoEnabled()) {
logger.info("ThumbnailGenerator is stopped.");
}
exitCode = Constants.EXIT_FAIL;
} catch (final Throwable t) {
logger.error("ThumbnailGenerator does not work correctly.", t);
exitCode = Constants.EXIT_FAIL;
} finally {
if (systemMonitorTask != null) {
systemMonitorTask.cancel();
}
destroyContainer();
}
logger.info("Finished ThumbnailGenerator.");
System.exit(exitCode);
}
private static void destroyContainer() {
synchronized (SingletonLaContainerFactory.class) {
SingletonLaContainerFactory.destroy();
}
}
private static int process(final Options options) {
final DynamicProperties systemProperties = ComponentUtil.getSystemProperties();
if (StringUtil.isNotBlank(options.propertiesPath)) {
systemProperties.reload(options.propertiesPath);
} else {
try {
final File propFile = File.createTempFile("thumbnail_", ".properties");
if (propFile.delete() && logger.isDebugEnabled()) {
logger.debug("Deleted a temp file: " + propFile.getAbsolutePath());
}
systemProperties.reload(propFile.getAbsolutePath());
propFile.deleteOnExit();
} catch (final IOException e) {
logger.warn("Failed to create system properties file.", e);
}
}
int totalCount = 0;
int count = 1;
while (count != 0) {
count = ComponentUtil.getThumbnailManager().generate();
totalCount += count;
}
logger.info("Created " + totalCount + " thumbnail files.");
return 0;
}
}

13
src/main/java/org/codelibs/fess/job/CrawlJob.java
View file

@ -227,9 +227,11 @@ public class CrawlJob {
buf.append(confPath);
buf.append(cpSeparator);
}
// WEB-INF/crawler/resources
// WEB-INF/env/crawler/resources
buf.append("WEB-INF");
buf.append(File.separator);
buf.append("env");
buf.append(File.separator);
buf.append("crawler");
buf.append(File.separator);
buf.append("resources");
@ -247,10 +249,11 @@ public class CrawlJob {
buf.append(targetClassesDir.getAbsolutePath());
}
// WEB-INF/lib
appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/lib")), "WEB-INF/lib" + File.separator);
// WEB-INF/crawler/lib
appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/crawler/lib")), "WEB-INF/crawler" + File.separator
+ "lib" + File.separator);
appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/lib")), "WEB-INF" + File.separator + "lib"
+ File.separator);
// WEB-INF/env/crawler/lib
appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/env/crawler/lib")), "WEB-INF" + File.separator
+ "env" + File.separator + "crawler" + File.separator + "lib" + File.separator);
final File targetLibDir = new File(targetDir, "fess" + File.separator + "WEB-INF" + File.separator + "lib");
if (targetLibDir.isDirectory()) {
appendJarFile(cpSeparator, buf, targetLibDir, targetLibDir.getAbsolutePath() + File.separator);

307
src/main/java/org/codelibs/fess/job/GenerateThumbnailJob.java
View file

@ -15,26 +15,317 @@
*/
package org.codelibs.fess.job;
import static org.codelibs.core.stream.StreamUtil.split;
import static org.codelibs.core.stream.StreamUtil.stream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FilenameFilter;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Properties;
import javax.servlet.ServletContext;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.SystemUtils;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.fess.Constants;
import org.codelibs.fess.exception.FessSystemException;
import org.codelibs.fess.exec.ThumbnailGenerator;
import org.codelibs.fess.helper.ProcessHelper;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.util.ComponentUtil;
import org.codelibs.fess.util.InputStreamThread;
import org.codelibs.fess.util.JobProcess;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class GenerateThumbnailJob {
private static final String REMOTE_DEBUG_OPTIONS = "-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=localhost:8000";
private static final Logger logger = LoggerFactory.getLogger(GenerateThumbnailJob.class);
protected JobExecutor jobExecutor;
protected String sessionId;
protected boolean useLocaleElasticsearch = true;
protected String logFilePath;
protected String logLevel;
protected String jvmOptions;
protected String lastaEnv;
public GenerateThumbnailJob jobExecutor(final JobExecutor jobExecutor) {
this.jobExecutor = jobExecutor;
return this;
}
public GenerateThumbnailJob sessionId(final String sessionId) {
this.sessionId = sessionId;
return this;
}
public GenerateThumbnailJob logFilePath(final String logFilePath) {
this.logFilePath = logFilePath;
return this;
}
public GenerateThumbnailJob logLevel(final String logLevel) {
this.logLevel = logLevel;
return this;
}
public GenerateThumbnailJob useLocaleElasticsearch(final boolean useLocaleElasticsearch) {
this.useLocaleElasticsearch = useLocaleElasticsearch;
return this;
}
public GenerateThumbnailJob remoteDebug() {
return jvmOptions(REMOTE_DEBUG_OPTIONS);
}
public GenerateThumbnailJob jvmOptions(final String option) {
this.jvmOptions = option;
return this;
}
public GenerateThumbnailJob lastaEnv(final String env) {
this.lastaEnv = env;
return this;
}
public String execute(final JobExecutor jobExecutor) {
jobExecutor(jobExecutor);
return execute();
}
public String execute() {
int totalCount = 0;
int count = 1;
final StringBuilder resultBuf = new StringBuilder();
if (sessionId == null) { // create session id
final SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHmmss");
sessionId = sdf.format(new Date());
}
resultBuf.append("Session Id: ").append(sessionId).append("\n");
if (jobExecutor != null) {
jobExecutor.addShutdownListener(() -> ComponentUtil.getProcessHelper().destroyProcess(sessionId));
}
try {
while (count != 0) {
count = ComponentUtil.getThumbnailManager().generate();
totalCount += count;
}
return "Created " + totalCount + " thumbnail files.";
executeThumbnailGenerator();
} catch (final Exception e) {
logger.error("Failed to purge user info.", e);
return e.getMessage();
resultBuf.append(e.getMessage()).append("\n");
}
return resultBuf.toString();
}
protected void executeThumbnailGenerator() {
final List<String> cmdList = new ArrayList<>();
final String cpSeparator = SystemUtils.IS_OS_WINDOWS ? ";" : ":";
final ServletContext servletContext = ComponentUtil.getComponent(ServletContext.class);
final ProcessHelper processHelper = ComponentUtil.getProcessHelper();
final FessConfig fessConfig = ComponentUtil.getFessConfig();
cmdList.add(fessConfig.getJavaCommandPath());
// -cp
cmdList.add("-cp");
final StringBuilder buf = new StringBuilder(100);
final String confPath = System.getProperty(Constants.FESS_CONF_PATH);
if (StringUtil.isNotBlank(confPath)) {
buf.append(confPath);
buf.append(cpSeparator);
}
// WEB-INF/env/thumbnail/resources
buf.append("WEB-INF");
buf.append(File.separator);
buf.append("env");
buf.append(File.separator);
buf.append("thumbnail");
buf.append(File.separator);
buf.append("resources");
buf.append(cpSeparator);
// WEB-INF/classes
buf.append("WEB-INF");
buf.append(File.separator);
buf.append("classes");
// target/classes
final String userDir = System.getProperty("user.dir");
final File targetDir = new File(userDir, "target");
final File targetClassesDir = new File(targetDir, "classes");
if (targetClassesDir.isDirectory()) {
buf.append(cpSeparator);
buf.append(targetClassesDir.getAbsolutePath());
}
// WEB-INF/lib
appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/lib")), "WEB-INF" + File.separator + "lib"
+ File.separator);
// WEB-INF/crawler/lib
appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/env/thumbnail/lib")), "WEB-INF" + File.separator
+ "env" + File.separator + "thumbnail" + File.separator + "lib" + File.separator);
final File targetLibDir = new File(targetDir, "fess" + File.separator + "WEB-INF" + File.separator + "lib");
if (targetLibDir.isDirectory()) {
appendJarFile(cpSeparator, buf, targetLibDir, targetLibDir.getAbsolutePath() + File.separator);
}
cmdList.add(buf.toString());
if (useLocaleElasticsearch) {
final String transportAddresses = System.getProperty(Constants.FESS_ES_TRANSPORT_ADDRESSES);
if (StringUtil.isNotBlank(transportAddresses)) {
cmdList.add("-D" + Constants.FESS_ES_TRANSPORT_ADDRESSES + "=" + transportAddresses);
}
}
final String clusterName = System.getProperty(Constants.FESS_ES_CLUSTER_NAME);
if (StringUtil.isNotBlank(clusterName)) {
cmdList.add("-D" + Constants.FESS_ES_CLUSTER_NAME + "=" + clusterName);
} else {
cmdList.add("-D" + Constants.FESS_ES_CLUSTER_NAME + "=" + fessConfig.getElasticsearchClusterName());
}
final String systemLastaEnv = System.getProperty("lasta.env");
if (StringUtil.isNotBlank(systemLastaEnv)) {
if (systemLastaEnv.equals("web")) {
cmdList.add("-Dlasta.env=thumbnail");
} else {
cmdList.add("-Dlasta.env=" + systemLastaEnv);
}
} else if (StringUtil.isNotBlank(lastaEnv)) {
cmdList.add("-Dlasta.env=" + lastaEnv);
}
cmdList.add("-Dfess.thumbnail.process=true");
if (logFilePath == null) {
final String value = System.getProperty("fess.log.path");
logFilePath = value != null ? value : new File(targetDir, "logs").getAbsolutePath();
}
cmdList.add("-Dfess.log.path=" + logFilePath);
addSystemProperty(cmdList, "fess.log.name", "fess-thumbnail", "-thumbnail");
if (logLevel == null) {
addSystemProperty(cmdList, "fess.log.level", null, null);
} else {
cmdList.add("-Dfess.log.level=" + logLevel);
}
stream(fessConfig.getJvmSuggestOptionsAsArray()).of(
stream -> stream.filter(StringUtil::isNotBlank).forEach(value -> cmdList.add(value)));
File ownTmpDir = null;
final String tmpDir = System.getProperty("java.io.tmpdir");
if (fessConfig.isUseOwnTmpDir() && StringUtil.isNotBlank(tmpDir)) {
ownTmpDir = new File(tmpDir, "fessTmpDir_" + sessionId);
if (ownTmpDir.mkdirs()) {
cmdList.add("-Djava.io.tmpdir=" + ownTmpDir.getAbsolutePath());
} else {
ownTmpDir = null;
}
}
if (StringUtil.isNotBlank(jvmOptions)) {
split(jvmOptions, " ").of(stream -> stream.filter(StringUtil::isNotBlank).forEach(s -> cmdList.add(s)));
}
cmdList.add(ThumbnailGenerator.class.getCanonicalName());
cmdList.add("--sessionId");
cmdList.add(sessionId);
File propFile = null;
try {
cmdList.add("-p");
propFile = File.createTempFile("thumbnail_", ".properties");
cmdList.add(propFile.getAbsolutePath());
try (FileOutputStream out = new FileOutputStream(propFile)) {
final Properties prop = new Properties();
prop.putAll(ComponentUtil.getSystemProperties());
prop.store(out, cmdList.toString());
}
final File baseDir = new File(servletContext.getRealPath("/WEB-INF")).getParentFile();
if (logger.isInfoEnabled()) {
logger.info("ThumbnailGenerator: \nDirectory=" + baseDir + "\nOptions=" + cmdList);
}
final JobProcess jobProcess = processHelper.startProcess(sessionId, cmdList, pb -> {
pb.directory(baseDir);
pb.redirectErrorStream(true);
});
final InputStreamThread it = jobProcess.getInputStreamThread();
it.start();
final Process currentProcess = jobProcess.getProcess();
currentProcess.waitFor();
it.join(5000);
final int exitValue = currentProcess.exitValue();
if (logger.isInfoEnabled()) {
logger.info("ThumbnailGenerator: Exit Code=" + exitValue + " - ThumbnailGenerator Process Output:\n" + it.getOutput());
}
if (exitValue != 0) {
throw new FessSystemException("Exit Code: " + exitValue + "\nOutput:\n" + it.getOutput());
}
ComponentUtil.getPopularWordHelper().clearCache();
} catch (final FessSystemException e) {
throw e;
} catch (final InterruptedException e) {
logger.warn("ThumbnailGenerator Process interrupted.");
} catch (final Exception e) {
throw new FessSystemException("ThumbnailGenerator Process terminated.", e);
} finally {
try {
processHelper.destroyProcess(sessionId);
} finally {
if (propFile != null && !propFile.delete()) {
logger.warn("Failed to delete {}.", propFile.getAbsolutePath());
}
deleteTempDir(ownTmpDir);
}
}
}
private void addSystemProperty(final List<String> crawlerCmdList, final String name, final String defaultValue, final String appendValue) {
final String value = System.getProperty(name);
if (value != null) {
final StringBuilder buf = new StringBuilder();
buf.append("-D").append(name).append("=").append(value);
if (appendValue != null) {
buf.append(appendValue);
}
crawlerCmdList.add(buf.toString());
} else if (defaultValue != null) {
crawlerCmdList.add("-D" + name + "=" + defaultValue);
}
}
protected void deleteTempDir(final File ownTmpDir) {
if (ownTmpDir == null) {
return;
}
if (!FileUtils.deleteQuietly(ownTmpDir)) {
logger.warn("Could not delete a temp dir: " + ownTmpDir.getAbsolutePath());
}
}
protected void appendJarFile(final String cpSeparator, final StringBuilder buf, final File libDir, final String basePath) {
final File[] jarFiles = libDir.listFiles((FilenameFilter) (dir, name) -> name.toLowerCase().endsWith(".jar"));
if (jarFiles != null) {
for (final File file : jarFiles) {
buf.append(cpSeparator);
buf.append(basePath);
buf.append(file.getName());
}
}
}
}

13
src/main/java/org/codelibs/fess/job/SuggestJob.java
View file

@ -146,9 +146,11 @@ public class SuggestJob {
buf.append(confPath);
buf.append(cpSeparator);
}
// WEB-INF/suggest/resources
// WEB-INF/env/suggest/resources
buf.append("WEB-INF");
buf.append(File.separator);
buf.append("env");
buf.append(File.separator);
buf.append("suggest");
buf.append(File.separator);
buf.append("resources");
@ -166,10 +168,11 @@ public class SuggestJob {
buf.append(targetClassesDir.getAbsolutePath());
}
// WEB-INF/lib
appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/lib")), "WEB-INF/lib" + File.separator);
appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/lib")), "WEB-INF" + File.separator + "lib"
+ File.separator);
// WEB-INF/crawler/lib
appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/suggest/lib")), "WEB-INF/suggest" + File.separator
+ "lib" + File.separator);
appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/env/suggest/lib")), "WEB-INF" + File.separator
+ "env" + File.separator + "suggest" + File.separator + "lib" + File.separator);
final File targetLibDir = new File(targetDir, "fess" + File.separator + "WEB-INF" + File.separator + "lib");
if (targetLibDir.isDirectory()) {
appendJarFile(cpSeparator, buf, targetLibDir, targetLibDir.getAbsolutePath() + File.separator);
@ -239,7 +242,7 @@ public class SuggestJob {
File propFile = null;
try {
cmdList.add("-p");
propFile = File.createTempFile("crawler_", ".properties");
propFile = File.createTempFile("suggest_", ".properties");
cmdList.add(propFile.getAbsolutePath());
try (FileOutputStream out = new FileOutputStream(propFile)) {
final Properties prop = new Properties();

69
src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java
View file

@ -103,6 +103,35 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
*/
String JVM_SUGGEST_OPTIONS = "jvm.suggest.options";
/** The key of the configuration. e.g. -Djava.awt.headless=true
-Dfile.encoding=UTF-8
-Djna.nosys=true
-Djdk.io.permissionsUseCanonicalPath=true
-server
-Xmx128m
-XX:MaxMetaspaceSize=128m
-XX:CompressedClassSpaceSize=32m
-XX:-UseGCOverheadLimit
-XX:+UseConcMarkSweepGC
-XX:CMSInitiatingOccupancyFraction=75
-XX:+UseCMSInitiatingOccupancyOnly
-XX:+UseTLAB
-XX:+DisableExplicitGC
-XX:+HeapDumpOnOutOfMemoryError
-XX:-OmitStackTraceInFastThrow
-Djcifs.smb.client.connTimeout=60000
-Djcifs.smb.client.soTimeout=35000
-Djcifs.smb.client.responseTimeout=30000
-Dgroovy.use.classvalue=true
-Dio.netty.noUnsafe=true
-Dio.netty.noKeySetOptimization=true
-Dio.netty.recycler.maxCapacityPerThread=0
-Dlog4j.shutdownHookEnabled=false
-Dlog4j2.disable.jmx=true
-Dlog4j.skipJansi=true
*/
String JVM_THUMBNAIL_OPTIONS = "jvm.thumbnail.options";
/** The key of the configuration. e.g. default_crawler */
String JOB_SYSTEM_JOB_IDS = "job.system.job.ids";
@ -1471,6 +1500,39 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
*/
String getJvmSuggestOptions();
/**
* Get the value for the key 'jvm.thumbnail.options'. <br>
* The value is, e.g. -Djava.awt.headless=true
-Dfile.encoding=UTF-8
-Djna.nosys=true
-Djdk.io.permissionsUseCanonicalPath=true
-server
-Xmx128m
-XX:MaxMetaspaceSize=128m
-XX:CompressedClassSpaceSize=32m
-XX:-UseGCOverheadLimit
-XX:+UseConcMarkSweepGC
-XX:CMSInitiatingOccupancyFraction=75
-XX:+UseCMSInitiatingOccupancyOnly
-XX:+UseTLAB
-XX:+DisableExplicitGC
-XX:+HeapDumpOnOutOfMemoryError
-XX:-OmitStackTraceInFastThrow
-Djcifs.smb.client.connTimeout=60000
-Djcifs.smb.client.soTimeout=35000
-Djcifs.smb.client.responseTimeout=30000
-Dgroovy.use.classvalue=true
-Dio.netty.noUnsafe=true
-Dio.netty.noKeySetOptimization=true
-Dio.netty.recycler.maxCapacityPerThread=0
-Dlog4j.shutdownHookEnabled=false
-Dlog4j2.disable.jmx=true
-Dlog4j.skipJansi=true
<br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getJvmThumbnailOptions();
/**
* Get the value for the key 'job.system.job.ids'. <br>
* The value is, e.g. default_crawler <br>
@ -5605,6 +5667,10 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
return get(FessConfig.JVM_SUGGEST_OPTIONS);
}
public String getJvmThumbnailOptions() {
return get(FessConfig.JVM_THUMBNAIL_OPTIONS);
}
public String getJobSystemJobIds() {
return get(FessConfig.JOB_SYSTEM_JOB_IDS);
}
@ -7815,6 +7881,9 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
defaultMap
.put(FessConfig.JVM_SUGGEST_OPTIONS,
"-Djava.awt.headless=true\n-Dfile.encoding=UTF-8\n-Djna.nosys=true\n-Djdk.io.permissionsUseCanonicalPath=true\n-server\n-Xmx256m\n-XX:MaxMetaspaceSize=128m\n-XX:CompressedClassSpaceSize=32m\n-XX:-UseGCOverheadLimit\n-XX:+UseConcMarkSweepGC\n-XX:CMSInitiatingOccupancyFraction=75\n-XX:+UseCMSInitiatingOccupancyOnly\n-XX:+UseTLAB\n-XX:+DisableExplicitGC\n-XX:+HeapDumpOnOutOfMemoryError\n-Dgroovy.use.classvalue=true\n-Dio.netty.noUnsafe=true\n-Dio.netty.noKeySetOptimization=true\n-Dio.netty.recycler.maxCapacityPerThread=0\n-Dlog4j.shutdownHookEnabled=false\n-Dlog4j2.disable.jmx=true\n-Dlog4j.skipJansi=true\n");
defaultMap
.put(FessConfig.JVM_THUMBNAIL_OPTIONS,
"-Djava.awt.headless=true\n-Dfile.encoding=UTF-8\n-Djna.nosys=true\n-Djdk.io.permissionsUseCanonicalPath=true\n-server\n-Xmx128m\n-XX:MaxMetaspaceSize=128m\n-XX:CompressedClassSpaceSize=32m\n-XX:-UseGCOverheadLimit\n-XX:+UseConcMarkSweepGC\n-XX:CMSInitiatingOccupancyFraction=75\n-XX:+UseCMSInitiatingOccupancyOnly\n-XX:+UseTLAB\n-XX:+DisableExplicitGC\n-XX:+HeapDumpOnOutOfMemoryError\n-XX:-OmitStackTraceInFastThrow\n-Djcifs.smb.client.connTimeout=60000\n-Djcifs.smb.client.soTimeout=35000\n-Djcifs.smb.client.responseTimeout=30000\n-Dgroovy.use.classvalue=true\n-Dio.netty.noUnsafe=true\n-Dio.netty.noKeySetOptimization=true\n-Dio.netty.recycler.maxCapacityPerThread=0\n-Dlog4j.shutdownHookEnabled=false\n-Dlog4j2.disable.jmx=true\n-Dlog4j.skipJansi=true\n");
defaultMap.put(FessConfig.JOB_SYSTEM_JOB_IDS, "default_crawler");
defaultMap.put(FessConfig.JOB_TEMPLATE_TITLE_WEB, "Web Crawler - {0}");
defaultMap.put(FessConfig.JOB_TEMPLATE_TITLE_FILE, "File Crawler - {0}");

2
src/main/java/org/codelibs/fess/thumbnail/ThumbnailManager.java
View file

@ -112,7 +112,7 @@ public class ThumbnailManager {
}
thumbnailTaskQueue = new LinkedBlockingQueue<>(thumbnailTaskQueueSize);
generating = true;
generating = !Constants.TRUE.equalsIgnoreCase(System.getProperty("fess.thumbnail.process"));
thumbnailQueueThread = new Thread((Runnable) () -> {
final List<Tuple3<String, String, String>> taskList = new ArrayList<>();
while (generating) {

5
src/main/java/org/codelibs/fess/thumbnail/impl/BaseThumbnailGenerator.java
View file

@ -194,6 +194,11 @@ public abstract class BaseThumbnailGenerator implements ThumbnailGenerator {
if (config == null) {
throw new ThumbnailGenerationException("No CrawlingConfig: " + configId);
}
if (logger.isInfoEnabled()) {
logger.info("Generating Thumbnail: " + url);
}
final CrawlerClientFactory crawlerClientFactory = ComponentUtil.getComponent(CrawlerClientFactory.class);
config.initializeClientFactory(crawlerClientFactory);
final CrawlerClient client = crawlerClientFactory.getClient(url);

7
src/main/java/org/codelibs/fess/thumbnail/impl/CommandGenerator.java
View file

@ -26,8 +26,6 @@ import java.util.TimerTask;
import java.util.concurrent.TimeUnit;
import javax.annotation.PostConstruct;
import javax.annotation.Resource;
import javax.servlet.ServletContext;
import org.codelibs.core.io.CloseableUtil;
import org.codelibs.core.io.CopyUtil;
@ -38,9 +36,6 @@ import org.slf4j.LoggerFactory;
public class CommandGenerator extends BaseThumbnailGenerator {
private static final Logger logger = LoggerFactory.getLogger(CommandGenerator.class);
@Resource
protected ServletContext application;
protected List<String> commandList;
protected long commandTimeout = 30 * 1000L;// 30sec
@ -54,7 +49,7 @@ public class CommandGenerator extends BaseThumbnailGenerator {
@PostConstruct
public void init() {
if (baseDir == null) {
baseDir = new File(application.getRealPath("/"));
baseDir = new File(System.getProperty("java.io.tmpdir"));
}
destoryTimer = new Timer("CommandGeneratorDestoryTimer-" + System.currentTimeMillis(), true);
}

29
src/main/resources/fess_config.properties
View file

@ -75,6 +75,35 @@ jvm.suggest.options=\
-Dlog4j.skipJansi=true\n\
jvm.thumbnail.options=\
-Djava.awt.headless=true\n\
-Dfile.encoding=UTF-8\n\
-Djna.nosys=true\n\
-Djdk.io.permissionsUseCanonicalPath=true\n\
-server\n\
-Xmx128m\n\
-XX:MaxMetaspaceSize=128m\n\
-XX:CompressedClassSpaceSize=32m\n\
-XX:-UseGCOverheadLimit\n\
-XX:+UseConcMarkSweepGC\n\
-XX:CMSInitiatingOccupancyFraction=75\n\
-XX:+UseCMSInitiatingOccupancyOnly\n\
-XX:+UseTLAB\n\
-XX:+DisableExplicitGC\n\
-XX:+HeapDumpOnOutOfMemoryError\n\
-XX:-OmitStackTraceInFastThrow\n\
-Djcifs.smb.client.connTimeout=60000\n\
-Djcifs.smb.client.soTimeout=35000\n\
-Djcifs.smb.client.responseTimeout=30000\n\
-Dgroovy.use.classvalue=true\n\
-Dio.netty.noUnsafe=true\n\
-Dio.netty.noKeySetOptimization=true\n\
-Dio.netty.recycler.maxCapacityPerThread=0\n\
-Dlog4j.shutdownHookEnabled=false\n\
-Dlog4j2.disable.jmx=true\n\
-Dlog4j.skipJansi=true\n\
#-Xdebug\n\
#-Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=127.0.0.1:8000\n\

47
src/main/resources/fess_env_thumbnail.properties Normal file
View file

@ -0,0 +1,47 @@
# _/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/_/
# Fess environment configuration for Local Development
# _/_/_/_/_/_/_/_/_/_/
# ========================================================================================
# Core
# ======
# The mode of Lasta Di's smart-deploy, should be cool in production (e.g. hot, cool, warm)
lasta_di.smart.deploy.mode = warm
# Is development environment here? (used for various purpose, you should set false if unknown)
development.here = false
# The title of environment (e.g. local or integration or production)
environment.title = Production
# Does it enable the Framework internal debug? (true only when emergency)
framework.debug = false
# one day: 86400000, three days: 259200000, five days: 432000000, one week: 604800000, one year: 31556926000
# special script :: absolute mode: $(2014/07/10), relative mode: addDay(3).addMonth(4)
# The milliseconds for (relative or absolute) adjust time (set only when test) @LongType *dynamic in development
time.adjust.time.millis = 0
# ----------------------------------------------------------
# Mail
# ------
# Does it send mock mail? (true: no send actually, logging only)
mail.send.mock = false
# SMTP server settings for main: host:port
mail.smtp.server.main.host.and.port = localhost:25
# The prefix of subject to show test environment or not
mail.subject.test.prefix =
# The common return path of all mail
mail.return.path = root@localhost
# ========================================================================================
# DB
# ====
# ========================================================================================
# Web
# =====

167
src/main/webapp/WEB-INF/crawler/resources/crawler_thumbnail.xml
View file

@ -1,167 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE components PUBLIC "-//DBFLUTE//DTD LastaDi 1.0//EN"
"http://dbflute.org/meta/lastadi10.dtd">
<components>
<component name="thumbnailManager" class="org.codelibs.fess.thumbnail.ThumbnailManager">
<postConstruct name="add">
<arg>htmlThumbnailGenerator</arg>
</postConstruct>
<postConstruct name="add">
<arg>msofficeThumbnailGenerator</arg>
</postConstruct>
<postConstruct name="add">
<arg>pdfThumbnailGenerator</arg>
</postConstruct>
<postConstruct name="add">
<arg>psThumbnailGenerator</arg>
</postConstruct>
<postConstruct name="add">
<arg>imageThumbnailGenerator</arg>
</postConstruct>
</component>
<component name="htmlThumbnailGenerator" class="org.codelibs.fess.thumbnail.impl.HtmlTagBasedGenerator">
<property name="name">"htmlThumbnailGenerator"</property>
<postConstruct name="addCondition">
<arg>"mimetype"</arg>
<arg>"text/html"</arg>
</postConstruct>
</component>
<component name="msofficeThumbnailGenerator" class="org.codelibs.fess.thumbnail.impl.EmptyGenerator">
<property name="name">"msofficeThumbnailGenerator"</property>
<property name="generatorList">
["${path}/generate-thumbnail"]
</property>
<postConstruct name="addCondition">
<arg>"mimetype"</arg>
<arg>"application/vnd.openxmlformats-officedocument.wordprocessingml.document"</arg>
</postConstruct>
<postConstruct name="addCondition">
<arg>"mimetype"</arg>
<arg>"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"</arg>
</postConstruct>
<postConstruct name="addCondition">
<arg>"mimetype"</arg>
<arg>"application/vnd.openxmlformats-officedocument.presentationml.presentation"</arg>
</postConstruct>
<postConstruct name="addCondition">
<arg>"mimetype"</arg>
<arg>"application/msword"</arg>
</postConstruct>
<postConstruct name="addCondition">
<arg>"mimetype"</arg>
<arg>"application/vnd.ms-excel"</arg>
</postConstruct>
<postConstruct name="addCondition">
<arg>"mimetype"</arg>
<arg>"application/vnd.ms-excel.sheet.2"</arg>
</postConstruct>
<postConstruct name="addCondition">
<arg>"mimetype"</arg>
<arg>"application/vnd.ms-excel.sheet.3"</arg>
</postConstruct>
<postConstruct name="addCondition">
<arg>"mimetype"</arg>
<arg>"application/vnd.ms-excel.sheet.4"</arg>
</postConstruct>
<postConstruct name="addCondition">
<arg>"mimetype"</arg>
<arg>"application/vnd.ms-excel.workspace.3"</arg>
</postConstruct>
<postConstruct name="addCondition">
<arg>"mimetype"</arg>
<arg>"application/vnd.ms-excel.workspace.4"</arg>
</postConstruct>
<postConstruct name="addCondition">
<arg>"mimetype"</arg>
<arg>"application/vnd.ms-powerpoint"
</arg>
</postConstruct>
<postConstruct name="addCondition">
<arg>"mimetype"</arg>
<arg>"application/rtf"</arg>
</postConstruct>
</component>
<component name="pdfThumbnailGenerator" class="org.codelibs.fess.thumbnail.impl.EmptyGenerator">
<property name="name">"pdfThumbnailGenerator"</property>
<property name="generatorList">
["${path}/generate-thumbnail"]
</property>
<postConstruct name="addCondition">
<arg>"mimetype"</arg>
<arg>"application/pdf"
</arg>
</postConstruct>
</component>
<component name="psThumbnailGenerator" class="org.codelibs.fess.thumbnail.impl.EmptyGenerator">
<property name="name">"psThumbnailGenerator"</property>
<property name="generatorList">
["${path}/generate-thumbnail"]
</property>
<postConstruct name="addCondition">
<arg>"mimetype"</arg>
<arg>"application/postscript"
</arg>
</postConstruct>
</component>
<component name="imageThumbnailGenerator" class="org.codelibs.fess.thumbnail.impl.EmptyGenerator">
<property name="name">"imageThumbnailGenerator"</property>
<property name="generatorList">
["${path}/generate-thumbnail"]
</property>
<postConstruct name="addCondition">
<arg>"mimetype"</arg>
<arg>"image/jpeg"
</arg>
</postConstruct>
<postConstruct name="addCondition">
<arg>"mimetype"</arg>
<arg>"image/tiff"
</arg>
</postConstruct>
<postConstruct name="addCondition">
<arg>"mimetype"</arg>
<arg>"image/bmp"
</arg>
</postConstruct>
<postConstruct name="addCondition">
<arg>"mimetype"</arg>
<arg>"image/x-windows-bmp"
</arg>
</postConstruct>
<postConstruct name="addCondition">
<arg>"mimetype"</arg>
<arg>"image/x-ms-bmp"
</arg>
</postConstruct>
<postConstruct name="addCondition">
<arg>"mimetype"</arg>
<arg>"image/gif"
</arg>
</postConstruct>
<postConstruct name="addCondition">
<arg>"mimetype"</arg>
<arg>"image/png"
</arg>
</postConstruct>
<postConstruct name="addCondition">
<arg>"mimetype"</arg>
<arg>"image/vnd.adobe.photoshop"
</arg>
</postConstruct>
<postConstruct name="addCondition">
<arg>"mimetype"</arg>
<arg>"image/photoshop"
</arg>
</postConstruct>
<postConstruct name="addCondition">
<arg>"mimetype"</arg>
<arg>"application/x-photoshop"
</arg>
</postConstruct>
<postConstruct name="addCondition">
<arg>"mimetype"</arg>
<arg>"application/photoshop"
</arg>
</postConstruct>
</component>
</components>

8
src/main/webapp/WEB-INF/crawler/resources/app.xml → src/main/webapp/WEB-INF/env/crawler/resources/app.xml vendored
View file

@ -7,7 +7,7 @@
<include path="fess.xml" />
<include path="crawler_es.xml" />
<include path="crawler_thumbnail.xml" />
<include path="fess_thumbnail.xml" />
<component name="labelTypeHelper" class="org.codelibs.fess.helper.LabelTypeHelper">
</component>
@ -18,7 +18,7 @@
<component name="duplicateHostHelper" class="org.codelibs.fess.helper.DuplicateHostHelper">
</component>
<component name="intervalControlHelper" class="org.codelibs.fess.helper.IntervalControlHelper">
<!--
<!--
<postConstruct name="addIntervalRule">
<arg>"5:00"</arg>
<arg>"10:00"</arg>
@ -29,7 +29,7 @@
</component>
<component name="indexUpdater" class="org.codelibs.fess.indexer.IndexUpdater"
instance="prototype">
<!--
<!--
<property name="maxDocumentCacheSize">5</property>
<property name="unprocessedDocumentSize">100</property>
<property name="threadDump">false</property>
@ -103,7 +103,7 @@
<postConstruct name="add">
<arg>"application/vnd.oasis.opendocument.presentation"</arg>
<arg>"odp"</arg>
</postConstruct>
</postConstruct>
<postConstruct name="add">
<arg>"application/pdf"</arg>
<arg>"pdf"</arg>

0
src/main/webapp/WEB-INF/crawler/resources/my_creator.xml → src/main/webapp/WEB-INF/env/crawler/resources/my_creator.xml vendored
View file

0
src/main/webapp/WEB-INF/suggest/resources/app.xml → src/main/webapp/WEB-INF/env/suggest/resources/app.xml vendored
View file

0
src/main/webapp/WEB-INF/suggest/resources/my_creator.xml → src/main/webapp/WEB-INF/env/suggest/resources/my_creator.xml vendored
View file

12
src/main/webapp/WEB-INF/env/thumbnail/resources/app.xml vendored Normal file
View file

@ -0,0 +1,12 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE components PUBLIC "-//DBFLUTE//DTD LastaDi 1.0//EN"
"http://dbflute.org/meta/lastadi10.dtd">
<components>
<include path="convention.xml" />
<include path="lastaflute_core.xml"/>
<include path="fess.xml" />
<include path="crawler_es.xml" />
<include path="fess_thumbnail.xml"/>
</components>

5
src/main/webapp/WEB-INF/env/thumbnail/resources/my_creator.xml vendored Normal file
View file

@ -0,0 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE components PUBLIC "-//DBFLUTE//DTD LastaDi 1.0//EN"
"http://dbflute.org/meta/lastadi10.dtd">
<components>
</components>