0ct0pu5/fess
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix #1926 fix jvm options

Browse source
This commit is contained in:
Shinsuke Sugaya 2018-12-03 06:39:12 +09:00
parent cb854feadb
commit 38eaea0b2d
4 changed files with 137 additions and 279 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

93
src/main/java/org/codelibs/fess/job/CrawlJob.java
View file

@ -20,7 +20,6 @@ import static org.codelibs.core.stream.StreamUtil.stream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FilenameFilter;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
@ -29,7 +28,6 @@ import java.util.Properties;
import javax.servlet.ServletContext;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.SystemUtils;
import org.codelibs.core.lang.StringUtil;
@ -46,15 +44,10 @@ import org.codelibs.fess.util.ResourceUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class CrawlJob {
private static final String REMOTE_DEBUG_OPTIONS = "-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=localhost:8000";
public class CrawlJob extends ExecJob {
private static final Logger logger = LoggerFactory.getLogger(CrawlJob.class);
protected JobExecutor jobExecutor;
protected String sessionId;
protected String namespace = Constants.CRAWLING_INFO_SYSTEM_NAME;
protected String[] webConfigIds;
@ -63,43 +56,13 @@ public class CrawlJob {
protected String[] dataConfigIds;
protected String logFilePath;
protected String logLevel;
protected int documentExpires = -2;
protected boolean useLocalElasticsearch = true;
protected String jvmOptions;
protected String lastaEnv;
public CrawlJob jobExecutor(final JobExecutor jobExecutor) {
this.jobExecutor = jobExecutor;
return this;
}
public CrawlJob sessionId(final String sessionId) {
this.sessionId = sessionId;
return this;
}
public CrawlJob namespace(final String namespace) {
this.namespace = namespace;
return this;
}
public CrawlJob logFilePath(final String logFilePath) {
this.logFilePath = logFilePath;
return this;
}
public CrawlJob logLevel(final String logLevel) {
this.logLevel = logLevel;
return this;
}
public CrawlJob documentExpires(final int documentExpires) {
this.documentExpires = documentExpires;
return this;
@ -120,30 +83,13 @@ public class CrawlJob {
return this;
}
public CrawlJob useLocaleElasticsearch(final boolean useLocaleElasticsearch) {
this.useLocalElasticsearch = useLocaleElasticsearch;
return this;
}
public CrawlJob remoteDebug() {
return jvmOptions(REMOTE_DEBUG_OPTIONS);
}
public CrawlJob jvmOptions(final String option) {
this.jvmOptions = option;
return this;
}
public CrawlJob lastaEnv(final String env) {
this.lastaEnv = env;
return this;
}
@Deprecated
public String execute(final JobExecutor jobExecutor) {
jobExecutor(jobExecutor);
return execute();
}
@Override
public String execute() {
final StringBuilder resultBuf = new StringBuilder(100);
final boolean runAll = webConfigIds == null && fileConfigIds == null && dataConfigIds == null;
@ -395,37 +341,4 @@ public class CrawlJob {
}
}
private void addSystemProperty(final List<String> crawlerCmdList, final String name, final String defaultValue, final String appendValue) {
final String value = System.getProperty(name);
if (value != null) {
final StringBuilder buf = new StringBuilder();
buf.append("-D").append(name).append("=").append(value);
if (appendValue != null) {
buf.append(appendValue);
}
crawlerCmdList.add(buf.toString());
} else if (defaultValue != null) {
crawlerCmdList.add("-D" + name + "=" + defaultValue);
}
}
protected void deleteTempDir(final File ownTmpDir) {
if (ownTmpDir == null) {
return;
}
if (!FileUtils.deleteQuietly(ownTmpDir)) {
logger.warn("Could not delete a temp dir: " + ownTmpDir.getAbsolutePath());
}
}
protected void appendJarFile(final String cpSeparator, final StringBuilder buf, final File libDir, final String basePath) {
final File[] jarFiles = libDir.listFiles((FilenameFilter) (dir, name) -> name.toLowerCase().endsWith(".jar"));
if (jarFiles != null) {
for (final File file : jarFiles) {
buf.append(cpSeparator);
buf.append(basePath);
buf.append(file.getName());
}
}
}
}

121
src/main/java/org/codelibs/fess/job/ExecJob.java Normal file
View file

@ -0,0 +1,121 @@
/*
* Copyright 2012-2018 CodeLibs Project and the Others.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
* either express or implied. See the License for the specific language
* governing permissions and limitations under the License.
*/
package org.codelibs.fess.job;
import java.io.File;
import java.io.FilenameFilter;
import java.util.List;
import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public abstract class ExecJob {
private static final Logger logger = LoggerFactory.getLogger(ExecJob.class);
protected static final String REMOTE_DEBUG_OPTIONS = "-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=localhost:8000";
protected JobExecutor jobExecutor;
protected String sessionId;
protected boolean useLocalElasticsearch = true;
protected String logFilePath;
protected String logLevel;
protected String jvmOptions;
protected String lastaEnv;
public abstract String execute();
public ExecJob jobExecutor(final JobExecutor jobExecutor) {
this.jobExecutor = jobExecutor;
return this;
}
public ExecJob sessionId(final String sessionId) {
this.sessionId = sessionId;
return this;
}
public ExecJob logFilePath(final String logFilePath) {
this.logFilePath = logFilePath;
return this;
}
public ExecJob logLevel(final String logLevel) {
this.logLevel = logLevel;
return this;
}
public ExecJob useLocalElasticsearch(final boolean useLocalElasticsearch) {
this.useLocalElasticsearch = useLocalElasticsearch;
return this;
}
public ExecJob remoteDebug() {
return jvmOptions(REMOTE_DEBUG_OPTIONS);
}
public ExecJob jvmOptions(final String option) {
this.jvmOptions = option;
return this;
}
public ExecJob lastaEnv(final String env) {
this.lastaEnv = env;
return this;
}
protected void addSystemProperty(final List<String> cmdList, final String name, final String defaultValue, final String appendValue) {
final String value = System.getProperty(name);
if (value != null) {
final StringBuilder buf = new StringBuilder();
buf.append("-D").append(name).append("=").append(value);
if (appendValue != null) {
buf.append(appendValue);
}
cmdList.add(buf.toString());
} else if (defaultValue != null) {
cmdList.add("-D" + name + "=" + defaultValue);
}
}
protected void deleteTempDir(final File ownTmpDir) {
if (ownTmpDir == null) {
return;
}
if (!FileUtils.deleteQuietly(ownTmpDir)) {
logger.warn("Could not delete a temp dir: " + ownTmpDir.getAbsolutePath());
}
}
protected void appendJarFile(final String cpSeparator, final StringBuilder buf, final File libDir, final String basePath) {
final File[] jarFiles = libDir.listFiles((FilenameFilter) (dir, name) -> name.toLowerCase().endsWith(".jar"));
if (jarFiles != null) {
for (final File file : jarFiles) {
buf.append(cpSeparator);
buf.append(basePath);
buf.append(file.getName());
}
}
}
}

105
src/main/java/org/codelibs/fess/job/GenerateThumbnailJob.java
View file

@ -20,14 +20,12 @@ import static org.codelibs.core.stream.StreamUtil.stream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FilenameFilter;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import javax.servlet.ServletContext;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.commons.lang3.SystemUtils;
import org.codelibs.core.lang.StringUtil;
@ -43,44 +41,13 @@ import org.codelibs.fess.util.ResourceUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class GenerateThumbnailJob {
private static final String REMOTE_DEBUG_OPTIONS = "-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=localhost:8000";
private static final Logger logger = LoggerFactory.getLogger(GenerateThumbnailJob.class);
protected JobExecutor jobExecutor;
protected String sessionId;
protected boolean useLocaleElasticsearch = true;
protected String logFilePath;
public class GenerateThumbnailJob extends ExecJob {
static final Logger logger = LoggerFactory.getLogger(GenerateThumbnailJob.class);
protected int numOfThreads = 1;
protected boolean cleanup = false;
protected String logLevel;
protected String jvmOptions;
protected String lastaEnv;
public GenerateThumbnailJob jobExecutor(final JobExecutor jobExecutor) {
this.jobExecutor = jobExecutor;
return this;
}
public GenerateThumbnailJob sessionId(final String sessionId) {
this.sessionId = sessionId;
return this;
}
public GenerateThumbnailJob logFilePath(final String logFilePath) {
this.logFilePath = logFilePath;
return this;
}
public GenerateThumbnailJob numOfThreads(final int numOfThreads) {
this.numOfThreads = numOfThreads;
return this;
@ -91,35 +58,13 @@ public class GenerateThumbnailJob {
return this;
}
public GenerateThumbnailJob logLevel(final String logLevel) {
this.logLevel = logLevel;
return this;
}
public GenerateThumbnailJob useLocaleElasticsearch(final boolean useLocaleElasticsearch) {
this.useLocaleElasticsearch = useLocaleElasticsearch;
return this;
}
public GenerateThumbnailJob remoteDebug() {
return jvmOptions(REMOTE_DEBUG_OPTIONS);
}
public GenerateThumbnailJob jvmOptions(final String option) {
this.jvmOptions = option;
return this;
}
public GenerateThumbnailJob lastaEnv(final String env) {
this.lastaEnv = env;
return this;
}
@Deprecated
public String execute(final JobExecutor jobExecutor) {
jobExecutor(jobExecutor);
return execute();
}
@Override
public String execute() {
final StringBuilder resultBuf = new StringBuilder();
@ -187,7 +132,7 @@ public class GenerateThumbnailJob {
// WEB-INF/lib
appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/lib")), "WEB-INF" + File.separator + "lib"
+ File.separator);
// WEB-INF/crawler/lib
// WEB-INF/env/thumbnail/lib
appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/env/thumbnail/lib")), "WEB-INF" + File.separator
+ "env" + File.separator + "thumbnail" + File.separator + "lib" + File.separator);
final File targetLibDir = new File(targetDir, "fess" + File.separator + "WEB-INF" + File.separator + "lib");
@ -196,7 +141,7 @@ public class GenerateThumbnailJob {
}
cmdList.add(buf.toString());
if (useLocaleElasticsearch) {
if (useLocalElasticsearch) {
final String httpAddress = System.getProperty(Constants.FESS_ES_HTTP_ADDRESS);
if (StringUtil.isNotBlank(httpAddress)) {
cmdList.add("-D" + Constants.FESS_ES_HTTP_ADDRESS + "=" + httpAddress);
@ -227,8 +172,8 @@ public class GenerateThumbnailJob {
if (logLevel != null) {
cmdList.add("-Dfess.log.level=" + logLevel);
}
stream(fessConfig.getJvmSuggestOptionsAsArray()).of(
stream -> stream.filter(StringUtil::isNotBlank).forEach(value -> cmdList.add(value)));
stream(fessConfig.getJvmThumbnailOptions())
.of(stream -> stream.filter(StringUtil::isNotBlank).forEach(value -> cmdList.add(value)));
File ownTmpDir = null;
final String tmpDir = System.getProperty("java.io.tmpdir");
@ -310,38 +255,4 @@ public class GenerateThumbnailJob {
}
}
}
private void addSystemProperty(final List<String> crawlerCmdList, final String name, final String defaultValue, final String appendValue) {
final String value = System.getProperty(name);
if (value != null) {
final StringBuilder buf = new StringBuilder();
buf.append("-D").append(name).append("=").append(value);
if (appendValue != null) {
buf.append(appendValue);
}
crawlerCmdList.add(buf.toString());
} else if (defaultValue != null) {
crawlerCmdList.add("-D" + name + "=" + defaultValue);
}
}
protected void deleteTempDir(final File ownTmpDir) {
if (ownTmpDir == null) {
return;
}
if (!FileUtils.deleteQuietly(ownTmpDir)) {
logger.warn("Could not delete a temp dir: " + ownTmpDir.getAbsolutePath());
}
}
protected void appendJarFile(final String cpSeparator, final StringBuilder buf, final File libDir, final String basePath) {
final File[] jarFiles = libDir.listFiles((FilenameFilter) (dir, name) -> name.toLowerCase().endsWith(".jar"));
if (jarFiles != null) {
for (final File file : jarFiles) {
buf.append(cpSeparator);
buf.append(basePath);
buf.append(file.getName());
}
}
}
}

97
src/main/java/org/codelibs/fess/job/SuggestJob.java
View file

@ -20,14 +20,12 @@ import static org.codelibs.core.stream.StreamUtil.stream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FilenameFilter;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import javax.servlet.ServletContext;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.commons.lang3.SystemUtils;
import org.codelibs.core.lang.StringUtil;
@ -43,69 +41,17 @@ import org.codelibs.fess.util.ResourceUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class SuggestJob {
private static final String REMOTE_DEBUG_OPTIONS = "-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=localhost:8000";
public class SuggestJob extends ExecJob {
private static final Logger logger = LoggerFactory.getLogger(SuggestJob.class);
protected JobExecutor jobExecutor;
protected String sessionId;
protected boolean useLocaleElasticsearch = true;
protected String logFilePath;
protected String logLevel;
protected String jvmOptions;
protected String lastaEnv;
public SuggestJob jobExecutor(final JobExecutor jobExecutor) {
this.jobExecutor = jobExecutor;
return this;
}
public SuggestJob sessionId(final String sessionId) {
this.sessionId = sessionId;
return this;
}
public SuggestJob logFilePath(final String logFilePath) {
this.logFilePath = logFilePath;
return this;
}
public SuggestJob logLevel(final String logLevel) {
this.logLevel = logLevel;
return this;
}
public SuggestJob useLocaleElasticsearch(final boolean useLocaleElasticsearch) {
this.useLocaleElasticsearch = useLocaleElasticsearch;
return this;
}
public SuggestJob remoteDebug() {
return jvmOptions(REMOTE_DEBUG_OPTIONS);
}
public SuggestJob jvmOptions(final String option) {
this.jvmOptions = option;
return this;
}
public SuggestJob lastaEnv(final String env) {
this.lastaEnv = env;
return this;
}
@Deprecated
public String execute(final JobExecutor jobExecutor) {
jobExecutor(jobExecutor);
return execute();
}
@Override
public String execute() {
final StringBuilder resultBuf = new StringBuilder();
@ -173,7 +119,7 @@ public class SuggestJob {
// WEB-INF/lib
appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/lib")), "WEB-INF" + File.separator + "lib"
+ File.separator);
// WEB-INF/crawler/lib
// WEB-INF/env/suggest/lib
appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/env/suggest/lib")), "WEB-INF" + File.separator
+ "env" + File.separator + "suggest" + File.separator + "lib" + File.separator);
final File targetLibDir = new File(targetDir, "fess" + File.separator + "WEB-INF" + File.separator + "lib");
@ -182,7 +128,7 @@ public class SuggestJob {
}
cmdList.add(buf.toString());
if (useLocaleElasticsearch) {
if (useLocalElasticsearch) {
final String httpAddress = System.getProperty(Constants.FESS_ES_HTTP_ADDRESS);
if (StringUtil.isNotBlank(httpAddress)) {
cmdList.add("-D" + Constants.FESS_ES_HTTP_ADDRESS + "=" + httpAddress);
@ -292,37 +238,4 @@ public class SuggestJob {
}
}
private void addSystemProperty(final List<String> crawlerCmdList, final String name, final String defaultValue, final String appendValue) {
final String value = System.getProperty(name);
if (value != null) {
final StringBuilder buf = new StringBuilder();
buf.append("-D").append(name).append("=").append(value);
if (appendValue != null) {
buf.append(appendValue);
}
crawlerCmdList.add(buf.toString());
} else if (defaultValue != null) {
crawlerCmdList.add("-D" + name + "=" + defaultValue);
}
}
protected void deleteTempDir(final File ownTmpDir) {
if (ownTmpDir == null) {
return;
}
if (!FileUtils.deleteQuietly(ownTmpDir)) {
logger.warn("Could not delete a temp dir: " + ownTmpDir.getAbsolutePath());
}
}
protected void appendJarFile(final String cpSeparator, final StringBuilder buf, final File libDir, final String basePath) {
final File[] jarFiles = libDir.listFiles((FilenameFilter) (dir, name) -> name.toLowerCase().endsWith(".jar"));
if (jarFiles != null) {
for (final File file : jarFiles) {
buf.append(cpSeparator);
buf.append(basePath);
buf.append(file.getName());
}
}
}
}