diff --git a/src/main/java/org/codelibs/fess/job/PythonJob.java b/src/main/java/org/codelibs/fess/job/PythonJob.java new file mode 100644 index 000000000..eb2423429 --- /dev/null +++ b/src/main/java/org/codelibs/fess/job/PythonJob.java @@ -0,0 +1,166 @@ +/* + * Copyright 2012-2020 CodeLibs Project and the Others. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ +package org.codelibs.fess.job; + +import static org.codelibs.core.stream.StreamUtil.stream; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; + +import javax.servlet.ServletContext; + +import org.apache.commons.lang3.RandomStringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.codelibs.core.lang.StringUtil; +import org.codelibs.core.timer.TimeoutTask; +import org.codelibs.fess.exception.JobProcessingException; +import org.codelibs.fess.helper.ProcessHelper; +import org.codelibs.fess.mylasta.direction.FessConfig; +import org.codelibs.fess.util.ComponentUtil; +import org.codelibs.fess.util.InputStreamThread; +import org.codelibs.fess.util.JobProcess; + +public class PythonJob extends ExecJob { + static final Logger logger = LogManager.getLogger(PythonJob.class); + + protected String filename; + + protected List argList = new ArrayList<>(); + + public PythonJob filename(final String filename) { + this.filename = filename; + return this; + } + + public PythonJob arg(final String value) { + argList.add(value); + return this; + } + + public PythonJob args(final String... values) { + stream(values).of(stream -> stream.forEach(argList::add)); + return this; + } + + @Override + public String execute() { + final StringBuilder resultBuf = new StringBuilder(); + + if (sessionId == null) { // create session id + sessionId = RandomStringUtils.randomAlphabetic(15); + } + resultBuf.append("Session Id: ").append(sessionId).append("\n"); + if (jobExecutor != null) { + jobExecutor.addShutdownListener(() -> ComponentUtil.getProcessHelper().destroyProcess(sessionId)); + } + + final TimeoutTask timeoutTask = createTimeoutTask(); + try { + executePython(); + } catch (final Exception e) { + logger.warn("Failed to run python command.", e); + resultBuf.append(e.getMessage()).append("\n"); + } finally { + if (timeoutTask != null && !timeoutTask.isCanceled()) { + timeoutTask.cancel(); + } + } + + return resultBuf.toString(); + + } + + protected void executePython() { + final List cmdList = new ArrayList<>(); + final ServletContext servletContext = ComponentUtil.getComponent(ServletContext.class); + final ProcessHelper processHelper = ComponentUtil.getProcessHelper(); + final FessConfig fessConfig = ComponentUtil.getFessConfig(); + + if (StringUtil.isBlank(filename)) { + throw new JobProcessingException("Python script is not specified."); + } + + cmdList.add(fessConfig.getPythonCommandPath()); + + cmdList.add(getPyFilePath()); + + cmdList.addAll(argList); + + try { + + final File baseDir = new File(servletContext.getRealPath("/WEB-INF")).getParentFile(); + + if (logger.isInfoEnabled()) { + logger.info("Python: \nDirectory={}\nOptions={}", baseDir, cmdList); + } + + final JobProcess jobProcess = processHelper.startProcess(sessionId, cmdList, pb -> { + pb.directory(baseDir); + pb.redirectErrorStream(true); + }); + + final InputStreamThread it = jobProcess.getInputStreamThread(); + it.start(); + + final Process currentProcess = jobProcess.getProcess(); + currentProcess.waitFor(); + it.join(5000); + + final int exitValue = currentProcess.exitValue(); + + if (logger.isInfoEnabled()) { + logger.info("Python: Exit Code={} - Process Output:\n{}", exitValue, it.getOutput()); + } + if (exitValue != 0) { + final StringBuilder out = new StringBuilder(); + if (processTimeout) { + out.append("Process is terminated due to ").append(timeout).append(" second exceeded.\n"); + } + out.append("Exit Code: ").append(exitValue).append("\nOutput:\n").append(it.getOutput()); + throw new JobProcessingException(out.toString()); + } + ComponentUtil.getPopularWordHelper().clearCache(); + } catch (final JobProcessingException e) { + throw e; + } catch (final Exception e) { + throw new JobProcessingException("Python Process terminated.", e); + } finally { + processHelper.destroyProcess(sessionId); + + } + } + + protected String getPyFilePath() { + final StringBuilder buf = new StringBuilder(100); + buf.append("WEB-INF"); + buf.append(File.separator); + buf.append("env"); + buf.append(File.separator); + buf.append(getExecuteType()); + buf.append(File.separator); + buf.append("resources"); + buf.append(File.separator); + buf.append(filename.replaceAll("\\.\\.+", "")); + return buf.toString(); + } + + @Override + protected String getExecuteType() { + return "python"; + } +} diff --git a/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java b/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java index 586a3218f..7fedc3fe8 100644 --- a/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java +++ b/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java @@ -181,6 +181,9 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction /** The key of the configuration. e.g. java */ String JAVA_COMMAND_PATH = "java.command.path"; + /** The key of the configuration. e.g. python */ + String PYTHON_COMMAND_PATH = "python.command.path"; + /** The key of the configuration. e.g. UTF-8 */ String PATH_ENCODING = "path.encoding"; @@ -1910,6 +1913,13 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction */ String getJavaCommandPath(); + /** + * Get the value for the key 'python.command.path'.
+ * The value is, e.g. python
+ * @return The value of found property. (NotNull: if not found, exception but basically no way) + */ + String getPythonCommandPath(); + /** * Get the value for the key 'path.encoding'.
* The value is, e.g. UTF-8
@@ -6861,6 +6871,10 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction return get(FessConfig.JAVA_COMMAND_PATH); } + public String getPythonCommandPath() { + return get(FessConfig.PYTHON_COMMAND_PATH); + } + public String getPathEncoding() { return get(FessConfig.PATH_ENCODING); } @@ -9474,6 +9488,7 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction defaultMap.put(FessConfig.JOB_MAX_CRAWLER_PROCESSES, "0"); defaultMap.put(FessConfig.PROCESSORS, "0"); defaultMap.put(FessConfig.JAVA_COMMAND_PATH, "java"); + defaultMap.put(FessConfig.PYTHON_COMMAND_PATH, "python"); defaultMap.put(FessConfig.PATH_ENCODING, "UTF-8"); defaultMap.put(FessConfig.USE_OWN_TMP_DIR, "true"); defaultMap.put(FessConfig.MAX_LOG_OUTPUT_LENGTH, "4000"); diff --git a/src/main/resources/fess_config.properties b/src/main/resources/fess_config.properties index f04996730..96bf49d60 100644 --- a/src/main/resources/fess_config.properties +++ b/src/main/resources/fess_config.properties @@ -147,6 +147,7 @@ job.max.crawler.processes=0 processors=0 java.command.path=java +python.command.path=python path.encoding=UTF-8 use.own.tmp.dir=true max.log.output.length=4000 diff --git a/src/main/resources/fess_job.xml b/src/main/resources/fess_job.xml index 4da408af9..9e0414867 100644 --- a/src/main/resources/fess_job.xml +++ b/src/main/resources/fess_job.xml @@ -24,4 +24,6 @@ + +