diff --git a/src/main/java/org/codelibs/fess/app/job/ScriptExecutorJob.java b/src/main/java/org/codelibs/fess/app/job/ScriptExecutorJob.java index 2cf7ffc79..1f6ed5a69 100644 --- a/src/main/java/org/codelibs/fess/app/job/ScriptExecutorJob.java +++ b/src/main/java/org/codelibs/fess/app/job/ScriptExecutorJob.java @@ -15,7 +15,6 @@ */ package org.codelibs.fess.app.job; -import org.codelibs.core.lang.StringUtil; import org.codelibs.core.timer.TimeoutTask; import org.codelibs.fess.Constants; import org.codelibs.fess.es.config.exentity.JobLog; @@ -48,7 +47,7 @@ public class ScriptExecutorJob implements LaJob { final ScheduledJob scheduledJob = (ScheduledJob) runtime.getParameterMap().get(Constants.SCHEDULED_JOB); final String id = scheduledJob.getId(); final String target = scheduledJob.getTarget(); - if (!isTarget(target)) { + if (!ComponentUtil.getFessConfig().isSchedulerTarget(target)) { logger.info("Ignore Job " + id + ":" + scheduledJob.getName() + " because of not target: " + scheduledJob.getTarget()); return; } @@ -123,23 +122,4 @@ public class ScriptExecutorJob implements LaJob { } } - protected boolean isTarget(final String target) { - if (StringUtil.isBlank(target)) { - return true; - } - - final String myName = ComponentUtil.getFessConfig().getSchedulerTargetName(); - - final String[] targets = target.split(","); - for (String name : targets) { - name = name.trim(); - if (Constants.DEFAULT_JOB_TARGET.equalsIgnoreCase(name)) { - return true; - } else if (StringUtil.isNotBlank(myName) && myName.equalsIgnoreCase(name)) { - return true; - } - } - return false; - } - } diff --git a/src/main/java/org/codelibs/fess/job/CrawlJob.java b/src/main/java/org/codelibs/fess/job/CrawlJob.java index a7dc46a85..1763ba2e3 100644 --- a/src/main/java/org/codelibs/fess/job/CrawlJob.java +++ b/src/main/java/org/codelibs/fess/job/CrawlJob.java @@ -25,6 +25,7 @@ import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.Properties; +import java.util.concurrent.atomic.AtomicInteger; import javax.servlet.ServletContext; @@ -32,6 +33,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.SystemUtils; import org.codelibs.core.lang.StringUtil; import org.codelibs.fess.Constants; +import org.codelibs.fess.es.config.exbhv.ScheduledJobBhv; import org.codelibs.fess.exception.FessSystemException; import org.codelibs.fess.exec.Crawler; import org.codelibs.fess.helper.ProcessHelper; @@ -91,6 +93,16 @@ public class CrawlJob extends ExecJob { @Override public String execute() { + // check # of crawler processes + final int maxCrawlerProcesses = ComponentUtil.getFessConfig().getJobMaxCrawlerProcessesAsInteger(); + if (maxCrawlerProcesses > 0) { + final int runningJobCount = getRunningJobCount(); + if (runningJobCount > maxCrawlerProcesses) { + throw new FessSystemException(runningJobCount + " crawler processes are running. Max processes are " + maxCrawlerProcesses + + "."); + } + } + final StringBuilder resultBuf = new StringBuilder(100); final boolean runAll = webConfigIds == null && fileConfigIds == null && dataConfigIds == null; @@ -156,6 +168,27 @@ public class CrawlJob extends ExecJob { } + protected int getRunningJobCount() { + final AtomicInteger counter = new AtomicInteger(0); + final FessConfig fessConfig = ComponentUtil.getFessConfig(); + ComponentUtil.getComponent(ScheduledJobBhv.class).selectCursor(cb -> { + cb.query().setAvailable_Equal(Constants.T); + cb.query().setCrawler_Equal(Constants.T); + }, scheduledJob -> { + if (fessConfig.isSchedulerTarget(scheduledJob.getTarget())) { + if (scheduledJob.isRunning()) { + if (logger.isDebugEnabled()) { + logger.debug(scheduledJob.getId() + " is running."); + } + counter.incrementAndGet(); + } else if (logger.isDebugEnabled()) { + logger.debug(scheduledJob.getId() + " is not running."); + } + } + }); + return counter.get(); + } + protected void executeCrawler() { final List cmdList = new ArrayList<>(); final String cpSeparator = SystemUtils.IS_OS_WINDOWS ? ";" : ":"; diff --git a/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java b/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java index 2a5588f0d..3887a9a9d 100644 --- a/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java +++ b/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java @@ -142,6 +142,9 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction /** The key of the configuration. e.g. return container.getComponent("crawlJob").logLevel("info").sessionId("{3}").webConfigIds([{0}] as String[]).fileConfigIds([{1}] as String[]).dataConfigIds([{2}] as String[]).jobExecutor(executor).execute(); */ String JOB_TEMPLATE_SCRIPT = "job.template.script"; + /** The key of the configuration. e.g. 0 */ + String JOB_MAX_CRAWLER_PROCESSES = "job.max.crawler.processes"; + /** The key of the configuration. e.g. java */ String JAVA_COMMAND_PATH = "java.command.path"; @@ -1614,6 +1617,21 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction */ String getJobTemplateScript(); + /** + * Get the value for the key 'job.max.crawler.processes'.
+ * The value is, e.g. 0
+ * @return The value of found property. (NotNull: if not found, exception but basically no way) + */ + String getJobMaxCrawlerProcesses(); + + /** + * Get the value for the key 'job.max.crawler.processes' as {@link Integer}.
+ * The value is, e.g. 0
+ * @return The value of found property. (NotNull: if not found, exception but basically no way) + * @throws NumberFormatException When the property is not integer. + */ + Integer getJobMaxCrawlerProcessesAsInteger(); + /** * Get the value for the key 'java.command.path'.
* The value is, e.g. java
@@ -5992,6 +6010,14 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction return get(FessConfig.JOB_TEMPLATE_SCRIPT); } + public String getJobMaxCrawlerProcesses() { + return get(FessConfig.JOB_MAX_CRAWLER_PROCESSES); + } + + public Integer getJobMaxCrawlerProcessesAsInteger() { + return getAsInteger(FessConfig.JOB_MAX_CRAWLER_PROCESSES); + } + public String getJavaCommandPath() { return get(FessConfig.JAVA_COMMAND_PATH); } @@ -8340,6 +8366,7 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction defaultMap .put(FessConfig.JOB_TEMPLATE_SCRIPT, "return container.getComponent(\"crawlJob\").logLevel(\"info\").sessionId(\"{3}\").webConfigIds([{0}] as String[]).fileConfigIds([{1}] as String[]).dataConfigIds([{2}] as String[]).jobExecutor(executor).execute();"); + defaultMap.put(FessConfig.JOB_MAX_CRAWLER_PROCESSES, "0"); defaultMap.put(FessConfig.JAVA_COMMAND_PATH, "java"); defaultMap.put(FessConfig.PATH_ENCODING, "UTF-8"); defaultMap.put(FessConfig.USE_OWN_TMP_DIR, "true"); diff --git a/src/main/java/org/codelibs/fess/mylasta/direction/FessProp.java b/src/main/java/org/codelibs/fess/mylasta/direction/FessProp.java index 0831b774f..fe4a83a14 100644 --- a/src/main/java/org/codelibs/fess/mylasta/direction/FessProp.java +++ b/src/main/java/org/codelibs/fess/mylasta/direction/FessProp.java @@ -1889,4 +1889,23 @@ public interface FessProp { } return fields; } + + String getSchedulerTargetName(); + + default boolean isSchedulerTarget(final String target) { + if (StringUtil.isBlank(target)) { + return true; + } + + final String myName = getSchedulerTargetName(); + + final String[] targets = target.split(","); + for (String name : targets) { + name = name.trim(); + if (Constants.DEFAULT_JOB_TARGET.equalsIgnoreCase(name) || StringUtil.isNotBlank(myName) && myName.equalsIgnoreCase(name)) { + return true; + } + } + return false; + } } diff --git a/src/main/resources/fess_config.properties b/src/main/resources/fess_config.properties index 84776258c..d94bc08b9 100644 --- a/src/main/resources/fess_config.properties +++ b/src/main/resources/fess_config.properties @@ -116,6 +116,7 @@ job.template.title.web=Web Crawler - {0} job.template.title.file=File Crawler - {0} job.template.title.data=Data Crawler - {0} job.template.script=return container.getComponent("crawlJob").logLevel("info").sessionId("{3}").webConfigIds([{0}] as String[]).fileConfigIds([{1}] as String[]).dataConfigIds([{2}] as String[]).jobExecutor(executor).execute(); +job.max.crawler.processes=0 java.command.path=java path.encoding=UTF-8