fix #1988 add job.max.crawler.processes

src/main/java/org/codelibs/fess/app/job/ScriptExecutorJob.java

@@ -15,7 +15,6 @@
  */
 package org.codelibs.fess.app.job;
 
-import org.codelibs.core.lang.StringUtil;
 import org.codelibs.core.timer.TimeoutTask;
 import org.codelibs.fess.Constants;
 import org.codelibs.fess.es.config.exentity.JobLog;
@@ -48,7 +47,7 @@ public class ScriptExecutorJob implements LaJob {
         final ScheduledJob scheduledJob = (ScheduledJob) runtime.getParameterMap().get(Constants.SCHEDULED_JOB);
         final String id = scheduledJob.getId();
         final String target = scheduledJob.getTarget();
-        if (!isTarget(target)) {
+        if (!ComponentUtil.getFessConfig().isSchedulerTarget(target)) {
             logger.info("Ignore Job " + id + ":" + scheduledJob.getName() + " because of not target: " + scheduledJob.getTarget());
             return;
         }
@@ -123,23 +122,4 @@ public class ScriptExecutorJob implements LaJob {
         }
     }
 
-    protected boolean isTarget(final String target) {
-        if (StringUtil.isBlank(target)) {
-            return true;
-        }
-
-        final String myName = ComponentUtil.getFessConfig().getSchedulerTargetName();
-
-        final String[] targets = target.split(",");
-        for (String name : targets) {
-            name = name.trim();
-            if (Constants.DEFAULT_JOB_TARGET.equalsIgnoreCase(name)) {
-                return true;
-            } else if (StringUtil.isNotBlank(myName) && myName.equalsIgnoreCase(name)) {
-                return true;
-            }
-        }
-        return false;
-    }
-
 }

src/main/java/org/codelibs/fess/job/CrawlJob.java

@@ -25,6 +25,7 @@ import java.util.ArrayList;
 import java.util.Date;
 import java.util.List;
 import java.util.Properties;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import javax.servlet.ServletContext;
 
@@ -32,6 +33,7 @@ import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.SystemUtils;
 import org.codelibs.core.lang.StringUtil;
 import org.codelibs.fess.Constants;
+import org.codelibs.fess.es.config.exbhv.ScheduledJobBhv;
 import org.codelibs.fess.exception.FessSystemException;
 import org.codelibs.fess.exec.Crawler;
 import org.codelibs.fess.helper.ProcessHelper;
@@ -91,6 +93,16 @@ public class CrawlJob extends ExecJob {
 
     @Override
     public String execute() {
+        //   check # of crawler processes
+        final int maxCrawlerProcesses = ComponentUtil.getFessConfig().getJobMaxCrawlerProcessesAsInteger();
+        if (maxCrawlerProcesses > 0) {
+            final int runningJobCount = getRunningJobCount();
+            if (runningJobCount > maxCrawlerProcesses) {
+                throw new FessSystemException(runningJobCount + " crawler processes are running. Max processes are " + maxCrawlerProcesses
+                        + ".");
+            }
+        }
+
         final StringBuilder resultBuf = new StringBuilder(100);
         final boolean runAll = webConfigIds == null && fileConfigIds == null && dataConfigIds == null;
 
@@ -156,6 +168,27 @@ public class CrawlJob extends ExecJob {
 
     }
 
+    protected int getRunningJobCount() {
+        final AtomicInteger counter = new AtomicInteger(0);
+        final FessConfig fessConfig = ComponentUtil.getFessConfig();
+        ComponentUtil.getComponent(ScheduledJobBhv.class).selectCursor(cb -> {
+            cb.query().setAvailable_Equal(Constants.T);
+            cb.query().setCrawler_Equal(Constants.T);
+        }, scheduledJob -> {
+            if (fessConfig.isSchedulerTarget(scheduledJob.getTarget())) {
+                if (scheduledJob.isRunning()) {
+                    if (logger.isDebugEnabled()) {
+                        logger.debug(scheduledJob.getId() + " is running.");
+                    }
+                    counter.incrementAndGet();
+                } else if (logger.isDebugEnabled()) {
+                    logger.debug(scheduledJob.getId() + " is not running.");
+                }
+            }
+        });
+        return counter.get();
+    }
+
     protected void executeCrawler() {
         final List<String> cmdList = new ArrayList<>();
         final String cpSeparator = SystemUtils.IS_OS_WINDOWS ? ";" : ":";

src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java

@@ -142,6 +142,9 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
     /** The key of the configuration. e.g. return container.getComponent("crawlJob").logLevel("info").sessionId("{3}").webConfigIds([{0}] as String[]).fileConfigIds([{1}] as String[]).dataConfigIds([{2}] as String[]).jobExecutor(executor).execute(); */
     String JOB_TEMPLATE_SCRIPT = "job.template.script";
 
+    /** The key of the configuration. e.g. 0 */
+    String JOB_MAX_CRAWLER_PROCESSES = "job.max.crawler.processes";
+
     /** The key of the configuration. e.g. java */
     String JAVA_COMMAND_PATH = "java.command.path";
 
@@ -1614,6 +1617,21 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
      */
     String getJobTemplateScript();
 
+    /**
+     * Get the value for the key 'job.max.crawler.processes'. <br>
+     * The value is, e.g. 0 <br>
+     * @return The value of found property. (NotNull: if not found, exception but basically no way)
+     */
+    String getJobMaxCrawlerProcesses();
+
+    /**
+     * Get the value for the key 'job.max.crawler.processes' as {@link Integer}. <br>
+     * The value is, e.g. 0 <br>
+     * @return The value of found property. (NotNull: if not found, exception but basically no way)
+     * @throws NumberFormatException When the property is not integer.
+     */
+    Integer getJobMaxCrawlerProcessesAsInteger();
+
     /**
      * Get the value for the key 'java.command.path'. <br>
      * The value is, e.g. java <br>
@@ -5992,6 +6010,14 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
             return get(FessConfig.JOB_TEMPLATE_SCRIPT);
         }
 
+        public String getJobMaxCrawlerProcesses() {
+            return get(FessConfig.JOB_MAX_CRAWLER_PROCESSES);
+        }
+
+        public Integer getJobMaxCrawlerProcessesAsInteger() {
+            return getAsInteger(FessConfig.JOB_MAX_CRAWLER_PROCESSES);
+        }
+
         public String getJavaCommandPath() {
             return get(FessConfig.JAVA_COMMAND_PATH);
         }
@@ -8340,6 +8366,7 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
             defaultMap
                     .put(FessConfig.JOB_TEMPLATE_SCRIPT,
                             "return container.getComponent(\"crawlJob\").logLevel(\"info\").sessionId(\"{3}\").webConfigIds([{0}] as String[]).fileConfigIds([{1}] as String[]).dataConfigIds([{2}] as String[]).jobExecutor(executor).execute();");
+            defaultMap.put(FessConfig.JOB_MAX_CRAWLER_PROCESSES, "0");
             defaultMap.put(FessConfig.JAVA_COMMAND_PATH, "java");
             defaultMap.put(FessConfig.PATH_ENCODING, "UTF-8");
             defaultMap.put(FessConfig.USE_OWN_TMP_DIR, "true");

src/main/java/org/codelibs/fess/mylasta/direction/FessProp.java

@@ -1889,4 +1889,23 @@ public interface FessProp {
         }
         return fields;
     }
+
+    String getSchedulerTargetName();
+
+    default boolean isSchedulerTarget(final String target) {
+        if (StringUtil.isBlank(target)) {
+            return true;
+        }
+
+        final String myName = getSchedulerTargetName();
+
+        final String[] targets = target.split(",");
+        for (String name : targets) {
+            name = name.trim();
+            if (Constants.DEFAULT_JOB_TARGET.equalsIgnoreCase(name) || StringUtil.isNotBlank(myName) && myName.equalsIgnoreCase(name)) {
+                return true;
+            }
+        }
+        return false;
+    }
 }

src/main/resources/fess_config.properties

@@ -116,6 +116,7 @@ job.template.title.web=Web Crawler - {0}
 job.template.title.file=File Crawler - {0}
 job.template.title.data=Data Crawler - {0}
 job.template.script=return container.getComponent("crawlJob").logLevel("info").sessionId("{3}").webConfigIds([{0}] as String[]).fileConfigIds([{1}] as String[]).dataConfigIds([{2}] as String[]).jobExecutor(executor).execute();
+job.max.crawler.processes=0
 
 java.command.path=java
 path.encoding=UTF-8