0ct0pu5/fess
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix #1988 add job.max.crawler.processes

Browse source
This commit is contained in:
Shinsuke Sugaya 2019-01-25 06:42:22 +09:00
parent 197d95b6a6
commit e85f00d568
5 changed files with 81 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
src/main/java/org/codelibs/fess/app/job/ScriptExecutorJob.java
View file

@ -15,7 +15,6 @@
*/
package org.codelibs.fess.app.job;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.core.timer.TimeoutTask;
import org.codelibs.fess.Constants;
import org.codelibs.fess.es.config.exentity.JobLog;
@ -48,7 +47,7 @@ public class ScriptExecutorJob implements LaJob {
final ScheduledJob scheduledJob = (ScheduledJob) runtime.getParameterMap().get(Constants.SCHEDULED_JOB);
final String id = scheduledJob.getId();
final String target = scheduledJob.getTarget();
if (!isTarget(target)) {
if (!ComponentUtil.getFessConfig().isSchedulerTarget(target)) {
logger.info("Ignore Job " + id + ":" + scheduledJob.getName() + " because of not target: " + scheduledJob.getTarget());
return;
}
@ -123,23 +122,4 @@ public class ScriptExecutorJob implements LaJob {
}
}
protected boolean isTarget(final String target) {
if (StringUtil.isBlank(target)) {
return true;
}
final String myName = ComponentUtil.getFessConfig().getSchedulerTargetName();
final String[] targets = target.split(",");
for (String name : targets) {
name = name.trim();
if (Constants.DEFAULT_JOB_TARGET.equalsIgnoreCase(name)) {
return true;
} else if (StringUtil.isNotBlank(myName) && myName.equalsIgnoreCase(name)) {
return true;
}
}
return false;
}
}

33
src/main/java/org/codelibs/fess/job/CrawlJob.java
View file

@ -25,6 +25,7 @@ import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Properties;
import java.util.concurrent.atomic.AtomicInteger;
import javax.servlet.ServletContext;
@ -32,6 +33,7 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.SystemUtils;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.fess.Constants;
import org.codelibs.fess.es.config.exbhv.ScheduledJobBhv;
import org.codelibs.fess.exception.FessSystemException;
import org.codelibs.fess.exec.Crawler;
import org.codelibs.fess.helper.ProcessHelper;
@ -91,6 +93,16 @@ public class CrawlJob extends ExecJob {
@Override
public String execute() {
// check # of crawler processes
final int maxCrawlerProcesses = ComponentUtil.getFessConfig().getJobMaxCrawlerProcessesAsInteger();
if (maxCrawlerProcesses > 0) {
final int runningJobCount = getRunningJobCount();
if (runningJobCount > maxCrawlerProcesses) {
throw new FessSystemException(runningJobCount + " crawler processes are running. Max processes are " + maxCrawlerProcesses
+ ".");
}
}
final StringBuilder resultBuf = new StringBuilder(100);
final boolean runAll = webConfigIds == null && fileConfigIds == null && dataConfigIds == null;
@ -156,6 +168,27 @@ public class CrawlJob extends ExecJob {
}
protected int getRunningJobCount() {
final AtomicInteger counter = new AtomicInteger(0);
final FessConfig fessConfig = ComponentUtil.getFessConfig();
ComponentUtil.getComponent(ScheduledJobBhv.class).selectCursor(cb -> {
cb.query().setAvailable_Equal(Constants.T);
cb.query().setCrawler_Equal(Constants.T);
}, scheduledJob -> {
if (fessConfig.isSchedulerTarget(scheduledJob.getTarget())) {
if (scheduledJob.isRunning()) {
if (logger.isDebugEnabled()) {
logger.debug(scheduledJob.getId() + " is running.");
}
counter.incrementAndGet();
} else if (logger.isDebugEnabled()) {
logger.debug(scheduledJob.getId() + " is not running.");
}
}
});
return counter.get();
}
protected void executeCrawler() {
final List<String> cmdList = new ArrayList<>();
final String cpSeparator = SystemUtils.IS_OS_WINDOWS ? ";" : ":";

27
src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java
View file

@ -142,6 +142,9 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
/** The key of the configuration. e.g. return container.getComponent("crawlJob").logLevel("info").sessionId("{3}").webConfigIds([{0}] as String[]).fileConfigIds([{1}] as String[]).dataConfigIds([{2}] as String[]).jobExecutor(executor).execute(); */
String JOB_TEMPLATE_SCRIPT = "job.template.script";
/** The key of the configuration. e.g. 0 */
String JOB_MAX_CRAWLER_PROCESSES = "job.max.crawler.processes";
/** The key of the configuration. e.g. java */
String JAVA_COMMAND_PATH = "java.command.path";
@ -1614,6 +1617,21 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
*/
String getJobTemplateScript();
/**
* Get the value for the key 'job.max.crawler.processes'. <br>
* The value is, e.g. 0 <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getJobMaxCrawlerProcesses();
/**
* Get the value for the key 'job.max.crawler.processes' as {@link Integer}. <br>
* The value is, e.g. 0 <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
* @throws NumberFormatException When the property is not integer.
*/
Integer getJobMaxCrawlerProcessesAsInteger();
/**
* Get the value for the key 'java.command.path'. <br>
* The value is, e.g. java <br>
@ -5992,6 +6010,14 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
return get(FessConfig.JOB_TEMPLATE_SCRIPT);
}
public String getJobMaxCrawlerProcesses() {
return get(FessConfig.JOB_MAX_CRAWLER_PROCESSES);
}
public Integer getJobMaxCrawlerProcessesAsInteger() {
return getAsInteger(FessConfig.JOB_MAX_CRAWLER_PROCESSES);
}
public String getJavaCommandPath() {
return get(FessConfig.JAVA_COMMAND_PATH);
}
@ -8340,6 +8366,7 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
defaultMap
.put(FessConfig.JOB_TEMPLATE_SCRIPT,
"return container.getComponent(\"crawlJob\").logLevel(\"info\").sessionId(\"{3}\").webConfigIds([{0}] as String[]).fileConfigIds([{1}] as String[]).dataConfigIds([{2}] as String[]).jobExecutor(executor).execute();");
defaultMap.put(FessConfig.JOB_MAX_CRAWLER_PROCESSES, "0");
defaultMap.put(FessConfig.JAVA_COMMAND_PATH, "java");
defaultMap.put(FessConfig.PATH_ENCODING, "UTF-8");
defaultMap.put(FessConfig.USE_OWN_TMP_DIR, "true");

19
src/main/java/org/codelibs/fess/mylasta/direction/FessProp.java
View file

@ -1889,4 +1889,23 @@ public interface FessProp {
}
return fields;
}
String getSchedulerTargetName();
default boolean isSchedulerTarget(final String target) {
if (StringUtil.isBlank(target)) {
return true;
}
final String myName = getSchedulerTargetName();
final String[] targets = target.split(",");
for (String name : targets) {
name = name.trim();
if (Constants.DEFAULT_JOB_TARGET.equalsIgnoreCase(name) || StringUtil.isNotBlank(myName) && myName.equalsIgnoreCase(name)) {
return true;
}
}
return false;
}
}

1
src/main/resources/fess_config.properties
View file

@ -116,6 +116,7 @@ job.template.title.web=Web Crawler - {0}
job.template.title.file=File Crawler - {0}
job.template.title.data=Data Crawler - {0}
job.template.script=return container.getComponent("crawlJob").logLevel("info").sessionId("{3}").webConfigIds([{0}] as String[]).fileConfigIds([{1}] as String[]).dataConfigIds([{2}] as String[]).jobExecutor(executor).execute();
job.max.crawler.processes=0
java.command.path=java
path.encoding=UTF-8