diff --git a/src/main/java/org/codelibs/fess/exec/Crawler.java b/src/main/java/org/codelibs/fess/exec/Crawler.java
index 82c763b75..7c3978be8 100644
--- a/src/main/java/org/codelibs/fess/exec/Crawler.java
+++ b/src/main/java/org/codelibs/fess/exec/Crawler.java
@@ -36,6 +36,8 @@ import javax.annotation.Resource;
import org.codelibs.core.CoreLibConstants;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.core.misc.DynamicProperties;
+import org.codelibs.core.timer.TimeoutManager;
+import org.codelibs.core.timer.TimeoutTask;
import org.codelibs.fess.Constants;
import org.codelibs.fess.app.service.CrawlingInfoService;
import org.codelibs.fess.app.service.PathMappingService;
@@ -49,7 +51,11 @@ import org.codelibs.fess.helper.PathMappingHelper;
import org.codelibs.fess.helper.WebFsIndexHelper;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.mylasta.mail.CrawlerPostcard;
+import org.codelibs.fess.timer.SystemMonitorTarget;
import org.codelibs.fess.util.ComponentUtil;
+import org.elasticsearch.monitor.jvm.JvmInfo;
+import org.elasticsearch.monitor.os.OsProbe;
+import org.elasticsearch.monitor.process.ProcessProbe;
import org.kohsuke.args4j.CmdLineException;
import org.kohsuke.args4j.CmdLineParser;
import org.kohsuke.args4j.Option;
@@ -161,6 +167,13 @@ public class Crawler {
}
+ static void initializeProbes() {
+ // Force probes to be loaded
+ ProcessProbe.getInstance();
+ OsProbe.getInstance();
+ JvmInfo.jvmInfo();
+ }
+
public static void main(final String[] args) {
final Options options = new Options();
@@ -185,6 +198,8 @@ public class Crawler {
}
}
+ initializeProbes();
+
final String transportAddresses = System.getProperty(Constants.FESS_ES_TRANSPORT_ADDRESSES);
if (StringUtil.isNotBlank(transportAddresses)) {
System.setProperty(EsClient.TRANSPORT_ADDRESSES, transportAddresses);
@@ -194,6 +209,7 @@ public class Crawler {
System.setProperty(EsClient.CLUSTER_NAME, clusterName);
}
+ TimeoutTask systemMonitorTask = null;
int exitCode;
try {
running.set(true);
@@ -211,6 +227,10 @@ public class Crawler {
};
Runtime.getRuntime().addShutdownHook(shutdownCallback);
+ systemMonitorTask =
+ TimeoutManager.getInstance().addTimeoutTarget(new SystemMonitorTarget(),
+ ComponentUtil.getFessConfig().getCrawlerSystemMonitorIntervalAsInteger(), true);
+
exitCode = process(options);
} catch (final ContainerNotAvailableException e) {
if (logger.isDebugEnabled()) {
@@ -223,6 +243,9 @@ public class Crawler {
logger.error("Crawler does not work correctly.", t);
exitCode = Constants.EXIT_FAIL;
} finally {
+ if (systemMonitorTask != null) {
+ systemMonitorTask.cancel();
+ }
destroyContainer();
}
diff --git a/src/main/java/org/codelibs/fess/mylasta/action/FessHtmlPath.java b/src/main/java/org/codelibs/fess/mylasta/action/FessHtmlPath.java
index 21cc4cce7..5cf591ac9 100644
--- a/src/main/java/org/codelibs/fess/mylasta/action/FessHtmlPath.java
+++ b/src/main/java/org/codelibs/fess/mylasta/action/FessHtmlPath.java
@@ -388,6 +388,51 @@ public interface FessHtmlPath {
/** The path of the HTML: /index.jsp */
HtmlNext path_IndexJsp = new HtmlNext("/index.jsp");
+ /** The path of the HTML: /local/error/badRequest.jsp */
+ HtmlNext path_LocalError_BadRequestJsp = new HtmlNext("/local/error/badRequest.jsp");
+
+ /** The path of the HTML: /local/error/error.jsp */
+ HtmlNext path_LocalError_ErrorJsp = new HtmlNext("/local/error/error.jsp");
+
+ /** The path of the HTML: /local/error/notFound.jsp */
+ HtmlNext path_LocalError_NotFoundJsp = new HtmlNext("/local/error/notFound.jsp");
+
+ /** The path of the HTML: /local/error/redirect.jsp */
+ HtmlNext path_LocalError_RedirectJsp = new HtmlNext("/local/error/redirect.jsp");
+
+ /** The path of the HTML: /local/error/system.jsp */
+ HtmlNext path_LocalError_SystemJsp = new HtmlNext("/local/error/system.jsp");
+
+ /** The path of the HTML: /local/footer.jsp */
+ HtmlNext path_Local_FooterJsp = new HtmlNext("/local/footer.jsp");
+
+ /** The path of the HTML: /local/header.jsp */
+ HtmlNext path_Local_HeaderJsp = new HtmlNext("/local/header.jsp");
+
+ /** The path of the HTML: /local/help.jsp */
+ HtmlNext path_Local_HelpJsp = new HtmlNext("/local/help.jsp");
+
+ /** The path of the HTML: /local/index.jsp */
+ HtmlNext path_Local_IndexJsp = new HtmlNext("/local/index.jsp");
+
+ /** The path of the HTML: /local/login/index.jsp */
+ HtmlNext path_LocalLogin_IndexJsp = new HtmlNext("/local/login/index.jsp");
+
+ /** The path of the HTML: /local/profile/index.jsp */
+ HtmlNext path_LocalProfile_IndexJsp = new HtmlNext("/local/profile/index.jsp");
+
+ /** The path of the HTML: /local/search.jsp */
+ HtmlNext path_Local_SearchJsp = new HtmlNext("/local/search.jsp");
+
+ /** The path of the HTML: /local/searchNoResult.jsp */
+ HtmlNext path_Local_SearchNoResultJsp = new HtmlNext("/local/searchNoResult.jsp");
+
+ /** The path of the HTML: /local/searchOptions.jsp */
+ HtmlNext path_Local_SearchOptionsJsp = new HtmlNext("/local/searchOptions.jsp");
+
+ /** The path of the HTML: /local/searchResults.jsp */
+ HtmlNext path_Local_SearchResultsJsp = new HtmlNext("/local/searchResults.jsp");
+
/** The path of the HTML: /login/index.jsp */
HtmlNext path_Login_IndexJsp = new HtmlNext("/login/index.jsp");
diff --git a/src/main/java/org/codelibs/fess/mylasta/action/FessLabels.java b/src/main/java/org/codelibs/fess/mylasta/action/FessLabels.java
index 9bf6cd191..a606e287d 100644
--- a/src/main/java/org/codelibs/fess/mylasta/action/FessLabels.java
+++ b/src/main/java/org/codelibs/fess/mylasta/action/FessLabels.java
@@ -827,6 +827,9 @@ public class FessLabels extends UserMessages {
/** The key of the message: WAV Audio */
public static final String LABELS_facet_filetype_wav = "{labels.facet_filetype_wav}";
+ /** The key of the message: WebP Audio */
+ public static final String LABELS_facet_filetype_webp = "{labels.facet_filetype_webp}";
+
/** The key of the message: AIF Audio */
public static final String LABELS_facet_filetype_aif = "{labels.facet_filetype_aif}";
diff --git a/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java b/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java
index e044ea037..7a27ce415 100644
--- a/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java
+++ b/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java
@@ -50,6 +50,9 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
String APP_DIGEST_ALGORISM = "app.digest.algorism";
/** The key of the configuration. e.g. -Djava.awt.headless=true
+ -Dfile.encoding=UTF-8
+ -Djna.nosys=true
+ -Djdk.io.permissionsUseCanonicalPath=true
-server
-Xmx512m
-XX:MaxMetaspaceSize=128m
@@ -57,7 +60,7 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
-XX:-UseGCOverheadLimit
-XX:+UseConcMarkSweepGC
-XX:CMSInitiatingOccupancyFraction=75
- -XX:+UseParNewGC
+ -XX:+UseCMSInitiatingOccupancyOnly
-XX:+UseTLAB
-XX:+DisableExplicitGC
-XX:+HeapDumpOnOutOfMemoryError
@@ -66,10 +69,19 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
-Djcifs.smb.client.soTimeout=35000
-Djcifs.smb.client.responseTimeout=30000
-Dgroovy.use.classvalue=true
+ -Dio.netty.noUnsafe=true
+ -Dio.netty.noKeySetOptimization=true
+ -Dio.netty.recycler.maxCapacityPerThread=0
+ -Dlog4j.shutdownHookEnabled=false
+ -Dlog4j2.disable.jmx=true
+ -Dlog4j.skipJansi=true
*/
String JVM_CRAWLER_OPTIONS = "jvm.crawler.options";
/** The key of the configuration. e.g. -Djava.awt.headless=true
+ -Dfile.encoding=UTF-8
+ -Djna.nosys=true
+ -Djdk.io.permissionsUseCanonicalPath=true
-server
-Xmx256m
-XX:MaxMetaspaceSize=128m
@@ -77,11 +89,17 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
-XX:-UseGCOverheadLimit
-XX:+UseConcMarkSweepGC
-XX:CMSInitiatingOccupancyFraction=75
- -XX:+UseParNewGC
+ -XX:+UseCMSInitiatingOccupancyOnly
-XX:+UseTLAB
-XX:+DisableExplicitGC
-XX:+HeapDumpOnOutOfMemoryError
-Dgroovy.use.classvalue=true
+ -Dio.netty.noUnsafe=true
+ -Dio.netty.noKeySetOptimization=true
+ -Dio.netty.recycler.maxCapacityPerThread=0
+ -Dlog4j.shutdownHookEnabled=false
+ -Dlog4j2.disable.jmx=true
+ -Dlog4j.skipJansi=true
*/
String JVM_SUGGEST_OPTIONS = "jvm.suggest.options";
@@ -199,6 +217,9 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
/** The key of the configuration. e.g. 404 */
String CRAWLER_FAILURE_URL_STATUS_CODES = "crawler.failure.url.status.codes";
+ /** The key of the configuration. e.g. 60 */
+ String CRAWLER_SYSTEM_MONITOR_INTERVAL = "crawler.system.monitor.interval";
+
/** The key of the configuration. e.g. resourceName,X-Parsed-By,Content-Encoding.*,Content-Type.* */
String CRAWLER_METADATA_CONTENT_EXCLUDES = "crawler.metadata.content.excludes";
@@ -1360,6 +1381,9 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
/**
* Get the value for the key 'jvm.crawler.options'.
* The value is, e.g. -Djava.awt.headless=true
+ -Dfile.encoding=UTF-8
+ -Djna.nosys=true
+ -Djdk.io.permissionsUseCanonicalPath=true
-server
-Xmx512m
-XX:MaxMetaspaceSize=128m
@@ -1367,7 +1391,7 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
-XX:-UseGCOverheadLimit
-XX:+UseConcMarkSweepGC
-XX:CMSInitiatingOccupancyFraction=75
- -XX:+UseParNewGC
+ -XX:+UseCMSInitiatingOccupancyOnly
-XX:+UseTLAB
-XX:+DisableExplicitGC
-XX:+HeapDumpOnOutOfMemoryError
@@ -1376,6 +1400,12 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
-Djcifs.smb.client.soTimeout=35000
-Djcifs.smb.client.responseTimeout=30000
-Dgroovy.use.classvalue=true
+ -Dio.netty.noUnsafe=true
+ -Dio.netty.noKeySetOptimization=true
+ -Dio.netty.recycler.maxCapacityPerThread=0
+ -Dlog4j.shutdownHookEnabled=false
+ -Dlog4j2.disable.jmx=true
+ -Dlog4j.skipJansi=true
* comment: JVM options
* @return The value of found property. (NotNull: if not found, exception but basically no way)
@@ -1385,6 +1415,9 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
/**
* Get the value for the key 'jvm.suggest.options'.
* The value is, e.g. -Djava.awt.headless=true
+ -Dfile.encoding=UTF-8
+ -Djna.nosys=true
+ -Djdk.io.permissionsUseCanonicalPath=true
-server
-Xmx256m
-XX:MaxMetaspaceSize=128m
@@ -1392,11 +1425,17 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
-XX:-UseGCOverheadLimit
-XX:+UseConcMarkSweepGC
-XX:CMSInitiatingOccupancyFraction=75
- -XX:+UseParNewGC
+ -XX:+UseCMSInitiatingOccupancyOnly
-XX:+UseTLAB
-XX:+DisableExplicitGC
-XX:+HeapDumpOnOutOfMemoryError
-Dgroovy.use.classvalue=true
+ -Dio.netty.noUnsafe=true
+ -Dio.netty.noKeySetOptimization=true
+ -Dio.netty.recycler.maxCapacityPerThread=0
+ -Dlog4j.shutdownHookEnabled=false
+ -Dlog4j2.disable.jmx=true
+ -Dlog4j.skipJansi=true
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
@@ -1815,6 +1854,21 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
*/
Integer getCrawlerFailureUrlStatusCodesAsInteger();
+ /**
+ * Get the value for the key 'crawler.system.monitor.interval'.
+ * The value is, e.g. 60
+ * @return The value of found property. (NotNull: if not found, exception but basically no way)
+ */
+ String getCrawlerSystemMonitorInterval();
+
+ /**
+ * Get the value for the key 'crawler.system.monitor.interval' as {@link Integer}.
+ * The value is, e.g. 60
+ * @return The value of found property. (NotNull: if not found, exception but basically no way)
+ * @throws NumberFormatException When the property is not integer.
+ */
+ Integer getCrawlerSystemMonitorIntervalAsInteger();
+
/**
* Get the value for the key 'crawler.metadata.content.excludes'.
* The value is, e.g. resourceName,X-Parsed-By,Content-Encoding.*,Content-Type.*
@@ -5607,6 +5661,14 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
return getAsInteger(FessConfig.CRAWLER_FAILURE_URL_STATUS_CODES);
}
+ public String getCrawlerSystemMonitorInterval() {
+ return get(FessConfig.CRAWLER_SYSTEM_MONITOR_INTERVAL);
+ }
+
+ public Integer getCrawlerSystemMonitorIntervalAsInteger() {
+ return getAsInteger(FessConfig.CRAWLER_SYSTEM_MONITOR_INTERVAL);
+ }
+
public String getCrawlerMetadataContentExcludes() {
return get(FessConfig.CRAWLER_METADATA_CONTENT_EXCLUDES);
}
@@ -7501,10 +7563,10 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
defaultMap.put(FessConfig.APP_DIGEST_ALGORISM, "sha256");
defaultMap
.put(FessConfig.JVM_CRAWLER_OPTIONS,
- "-Djava.awt.headless=true\n-server\n-Xmx512m\n-XX:MaxMetaspaceSize=128m\n-XX:CompressedClassSpaceSize=32m\n-XX:-UseGCOverheadLimit\n-XX:+UseConcMarkSweepGC\n-XX:CMSInitiatingOccupancyFraction=75\n-XX:+UseParNewGC\n-XX:+UseTLAB\n-XX:+DisableExplicitGC\n-XX:+HeapDumpOnOutOfMemoryError\n-XX:-OmitStackTraceInFastThrow\n-Djcifs.smb.client.connTimeout=60000\n-Djcifs.smb.client.soTimeout=35000\n-Djcifs.smb.client.responseTimeout=30000\n-Dgroovy.use.classvalue=true\n");
+ "-Djava.awt.headless=true\n-Dfile.encoding=UTF-8\n-Djna.nosys=true\n-Djdk.io.permissionsUseCanonicalPath=true\n-server\n-Xmx512m\n-XX:MaxMetaspaceSize=128m\n-XX:CompressedClassSpaceSize=32m\n-XX:-UseGCOverheadLimit\n-XX:+UseConcMarkSweepGC\n-XX:CMSInitiatingOccupancyFraction=75\n-XX:+UseCMSInitiatingOccupancyOnly\n-XX:+UseTLAB\n-XX:+DisableExplicitGC\n-XX:+HeapDumpOnOutOfMemoryError\n-XX:-OmitStackTraceInFastThrow\n-Djcifs.smb.client.connTimeout=60000\n-Djcifs.smb.client.soTimeout=35000\n-Djcifs.smb.client.responseTimeout=30000\n-Dgroovy.use.classvalue=true\n-Dio.netty.noUnsafe=true\n-Dio.netty.noKeySetOptimization=true\n-Dio.netty.recycler.maxCapacityPerThread=0\n-Dlog4j.shutdownHookEnabled=false\n-Dlog4j2.disable.jmx=true\n-Dlog4j.skipJansi=true\n");
defaultMap
.put(FessConfig.JVM_SUGGEST_OPTIONS,
- "-Djava.awt.headless=true\n-server\n-Xmx256m\n-XX:MaxMetaspaceSize=128m\n-XX:CompressedClassSpaceSize=32m\n-XX:-UseGCOverheadLimit\n-XX:+UseConcMarkSweepGC\n-XX:CMSInitiatingOccupancyFraction=75\n-XX:+UseParNewGC\n-XX:+UseTLAB\n-XX:+DisableExplicitGC\n-XX:+HeapDumpOnOutOfMemoryError\n-Dgroovy.use.classvalue=true\n");
+ "-Djava.awt.headless=true\n-Dfile.encoding=UTF-8\n-Djna.nosys=true\n-Djdk.io.permissionsUseCanonicalPath=true\n-server\n-Xmx256m\n-XX:MaxMetaspaceSize=128m\n-XX:CompressedClassSpaceSize=32m\n-XX:-UseGCOverheadLimit\n-XX:+UseConcMarkSweepGC\n-XX:CMSInitiatingOccupancyFraction=75\n-XX:+UseCMSInitiatingOccupancyOnly\n-XX:+UseTLAB\n-XX:+DisableExplicitGC\n-XX:+HeapDumpOnOutOfMemoryError\n-Dgroovy.use.classvalue=true\n-Dio.netty.noUnsafe=true\n-Dio.netty.noKeySetOptimization=true\n-Dio.netty.recycler.maxCapacityPerThread=0\n-Dlog4j.shutdownHookEnabled=false\n-Dlog4j2.disable.jmx=true\n-Dlog4j.skipJansi=true\n");
defaultMap.put(FessConfig.JOB_SYSTEM_JOB_IDS, "default_crawler");
defaultMap.put(FessConfig.JOB_TEMPLATE_TITLE_WEB, "Web Crawler - {0}");
defaultMap.put(FessConfig.JOB_TEMPLATE_TITLE_FILE, "File Crawler - {0}");
@@ -7549,6 +7611,7 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
defaultMap.put(FessConfig.CRAWLER_IGNORE_META_ROBOTS, "false");
defaultMap.put(FessConfig.CRAWLER_IGNORE_CONTENT_EXCEPTION, "true");
defaultMap.put(FessConfig.CRAWLER_FAILURE_URL_STATUS_CODES, "404");
+ defaultMap.put(FessConfig.CRAWLER_SYSTEM_MONITOR_INTERVAL, "60");
defaultMap.put(FessConfig.CRAWLER_METADATA_CONTENT_EXCLUDES, "resourceName,X-Parsed-By,Content-Encoding.*,Content-Type.*");
defaultMap.put(FessConfig.CRAWLER_METADATA_NAME_MAPPING, "title=title:string\nTitle=title:string\n");
defaultMap.put(FessConfig.CRAWLER_DOCUMENT_HTML_CONTENT_XPATH, "//BODY");
diff --git a/src/main/java/org/codelibs/fess/timer/SystemMonitorTarget.java b/src/main/java/org/codelibs/fess/timer/SystemMonitorTarget.java
new file mode 100644
index 000000000..990b94f99
--- /dev/null
+++ b/src/main/java/org/codelibs/fess/timer/SystemMonitorTarget.java
@@ -0,0 +1,106 @@
+/*
+ * Copyright 2012-2017 CodeLibs Project and the Others.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+ * either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+package org.codelibs.fess.timer;
+
+import java.util.Arrays;
+import java.util.function.Supplier;
+
+import org.apache.commons.text.StringEscapeUtils;
+import org.codelibs.core.timer.TimeoutTarget;
+import org.elasticsearch.monitor.os.OsProbe;
+import org.elasticsearch.monitor.os.OsStats;
+import org.elasticsearch.monitor.process.ProcessProbe;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class SystemMonitorTarget implements TimeoutTarget {
+ private static final Logger logger = LoggerFactory.getLogger(SystemMonitorTarget.class);
+
+ protected StringBuilder append(final StringBuilder buf, final String key, final Supplier