fix #1308 add system monitoring logs

src/main/java/org/codelibs/fess/exec/Crawler.java

@@ -36,6 +36,8 @@ import javax.annotation.Resource;
 import org.codelibs.core.CoreLibConstants;
 import org.codelibs.core.lang.StringUtil;
 import org.codelibs.core.misc.DynamicProperties;
+import org.codelibs.core.timer.TimeoutManager;
+import org.codelibs.core.timer.TimeoutTask;
 import org.codelibs.fess.Constants;
 import org.codelibs.fess.app.service.CrawlingInfoService;
 import org.codelibs.fess.app.service.PathMappingService;
@@ -49,7 +51,11 @@ import org.codelibs.fess.helper.PathMappingHelper;
 import org.codelibs.fess.helper.WebFsIndexHelper;
 import org.codelibs.fess.mylasta.direction.FessConfig;
 import org.codelibs.fess.mylasta.mail.CrawlerPostcard;
+import org.codelibs.fess.timer.SystemMonitorTarget;
 import org.codelibs.fess.util.ComponentUtil;
+import org.elasticsearch.monitor.jvm.JvmInfo;
+import org.elasticsearch.monitor.os.OsProbe;
+import org.elasticsearch.monitor.process.ProcessProbe;
 import org.kohsuke.args4j.CmdLineException;
 import org.kohsuke.args4j.CmdLineParser;
 import org.kohsuke.args4j.Option;
@@ -161,6 +167,13 @@ public class Crawler {
 
     }
 
+    static void initializeProbes() {
+        // Force probes to be loaded
+        ProcessProbe.getInstance();
+        OsProbe.getInstance();
+        JvmInfo.jvmInfo();
+    }
+
     public static void main(final String[] args) {
         final Options options = new Options();
 
@@ -185,6 +198,8 @@ public class Crawler {
             }
         }
 
+        initializeProbes();
+
         final String transportAddresses = System.getProperty(Constants.FESS_ES_TRANSPORT_ADDRESSES);
         if (StringUtil.isNotBlank(transportAddresses)) {
             System.setProperty(EsClient.TRANSPORT_ADDRESSES, transportAddresses);
@@ -194,6 +209,7 @@ public class Crawler {
             System.setProperty(EsClient.CLUSTER_NAME, clusterName);
         }
 
+        TimeoutTask systemMonitorTask = null;
         int exitCode;
         try {
             running.set(true);
@@ -211,6 +227,10 @@ public class Crawler {
             };
             Runtime.getRuntime().addShutdownHook(shutdownCallback);
 
+            systemMonitorTask =
+                    TimeoutManager.getInstance().addTimeoutTarget(new SystemMonitorTarget(),
+                            ComponentUtil.getFessConfig().getCrawlerSystemMonitorIntervalAsInteger(), true);
+
             exitCode = process(options);
         } catch (final ContainerNotAvailableException e) {
             if (logger.isDebugEnabled()) {
@@ -223,6 +243,9 @@ public class Crawler {
             logger.error("Crawler does not work correctly.", t);
             exitCode = Constants.EXIT_FAIL;
         } finally {
+            if (systemMonitorTask != null) {
+                systemMonitorTask.cancel();
+            }
             destroyContainer();
         }
 

src/main/java/org/codelibs/fess/mylasta/action/FessHtmlPath.java

@@ -388,6 +388,51 @@ public interface FessHtmlPath {
     /** The path of the HTML: /index.jsp */
     HtmlNext path_IndexJsp = new HtmlNext("/index.jsp");
 
+    /** The path of the HTML: /local/error/badRequest.jsp */
+    HtmlNext path_LocalError_BadRequestJsp = new HtmlNext("/local/error/badRequest.jsp");
+
+    /** The path of the HTML: /local/error/error.jsp */
+    HtmlNext path_LocalError_ErrorJsp = new HtmlNext("/local/error/error.jsp");
+
+    /** The path of the HTML: /local/error/notFound.jsp */
+    HtmlNext path_LocalError_NotFoundJsp = new HtmlNext("/local/error/notFound.jsp");
+
+    /** The path of the HTML: /local/error/redirect.jsp */
+    HtmlNext path_LocalError_RedirectJsp = new HtmlNext("/local/error/redirect.jsp");
+
+    /** The path of the HTML: /local/error/system.jsp */
+    HtmlNext path_LocalError_SystemJsp = new HtmlNext("/local/error/system.jsp");
+
+    /** The path of the HTML: /local/footer.jsp */
+    HtmlNext path_Local_FooterJsp = new HtmlNext("/local/footer.jsp");
+
+    /** The path of the HTML: /local/header.jsp */
+    HtmlNext path_Local_HeaderJsp = new HtmlNext("/local/header.jsp");
+
+    /** The path of the HTML: /local/help.jsp */
+    HtmlNext path_Local_HelpJsp = new HtmlNext("/local/help.jsp");
+
+    /** The path of the HTML: /local/index.jsp */
+    HtmlNext path_Local_IndexJsp = new HtmlNext("/local/index.jsp");
+
+    /** The path of the HTML: /local/login/index.jsp */
+    HtmlNext path_LocalLogin_IndexJsp = new HtmlNext("/local/login/index.jsp");
+
+    /** The path of the HTML: /local/profile/index.jsp */
+    HtmlNext path_LocalProfile_IndexJsp = new HtmlNext("/local/profile/index.jsp");
+
+    /** The path of the HTML: /local/search.jsp */
+    HtmlNext path_Local_SearchJsp = new HtmlNext("/local/search.jsp");
+
+    /** The path of the HTML: /local/searchNoResult.jsp */
+    HtmlNext path_Local_SearchNoResultJsp = new HtmlNext("/local/searchNoResult.jsp");
+
+    /** The path of the HTML: /local/searchOptions.jsp */
+    HtmlNext path_Local_SearchOptionsJsp = new HtmlNext("/local/searchOptions.jsp");
+
+    /** The path of the HTML: /local/searchResults.jsp */
+    HtmlNext path_Local_SearchResultsJsp = new HtmlNext("/local/searchResults.jsp");
+
     /** The path of the HTML: /login/index.jsp */
     HtmlNext path_Login_IndexJsp = new HtmlNext("/login/index.jsp");
 

src/main/java/org/codelibs/fess/mylasta/action/FessLabels.java

@@ -827,6 +827,9 @@ public class FessLabels extends UserMessages {
     /** The key of the message: WAV Audio */
     public static final String LABELS_facet_filetype_wav = "{labels.facet_filetype_wav}";
 
+    /** The key of the message: WebP Audio */
+    public static final String LABELS_facet_filetype_webp = "{labels.facet_filetype_webp}";
+
     /** The key of the message: AIF Audio */
     public static final String LABELS_facet_filetype_aif = "{labels.facet_filetype_aif}";
 

src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java

@@ -50,6 +50,9 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
     String APP_DIGEST_ALGORISM = "app.digest.algorism";
 
     /** The key of the configuration. e.g. -Djava.awt.headless=true
+    -Dfile.encoding=UTF-8
+    -Djna.nosys=true
+    -Djdk.io.permissionsUseCanonicalPath=true
     -server
     -Xmx512m
     -XX:MaxMetaspaceSize=128m
@@ -57,7 +60,7 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
     -XX:-UseGCOverheadLimit
     -XX:+UseConcMarkSweepGC
     -XX:CMSInitiatingOccupancyFraction=75
-    -XX:+UseParNewGC
+    -XX:+UseCMSInitiatingOccupancyOnly
     -XX:+UseTLAB
     -XX:+DisableExplicitGC
     -XX:+HeapDumpOnOutOfMemoryError
@@ -66,10 +69,19 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
     -Djcifs.smb.client.soTimeout=35000
     -Djcifs.smb.client.responseTimeout=30000
     -Dgroovy.use.classvalue=true
+    -Dio.netty.noUnsafe=true
+    -Dio.netty.noKeySetOptimization=true
+    -Dio.netty.recycler.maxCapacityPerThread=0
+    -Dlog4j.shutdownHookEnabled=false
+    -Dlog4j2.disable.jmx=true
+    -Dlog4j.skipJansi=true
     */
     String JVM_CRAWLER_OPTIONS = "jvm.crawler.options";
 
     /** The key of the configuration. e.g. -Djava.awt.headless=true
+    -Dfile.encoding=UTF-8
+    -Djna.nosys=true
+    -Djdk.io.permissionsUseCanonicalPath=true
     -server
     -Xmx256m
     -XX:MaxMetaspaceSize=128m
@@ -77,11 +89,17 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
     -XX:-UseGCOverheadLimit
     -XX:+UseConcMarkSweepGC
     -XX:CMSInitiatingOccupancyFraction=75
-    -XX:+UseParNewGC
+    -XX:+UseCMSInitiatingOccupancyOnly
     -XX:+UseTLAB
     -XX:+DisableExplicitGC
     -XX:+HeapDumpOnOutOfMemoryError
     -Dgroovy.use.classvalue=true
+    -Dio.netty.noUnsafe=true
+    -Dio.netty.noKeySetOptimization=true
+    -Dio.netty.recycler.maxCapacityPerThread=0
+    -Dlog4j.shutdownHookEnabled=false
+    -Dlog4j2.disable.jmx=true
+    -Dlog4j.skipJansi=true
     */
     String JVM_SUGGEST_OPTIONS = "jvm.suggest.options";
 
@@ -199,6 +217,9 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
     /** The key of the configuration. e.g. 404 */
     String CRAWLER_FAILURE_URL_STATUS_CODES = "crawler.failure.url.status.codes";
 
+    /** The key of the configuration. e.g. 60 */
+    String CRAWLER_SYSTEM_MONITOR_INTERVAL = "crawler.system.monitor.interval";
+
     /** The key of the configuration. e.g. resourceName,X-Parsed-By,Content-Encoding.*,Content-Type.* */
     String CRAWLER_METADATA_CONTENT_EXCLUDES = "crawler.metadata.content.excludes";
 
@@ -1360,6 +1381,9 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
     /**
      * Get the value for the key 'jvm.crawler.options'. <br>
      * The value is, e.g. -Djava.awt.headless=true
+    -Dfile.encoding=UTF-8
+    -Djna.nosys=true
+    -Djdk.io.permissionsUseCanonicalPath=true
     -server
     -Xmx512m
     -XX:MaxMetaspaceSize=128m
@@ -1367,7 +1391,7 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
     -XX:-UseGCOverheadLimit
     -XX:+UseConcMarkSweepGC
     -XX:CMSInitiatingOccupancyFraction=75
-    -XX:+UseParNewGC
+    -XX:+UseCMSInitiatingOccupancyOnly
     -XX:+UseTLAB
     -XX:+DisableExplicitGC
     -XX:+HeapDumpOnOutOfMemoryError
@@ -1376,6 +1400,12 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
     -Djcifs.smb.client.soTimeout=35000
     -Djcifs.smb.client.responseTimeout=30000
     -Dgroovy.use.classvalue=true
+    -Dio.netty.noUnsafe=true
+    -Dio.netty.noKeySetOptimization=true
+    -Dio.netty.recycler.maxCapacityPerThread=0
+    -Dlog4j.shutdownHookEnabled=false
+    -Dlog4j2.disable.jmx=true
+    -Dlog4j.skipJansi=true
     <br>
      * comment: JVM options
      * @return The value of found property. (NotNull: if not found, exception but basically no way)
@@ -1385,6 +1415,9 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
     /**
      * Get the value for the key 'jvm.suggest.options'. <br>
      * The value is, e.g. -Djava.awt.headless=true
+    -Dfile.encoding=UTF-8
+    -Djna.nosys=true
+    -Djdk.io.permissionsUseCanonicalPath=true
     -server
     -Xmx256m
     -XX:MaxMetaspaceSize=128m
@@ -1392,11 +1425,17 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
     -XX:-UseGCOverheadLimit
     -XX:+UseConcMarkSweepGC
     -XX:CMSInitiatingOccupancyFraction=75
-    -XX:+UseParNewGC
+    -XX:+UseCMSInitiatingOccupancyOnly
     -XX:+UseTLAB
     -XX:+DisableExplicitGC
     -XX:+HeapDumpOnOutOfMemoryError
     -Dgroovy.use.classvalue=true
+    -Dio.netty.noUnsafe=true
+    -Dio.netty.noKeySetOptimization=true
+    -Dio.netty.recycler.maxCapacityPerThread=0
+    -Dlog4j.shutdownHookEnabled=false
+    -Dlog4j2.disable.jmx=true
+    -Dlog4j.skipJansi=true
     <br>
      * @return The value of found property. (NotNull: if not found, exception but basically no way)
      */
@@ -1815,6 +1854,21 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
      */
     Integer getCrawlerFailureUrlStatusCodesAsInteger();
 
+    /**
+     * Get the value for the key 'crawler.system.monitor.interval'. <br>
+     * The value is, e.g. 60 <br>
+     * @return The value of found property. (NotNull: if not found, exception but basically no way)
+     */
+    String getCrawlerSystemMonitorInterval();
+
+    /**
+     * Get the value for the key 'crawler.system.monitor.interval' as {@link Integer}. <br>
+     * The value is, e.g. 60 <br>
+     * @return The value of found property. (NotNull: if not found, exception but basically no way)
+     * @throws NumberFormatException When the property is not integer.
+     */
+    Integer getCrawlerSystemMonitorIntervalAsInteger();
+
     /**
      * Get the value for the key 'crawler.metadata.content.excludes'. <br>
      * The value is, e.g. resourceName,X-Parsed-By,Content-Encoding.*,Content-Type.* <br>
@@ -5607,6 +5661,14 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
             return getAsInteger(FessConfig.CRAWLER_FAILURE_URL_STATUS_CODES);
         }
 
+        public String getCrawlerSystemMonitorInterval() {
+            return get(FessConfig.CRAWLER_SYSTEM_MONITOR_INTERVAL);
+        }
+
+        public Integer getCrawlerSystemMonitorIntervalAsInteger() {
+            return getAsInteger(FessConfig.CRAWLER_SYSTEM_MONITOR_INTERVAL);
+        }
+
         public String getCrawlerMetadataContentExcludes() {
             return get(FessConfig.CRAWLER_METADATA_CONTENT_EXCLUDES);
         }
@@ -7501,10 +7563,10 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
             defaultMap.put(FessConfig.APP_DIGEST_ALGORISM, "sha256");
             defaultMap
                     .put(FessConfig.JVM_CRAWLER_OPTIONS,
-                            "-Djava.awt.headless=true\n-server\n-Xmx512m\n-XX:MaxMetaspaceSize=128m\n-XX:CompressedClassSpaceSize=32m\n-XX:-UseGCOverheadLimit\n-XX:+UseConcMarkSweepGC\n-XX:CMSInitiatingOccupancyFraction=75\n-XX:+UseParNewGC\n-XX:+UseTLAB\n-XX:+DisableExplicitGC\n-XX:+HeapDumpOnOutOfMemoryError\n-XX:-OmitStackTraceInFastThrow\n-Djcifs.smb.client.connTimeout=60000\n-Djcifs.smb.client.soTimeout=35000\n-Djcifs.smb.client.responseTimeout=30000\n-Dgroovy.use.classvalue=true\n");
+                            "-Djava.awt.headless=true\n-Dfile.encoding=UTF-8\n-Djna.nosys=true\n-Djdk.io.permissionsUseCanonicalPath=true\n-server\n-Xmx512m\n-XX:MaxMetaspaceSize=128m\n-XX:CompressedClassSpaceSize=32m\n-XX:-UseGCOverheadLimit\n-XX:+UseConcMarkSweepGC\n-XX:CMSInitiatingOccupancyFraction=75\n-XX:+UseCMSInitiatingOccupancyOnly\n-XX:+UseTLAB\n-XX:+DisableExplicitGC\n-XX:+HeapDumpOnOutOfMemoryError\n-XX:-OmitStackTraceInFastThrow\n-Djcifs.smb.client.connTimeout=60000\n-Djcifs.smb.client.soTimeout=35000\n-Djcifs.smb.client.responseTimeout=30000\n-Dgroovy.use.classvalue=true\n-Dio.netty.noUnsafe=true\n-Dio.netty.noKeySetOptimization=true\n-Dio.netty.recycler.maxCapacityPerThread=0\n-Dlog4j.shutdownHookEnabled=false\n-Dlog4j2.disable.jmx=true\n-Dlog4j.skipJansi=true\n");
             defaultMap
                     .put(FessConfig.JVM_SUGGEST_OPTIONS,
-                            "-Djava.awt.headless=true\n-server\n-Xmx256m\n-XX:MaxMetaspaceSize=128m\n-XX:CompressedClassSpaceSize=32m\n-XX:-UseGCOverheadLimit\n-XX:+UseConcMarkSweepGC\n-XX:CMSInitiatingOccupancyFraction=75\n-XX:+UseParNewGC\n-XX:+UseTLAB\n-XX:+DisableExplicitGC\n-XX:+HeapDumpOnOutOfMemoryError\n-Dgroovy.use.classvalue=true\n");
+                            "-Djava.awt.headless=true\n-Dfile.encoding=UTF-8\n-Djna.nosys=true\n-Djdk.io.permissionsUseCanonicalPath=true\n-server\n-Xmx256m\n-XX:MaxMetaspaceSize=128m\n-XX:CompressedClassSpaceSize=32m\n-XX:-UseGCOverheadLimit\n-XX:+UseConcMarkSweepGC\n-XX:CMSInitiatingOccupancyFraction=75\n-XX:+UseCMSInitiatingOccupancyOnly\n-XX:+UseTLAB\n-XX:+DisableExplicitGC\n-XX:+HeapDumpOnOutOfMemoryError\n-Dgroovy.use.classvalue=true\n-Dio.netty.noUnsafe=true\n-Dio.netty.noKeySetOptimization=true\n-Dio.netty.recycler.maxCapacityPerThread=0\n-Dlog4j.shutdownHookEnabled=false\n-Dlog4j2.disable.jmx=true\n-Dlog4j.skipJansi=true\n");
             defaultMap.put(FessConfig.JOB_SYSTEM_JOB_IDS, "default_crawler");
             defaultMap.put(FessConfig.JOB_TEMPLATE_TITLE_WEB, "Web Crawler - {0}");
             defaultMap.put(FessConfig.JOB_TEMPLATE_TITLE_FILE, "File Crawler - {0}");
@@ -7549,6 +7611,7 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
             defaultMap.put(FessConfig.CRAWLER_IGNORE_META_ROBOTS, "false");
             defaultMap.put(FessConfig.CRAWLER_IGNORE_CONTENT_EXCEPTION, "true");
             defaultMap.put(FessConfig.CRAWLER_FAILURE_URL_STATUS_CODES, "404");
+            defaultMap.put(FessConfig.CRAWLER_SYSTEM_MONITOR_INTERVAL, "60");
             defaultMap.put(FessConfig.CRAWLER_METADATA_CONTENT_EXCLUDES, "resourceName,X-Parsed-By,Content-Encoding.*,Content-Type.*");
             defaultMap.put(FessConfig.CRAWLER_METADATA_NAME_MAPPING, "title=title:string\nTitle=title:string\n");
             defaultMap.put(FessConfig.CRAWLER_DOCUMENT_HTML_CONTENT_XPATH, "//BODY");

src/main/java/org/codelibs/fess/timer/SystemMonitorTarget.java

@@ -0,0 +1,106 @@
+/*
+ * Copyright 2012-2017 CodeLibs Project and the Others.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+ * either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+package org.codelibs.fess.timer;
+
+import java.util.Arrays;
+import java.util.function.Supplier;
+
+import org.apache.commons.text.StringEscapeUtils;
+import org.codelibs.core.timer.TimeoutTarget;
+import org.elasticsearch.monitor.os.OsProbe;
+import org.elasticsearch.monitor.os.OsStats;
+import org.elasticsearch.monitor.process.ProcessProbe;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class SystemMonitorTarget implements TimeoutTarget {
+    private static final Logger logger = LoggerFactory.getLogger(SystemMonitorTarget.class);
+
+    protected StringBuilder append(final StringBuilder buf, final String key, final Supplier<Object> supplier) {
+        buf.append('"').append(key).append("\":");
+        try {
+            final Object value = supplier.get();
+            if (value == null) {
+                buf.append("null");
+            } else if (value instanceof Long) {
+                buf.append(((Long) value).longValue());
+            } else if (value instanceof Short) {
+                buf.append(((Short) value).shortValue());
+            } else if (value instanceof double[]) {
+                buf.append(Arrays.toString((double[]) value));
+            } else {
+                buf.append('"').append(StringEscapeUtils.escapeJson(value.toString())).append('"');
+            }
+        } catch (final Exception e) {
+            buf.append("null");
+        }
+        return buf;
+    }
+
+    @Override
+    public void expired() {
+        StringBuilder buf = new StringBuilder();
+
+        buf.append("[SYSTEM MONITOR] ");
+        buf.append('{');
+
+        buf.append("\"os\":{");
+        final OsProbe osProbe = OsProbe.getInstance();
+        buf.append("\"memory\":{");
+        buf.append("\"physical\":{");
+        append(buf, "free", () -> osProbe.getFreePhysicalMemorySize()).append(',');
+        append(buf, "total", () -> osProbe.getTotalPhysicalMemorySize());
+        buf.append("},");
+        buf.append("\"swap_space\":{");
+        append(buf, "free", () -> osProbe.getFreeSwapSpaceSize()).append(',');
+        append(buf, "total", () -> osProbe.getTotalSwapSpaceSize());
+        buf.append('}');
+        buf.append("},");
+        buf.append("\"cpu\":{");
+        append(buf, "percent", () -> osProbe.getSystemCpuPercent());
+        OsStats osStats = osProbe.osStats();
+        buf.append("},");
+        append(buf, "load_averages", () -> osStats.getCpu().getLoadAverage());
+        buf.append("},");
+
+        buf.append("\"process\":{");
+        final ProcessProbe processProbe = ProcessProbe.getInstance();
+        buf.append("\"file_descriptor\":{");
+        append(buf, "open", () -> processProbe.getOpenFileDescriptorCount()).append(',');
+        append(buf, "max", () -> processProbe.getMaxFileDescriptorCount());
+        buf.append("},");
+        buf.append("\"cpu\":{");
+        append(buf, "percent", () -> processProbe.getProcessCpuPercent()).append(',');
+        append(buf, "time", () -> processProbe.getProcessCpuTotalTime());
+        buf.append("},");
+        buf.append("\"virtual_memory\":{");
+        append(buf, "total", () -> processProbe.getTotalVirtualMemorySize());
+        buf.append('}');
+        buf.append("},");
+
+        buf.append("\"jvm\":{");
+        final Runtime runtime = Runtime.getRuntime();
+        buf.append("\"memory\":{");
+        append(buf, "free", () -> runtime.freeMemory()).append(',');
+        append(buf, "max", () -> runtime.maxMemory()).append(',');
+        append(buf, "total", () -> runtime.totalMemory());
+        buf.append('}');
+        buf.append('}');
+        buf.append('}');
+
+        logger.info(buf.toString());
+    }
+}

src/main/resources/fess_config.properties

@@ -127,6 +127,7 @@ crawler.ignore.robots.txt=false
 crawler.ignore.meta.robots=false
 crawler.ignore.content.exception=true
 crawler.failure.url.status.codes=404
+crawler.system.monitor.interval=60
 crawler.metadata.content.excludes=resourceName,X-Parsed-By,Content-Encoding.*,Content-Type.*
 crawler.metadata.name.mapping=\
 title=title:string\n\