diff --git a/src/main/java/org/codelibs/fess/exec/Crawler.java b/src/main/java/org/codelibs/fess/exec/Crawler.java index 82c763b75..7c3978be8 100644 --- a/src/main/java/org/codelibs/fess/exec/Crawler.java +++ b/src/main/java/org/codelibs/fess/exec/Crawler.java @@ -36,6 +36,8 @@ import javax.annotation.Resource; import org.codelibs.core.CoreLibConstants; import org.codelibs.core.lang.StringUtil; import org.codelibs.core.misc.DynamicProperties; +import org.codelibs.core.timer.TimeoutManager; +import org.codelibs.core.timer.TimeoutTask; import org.codelibs.fess.Constants; import org.codelibs.fess.app.service.CrawlingInfoService; import org.codelibs.fess.app.service.PathMappingService; @@ -49,7 +51,11 @@ import org.codelibs.fess.helper.PathMappingHelper; import org.codelibs.fess.helper.WebFsIndexHelper; import org.codelibs.fess.mylasta.direction.FessConfig; import org.codelibs.fess.mylasta.mail.CrawlerPostcard; +import org.codelibs.fess.timer.SystemMonitorTarget; import org.codelibs.fess.util.ComponentUtil; +import org.elasticsearch.monitor.jvm.JvmInfo; +import org.elasticsearch.monitor.os.OsProbe; +import org.elasticsearch.monitor.process.ProcessProbe; import org.kohsuke.args4j.CmdLineException; import org.kohsuke.args4j.CmdLineParser; import org.kohsuke.args4j.Option; @@ -161,6 +167,13 @@ public class Crawler { } + static void initializeProbes() { + // Force probes to be loaded + ProcessProbe.getInstance(); + OsProbe.getInstance(); + JvmInfo.jvmInfo(); + } + public static void main(final String[] args) { final Options options = new Options(); @@ -185,6 +198,8 @@ public class Crawler { } } + initializeProbes(); + final String transportAddresses = System.getProperty(Constants.FESS_ES_TRANSPORT_ADDRESSES); if (StringUtil.isNotBlank(transportAddresses)) { System.setProperty(EsClient.TRANSPORT_ADDRESSES, transportAddresses); @@ -194,6 +209,7 @@ public class Crawler { System.setProperty(EsClient.CLUSTER_NAME, clusterName); } + TimeoutTask systemMonitorTask = null; int exitCode; try { running.set(true); @@ -211,6 +227,10 @@ public class Crawler { }; Runtime.getRuntime().addShutdownHook(shutdownCallback); + systemMonitorTask = + TimeoutManager.getInstance().addTimeoutTarget(new SystemMonitorTarget(), + ComponentUtil.getFessConfig().getCrawlerSystemMonitorIntervalAsInteger(), true); + exitCode = process(options); } catch (final ContainerNotAvailableException e) { if (logger.isDebugEnabled()) { @@ -223,6 +243,9 @@ public class Crawler { logger.error("Crawler does not work correctly.", t); exitCode = Constants.EXIT_FAIL; } finally { + if (systemMonitorTask != null) { + systemMonitorTask.cancel(); + } destroyContainer(); } diff --git a/src/main/java/org/codelibs/fess/mylasta/action/FessHtmlPath.java b/src/main/java/org/codelibs/fess/mylasta/action/FessHtmlPath.java index 21cc4cce7..5cf591ac9 100644 --- a/src/main/java/org/codelibs/fess/mylasta/action/FessHtmlPath.java +++ b/src/main/java/org/codelibs/fess/mylasta/action/FessHtmlPath.java @@ -388,6 +388,51 @@ public interface FessHtmlPath { /** The path of the HTML: /index.jsp */ HtmlNext path_IndexJsp = new HtmlNext("/index.jsp"); + /** The path of the HTML: /local/error/badRequest.jsp */ + HtmlNext path_LocalError_BadRequestJsp = new HtmlNext("/local/error/badRequest.jsp"); + + /** The path of the HTML: /local/error/error.jsp */ + HtmlNext path_LocalError_ErrorJsp = new HtmlNext("/local/error/error.jsp"); + + /** The path of the HTML: /local/error/notFound.jsp */ + HtmlNext path_LocalError_NotFoundJsp = new HtmlNext("/local/error/notFound.jsp"); + + /** The path of the HTML: /local/error/redirect.jsp */ + HtmlNext path_LocalError_RedirectJsp = new HtmlNext("/local/error/redirect.jsp"); + + /** The path of the HTML: /local/error/system.jsp */ + HtmlNext path_LocalError_SystemJsp = new HtmlNext("/local/error/system.jsp"); + + /** The path of the HTML: /local/footer.jsp */ + HtmlNext path_Local_FooterJsp = new HtmlNext("/local/footer.jsp"); + + /** The path of the HTML: /local/header.jsp */ + HtmlNext path_Local_HeaderJsp = new HtmlNext("/local/header.jsp"); + + /** The path of the HTML: /local/help.jsp */ + HtmlNext path_Local_HelpJsp = new HtmlNext("/local/help.jsp"); + + /** The path of the HTML: /local/index.jsp */ + HtmlNext path_Local_IndexJsp = new HtmlNext("/local/index.jsp"); + + /** The path of the HTML: /local/login/index.jsp */ + HtmlNext path_LocalLogin_IndexJsp = new HtmlNext("/local/login/index.jsp"); + + /** The path of the HTML: /local/profile/index.jsp */ + HtmlNext path_LocalProfile_IndexJsp = new HtmlNext("/local/profile/index.jsp"); + + /** The path of the HTML: /local/search.jsp */ + HtmlNext path_Local_SearchJsp = new HtmlNext("/local/search.jsp"); + + /** The path of the HTML: /local/searchNoResult.jsp */ + HtmlNext path_Local_SearchNoResultJsp = new HtmlNext("/local/searchNoResult.jsp"); + + /** The path of the HTML: /local/searchOptions.jsp */ + HtmlNext path_Local_SearchOptionsJsp = new HtmlNext("/local/searchOptions.jsp"); + + /** The path of the HTML: /local/searchResults.jsp */ + HtmlNext path_Local_SearchResultsJsp = new HtmlNext("/local/searchResults.jsp"); + /** The path of the HTML: /login/index.jsp */ HtmlNext path_Login_IndexJsp = new HtmlNext("/login/index.jsp"); diff --git a/src/main/java/org/codelibs/fess/mylasta/action/FessLabels.java b/src/main/java/org/codelibs/fess/mylasta/action/FessLabels.java index 9bf6cd191..a606e287d 100644 --- a/src/main/java/org/codelibs/fess/mylasta/action/FessLabels.java +++ b/src/main/java/org/codelibs/fess/mylasta/action/FessLabels.java @@ -827,6 +827,9 @@ public class FessLabels extends UserMessages { /** The key of the message: WAV Audio */ public static final String LABELS_facet_filetype_wav = "{labels.facet_filetype_wav}"; + /** The key of the message: WebP Audio */ + public static final String LABELS_facet_filetype_webp = "{labels.facet_filetype_webp}"; + /** The key of the message: AIF Audio */ public static final String LABELS_facet_filetype_aif = "{labels.facet_filetype_aif}"; diff --git a/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java b/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java index e044ea037..7a27ce415 100644 --- a/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java +++ b/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java @@ -50,6 +50,9 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction String APP_DIGEST_ALGORISM = "app.digest.algorism"; /** The key of the configuration. e.g. -Djava.awt.headless=true + -Dfile.encoding=UTF-8 + -Djna.nosys=true + -Djdk.io.permissionsUseCanonicalPath=true -server -Xmx512m -XX:MaxMetaspaceSize=128m @@ -57,7 +60,7 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction -XX:-UseGCOverheadLimit -XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=75 - -XX:+UseParNewGC + -XX:+UseCMSInitiatingOccupancyOnly -XX:+UseTLAB -XX:+DisableExplicitGC -XX:+HeapDumpOnOutOfMemoryError @@ -66,10 +69,19 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction -Djcifs.smb.client.soTimeout=35000 -Djcifs.smb.client.responseTimeout=30000 -Dgroovy.use.classvalue=true + -Dio.netty.noUnsafe=true + -Dio.netty.noKeySetOptimization=true + -Dio.netty.recycler.maxCapacityPerThread=0 + -Dlog4j.shutdownHookEnabled=false + -Dlog4j2.disable.jmx=true + -Dlog4j.skipJansi=true */ String JVM_CRAWLER_OPTIONS = "jvm.crawler.options"; /** The key of the configuration. e.g. -Djava.awt.headless=true + -Dfile.encoding=UTF-8 + -Djna.nosys=true + -Djdk.io.permissionsUseCanonicalPath=true -server -Xmx256m -XX:MaxMetaspaceSize=128m @@ -77,11 +89,17 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction -XX:-UseGCOverheadLimit -XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=75 - -XX:+UseParNewGC + -XX:+UseCMSInitiatingOccupancyOnly -XX:+UseTLAB -XX:+DisableExplicitGC -XX:+HeapDumpOnOutOfMemoryError -Dgroovy.use.classvalue=true + -Dio.netty.noUnsafe=true + -Dio.netty.noKeySetOptimization=true + -Dio.netty.recycler.maxCapacityPerThread=0 + -Dlog4j.shutdownHookEnabled=false + -Dlog4j2.disable.jmx=true + -Dlog4j.skipJansi=true */ String JVM_SUGGEST_OPTIONS = "jvm.suggest.options"; @@ -199,6 +217,9 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction /** The key of the configuration. e.g. 404 */ String CRAWLER_FAILURE_URL_STATUS_CODES = "crawler.failure.url.status.codes"; + /** The key of the configuration. e.g. 60 */ + String CRAWLER_SYSTEM_MONITOR_INTERVAL = "crawler.system.monitor.interval"; + /** The key of the configuration. e.g. resourceName,X-Parsed-By,Content-Encoding.*,Content-Type.* */ String CRAWLER_METADATA_CONTENT_EXCLUDES = "crawler.metadata.content.excludes"; @@ -1360,6 +1381,9 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction /** * Get the value for the key 'jvm.crawler.options'.
* The value is, e.g. -Djava.awt.headless=true + -Dfile.encoding=UTF-8 + -Djna.nosys=true + -Djdk.io.permissionsUseCanonicalPath=true -server -Xmx512m -XX:MaxMetaspaceSize=128m @@ -1367,7 +1391,7 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction -XX:-UseGCOverheadLimit -XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=75 - -XX:+UseParNewGC + -XX:+UseCMSInitiatingOccupancyOnly -XX:+UseTLAB -XX:+DisableExplicitGC -XX:+HeapDumpOnOutOfMemoryError @@ -1376,6 +1400,12 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction -Djcifs.smb.client.soTimeout=35000 -Djcifs.smb.client.responseTimeout=30000 -Dgroovy.use.classvalue=true + -Dio.netty.noUnsafe=true + -Dio.netty.noKeySetOptimization=true + -Dio.netty.recycler.maxCapacityPerThread=0 + -Dlog4j.shutdownHookEnabled=false + -Dlog4j2.disable.jmx=true + -Dlog4j.skipJansi=true
* comment: JVM options * @return The value of found property. (NotNull: if not found, exception but basically no way) @@ -1385,6 +1415,9 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction /** * Get the value for the key 'jvm.suggest.options'.
* The value is, e.g. -Djava.awt.headless=true + -Dfile.encoding=UTF-8 + -Djna.nosys=true + -Djdk.io.permissionsUseCanonicalPath=true -server -Xmx256m -XX:MaxMetaspaceSize=128m @@ -1392,11 +1425,17 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction -XX:-UseGCOverheadLimit -XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=75 - -XX:+UseParNewGC + -XX:+UseCMSInitiatingOccupancyOnly -XX:+UseTLAB -XX:+DisableExplicitGC -XX:+HeapDumpOnOutOfMemoryError -Dgroovy.use.classvalue=true + -Dio.netty.noUnsafe=true + -Dio.netty.noKeySetOptimization=true + -Dio.netty.recycler.maxCapacityPerThread=0 + -Dlog4j.shutdownHookEnabled=false + -Dlog4j2.disable.jmx=true + -Dlog4j.skipJansi=true
* @return The value of found property. (NotNull: if not found, exception but basically no way) */ @@ -1815,6 +1854,21 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction */ Integer getCrawlerFailureUrlStatusCodesAsInteger(); + /** + * Get the value for the key 'crawler.system.monitor.interval'.
+ * The value is, e.g. 60
+ * @return The value of found property. (NotNull: if not found, exception but basically no way) + */ + String getCrawlerSystemMonitorInterval(); + + /** + * Get the value for the key 'crawler.system.monitor.interval' as {@link Integer}.
+ * The value is, e.g. 60
+ * @return The value of found property. (NotNull: if not found, exception but basically no way) + * @throws NumberFormatException When the property is not integer. + */ + Integer getCrawlerSystemMonitorIntervalAsInteger(); + /** * Get the value for the key 'crawler.metadata.content.excludes'.
* The value is, e.g. resourceName,X-Parsed-By,Content-Encoding.*,Content-Type.*
@@ -5607,6 +5661,14 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction return getAsInteger(FessConfig.CRAWLER_FAILURE_URL_STATUS_CODES); } + public String getCrawlerSystemMonitorInterval() { + return get(FessConfig.CRAWLER_SYSTEM_MONITOR_INTERVAL); + } + + public Integer getCrawlerSystemMonitorIntervalAsInteger() { + return getAsInteger(FessConfig.CRAWLER_SYSTEM_MONITOR_INTERVAL); + } + public String getCrawlerMetadataContentExcludes() { return get(FessConfig.CRAWLER_METADATA_CONTENT_EXCLUDES); } @@ -7501,10 +7563,10 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction defaultMap.put(FessConfig.APP_DIGEST_ALGORISM, "sha256"); defaultMap .put(FessConfig.JVM_CRAWLER_OPTIONS, - "-Djava.awt.headless=true\n-server\n-Xmx512m\n-XX:MaxMetaspaceSize=128m\n-XX:CompressedClassSpaceSize=32m\n-XX:-UseGCOverheadLimit\n-XX:+UseConcMarkSweepGC\n-XX:CMSInitiatingOccupancyFraction=75\n-XX:+UseParNewGC\n-XX:+UseTLAB\n-XX:+DisableExplicitGC\n-XX:+HeapDumpOnOutOfMemoryError\n-XX:-OmitStackTraceInFastThrow\n-Djcifs.smb.client.connTimeout=60000\n-Djcifs.smb.client.soTimeout=35000\n-Djcifs.smb.client.responseTimeout=30000\n-Dgroovy.use.classvalue=true\n"); + "-Djava.awt.headless=true\n-Dfile.encoding=UTF-8\n-Djna.nosys=true\n-Djdk.io.permissionsUseCanonicalPath=true\n-server\n-Xmx512m\n-XX:MaxMetaspaceSize=128m\n-XX:CompressedClassSpaceSize=32m\n-XX:-UseGCOverheadLimit\n-XX:+UseConcMarkSweepGC\n-XX:CMSInitiatingOccupancyFraction=75\n-XX:+UseCMSInitiatingOccupancyOnly\n-XX:+UseTLAB\n-XX:+DisableExplicitGC\n-XX:+HeapDumpOnOutOfMemoryError\n-XX:-OmitStackTraceInFastThrow\n-Djcifs.smb.client.connTimeout=60000\n-Djcifs.smb.client.soTimeout=35000\n-Djcifs.smb.client.responseTimeout=30000\n-Dgroovy.use.classvalue=true\n-Dio.netty.noUnsafe=true\n-Dio.netty.noKeySetOptimization=true\n-Dio.netty.recycler.maxCapacityPerThread=0\n-Dlog4j.shutdownHookEnabled=false\n-Dlog4j2.disable.jmx=true\n-Dlog4j.skipJansi=true\n"); defaultMap .put(FessConfig.JVM_SUGGEST_OPTIONS, - "-Djava.awt.headless=true\n-server\n-Xmx256m\n-XX:MaxMetaspaceSize=128m\n-XX:CompressedClassSpaceSize=32m\n-XX:-UseGCOverheadLimit\n-XX:+UseConcMarkSweepGC\n-XX:CMSInitiatingOccupancyFraction=75\n-XX:+UseParNewGC\n-XX:+UseTLAB\n-XX:+DisableExplicitGC\n-XX:+HeapDumpOnOutOfMemoryError\n-Dgroovy.use.classvalue=true\n"); + "-Djava.awt.headless=true\n-Dfile.encoding=UTF-8\n-Djna.nosys=true\n-Djdk.io.permissionsUseCanonicalPath=true\n-server\n-Xmx256m\n-XX:MaxMetaspaceSize=128m\n-XX:CompressedClassSpaceSize=32m\n-XX:-UseGCOverheadLimit\n-XX:+UseConcMarkSweepGC\n-XX:CMSInitiatingOccupancyFraction=75\n-XX:+UseCMSInitiatingOccupancyOnly\n-XX:+UseTLAB\n-XX:+DisableExplicitGC\n-XX:+HeapDumpOnOutOfMemoryError\n-Dgroovy.use.classvalue=true\n-Dio.netty.noUnsafe=true\n-Dio.netty.noKeySetOptimization=true\n-Dio.netty.recycler.maxCapacityPerThread=0\n-Dlog4j.shutdownHookEnabled=false\n-Dlog4j2.disable.jmx=true\n-Dlog4j.skipJansi=true\n"); defaultMap.put(FessConfig.JOB_SYSTEM_JOB_IDS, "default_crawler"); defaultMap.put(FessConfig.JOB_TEMPLATE_TITLE_WEB, "Web Crawler - {0}"); defaultMap.put(FessConfig.JOB_TEMPLATE_TITLE_FILE, "File Crawler - {0}"); @@ -7549,6 +7611,7 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction defaultMap.put(FessConfig.CRAWLER_IGNORE_META_ROBOTS, "false"); defaultMap.put(FessConfig.CRAWLER_IGNORE_CONTENT_EXCEPTION, "true"); defaultMap.put(FessConfig.CRAWLER_FAILURE_URL_STATUS_CODES, "404"); + defaultMap.put(FessConfig.CRAWLER_SYSTEM_MONITOR_INTERVAL, "60"); defaultMap.put(FessConfig.CRAWLER_METADATA_CONTENT_EXCLUDES, "resourceName,X-Parsed-By,Content-Encoding.*,Content-Type.*"); defaultMap.put(FessConfig.CRAWLER_METADATA_NAME_MAPPING, "title=title:string\nTitle=title:string\n"); defaultMap.put(FessConfig.CRAWLER_DOCUMENT_HTML_CONTENT_XPATH, "//BODY"); diff --git a/src/main/java/org/codelibs/fess/timer/SystemMonitorTarget.java b/src/main/java/org/codelibs/fess/timer/SystemMonitorTarget.java new file mode 100644 index 000000000..990b94f99 --- /dev/null +++ b/src/main/java/org/codelibs/fess/timer/SystemMonitorTarget.java @@ -0,0 +1,106 @@ +/* + * Copyright 2012-2017 CodeLibs Project and the Others. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ +package org.codelibs.fess.timer; + +import java.util.Arrays; +import java.util.function.Supplier; + +import org.apache.commons.text.StringEscapeUtils; +import org.codelibs.core.timer.TimeoutTarget; +import org.elasticsearch.monitor.os.OsProbe; +import org.elasticsearch.monitor.os.OsStats; +import org.elasticsearch.monitor.process.ProcessProbe; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class SystemMonitorTarget implements TimeoutTarget { + private static final Logger logger = LoggerFactory.getLogger(SystemMonitorTarget.class); + + protected StringBuilder append(final StringBuilder buf, final String key, final Supplier supplier) { + buf.append('"').append(key).append("\":"); + try { + final Object value = supplier.get(); + if (value == null) { + buf.append("null"); + } else if (value instanceof Long) { + buf.append(((Long) value).longValue()); + } else if (value instanceof Short) { + buf.append(((Short) value).shortValue()); + } else if (value instanceof double[]) { + buf.append(Arrays.toString((double[]) value)); + } else { + buf.append('"').append(StringEscapeUtils.escapeJson(value.toString())).append('"'); + } + } catch (final Exception e) { + buf.append("null"); + } + return buf; + } + + @Override + public void expired() { + StringBuilder buf = new StringBuilder(); + + buf.append("[SYSTEM MONITOR] "); + buf.append('{'); + + buf.append("\"os\":{"); + final OsProbe osProbe = OsProbe.getInstance(); + buf.append("\"memory\":{"); + buf.append("\"physical\":{"); + append(buf, "free", () -> osProbe.getFreePhysicalMemorySize()).append(','); + append(buf, "total", () -> osProbe.getTotalPhysicalMemorySize()); + buf.append("},"); + buf.append("\"swap_space\":{"); + append(buf, "free", () -> osProbe.getFreeSwapSpaceSize()).append(','); + append(buf, "total", () -> osProbe.getTotalSwapSpaceSize()); + buf.append('}'); + buf.append("},"); + buf.append("\"cpu\":{"); + append(buf, "percent", () -> osProbe.getSystemCpuPercent()); + OsStats osStats = osProbe.osStats(); + buf.append("},"); + append(buf, "load_averages", () -> osStats.getCpu().getLoadAverage()); + buf.append("},"); + + buf.append("\"process\":{"); + final ProcessProbe processProbe = ProcessProbe.getInstance(); + buf.append("\"file_descriptor\":{"); + append(buf, "open", () -> processProbe.getOpenFileDescriptorCount()).append(','); + append(buf, "max", () -> processProbe.getMaxFileDescriptorCount()); + buf.append("},"); + buf.append("\"cpu\":{"); + append(buf, "percent", () -> processProbe.getProcessCpuPercent()).append(','); + append(buf, "time", () -> processProbe.getProcessCpuTotalTime()); + buf.append("},"); + buf.append("\"virtual_memory\":{"); + append(buf, "total", () -> processProbe.getTotalVirtualMemorySize()); + buf.append('}'); + buf.append("},"); + + buf.append("\"jvm\":{"); + final Runtime runtime = Runtime.getRuntime(); + buf.append("\"memory\":{"); + append(buf, "free", () -> runtime.freeMemory()).append(','); + append(buf, "max", () -> runtime.maxMemory()).append(','); + append(buf, "total", () -> runtime.totalMemory()); + buf.append('}'); + buf.append('}'); + buf.append('}'); + + logger.info(buf.toString()); + } +} diff --git a/src/main/resources/fess_config.properties b/src/main/resources/fess_config.properties index 1953dc8e7..d3ccb45e4 100644 --- a/src/main/resources/fess_config.properties +++ b/src/main/resources/fess_config.properties @@ -127,6 +127,7 @@ crawler.ignore.robots.txt=false crawler.ignore.meta.robots=false crawler.ignore.content.exception=true crawler.failure.url.status.codes=404 +crawler.system.monitor.interval=60 crawler.metadata.content.excludes=resourceName,X-Parsed-By,Content-Encoding.*,Content-Type.* crawler.metadata.name.mapping=\ title=title:string\n\