0ct0pu5/fess
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix #1308 add system monitoring logs

Browse source
This commit is contained in:
Shinsuke Sugaya 2017-10-14 05:51:22 +09:00
parent c68d8967ad
commit fd9e822302
6 changed files with 247 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
src/main/java/org/codelibs/fess/exec/Crawler.java
View file

@ -36,6 +36,8 @@ import javax.annotation.Resource;
import org.codelibs.core.CoreLibConstants;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.core.misc.DynamicProperties;
import org.codelibs.core.timer.TimeoutManager;
import org.codelibs.core.timer.TimeoutTask;
import org.codelibs.fess.Constants;
import org.codelibs.fess.app.service.CrawlingInfoService;
import org.codelibs.fess.app.service.PathMappingService;
@ -49,7 +51,11 @@ import org.codelibs.fess.helper.PathMappingHelper;
import org.codelibs.fess.helper.WebFsIndexHelper;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.mylasta.mail.CrawlerPostcard;
import org.codelibs.fess.timer.SystemMonitorTarget;
import org.codelibs.fess.util.ComponentUtil;
import org.elasticsearch.monitor.jvm.JvmInfo;
import org.elasticsearch.monitor.os.OsProbe;
import org.elasticsearch.monitor.process.ProcessProbe;
import org.kohsuke.args4j.CmdLineException;
import org.kohsuke.args4j.CmdLineParser;
import org.kohsuke.args4j.Option;
@ -161,6 +167,13 @@ public class Crawler {
}
static void initializeProbes() {
// Force probes to be loaded
ProcessProbe.getInstance();
OsProbe.getInstance();
JvmInfo.jvmInfo();
}
public static void main(final String[] args) {
final Options options = new Options();
@ -185,6 +198,8 @@ public class Crawler {
}
}
initializeProbes();
final String transportAddresses = System.getProperty(Constants.FESS_ES_TRANSPORT_ADDRESSES);
if (StringUtil.isNotBlank(transportAddresses)) {
System.setProperty(EsClient.TRANSPORT_ADDRESSES, transportAddresses);
@ -194,6 +209,7 @@ public class Crawler {
System.setProperty(EsClient.CLUSTER_NAME, clusterName);
}
TimeoutTask systemMonitorTask = null;
int exitCode;
try {
running.set(true);
@ -211,6 +227,10 @@ public class Crawler {
};
Runtime.getRuntime().addShutdownHook(shutdownCallback);
systemMonitorTask =
TimeoutManager.getInstance().addTimeoutTarget(new SystemMonitorTarget(),
ComponentUtil.getFessConfig().getCrawlerSystemMonitorIntervalAsInteger(), true);
exitCode = process(options);
} catch (final ContainerNotAvailableException e) {
if (logger.isDebugEnabled()) {
@ -223,6 +243,9 @@ public class Crawler {
logger.error("Crawler does not work correctly.", t);
exitCode = Constants.EXIT_FAIL;
} finally {
if (systemMonitorTask != null) {
systemMonitorTask.cancel();
}
destroyContainer();
}

45
src/main/java/org/codelibs/fess/mylasta/action/FessHtmlPath.java
View file

@ -388,6 +388,51 @@ public interface FessHtmlPath {
/** The path of the HTML: /index.jsp */
HtmlNext path_IndexJsp = new HtmlNext("/index.jsp");
/** The path of the HTML: /local/error/badRequest.jsp */
HtmlNext path_LocalError_BadRequestJsp = new HtmlNext("/local/error/badRequest.jsp");
/** The path of the HTML: /local/error/error.jsp */
HtmlNext path_LocalError_ErrorJsp = new HtmlNext("/local/error/error.jsp");
/** The path of the HTML: /local/error/notFound.jsp */
HtmlNext path_LocalError_NotFoundJsp = new HtmlNext("/local/error/notFound.jsp");
/** The path of the HTML: /local/error/redirect.jsp */
HtmlNext path_LocalError_RedirectJsp = new HtmlNext("/local/error/redirect.jsp");
/** The path of the HTML: /local/error/system.jsp */
HtmlNext path_LocalError_SystemJsp = new HtmlNext("/local/error/system.jsp");
/** The path of the HTML: /local/footer.jsp */
HtmlNext path_Local_FooterJsp = new HtmlNext("/local/footer.jsp");
/** The path of the HTML: /local/header.jsp */
HtmlNext path_Local_HeaderJsp = new HtmlNext("/local/header.jsp");
/** The path of the HTML: /local/help.jsp */
HtmlNext path_Local_HelpJsp = new HtmlNext("/local/help.jsp");
/** The path of the HTML: /local/index.jsp */
HtmlNext path_Local_IndexJsp = new HtmlNext("/local/index.jsp");
/** The path of the HTML: /local/login/index.jsp */
HtmlNext path_LocalLogin_IndexJsp = new HtmlNext("/local/login/index.jsp");
/** The path of the HTML: /local/profile/index.jsp */
HtmlNext path_LocalProfile_IndexJsp = new HtmlNext("/local/profile/index.jsp");
/** The path of the HTML: /local/search.jsp */
HtmlNext path_Local_SearchJsp = new HtmlNext("/local/search.jsp");
/** The path of the HTML: /local/searchNoResult.jsp */
HtmlNext path_Local_SearchNoResultJsp = new HtmlNext("/local/searchNoResult.jsp");
/** The path of the HTML: /local/searchOptions.jsp */
HtmlNext path_Local_SearchOptionsJsp = new HtmlNext("/local/searchOptions.jsp");
/** The path of the HTML: /local/searchResults.jsp */
HtmlNext path_Local_SearchResultsJsp = new HtmlNext("/local/searchResults.jsp");
/** The path of the HTML: /login/index.jsp */
HtmlNext path_Login_IndexJsp = new HtmlNext("/login/index.jsp");

3
src/main/java/org/codelibs/fess/mylasta/action/FessLabels.java
View file

@ -827,6 +827,9 @@ public class FessLabels extends UserMessages {
/** The key of the message: WAV Audio */
public static final String LABELS_facet_filetype_wav = "{labels.facet_filetype_wav}";
/** The key of the message: WebP Audio */
public static final String LABELS_facet_filetype_webp = "{labels.facet_filetype_webp}";
/** The key of the message: AIF Audio */
public static final String LABELS_facet_filetype_aif = "{labels.facet_filetype_aif}";

75
src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java
View file

@ -50,6 +50,9 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
String APP_DIGEST_ALGORISM = "app.digest.algorism";
/** The key of the configuration. e.g. -Djava.awt.headless=true
-Dfile.encoding=UTF-8
-Djna.nosys=true
-Djdk.io.permissionsUseCanonicalPath=true
-server
-Xmx512m
-XX:MaxMetaspaceSize=128m
@ -57,7 +60,7 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
-XX:-UseGCOverheadLimit
-XX:+UseConcMarkSweepGC
-XX:CMSInitiatingOccupancyFraction=75
-XX:+UseParNewGC
-XX:+UseCMSInitiatingOccupancyOnly
-XX:+UseTLAB
-XX:+DisableExplicitGC
-XX:+HeapDumpOnOutOfMemoryError
@ -66,10 +69,19 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
-Djcifs.smb.client.soTimeout=35000
-Djcifs.smb.client.responseTimeout=30000
-Dgroovy.use.classvalue=true
-Dio.netty.noUnsafe=true
-Dio.netty.noKeySetOptimization=true
-Dio.netty.recycler.maxCapacityPerThread=0
-Dlog4j.shutdownHookEnabled=false
-Dlog4j2.disable.jmx=true
-Dlog4j.skipJansi=true
*/
String JVM_CRAWLER_OPTIONS = "jvm.crawler.options";
/** The key of the configuration. e.g. -Djava.awt.headless=true
-Dfile.encoding=UTF-8
-Djna.nosys=true
-Djdk.io.permissionsUseCanonicalPath=true
-server
-Xmx256m
-XX:MaxMetaspaceSize=128m
@ -77,11 +89,17 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
-XX:-UseGCOverheadLimit
-XX:+UseConcMarkSweepGC
-XX:CMSInitiatingOccupancyFraction=75
-XX:+UseParNewGC
-XX:+UseCMSInitiatingOccupancyOnly
-XX:+UseTLAB
-XX:+DisableExplicitGC
-XX:+HeapDumpOnOutOfMemoryError
-Dgroovy.use.classvalue=true
-Dio.netty.noUnsafe=true
-Dio.netty.noKeySetOptimization=true
-Dio.netty.recycler.maxCapacityPerThread=0
-Dlog4j.shutdownHookEnabled=false
-Dlog4j2.disable.jmx=true
-Dlog4j.skipJansi=true
*/
String JVM_SUGGEST_OPTIONS = "jvm.suggest.options";
@ -199,6 +217,9 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
/** The key of the configuration. e.g. 404 */
String CRAWLER_FAILURE_URL_STATUS_CODES = "crawler.failure.url.status.codes";
/** The key of the configuration. e.g. 60 */
String CRAWLER_SYSTEM_MONITOR_INTERVAL = "crawler.system.monitor.interval";
/** The key of the configuration. e.g. resourceName,X-Parsed-By,Content-Encoding.*,Content-Type.* */
String CRAWLER_METADATA_CONTENT_EXCLUDES = "crawler.metadata.content.excludes";
@ -1360,6 +1381,9 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
/**
* Get the value for the key 'jvm.crawler.options'. <br>
* The value is, e.g. -Djava.awt.headless=true
-Dfile.encoding=UTF-8
-Djna.nosys=true
-Djdk.io.permissionsUseCanonicalPath=true
-server
-Xmx512m
-XX:MaxMetaspaceSize=128m
@ -1367,7 +1391,7 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
-XX:-UseGCOverheadLimit
-XX:+UseConcMarkSweepGC
-XX:CMSInitiatingOccupancyFraction=75
-XX:+UseParNewGC
-XX:+UseCMSInitiatingOccupancyOnly
-XX:+UseTLAB
-XX:+DisableExplicitGC
-XX:+HeapDumpOnOutOfMemoryError
@ -1376,6 +1400,12 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
-Djcifs.smb.client.soTimeout=35000
-Djcifs.smb.client.responseTimeout=30000
-Dgroovy.use.classvalue=true
-Dio.netty.noUnsafe=true
-Dio.netty.noKeySetOptimization=true
-Dio.netty.recycler.maxCapacityPerThread=0
-Dlog4j.shutdownHookEnabled=false
-Dlog4j2.disable.jmx=true
-Dlog4j.skipJansi=true
<br>
* comment: JVM options
* @return The value of found property. (NotNull: if not found, exception but basically no way)
@ -1385,6 +1415,9 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
/**
* Get the value for the key 'jvm.suggest.options'. <br>
* The value is, e.g. -Djava.awt.headless=true
-Dfile.encoding=UTF-8
-Djna.nosys=true
-Djdk.io.permissionsUseCanonicalPath=true
-server
-Xmx256m
-XX:MaxMetaspaceSize=128m
@ -1392,11 +1425,17 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
-XX:-UseGCOverheadLimit
-XX:+UseConcMarkSweepGC
-XX:CMSInitiatingOccupancyFraction=75
-XX:+UseParNewGC
-XX:+UseCMSInitiatingOccupancyOnly
-XX:+UseTLAB
-XX:+DisableExplicitGC
-XX:+HeapDumpOnOutOfMemoryError
-Dgroovy.use.classvalue=true
-Dio.netty.noUnsafe=true
-Dio.netty.noKeySetOptimization=true
-Dio.netty.recycler.maxCapacityPerThread=0
-Dlog4j.shutdownHookEnabled=false
-Dlog4j2.disable.jmx=true
-Dlog4j.skipJansi=true
<br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
@ -1815,6 +1854,21 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
*/
Integer getCrawlerFailureUrlStatusCodesAsInteger();
/**
* Get the value for the key 'crawler.system.monitor.interval'. <br>
* The value is, e.g. 60 <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getCrawlerSystemMonitorInterval();
/**
* Get the value for the key 'crawler.system.monitor.interval' as {@link Integer}. <br>
* The value is, e.g. 60 <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
* @throws NumberFormatException When the property is not integer.
*/
Integer getCrawlerSystemMonitorIntervalAsInteger();
/**
* Get the value for the key 'crawler.metadata.content.excludes'. <br>
* The value is, e.g. resourceName,X-Parsed-By,Content-Encoding.*,Content-Type.* <br>
@ -5607,6 +5661,14 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
return getAsInteger(FessConfig.CRAWLER_FAILURE_URL_STATUS_CODES);
}
public String getCrawlerSystemMonitorInterval() {
return get(FessConfig.CRAWLER_SYSTEM_MONITOR_INTERVAL);
}
public Integer getCrawlerSystemMonitorIntervalAsInteger() {
return getAsInteger(FessConfig.CRAWLER_SYSTEM_MONITOR_INTERVAL);
}
public String getCrawlerMetadataContentExcludes() {
return get(FessConfig.CRAWLER_METADATA_CONTENT_EXCLUDES);
}
@ -7501,10 +7563,10 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
defaultMap.put(FessConfig.APP_DIGEST_ALGORISM, "sha256");
defaultMap
.put(FessConfig.JVM_CRAWLER_OPTIONS,
"-Djava.awt.headless=true\n-server\n-Xmx512m\n-XX:MaxMetaspaceSize=128m\n-XX:CompressedClassSpaceSize=32m\n-XX:-UseGCOverheadLimit\n-XX:+UseConcMarkSweepGC\n-XX:CMSInitiatingOccupancyFraction=75\n-XX:+UseParNewGC\n-XX:+UseTLAB\n-XX:+DisableExplicitGC\n-XX:+HeapDumpOnOutOfMemoryError\n-XX:-OmitStackTraceInFastThrow\n-Djcifs.smb.client.connTimeout=60000\n-Djcifs.smb.client.soTimeout=35000\n-Djcifs.smb.client.responseTimeout=30000\n-Dgroovy.use.classvalue=true\n");
"-Djava.awt.headless=true\n-Dfile.encoding=UTF-8\n-Djna.nosys=true\n-Djdk.io.permissionsUseCanonicalPath=true\n-server\n-Xmx512m\n-XX:MaxMetaspaceSize=128m\n-XX:CompressedClassSpaceSize=32m\n-XX:-UseGCOverheadLimit\n-XX:+UseConcMarkSweepGC\n-XX:CMSInitiatingOccupancyFraction=75\n-XX:+UseCMSInitiatingOccupancyOnly\n-XX:+UseTLAB\n-XX:+DisableExplicitGC\n-XX:+HeapDumpOnOutOfMemoryError\n-XX:-OmitStackTraceInFastThrow\n-Djcifs.smb.client.connTimeout=60000\n-Djcifs.smb.client.soTimeout=35000\n-Djcifs.smb.client.responseTimeout=30000\n-Dgroovy.use.classvalue=true\n-Dio.netty.noUnsafe=true\n-Dio.netty.noKeySetOptimization=true\n-Dio.netty.recycler.maxCapacityPerThread=0\n-Dlog4j.shutdownHookEnabled=false\n-Dlog4j2.disable.jmx=true\n-Dlog4j.skipJansi=true\n");
defaultMap
.put(FessConfig.JVM_SUGGEST_OPTIONS,
"-Djava.awt.headless=true\n-server\n-Xmx256m\n-XX:MaxMetaspaceSize=128m\n-XX:CompressedClassSpaceSize=32m\n-XX:-UseGCOverheadLimit\n-XX:+UseConcMarkSweepGC\n-XX:CMSInitiatingOccupancyFraction=75\n-XX:+UseParNewGC\n-XX:+UseTLAB\n-XX:+DisableExplicitGC\n-XX:+HeapDumpOnOutOfMemoryError\n-Dgroovy.use.classvalue=true\n");
"-Djava.awt.headless=true\n-Dfile.encoding=UTF-8\n-Djna.nosys=true\n-Djdk.io.permissionsUseCanonicalPath=true\n-server\n-Xmx256m\n-XX:MaxMetaspaceSize=128m\n-XX:CompressedClassSpaceSize=32m\n-XX:-UseGCOverheadLimit\n-XX:+UseConcMarkSweepGC\n-XX:CMSInitiatingOccupancyFraction=75\n-XX:+UseCMSInitiatingOccupancyOnly\n-XX:+UseTLAB\n-XX:+DisableExplicitGC\n-XX:+HeapDumpOnOutOfMemoryError\n-Dgroovy.use.classvalue=true\n-Dio.netty.noUnsafe=true\n-Dio.netty.noKeySetOptimization=true\n-Dio.netty.recycler.maxCapacityPerThread=0\n-Dlog4j.shutdownHookEnabled=false\n-Dlog4j2.disable.jmx=true\n-Dlog4j.skipJansi=true\n");
defaultMap.put(FessConfig.JOB_SYSTEM_JOB_IDS, "default_crawler");
defaultMap.put(FessConfig.JOB_TEMPLATE_TITLE_WEB, "Web Crawler - {0}");
defaultMap.put(FessConfig.JOB_TEMPLATE_TITLE_FILE, "File Crawler - {0}");
@ -7549,6 +7611,7 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
defaultMap.put(FessConfig.CRAWLER_IGNORE_META_ROBOTS, "false");
defaultMap.put(FessConfig.CRAWLER_IGNORE_CONTENT_EXCEPTION, "true");
defaultMap.put(FessConfig.CRAWLER_FAILURE_URL_STATUS_CODES, "404");
defaultMap.put(FessConfig.CRAWLER_SYSTEM_MONITOR_INTERVAL, "60");
defaultMap.put(FessConfig.CRAWLER_METADATA_CONTENT_EXCLUDES, "resourceName,X-Parsed-By,Content-Encoding.*,Content-Type.*");
defaultMap.put(FessConfig.CRAWLER_METADATA_NAME_MAPPING, "title=title:string\nTitle=title:string\n");
defaultMap.put(FessConfig.CRAWLER_DOCUMENT_HTML_CONTENT_XPATH, "//BODY");

106
src/main/java/org/codelibs/fess/timer/SystemMonitorTarget.java Normal file
View file

@ -0,0 +1,106 @@
/*
* Copyright 2012-2017 CodeLibs Project and the Others.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
* either express or implied. See the License for the specific language
* governing permissions and limitations under the License.
*/
package org.codelibs.fess.timer;
import java.util.Arrays;
import java.util.function.Supplier;
import org.apache.commons.text.StringEscapeUtils;
import org.codelibs.core.timer.TimeoutTarget;
import org.elasticsearch.monitor.os.OsProbe;
import org.elasticsearch.monitor.os.OsStats;
import org.elasticsearch.monitor.process.ProcessProbe;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class SystemMonitorTarget implements TimeoutTarget {
private static final Logger logger = LoggerFactory.getLogger(SystemMonitorTarget.class);
protected StringBuilder append(final StringBuilder buf, final String key, final Supplier<Object> supplier) {
buf.append('"').append(key).append("\":");
try {
final Object value = supplier.get();
if (value == null) {
buf.append("null");
} else if (value instanceof Long) {
buf.append(((Long) value).longValue());
} else if (value instanceof Short) {
buf.append(((Short) value).shortValue());
} else if (value instanceof double[]) {
buf.append(Arrays.toString((double[]) value));
} else {
buf.append('"').append(StringEscapeUtils.escapeJson(value.toString())).append('"');
}
} catch (final Exception e) {
buf.append("null");
}
return buf;
}
@Override
public void expired() {
StringBuilder buf = new StringBuilder();
buf.append("[SYSTEM MONITOR] ");
buf.append('{');
buf.append("\"os\":{");
final OsProbe osProbe = OsProbe.getInstance();
buf.append("\"memory\":{");
buf.append("\"physical\":{");
append(buf, "free", () -> osProbe.getFreePhysicalMemorySize()).append(',');
append(buf, "total", () -> osProbe.getTotalPhysicalMemorySize());
buf.append("},");
buf.append("\"swap_space\":{");
append(buf, "free", () -> osProbe.getFreeSwapSpaceSize()).append(',');
append(buf, "total", () -> osProbe.getTotalSwapSpaceSize());
buf.append('}');
buf.append("},");
buf.append("\"cpu\":{");
append(buf, "percent", () -> osProbe.getSystemCpuPercent());
OsStats osStats = osProbe.osStats();
buf.append("},");
append(buf, "load_averages", () -> osStats.getCpu().getLoadAverage());
buf.append("},");
buf.append("\"process\":{");
final ProcessProbe processProbe = ProcessProbe.getInstance();
buf.append("\"file_descriptor\":{");
append(buf, "open", () -> processProbe.getOpenFileDescriptorCount()).append(',');
append(buf, "max", () -> processProbe.getMaxFileDescriptorCount());
buf.append("},");
buf.append("\"cpu\":{");
append(buf, "percent", () -> processProbe.getProcessCpuPercent()).append(',');
append(buf, "time", () -> processProbe.getProcessCpuTotalTime());
buf.append("},");
buf.append("\"virtual_memory\":{");
append(buf, "total", () -> processProbe.getTotalVirtualMemorySize());
buf.append('}');
buf.append("},");
buf.append("\"jvm\":{");
final Runtime runtime = Runtime.getRuntime();
buf.append("\"memory\":{");
append(buf, "free", () -> runtime.freeMemory()).append(',');
append(buf, "max", () -> runtime.maxMemory()).append(',');
append(buf, "total", () -> runtime.totalMemory());
buf.append('}');
buf.append('}');
buf.append('}');
logger.info(buf.toString());
}
}

1
src/main/resources/fess_config.properties
View file

@ -127,6 +127,7 @@ crawler.ignore.robots.txt=false
crawler.ignore.meta.robots=false
crawler.ignore.content.exception=true
crawler.failure.url.status.codes=404
crawler.system.monitor.interval=60
crawler.metadata.content.excludes=resourceName,X-Parsed-By,Content-Encoding.*,Content-Type.*
crawler.metadata.name.mapping=\
title=title:string\n\