0ct0pu5/fess
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix #1310 print thread dump on crawler

Browse source
This commit is contained in:
Shinsuke Sugaya 2017-10-14 18:27:51 +09:00
parent 1466e9ac4d
commit 685c2de26d
17 changed files with 197 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/main/java/org/codelibs/fess/Constants.java
View file

@ -416,4 +416,6 @@ public class Constants extends CoreLibConstants {
public static final String SYSTEM_USER = "system";
public static final String EMPTY_USER_ID = "<empty>";
public static final String CRAWLER_PROCESS_COMMAND_THREAD_DUMP = "thread_dump";
}

40
src/main/java/org/codelibs/fess/app/web/admin/crawlinginfo/AdminCrawlinginfoAction.java
View file

@ -28,6 +28,8 @@ import org.lastaflute.web.Execute;
import org.lastaflute.web.response.HtmlResponse;
import org.lastaflute.web.response.render.RenderData;
import org.lastaflute.web.ruts.process.ActionRuntime;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* @author shinsuke
@ -35,6 +37,8 @@ import org.lastaflute.web.ruts.process.ActionRuntime;
*/
public class AdminCrawlinginfoAction extends FessAdminAction {
private static final Logger logger = LoggerFactory.getLogger(AdminCrawlinginfoAction.class);
// ===================================================================================
// Attribute
// =========
@ -116,22 +120,44 @@ public class AdminCrawlinginfoAction extends FessAdminAction {
public HtmlResponse details(final int crudMode, final String id) {
verifyCrudMode(crudMode, CrudMode.DETAILS);
saveToken();
return asHtml(path_AdminCrawlinginfo_AdminCrawlinginfoDetailsJsp).useForm(EditForm.class, op -> {
op.setup(form -> {
crawlingInfoService.getCrawlingInfo(id).ifPresent(entity -> {
return crawlingInfoService.getCrawlingInfo(id).map(entity -> {
return asHtml(path_AdminCrawlinginfo_AdminCrawlinginfoDetailsJsp).useForm(EditForm.class, op -> {
op.setup(form -> {
copyBeanToBean(entity, form, copyOp -> {
copyOp.excludeNull();
});
form.crudMode = crudMode;
}).orElse(() -> {
throwValidationError(messages -> messages.addErrorsCrudCouldNotFindCrudTable(GLOBAL, id), () -> asListHtml());
});
}).renderWith(data -> {
RenderDataUtil.register(data, "crawlingInfoParamItems", crawlingInfoService.getCrawlingInfoParamList(id));
RenderDataUtil.register(data, "running", processHelper.isProcessRunning(entity.getSessionId()));
});
}).renderWith(data -> {
RenderDataUtil.register(data, "crawlingInfoParamItems", crawlingInfoService.getCrawlingInfoParamList(id));
}).orElseGet(() -> {
throwValidationError(messages -> messages.addErrorsCrudCouldNotFindCrudTable(GLOBAL, id), () -> asListHtml());
return null;
});
}
@Execute
public HtmlResponse threaddump(final EditForm form) {
verifyCrudMode(form.crudMode, CrudMode.DETAILS);
validate(form, messages -> {}, () -> asDetailsHtml());
verifyToken(() -> asDetailsHtml());
final String id = form.id;
crawlingInfoService.getCrawlingInfo(id).ifPresent(entity -> {
try {
processHelper.sendCommand(entity.getSessionId(), Constants.CRAWLER_PROCESS_COMMAND_THREAD_DUMP);
saveInfo(messages -> messages.addSuccessPrintThreadDump(GLOBAL));
} catch (Exception e) {
logger.warn("Failed to print a thread dump.", e);
throwValidationError(messages -> messages.addErrorsFailedToPrintThreadDump(GLOBAL), () -> asListHtml());
}
}).orElse(() -> {
throwValidationError(messages -> messages.addErrorsCrudCouldNotFindCrudTable(GLOBAL, id), () -> asListHtml());
});
return redirect(getClass());
}
// -----------------------------------------------------
// Actually Crud
// -------------

4
src/main/java/org/codelibs/fess/exception/JobNotFoundException.java
View file

@ -25,4 +25,8 @@ public class JobNotFoundException extends FessSystemException {
super(scheduledJob.toString());
}
public JobNotFoundException(String message) {
super(message);
}
}

6
src/main/java/org/codelibs/fess/exception/JobProcessingException.java
View file

@ -15,6 +15,8 @@
*/
package org.codelibs.fess.exception;
import java.io.IOException;
public class JobProcessingException extends FessSystemException {
private static final long serialVersionUID = 1L;
@ -23,4 +25,8 @@ public class JobProcessingException extends FessSystemException {
super(e);
}
public JobProcessingException(String message, IOException e) {
super(message, e);
}
}

37
src/main/java/org/codelibs/fess/exec/Crawler.java
View file

@ -17,8 +17,10 @@ package org.codelibs.fess.exec;
import static org.codelibs.core.stream.StreamUtil.stream;
import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
import java.lang.management.ManagementFactory;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
@ -53,6 +55,7 @@ import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.mylasta.mail.CrawlerPostcard;
import org.codelibs.fess.timer.SystemMonitorTarget;
import org.codelibs.fess.util.ComponentUtil;
import org.codelibs.fess.util.ThreadDumpUtil;
import org.elasticsearch.monitor.jvm.JvmInfo;
import org.elasticsearch.monitor.os.OsProbe;
import org.elasticsearch.monitor.process.ProcessProbe;
@ -210,6 +213,7 @@ public class Crawler {
}
TimeoutTask systemMonitorTask = null;
Thread commandThread = null;
int exitCode;
try {
running.set(true);
@ -227,6 +231,36 @@ public class Crawler {
};
Runtime.getRuntime().addShutdownHook(shutdownCallback);
commandThread = new Thread(() -> {
try (BufferedReader reader = new BufferedReader(new InputStreamReader(System.in))) {
String command;
while (true) {
try {
while (!reader.ready()) {
Thread.sleep(1000L);
}
command = reader.readLine().trim();
if (logger.isDebugEnabled()) {
logger.debug("Process command: " + command);
}
if (Constants.CRAWLER_PROCESS_COMMAND_THREAD_DUMP.equals(command)) {
ThreadDumpUtil.printThreadDump();
} else {
logger.warn("Unknown process command: " + command);
}
if (Thread.interrupted()) {
return;
}
} catch (InterruptedException e) {
return;
}
}
} catch (IOException e) {
logger.debug("I/O exception.", e);
}
}, "ProcessCommand");
commandThread.start();
systemMonitorTask =
TimeoutManager.getInstance().addTimeoutTarget(new SystemMonitorTarget(),
ComponentUtil.getFessConfig().getCrawlerSystemMonitorIntervalAsInteger(), true);
@ -243,6 +277,9 @@ public class Crawler {
logger.error("Crawler does not work correctly.", t);
exitCode = Constants.EXIT_FAIL;
} finally {
if (commandThread != null && commandThread.isAlive()) {
commandThread.interrupt();
}
if (systemMonitorTask != null) {
systemMonitorTask.cancel();
}

26
src/main/java/org/codelibs/fess/helper/ProcessHelper.java
View file

@ -16,6 +16,7 @@
package org.codelibs.fess.helper;
import java.io.IOException;
import java.io.OutputStream;
import java.util.List;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
@ -26,7 +27,9 @@ import java.util.function.Consumer;
import javax.annotation.PreDestroy;
import org.apache.commons.io.IOUtils;
import org.codelibs.fess.exception.FessSystemException;
import org.codelibs.fess.Constants;
import org.codelibs.fess.exception.JobNotFoundException;
import org.codelibs.fess.exception.JobProcessingException;
import org.codelibs.fess.util.InputStreamThread;
import org.codelibs.fess.util.JobProcess;
import org.slf4j.Logger;
@ -63,7 +66,7 @@ public class ProcessHelper {
destroyProcess(sessionId, runningProcessMap.putIfAbsent(sessionId, jobProcess));
return jobProcess;
} catch (final IOException e) {
throw new FessSystemException("Crawler Process terminated.", e);
throw new JobProcessingException("Crawler Process terminated.", e);
}
}
@ -76,6 +79,11 @@ public class ProcessHelper {
return !runningProcessMap.isEmpty();
}
public boolean isProcessRunning(String sessionId) {
JobProcess jobProcess = runningProcessMap.get(sessionId);
return jobProcess != null && jobProcess.getProcess().isAlive();
}
protected int destroyProcess(final String sessionId, final JobProcess jobProcess) {
if (jobProcess != null) {
final InputStreamThread ist = jobProcess.getInputStreamThread();
@ -138,4 +146,18 @@ public class ProcessHelper {
this.processDestroyTimeout = processDestroyTimeout;
}
public void sendCommand(String sessionId, String command) {
JobProcess jobProcess = runningProcessMap.get(sessionId);
if (jobProcess != null) {
try {
final OutputStream out = jobProcess.getProcess().getOutputStream();
IOUtils.write(command + "\n", out, Constants.CHARSET_UTF_8);
out.flush();
} catch (IOException e) {
throw new JobProcessingException(e);
}
} else {
throw new JobNotFoundException("Job for " + sessionId + " is not found.");
}
}
}

13
src/main/java/org/codelibs/fess/indexer/IndexUpdater.java
View file

@ -49,6 +49,7 @@ import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.util.ComponentUtil;
import org.codelibs.fess.util.DocList;
import org.codelibs.fess.util.MemoryUtil;
import org.codelibs.fess.util.ThreadDumpUtil;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
@ -270,7 +271,7 @@ public class IndexUpdater extends Thread {
finishCrawling = true;
forceStop();
if (fessConfig.getIndexerThreadDumpEnabledAsBoolean()) {
printThreadDump();
ThreadDumpUtil.printThreadDump();
}
org.codelibs.fess.exec.Crawler.addError("QueueTimeout");
}
@ -313,16 +314,6 @@ public class IndexUpdater extends Thread {
}
private void printThreadDump() {
for (final Map.Entry<Thread, StackTraceElement[]> entry : Thread.getAllStackTraces().entrySet()) {
logger.info("Thread: " + entry.getKey());
final StackTraceElement[] trace = entry.getValue();
for (final StackTraceElement element : trace) {
logger.info("\tat " + element);
}
}
}
private void processAccessResults(final DocList docList, final List<EsAccessResult> accessResultList, final List<EsAccessResult> arList) {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final long maxDocumentRequestSize = fessConfig.getIndexerWebfsMaxDocumentRequestSizeAsInteger().longValue();

3
src/main/java/org/codelibs/fess/mylasta/action/FessLabels.java
View file

@ -1283,6 +1283,9 @@ public class FessLabels extends UserMessages {
/** The key of the message: Cancel */
public static final String LABELS_crawling_info_delete_all_cancel = "{labels.crawling_info_delete_all_cancel}";
/** The key of the message: Thread Dump */
public static final String LABELS_crawling_info_thread_dump = "{labels.crawling_info_thread_dump}";
/** The key of the message: Crawler start time */
public static final String LABELS_crawling_info_CrawlerStartTime = "{labels.crawling_info_CrawlerStartTime}";

34
src/main/java/org/codelibs/fess/mylasta/action/FessMessages.java
View file

@ -329,6 +329,9 @@ public class FessMessages extends FessLabels {
/** The key of the message: Unauthorized request. */
public static final String ERRORS_unauthorized_request = "{errors.unauthorized_request}";
/** The key of the message: Failed to print thread dump. */
public static final String ERRORS_failed_to_print_thread_dump = "{errors.failed_to_print_thread_dump}";
/** The key of the message: The given query has unknown condition. */
public static final String ERRORS_invalid_query_unknown = "{errors.invalid_query_unknown}";
@ -446,6 +449,9 @@ public class FessMessages extends FessLabels {
/** The key of the message: Bulk process is started. */
public static final String SUCCESS_bulk_process_started = "{success.bulk_process_started}";
/** The key of the message: Printed thread dump to log file. */
public static final String SUCCESS_print_thread_dump = "{success.print_thread_dump}";
/** The key of the message: Created data. */
public static final String SUCCESS_crud_create_crud_table = "{success.crud_create_crud_table}";
@ -1913,6 +1919,20 @@ public class FessMessages extends FessLabels {
return this;
}
/**
* Add the created action message for the key 'errors.failed_to_print_thread_dump' with parameters.
* <pre>
* message: Failed to print thread dump.
* </pre>
* @param property The property name for the message. (NotNull)
* @return this. (NotNull)
*/
public FessMessages addErrorsFailedToPrintThreadDump(String property) {
assertPropertyNotNull(property);
add(property, new UserMessage(ERRORS_failed_to_print_thread_dump));
return this;
}
/**
* Add the created action message for the key 'errors.invalid_query_unknown' with parameters.
* <pre>
@ -2480,6 +2500,20 @@ public class FessMessages extends FessLabels {
return this;
}
/**
* Add the created action message for the key 'success.print_thread_dump' with parameters.
* <pre>
* message: Printed thread dump to log file.
* </pre>
* @param property The property name for the message. (NotNull)
* @return this. (NotNull)
*/
public FessMessages addSuccessPrintThreadDump(String property) {
assertPropertyNotNull(property);
add(property, new UserMessage(SUCCESS_print_thread_dump));
return this;
}
/**
* Add the created action message for the key 'success.crud_create_crud_table' with parameters.
* <pre>

36
src/main/java/org/codelibs/fess/util/ThreadDumpUtil.java Normal file
View file

@ -0,0 +1,36 @@
/*
* Copyright 2012-2017 CodeLibs Project and the Others.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
* either express or implied. See the License for the specific language
* governing permissions and limitations under the License.
*/
package org.codelibs.fess.util;
import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class ThreadDumpUtil {
private static final Logger logger = LoggerFactory.getLogger(ThreadDumpUtil.class);
public static void printThreadDump() {
for (final Map.Entry<Thread, StackTraceElement[]> entry : Thread.getAllStackTraces().entrySet()) {
logger.info("Thread: " + entry.getKey());
final StackTraceElement[] trace = entry.getValue();
for (final StackTraceElement element : trace) {
logger.info("\tat " + element);
}
}
}
}

1
src/main/resources/fess_label.properties
View file

@ -418,6 +418,7 @@ labels.crawling_info_created_time=Created
labels.crawling_info_delete_all_link=Delete All
labels.crawling_info_delete_all_confirmation=Do you really want to delete all?
labels.crawling_info_delete_all_cancel=Cancel
labels.crawling_info_thread_dump=Thread Dump
labels.crawling_info_CrawlerStartTime=Crawler start time
labels.crawling_info_CrawlerEndTime=Crawler end time
labels.crawling_info_CrawlerExecTime=Crawler exec time

1
src/main/resources/fess_label_en.properties
View file

@ -418,6 +418,7 @@ labels.crawling_info_created_time=Created
labels.crawling_info_delete_all_link=Delete All
labels.crawling_info_delete_all_confirmation=Do you really want to delete all?
labels.crawling_info_delete_all_cancel=Cancel
labels.crawling_info_thread_dump=Thread Dump
labels.crawling_info_CrawlerStartTime=Crawler start time
labels.crawling_info_CrawlerEndTime=Crawler end time
labels.crawling_info_CrawlerExecTime=Crawler exec time

1
src/main/resources/fess_label_ja.properties
View file

@ -411,6 +411,7 @@ labels.crawling_info_created_time=\u4f5c\u6210
labels.crawling_info_delete_all_link=\u3059\u3079\u3066\u524a\u9664
labels.crawling_info_delete_all_confirmation=\u672c\u5f53\u306b\u3059\u3079\u3066\u3092\u524a\u9664\u3057\u307e\u3059\u304b\uff1f
labels.crawling_info_delete_all_cancel=\u30ad\u30e3\u30f3\u30bb\u30eb
labels.crawling_info_thread_dump=\u30b9\u30ec\u30c3\u30c9\u30c0\u30f3\u30d7
labels.crawling_info_CrawlerStartTime=\u30af\u30ed\u30fc\u30e9\u30fc\u306e\u958b\u59cb\u6642\u9593
labels.crawling_info_CrawlerEndTime=\u30af\u30ed\u30fc\u30e9\u30fc\u306e\u7d42\u4e86\u6642\u9593
labels.crawling_info_CrawlerExecTime=\u30af\u30ed\u30fc\u30e9\u30fc\u306e\u5b9f\u884c\u6642\u9593

2
src/main/resources/fess_message.properties
View file

@ -131,6 +131,7 @@ errors.failed_to_read_request_file=Failed to read request file: {0}
errors.invalid_header_for_request_file=Invalid header: {0}
errors.could_not_delete_logged_in_user=Could not delete logged in user.
errors.unauthorized_request=Unauthorized request.
errors.failed_to_print_thread_dump=Failed to print thread dump.
errors.invalid_query_unknown=The given query has unknown condition.
errors.invalid_query_parse_error=The given query is invalid.
@ -174,6 +175,7 @@ success.changed_password=Changed your password.
success.started_data_update=Started data update process.
success.reindex_started=Started reindexing.
success.bulk_process_started=Bulk process is started.
success.print_thread_dump=Printed thread dump to log file.
success.crud_create_crud_table=Created data.
success.crud_update_crud_table=Updated data.

2
src/main/resources/fess_message_en.properties
View file

@ -127,6 +127,7 @@ errors.failed_to_read_request_file=Failed to read request file: {0}
errors.invalid_header_for_request_file=Invalid header: {0}
errors.could_not_delete_logged_in_user=Could not delete logged in user.
errors.unauthorized_request=Unauthorized request.
errors.failed_to_print_thread_dump=Failed to print thread dump.
errors.invalid_query_unknown=The given query has unknown condition.
errors.invalid_query_parse_error=The given query is invalid.
@ -170,6 +171,7 @@ success.changed_password=Changed your password.
success.started_data_update=Started data update process.
success.reindex_started=Started reindexing.
success.bulk_process_started=Bulk process is started.
success.print_thread_dump=Printed thread dump to log file.
success.crud_create_crud_table=Created data.
success.crud_update_crud_table=Updated data.

2
src/main/resources/fess_message_ja.properties
View file

@ -133,6 +133,7 @@ errors.failed_to_reindex={0}\u304b\u3089{1}\u3078\u306e\u518d\u30a4\u30f3\u30c7\
errors.failed_to_read_request_file=\u30ea\u30af\u30a8\u30b9\u30c8\u30d5\u30a1\u30a4\u30eb\u306e\u8aad\u307f\u8fbc\u307f\u306b\u5931\u6557\u3057\u307e\u3057\u305f: {0}
errors.invalid_header_for_request_file=\u30d8\u30c3\u30c0\u30fc\u884c\u304c\u6b63\u3057\u304f\u3042\u308a\u307e\u305b\u3093: {0}
errors.could_not_delete_logged_in_user=\u30ed\u30b0\u30a4\u30f3\u3057\u3066\u3044\u308b\u30e6\u30fc\u30b6\u30fc\u306f\u524a\u9664\u3067\u304d\u307e\u305b\u3093\u3002
errors.failed_to_print_thread_dump=\u30b9\u30ec\u30c3\u30c9\u30c0\u30f3\u30d7\u306e\u51fa\u529b\u306b\u5931\u6557\u3057\u307e\u3057\u305f\u3002
errors.property_required={0}\u306f\u5fc5\u9808\u3067\u3059\u3002
errors.property_type_integer={0}\u306f\u6570\u5024\u3067\u3059\u3002
@ -163,6 +164,7 @@ success.changed_password=\u30d1\u30b9\u30ef\u30fc\u30c9\u3092\u5909\u66f4\u3057\
success.started_data_update=\u30c7\u30fc\u30bf\u66f4\u65b0\u51e6\u7406\u3092\u958b\u59cb\u3057\u307e\u3057\u305f\u3002
success.reindex_started=\u518d\u30a4\u30f3\u30c7\u30af\u30b7\u30f3\u30b0\u3092\u958b\u59cb\u3057\u307e\u3057\u305f\u3002
success.bulk_process_started=\u30d0\u30eb\u30af\u51e6\u7406\u3092\u958b\u59cb\u3057\u307e\u3057\u305f\u3002
success.print_thread_dump=\u30b9\u30ec\u30c3\u30c9\u30c0\u30f3\u30d7\u3092\u30ed\u30b0\u30d5\u30a1\u30a4\u30eb\u306b\u51fa\u529b\u3057\u307e\u3057\u305f\u3002
success.crud_create_crud_table = \u30c7\u30fc\u30bf\u3092\u4f5c\u6210\u3057\u307e\u3057\u305f\u3002
success.crud_update_crud_table = \u30c7\u30fc\u30bf\u3092\u66f4\u65b0\u3057\u307e\u3057\u305f\u3002

7
src/main/webapp/WEB-INF/view/admin/crawlinginfo/admin_crawlinginfo_details.jsp
View file

@ -135,6 +135,13 @@
</div>
</div>
</div>
<c:if test="${running}">
<button type="submit" class="btn btn-warning" name="threaddump"
value="<la:message key="labels.crawling_info_thread_dump" />">
<i class="fa fa-bolt"></i>
<la:message key="labels.crawling_info_thread_dump" />
</button>
</c:if>
</c:if>
</div>
<!-- /.box-footer -->