fix #1310 print thread dump on crawler

src/main/java/org/codelibs/fess/Constants.java

@@ -416,4 +416,6 @@ public class Constants extends CoreLibConstants {
     public static final String SYSTEM_USER = "system";
 
     public static final String EMPTY_USER_ID = "<empty>";
+
+    public static final String CRAWLER_PROCESS_COMMAND_THREAD_DUMP = "thread_dump";
 }

src/main/java/org/codelibs/fess/app/web/admin/crawlinginfo/AdminCrawlinginfoAction.java

@@ -28,6 +28,8 @@ import org.lastaflute.web.Execute;
 import org.lastaflute.web.response.HtmlResponse;
 import org.lastaflute.web.response.render.RenderData;
 import org.lastaflute.web.ruts.process.ActionRuntime;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * @author shinsuke
@@ -35,6 +37,8 @@ import org.lastaflute.web.ruts.process.ActionRuntime;
  */
 public class AdminCrawlinginfoAction extends FessAdminAction {
 
+    private static final Logger logger = LoggerFactory.getLogger(AdminCrawlinginfoAction.class);
+
     // ===================================================================================
     //                                                                           Attribute
     //                                                                           =========
@@ -116,20 +120,42 @@ public class AdminCrawlinginfoAction extends FessAdminAction {
     public HtmlResponse details(final int crudMode, final String id) {
         verifyCrudMode(crudMode, CrudMode.DETAILS);
         saveToken();
-        return asHtml(path_AdminCrawlinginfo_AdminCrawlinginfoDetailsJsp).useForm(EditForm.class, op -> {
-            op.setup(form -> {
-                crawlingInfoService.getCrawlingInfo(id).ifPresent(entity -> {
+        return crawlingInfoService.getCrawlingInfo(id).map(entity -> {
+            return asHtml(path_AdminCrawlinginfo_AdminCrawlinginfoDetailsJsp).useForm(EditForm.class, op -> {
+                op.setup(form -> {
                     copyBeanToBean(entity, form, copyOp -> {
                         copyOp.excludeNull();
                     });
                     form.crudMode = crudMode;
-                }).orElse(() -> {
-                    throwValidationError(messages -> messages.addErrorsCrudCouldNotFindCrudTable(GLOBAL, id), () -> asListHtml());
                 });
+            }).renderWith(data -> {
+                RenderDataUtil.register(data, "crawlingInfoParamItems", crawlingInfoService.getCrawlingInfoParamList(id));
+                RenderDataUtil.register(data, "running", processHelper.isProcessRunning(entity.getSessionId()));
             });
-        }).renderWith(data -> {
-            RenderDataUtil.register(data, "crawlingInfoParamItems", crawlingInfoService.getCrawlingInfoParamList(id));
+        }).orElseGet(() -> {
+            throwValidationError(messages -> messages.addErrorsCrudCouldNotFindCrudTable(GLOBAL, id), () -> asListHtml());
+            return null;
+        });
+    }
+
+    @Execute
+    public HtmlResponse threaddump(final EditForm form) {
+        verifyCrudMode(form.crudMode, CrudMode.DETAILS);
+        validate(form, messages -> {}, () -> asDetailsHtml());
+        verifyToken(() -> asDetailsHtml());
+        final String id = form.id;
+        crawlingInfoService.getCrawlingInfo(id).ifPresent(entity -> {
+            try {
+                processHelper.sendCommand(entity.getSessionId(), Constants.CRAWLER_PROCESS_COMMAND_THREAD_DUMP);
+                saveInfo(messages -> messages.addSuccessPrintThreadDump(GLOBAL));
+            } catch (Exception e) {
+                logger.warn("Failed to print a thread dump.", e);
+                throwValidationError(messages -> messages.addErrorsFailedToPrintThreadDump(GLOBAL), () -> asListHtml());
+            }
+        }).orElse(() -> {
+            throwValidationError(messages -> messages.addErrorsCrudCouldNotFindCrudTable(GLOBAL, id), () -> asListHtml());
         });
+        return redirect(getClass());
     }
 
     // -----------------------------------------------------

src/main/java/org/codelibs/fess/exception/JobNotFoundException.java

@@ -25,4 +25,8 @@ public class JobNotFoundException extends FessSystemException {
         super(scheduledJob.toString());
     }
 
+    public JobNotFoundException(String message) {
+        super(message);
+    }
+
 }

src/main/java/org/codelibs/fess/exception/JobProcessingException.java

@@ -15,6 +15,8 @@
  */
 package org.codelibs.fess.exception;
 
+import java.io.IOException;
+
 public class JobProcessingException extends FessSystemException {
 
     private static final long serialVersionUID = 1L;
@@ -23,4 +25,8 @@ public class JobProcessingException extends FessSystemException {
         super(e);
     }
 
+    public JobProcessingException(String message, IOException e) {
+        super(message, e);
+    }
+
 }

src/main/java/org/codelibs/fess/exec/Crawler.java

@@ -17,8 +17,10 @@ package org.codelibs.fess.exec;
 
 import static org.codelibs.core.stream.StreamUtil.stream;
 
+import java.io.BufferedReader;
 import java.io.File;
 import java.io.IOException;
+import java.io.InputStreamReader;
 import java.lang.management.ManagementFactory;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
@@ -53,6 +55,7 @@ import org.codelibs.fess.mylasta.direction.FessConfig;
 import org.codelibs.fess.mylasta.mail.CrawlerPostcard;
 import org.codelibs.fess.timer.SystemMonitorTarget;
 import org.codelibs.fess.util.ComponentUtil;
+import org.codelibs.fess.util.ThreadDumpUtil;
 import org.elasticsearch.monitor.jvm.JvmInfo;
 import org.elasticsearch.monitor.os.OsProbe;
 import org.elasticsearch.monitor.process.ProcessProbe;
@@ -210,6 +213,7 @@ public class Crawler {
         }
 
         TimeoutTask systemMonitorTask = null;
+        Thread commandThread = null;
         int exitCode;
         try {
             running.set(true);
@@ -227,6 +231,36 @@ public class Crawler {
             };
             Runtime.getRuntime().addShutdownHook(shutdownCallback);
 
+            commandThread = new Thread(() -> {
+                try (BufferedReader reader = new BufferedReader(new InputStreamReader(System.in))) {
+                    String command;
+                    while (true) {
+                        try {
+                            while (!reader.ready()) {
+                                Thread.sleep(1000L);
+                            }
+                            command = reader.readLine().trim();
+                            if (logger.isDebugEnabled()) {
+                                logger.debug("Process command: " + command);
+                            }
+                            if (Constants.CRAWLER_PROCESS_COMMAND_THREAD_DUMP.equals(command)) {
+                                ThreadDumpUtil.printThreadDump();
+                            } else {
+                                logger.warn("Unknown process command: " + command);
+                            }
+                            if (Thread.interrupted()) {
+                                return;
+                            }
+                        } catch (InterruptedException e) {
+                            return;
+                        }
+                    }
+                } catch (IOException e) {
+                    logger.debug("I/O exception.", e);
+                }
+            }, "ProcessCommand");
+            commandThread.start();
+
             systemMonitorTask =
                     TimeoutManager.getInstance().addTimeoutTarget(new SystemMonitorTarget(),
                             ComponentUtil.getFessConfig().getCrawlerSystemMonitorIntervalAsInteger(), true);
@@ -243,6 +277,9 @@ public class Crawler {
             logger.error("Crawler does not work correctly.", t);
             exitCode = Constants.EXIT_FAIL;
         } finally {
+            if (commandThread != null && commandThread.isAlive()) {
+                commandThread.interrupt();
+            }
             if (systemMonitorTask != null) {
                 systemMonitorTask.cancel();
             }

src/main/java/org/codelibs/fess/helper/ProcessHelper.java

@@ -16,6 +16,7 @@
 package org.codelibs.fess.helper;
 
 import java.io.IOException;
+import java.io.OutputStream;
 import java.util.List;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
@@ -26,7 +27,9 @@ import java.util.function.Consumer;
 import javax.annotation.PreDestroy;
 
 import org.apache.commons.io.IOUtils;
-import org.codelibs.fess.exception.FessSystemException;
+import org.codelibs.fess.Constants;
+import org.codelibs.fess.exception.JobNotFoundException;
+import org.codelibs.fess.exception.JobProcessingException;
 import org.codelibs.fess.util.InputStreamThread;
 import org.codelibs.fess.util.JobProcess;
 import org.slf4j.Logger;
@@ -63,7 +66,7 @@ public class ProcessHelper {
             destroyProcess(sessionId, runningProcessMap.putIfAbsent(sessionId, jobProcess));
             return jobProcess;
         } catch (final IOException e) {
-            throw new FessSystemException("Crawler Process terminated.", e);
+            throw new JobProcessingException("Crawler Process terminated.", e);
         }
     }
 
@@ -76,6 +79,11 @@ public class ProcessHelper {
         return !runningProcessMap.isEmpty();
     }
 
+    public boolean isProcessRunning(String sessionId) {
+        JobProcess jobProcess = runningProcessMap.get(sessionId);
+        return jobProcess != null && jobProcess.getProcess().isAlive();
+    }
+
     protected int destroyProcess(final String sessionId, final JobProcess jobProcess) {
         if (jobProcess != null) {
             final InputStreamThread ist = jobProcess.getInputStreamThread();
@@ -138,4 +146,18 @@ public class ProcessHelper {
         this.processDestroyTimeout = processDestroyTimeout;
     }
 
+    public void sendCommand(String sessionId, String command) {
+        JobProcess jobProcess = runningProcessMap.get(sessionId);
+        if (jobProcess != null) {
+            try {
+                final OutputStream out = jobProcess.getProcess().getOutputStream();
+                IOUtils.write(command + "\n", out, Constants.CHARSET_UTF_8);
+                out.flush();
+            } catch (IOException e) {
+                throw new JobProcessingException(e);
+            }
+        } else {
+            throw new JobNotFoundException("Job for " + sessionId + " is not found.");
+        }
+    }
 }

src/main/java/org/codelibs/fess/indexer/IndexUpdater.java

@@ -49,6 +49,7 @@ import org.codelibs.fess.mylasta.direction.FessConfig;
 import org.codelibs.fess.util.ComponentUtil;
 import org.codelibs.fess.util.DocList;
 import org.codelibs.fess.util.MemoryUtil;
+import org.codelibs.fess.util.ThreadDumpUtil;
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
@@ -270,7 +271,7 @@ public class IndexUpdater extends Thread {
                     finishCrawling = true;
                     forceStop();
                     if (fessConfig.getIndexerThreadDumpEnabledAsBoolean()) {
-                        printThreadDump();
+                        ThreadDumpUtil.printThreadDump();
                     }
                     org.codelibs.fess.exec.Crawler.addError("QueueTimeout");
                 }
@@ -313,16 +314,6 @@ public class IndexUpdater extends Thread {
 
     }
 
-    private void printThreadDump() {
-        for (final Map.Entry<Thread, StackTraceElement[]> entry : Thread.getAllStackTraces().entrySet()) {
-            logger.info("Thread: " + entry.getKey());
-            final StackTraceElement[] trace = entry.getValue();
-            for (final StackTraceElement element : trace) {
-                logger.info("\tat " + element);
-            }
-        }
-    }
-
     private void processAccessResults(final DocList docList, final List<EsAccessResult> accessResultList, final List<EsAccessResult> arList) {
         final FessConfig fessConfig = ComponentUtil.getFessConfig();
         final long maxDocumentRequestSize = fessConfig.getIndexerWebfsMaxDocumentRequestSizeAsInteger().longValue();

src/main/java/org/codelibs/fess/mylasta/action/FessLabels.java

@@ -1283,6 +1283,9 @@ public class FessLabels extends UserMessages {
     /** The key of the message: Cancel */
     public static final String LABELS_crawling_info_delete_all_cancel = "{labels.crawling_info_delete_all_cancel}";
 
+    /** The key of the message: Thread Dump */
+    public static final String LABELS_crawling_info_thread_dump = "{labels.crawling_info_thread_dump}";
+
     /** The key of the message: Crawler start time */
     public static final String LABELS_crawling_info_CrawlerStartTime = "{labels.crawling_info_CrawlerStartTime}";
 

src/main/java/org/codelibs/fess/mylasta/action/FessMessages.java

@@ -329,6 +329,9 @@ public class FessMessages extends FessLabels {
     /** The key of the message: Unauthorized request. */
     public static final String ERRORS_unauthorized_request = "{errors.unauthorized_request}";
 
+    /** The key of the message: Failed to print thread dump. */
+    public static final String ERRORS_failed_to_print_thread_dump = "{errors.failed_to_print_thread_dump}";
+
     /** The key of the message: The given query has unknown condition. */
     public static final String ERRORS_invalid_query_unknown = "{errors.invalid_query_unknown}";
 
@@ -446,6 +449,9 @@ public class FessMessages extends FessLabels {
     /** The key of the message: Bulk process is started. */
     public static final String SUCCESS_bulk_process_started = "{success.bulk_process_started}";
 
+    /** The key of the message: Printed thread dump to log file. */
+    public static final String SUCCESS_print_thread_dump = "{success.print_thread_dump}";
+
     /** The key of the message: Created data. */
     public static final String SUCCESS_crud_create_crud_table = "{success.crud_create_crud_table}";
 
@@ -1913,6 +1919,20 @@ public class FessMessages extends FessLabels {
         return this;
     }
 
+    /**
+     * Add the created action message for the key 'errors.failed_to_print_thread_dump' with parameters.
+     * <pre>
+     * message: Failed to print thread dump.
+     * </pre>
+     * @param property The property name for the message. (NotNull)
+     * @return this. (NotNull)
+     */
+    public FessMessages addErrorsFailedToPrintThreadDump(String property) {
+        assertPropertyNotNull(property);
+        add(property, new UserMessage(ERRORS_failed_to_print_thread_dump));
+        return this;
+    }
+
     /**
      * Add the created action message for the key 'errors.invalid_query_unknown' with parameters.
      * <pre>
@@ -2480,6 +2500,20 @@ public class FessMessages extends FessLabels {
         return this;
     }
 
+    /**
+     * Add the created action message for the key 'success.print_thread_dump' with parameters.
+     * <pre>
+     * message: Printed thread dump to log file.
+     * </pre>
+     * @param property The property name for the message. (NotNull)
+     * @return this. (NotNull)
+     */
+    public FessMessages addSuccessPrintThreadDump(String property) {
+        assertPropertyNotNull(property);
+        add(property, new UserMessage(SUCCESS_print_thread_dump));
+        return this;
+    }
+
     /**
      * Add the created action message for the key 'success.crud_create_crud_table' with parameters.
      * <pre>

src/main/java/org/codelibs/fess/util/ThreadDumpUtil.java

@@ -0,0 +1,36 @@
+/*
+ * Copyright 2012-2017 CodeLibs Project and the Others.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+ * either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+package org.codelibs.fess.util;
+
+import java.util.Map;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ThreadDumpUtil {
+    private static final Logger logger = LoggerFactory.getLogger(ThreadDumpUtil.class);
+
+    public static void printThreadDump() {
+        for (final Map.Entry<Thread, StackTraceElement[]> entry : Thread.getAllStackTraces().entrySet()) {
+            logger.info("Thread: " + entry.getKey());
+            final StackTraceElement[] trace = entry.getValue();
+            for (final StackTraceElement element : trace) {
+                logger.info("\tat " + element);
+            }
+        }
+    }
+
+}

src/main/resources/fess_label.properties

@@ -418,6 +418,7 @@ labels.crawling_info_created_time=Created
 labels.crawling_info_delete_all_link=Delete All
 labels.crawling_info_delete_all_confirmation=Do you really want to delete all?
 labels.crawling_info_delete_all_cancel=Cancel
+labels.crawling_info_thread_dump=Thread Dump
 labels.crawling_info_CrawlerStartTime=Crawler start time
 labels.crawling_info_CrawlerEndTime=Crawler end time
 labels.crawling_info_CrawlerExecTime=Crawler exec time

src/main/resources/fess_label_en.properties

@@ -418,6 +418,7 @@ labels.crawling_info_created_time=Created
 labels.crawling_info_delete_all_link=Delete All
 labels.crawling_info_delete_all_confirmation=Do you really want to delete all?
 labels.crawling_info_delete_all_cancel=Cancel
+labels.crawling_info_thread_dump=Thread Dump
 labels.crawling_info_CrawlerStartTime=Crawler start time
 labels.crawling_info_CrawlerEndTime=Crawler end time
 labels.crawling_info_CrawlerExecTime=Crawler exec time

src/main/resources/fess_label_ja.properties

@@ -411,6 +411,7 @@ labels.crawling_info_created_time=\u4f5c\u6210
 labels.crawling_info_delete_all_link=\u3059\u3079\u3066\u524a\u9664
 labels.crawling_info_delete_all_confirmation=\u672c\u5f53\u306b\u3059\u3079\u3066\u3092\u524a\u9664\u3057\u307e\u3059\u304b\uff1f
 labels.crawling_info_delete_all_cancel=\u30ad\u30e3\u30f3\u30bb\u30eb
+labels.crawling_info_thread_dump=\u30b9\u30ec\u30c3\u30c9\u30c0\u30f3\u30d7
 labels.crawling_info_CrawlerStartTime=\u30af\u30ed\u30fc\u30e9\u30fc\u306e\u958b\u59cb\u6642\u9593
 labels.crawling_info_CrawlerEndTime=\u30af\u30ed\u30fc\u30e9\u30fc\u306e\u7d42\u4e86\u6642\u9593
 labels.crawling_info_CrawlerExecTime=\u30af\u30ed\u30fc\u30e9\u30fc\u306e\u5b9f\u884c\u6642\u9593

src/main/resources/fess_message.properties

@@ -131,6 +131,7 @@ errors.failed_to_read_request_file=Failed to read request file: {0}
 errors.invalid_header_for_request_file=Invalid header: {0}
 errors.could_not_delete_logged_in_user=Could not delete logged in user.
 errors.unauthorized_request=Unauthorized request.
+errors.failed_to_print_thread_dump=Failed to print thread dump.
 
 errors.invalid_query_unknown=The given query has unknown condition.
 errors.invalid_query_parse_error=The given query is invalid.
@@ -174,6 +175,7 @@ success.changed_password=Changed your password.
 success.started_data_update=Started data update process.
 success.reindex_started=Started reindexing.
 success.bulk_process_started=Bulk process is started.
+success.print_thread_dump=Printed thread dump to log file.
 
 success.crud_create_crud_table=Created data.
 success.crud_update_crud_table=Updated data.

src/main/resources/fess_message_en.properties

@@ -127,6 +127,7 @@ errors.failed_to_read_request_file=Failed to read request file: {0}
 errors.invalid_header_for_request_file=Invalid header: {0}
 errors.could_not_delete_logged_in_user=Could not delete logged in user.
 errors.unauthorized_request=Unauthorized request.
+errors.failed_to_print_thread_dump=Failed to print thread dump.
 
 errors.invalid_query_unknown=The given query has unknown condition.
 errors.invalid_query_parse_error=The given query is invalid.
@@ -170,6 +171,7 @@ success.changed_password=Changed your password.
 success.started_data_update=Started data update process.
 success.reindex_started=Started reindexing.
 success.bulk_process_started=Bulk process is started.
+success.print_thread_dump=Printed thread dump to log file.
 
 success.crud_create_crud_table=Created data.
 success.crud_update_crud_table=Updated data.

src/main/resources/fess_message_ja.properties

@@ -133,6 +133,7 @@ errors.failed_to_reindex={0}\u304b\u3089{1}\u3078\u306e\u518d\u30a4\u30f3\u30c7\
 errors.failed_to_read_request_file=\u30ea\u30af\u30a8\u30b9\u30c8\u30d5\u30a1\u30a4\u30eb\u306e\u8aad\u307f\u8fbc\u307f\u306b\u5931\u6557\u3057\u307e\u3057\u305f: {0}
 errors.invalid_header_for_request_file=\u30d8\u30c3\u30c0\u30fc\u884c\u304c\u6b63\u3057\u304f\u3042\u308a\u307e\u305b\u3093: {0}
 errors.could_not_delete_logged_in_user=\u30ed\u30b0\u30a4\u30f3\u3057\u3066\u3044\u308b\u30e6\u30fc\u30b6\u30fc\u306f\u524a\u9664\u3067\u304d\u307e\u305b\u3093\u3002
+errors.failed_to_print_thread_dump=\u30b9\u30ec\u30c3\u30c9\u30c0\u30f3\u30d7\u306e\u51fa\u529b\u306b\u5931\u6557\u3057\u307e\u3057\u305f\u3002
 
 errors.property_required={0}\u306f\u5fc5\u9808\u3067\u3059\u3002
 errors.property_type_integer={0}\u306f\u6570\u5024\u3067\u3059\u3002
@@ -163,6 +164,7 @@ success.changed_password=\u30d1\u30b9\u30ef\u30fc\u30c9\u3092\u5909\u66f4\u3057\
 success.started_data_update=\u30c7\u30fc\u30bf\u66f4\u65b0\u51e6\u7406\u3092\u958b\u59cb\u3057\u307e\u3057\u305f\u3002
 success.reindex_started=\u518d\u30a4\u30f3\u30c7\u30af\u30b7\u30f3\u30b0\u3092\u958b\u59cb\u3057\u307e\u3057\u305f\u3002
 success.bulk_process_started=\u30d0\u30eb\u30af\u51e6\u7406\u3092\u958b\u59cb\u3057\u307e\u3057\u305f\u3002
+success.print_thread_dump=\u30b9\u30ec\u30c3\u30c9\u30c0\u30f3\u30d7\u3092\u30ed\u30b0\u30d5\u30a1\u30a4\u30eb\u306b\u51fa\u529b\u3057\u307e\u3057\u305f\u3002
 
 success.crud_create_crud_table = \u30c7\u30fc\u30bf\u3092\u4f5c\u6210\u3057\u307e\u3057\u305f\u3002
 success.crud_update_crud_table = \u30c7\u30fc\u30bf\u3092\u66f4\u65b0\u3057\u307e\u3057\u305f\u3002

src/main/webapp/WEB-INF/view/admin/crawlinginfo/admin_crawlinginfo_details.jsp

@@ -135,6 +135,13 @@
 												</div>
 											</div>
 										</div>
+										<c:if test="${running}">
+										<button type="submit" class="btn btn-warning" name="threaddump"
+											value="<la:message key="labels.crawling_info_thread_dump" />">
+											<i class="fa fa-bolt"></i>
+											<la:message key="labels.crawling_info_thread_dump" />
+										</button>
+										</c:if>
 									</c:if>
 								</div>
 								<!-- /.box-footer -->