diff --git a/src/main/java/org/codelibs/fess/indexer/IndexUpdater.java b/src/main/java/org/codelibs/fess/indexer/IndexUpdater.java index 6e8947039..9a22aa43a 100644 --- a/src/main/java/org/codelibs/fess/indexer/IndexUpdater.java +++ b/src/main/java/org/codelibs/fess/indexer/IndexUpdater.java @@ -93,8 +93,6 @@ public class IndexUpdater extends Thread { protected int maxErrorCount = 2; - protected int unprocessedDocumentSize = 100; - protected List finishedSessionIdList = new ArrayList<>(); private final List docBoostMatcherList = new ArrayList<>(); @@ -181,7 +179,7 @@ public class IndexUpdater extends Thread { if (interval > 0) { // sleep try { - Thread.sleep(interval); // 1 min (default) + Thread.sleep(interval); // 10 sec (default) } catch (final InterruptedException e) { logger.warn("Interrupted index update.", e); } @@ -206,15 +204,9 @@ public class IndexUpdater extends Thread { } while (!arList.isEmpty()) { processAccessResults(docList, accessResultList, arList); - cleanupAccessResults(accessResultList); - - if (logger.isDebugEnabled()) { - logger.debug("Getting documents in IndexUpdater queue."); - } arList = getAccessResultList(cb); } - if (!docList.isEmpty()) { indexingHelper.sendDocuments(fessEsClient, docList); } @@ -445,10 +437,13 @@ public class IndexUpdater extends Thread { } private List getAccessResultList(final Consumer cb) { + if (logger.isDebugEnabled()) { + logger.debug("Getting documents in IndexUpdater queue."); + } final long execTime = System.currentTimeMillis(); final List arList = ((EsDataService) dataService).getAccessResultList(cb); + final FessConfig fessConfig = ComponentUtil.getFessConfig(); if (!arList.isEmpty()) { - final FessConfig fessConfig = ComponentUtil.getFessConfig(); final long commitMarginTime = fessConfig.getIndexerWebfsCommitMarginTimeAsInteger().longValue(); for (final AccessResult ar : arList.toArray(new AccessResult[arList.size()])) { if (ar.getCreateTime().longValue() > execTime - commitMarginTime) { @@ -460,6 +455,7 @@ public class IndexUpdater extends Thread { if (logger.isInfoEnabled()) { logger.info("Processing " + arList.size() + "/" + totalHits + " docs (" + (System.currentTimeMillis() - execTime) + "ms)"); } + final long unprocessedDocumentSize = fessConfig.getIndexerUnprocessedDocumentSizeAsInteger().longValue(); if (totalHits > unprocessedDocumentSize) { if (logger.isInfoEnabled()) { logger.info("Stopped all crawler threads. " + " You have " + totalHits + " (>" + unprocessedDocumentSize + ") " @@ -532,10 +528,6 @@ public class IndexUpdater extends Thread { this.maxIndexerErrorCount = maxIndexerErrorCount; } - public void setUnprocessedDocumentSize(final int unprocessedDocumentSize) { - this.unprocessedDocumentSize = unprocessedDocumentSize; - } - public void addDocBoostMatcher(final DocBoostMatcher rule) { docBoostMatcherList.add(rule); } diff --git a/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java b/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java index af5550382..eb3bca9a2 100644 --- a/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java +++ b/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java @@ -63,7 +63,11 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction -XX:CMSInitiatingOccupancyFraction=75 -XX:+UseParNewGC -XX:+UseTLAB - -XX:+DisableExplicitGC */ + -XX:+DisableExplicitGC + -Djcifs.smb.client.connTimeout=60000 + -Djcifs.smb.client.soTimeout=35000 + -Djcifs.smb.client.responseTimeout=30000 + */ String JVM_SUGGEST_OPTIONS = "jvm.suggest.options"; /** The key of the configuration. e.g. default_crawler */ @@ -200,19 +204,22 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction /** The key of the configuration. e.g. true */ String INDEXER_THREAD_DUMP_ENABLED = "indexer.thread.dump.enabled"; + /** The key of the configuration. e.g. 1000 */ + String INDEXER_UNPROCESSED_DOCUMENT_SIZE = "indexer.unprocessed.document.size"; + /** The key of the configuration. e.g. true */ String INDEXER_CLICK_COUNT_ENABLED = "indexer.click.count.enabled"; /** The key of the configuration. e.g. true */ String INDEXER_FAVORITE_COUNT_ENABLED = "indexer.favorite.count.enabled"; - /** The key of the configuration. e.g. 10000 */ + /** The key of the configuration. e.g. 1000 */ String INDEXER_WEBFS_COMMIT_MARGIN_TIME = "indexer.webfs.commit.margin.time"; /** The key of the configuration. e.g. 60 */ String INDEXER_WEBFS_MAX_EMPTY_LIST_CONUNT = "indexer.webfs.max.empty.list.conunt"; - /** The key of the configuration. e.g. 60000 */ + /** The key of the configuration. e.g. 10000 */ String INDEXER_WEBFS_UPDATE_INTERVAL = "indexer.webfs.update.interval"; /** The key of the configuration. e.g. 5 */ @@ -777,7 +784,11 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction -XX:CMSInitiatingOccupancyFraction=75 -XX:+UseParNewGC -XX:+UseTLAB - -XX:+DisableExplicitGC
+ -XX:+DisableExplicitGC + -Djcifs.smb.client.connTimeout=60000 + -Djcifs.smb.client.soTimeout=35000 + -Djcifs.smb.client.responseTimeout=30000 +
* @return The value of found property. (NotNull: if not found, exception but basically no way) */ String getJvmSuggestOptions(); @@ -1231,6 +1242,21 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction */ boolean isIndexerThreadDumpEnabled(); + /** + * Get the value for the key 'indexer.unprocessed.document.size'.
+ * The value is, e.g. 1000
+ * @return The value of found property. (NotNull: if not found, exception but basically no way) + */ + String getIndexerUnprocessedDocumentSize(); + + /** + * Get the value for the key 'indexer.unprocessed.document.size' as {@link Integer}.
+ * The value is, e.g. 1000
+ * @return The value of found property. (NotNull: if not found, exception but basically no way) + * @throws NumberFormatException When the property is not integer. + */ + Integer getIndexerUnprocessedDocumentSizeAsInteger(); + /** * Get the value for the key 'indexer.click.count.enabled'.
* The value is, e.g. true
@@ -1261,14 +1287,14 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction /** * Get the value for the key 'indexer.webfs.commit.margin.time'.
- * The value is, e.g. 10000
+ * The value is, e.g. 1000
* @return The value of found property. (NotNull: if not found, exception but basically no way) */ String getIndexerWebfsCommitMarginTime(); /** * Get the value for the key 'indexer.webfs.commit.margin.time' as {@link Integer}.
- * The value is, e.g. 10000
+ * The value is, e.g. 1000
* @return The value of found property. (NotNull: if not found, exception but basically no way) * @throws NumberFormatException When the property is not integer. */ @@ -1291,14 +1317,14 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction /** * Get the value for the key 'indexer.webfs.update.interval'.
- * The value is, e.g. 60000
+ * The value is, e.g. 10000
* @return The value of found property. (NotNull: if not found, exception but basically no way) */ String getIndexerWebfsUpdateInterval(); /** * Get the value for the key 'indexer.webfs.update.interval' as {@link Integer}.
- * The value is, e.g. 60000
+ * The value is, e.g. 10000
* @return The value of found property. (NotNull: if not found, exception but basically no way) * @throws NumberFormatException When the property is not integer. */ @@ -3010,6 +3036,14 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction return is(FessConfig.INDEXER_THREAD_DUMP_ENABLED); } + public String getIndexerUnprocessedDocumentSize() { + return get(FessConfig.INDEXER_UNPROCESSED_DOCUMENT_SIZE); + } + + public Integer getIndexerUnprocessedDocumentSizeAsInteger() { + return getAsInteger(FessConfig.INDEXER_UNPROCESSED_DOCUMENT_SIZE); + } + public String getIndexerClickCountEnabled() { return get(FessConfig.INDEXER_CLICK_COUNT_ENABLED); } diff --git a/src/main/java/org/codelibs/fess/mylasta/direction/FessProp.java b/src/main/java/org/codelibs/fess/mylasta/direction/FessProp.java index 70626eec7..7dbc37468 100644 --- a/src/main/java/org/codelibs/fess/mylasta/direction/FessProp.java +++ b/src/main/java/org/codelibs/fess/mylasta/direction/FessProp.java @@ -25,7 +25,6 @@ import java.util.stream.Collectors; import javax.naming.directory.Attribute; import javax.naming.directory.BasicAttribute; -import javax.servlet.http.HttpServletRequest; import org.codelibs.core.exception.ClassNotFoundRuntimeException; import org.codelibs.core.lang.StringUtil; diff --git a/src/main/resources/fess_config.properties b/src/main/resources/fess_config.properties index 21b6933a8..d7e7c1aa5 100644 --- a/src/main/resources/fess_config.properties +++ b/src/main/resources/fess_config.properties @@ -113,9 +113,10 @@ crawler.document.cache.html.mimetypes=text/html # indexer indexer.thread.dump.enabled=true +indexer.unprocessed.document.size=1000 indexer.click.count.enabled=true indexer.favorite.count.enabled=true -indexer.webfs.commit.margin.time=10000 +indexer.webfs.commit.margin.time=1000 indexer.webfs.max.empty.list.conunt=60 indexer.webfs.update.interval=10000 indexer.webfs.max.document.cache.size=5