fix #448 : add indexer.data.max.document.request.size

src/main/java/org/codelibs/fess/ds/impl/IndexUpdateCallbackImpl.java

@@ -18,6 +18,8 @@ package org.codelibs.fess.ds.impl;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicLong;
 
+import javax.annotation.PostConstruct;
+
 import org.codelibs.fess.ds.IndexUpdateCallback;
 import org.codelibs.fess.es.client.FessEsClient;
 import org.codelibs.fess.exception.FessSystemException;
@@ -28,23 +30,31 @@ import org.codelibs.fess.helper.SystemHelper;
 import org.codelibs.fess.mylasta.direction.FessConfig;
 import org.codelibs.fess.util.ComponentUtil;
 import org.codelibs.fess.util.DocList;
+import org.codelibs.fess.util.DocumentUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 public class IndexUpdateCallbackImpl implements IndexUpdateCallback {
     private static final Logger logger = LoggerFactory.getLogger(IndexUpdateCallbackImpl.class);
 
-    protected volatile AtomicLong documentSize = new AtomicLong(0);
+    protected AtomicLong documentSize = new AtomicLong(0);
 
     protected volatile long executeTime = 0;
 
-    final DocList docList = new DocList();
+    protected final DocList docList = new DocList();
+
+    protected long maxDocumentRequestSize;
+
+    @PostConstruct
+    public void init() {
+        maxDocumentRequestSize = ComponentUtil.getFessConfig().getIndexerWebfsMaxDocumentRequestSizeAsInteger().longValue();
+    }
 
     /* (non-Javadoc)
      * @see org.codelibs.fess.ds.impl.IndexUpdateCallback#store(java.util.Map)
      */
     @Override
-    public synchronized boolean store(final Map<String, String> paramMap, final Map<String, Object> dataMap) {
+    public boolean store(final Map<String, String> paramMap, final Map<String, Object> dataMap) {
         final long startTime = System.currentTimeMillis();
         final FessConfig fessConfig = ComponentUtil.getFessConfig();
         final FessEsClient fessEsClient = ComponentUtil.getElasticsearchClient();
@@ -78,30 +88,41 @@ public class IndexUpdateCallbackImpl implements IndexUpdateCallback {
             dataMap.put(fessConfig.getIndexFieldDocId(), systemHelper.generateDocId(dataMap));
         }
 
-        docList.add(dataMap);
-        if (logger.isDebugEnabled()) {
-            logger.debug("Added the document. " + "The number of a document cache is " + docList.size() + ".");
+        synchronized (docList) {
+            docList.add(dataMap);
+            if (logger.isDebugEnabled()) {
+                logger.debug("Added the document. " + "The number of a document cache is " + docList.size() + ".");
+            }
+
+            final Long contentLength = DocumentUtil.getValue(dataMap, fessConfig.getIndexFieldContentLength(), Long.class);
+            if (contentLength != null) {
+                docList.addContentSize(contentLength.longValue());
+                if (docList.getContentSize() >= maxDocumentRequestSize) {
+                    indexingHelper.sendDocuments(fessEsClient, docList);
+                }
+            } else if (docList.size() >= fessConfig.getIndexerDataMaxDocumentCacheSizeAsInteger().intValue()) {
+                indexingHelper.sendDocuments(fessEsClient, docList);
+            }
+            executeTime += System.currentTimeMillis() - startTime;
         }
 
-        if (docList.size() >= fessConfig.getIndexerDataMaxDocumentCacheSizeAsInteger().intValue()) {
-            indexingHelper.sendDocuments(fessEsClient, docList);
-        }
         documentSize.getAndIncrement();
 
         if (logger.isDebugEnabled()) {
             logger.debug("The number of an added document is " + documentSize.get() + ".");
         }
 
-        executeTime += System.currentTimeMillis() - startTime;
         return true;
     }
 
     @Override
     public void commit() {
-        if (!docList.isEmpty()) {
-            final IndexingHelper indexingHelper = ComponentUtil.getIndexingHelper();
-            final FessEsClient fessEsClient = ComponentUtil.getElasticsearchClient();
-            indexingHelper.sendDocuments(fessEsClient, docList);
+        synchronized (docList) {
+            if (!docList.isEmpty()) {
+                final IndexingHelper indexingHelper = ComponentUtil.getIndexingHelper();
+                final FessEsClient fessEsClient = ComponentUtil.getElasticsearchClient();
+                indexingHelper.sendDocuments(fessEsClient, docList);
+            }
         }
     }
 

src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java

@@ -234,6 +234,9 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
     /** The key of the configuration. e.g. 5 */
     String INDEXER_DATA_MAX_DOCUMENT_CACHE_SIZE = "indexer.data.max.document.cache.size";
 
+    /** The key of the configuration. e.g. 10485760 */
+    String INDEXER_DATA_MAX_DOCUMENT_REQUEST_SIZE = "indexer.data.max.document.request.size";
+
     /** The key of the configuration. e.g. favorite_count */
     String INDEX_FIELD_favorite_count = "index.field.favorite_count";
 
@@ -1468,6 +1471,21 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
      */
     Integer getIndexerDataMaxDocumentCacheSizeAsInteger();
 
+    /**
+     * Get the value for the key 'indexer.data.max.document.request.size'. <br>
+     * The value is, e.g. 10485760 <br>
+     * @return The value of found property. (NotNull: if not found, exception but basically no way)
+     */
+    String getIndexerDataMaxDocumentRequestSize();
+
+    /**
+     * Get the value for the key 'indexer.data.max.document.request.size' as {@link Integer}. <br>
+     * The value is, e.g. 10485760 <br>
+     * @return The value of found property. (NotNull: if not found, exception but basically no way)
+     * @throws NumberFormatException When the property is not integer.
+     */
+    Integer getIndexerDataMaxDocumentRequestSizeAsInteger();
+
     /**
      * Get the value for the key 'index.field.favorite_count'. <br>
      * The value is, e.g. favorite_count <br>
@@ -3529,6 +3547,14 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
             return getAsInteger(FessConfig.INDEXER_DATA_MAX_DOCUMENT_CACHE_SIZE);
         }
 
+        public String getIndexerDataMaxDocumentRequestSize() {
+            return get(FessConfig.INDEXER_DATA_MAX_DOCUMENT_REQUEST_SIZE);
+        }
+
+        public Integer getIndexerDataMaxDocumentRequestSizeAsInteger() {
+            return getAsInteger(FessConfig.INDEXER_DATA_MAX_DOCUMENT_REQUEST_SIZE);
+        }
+
         public String getIndexFieldFavoriteCount() {
             return get(FessConfig.INDEX_FIELD_favorite_count);
         }

src/main/resources/fess_config.properties

@@ -124,6 +124,7 @@ indexer.webfs.update.interval=10000
 indexer.webfs.max.document.cache.size=100
 indexer.webfs.max.document.request.size=10485760
 indexer.data.max.document.cache.size=5
+indexer.data.max.document.request.size=10485760
 
 # field names
 index.field.favorite_count=favorite_count