fix #2423 use sizeOf

src/main/java/org/codelibs/fess/ds/callback/IndexUpdateCallbackImpl.java

@@ -35,6 +35,7 @@ import org.codelibs.fess.mylasta.direction.FessConfig;
 import org.codelibs.fess.util.ComponentUtil;
 import org.codelibs.fess.util.DocList;
 import org.codelibs.fess.util.DocumentUtil;
+import org.codelibs.fess.util.MemoryUtil;
 
 public class IndexUpdateCallbackImpl implements IndexUpdateCallback {
     private static final Logger logger = LogManager.getLogger(IndexUpdateCallbackImpl.class);
@@ -47,12 +48,15 @@ public class IndexUpdateCallbackImpl implements IndexUpdateCallback {
 
     protected long maxDocumentRequestSize;
 
+    protected int maxDocumentCacheSize;
+
     @PostConstruct
     public void init() {
         if (logger.isDebugEnabled()) {
             logger.debug("Initialize {}", this.getClass().getSimpleName());
         }
-        maxDocumentRequestSize = ComponentUtil.getFessConfig().getIndexerDataMaxDocumentRequestSizeAsInteger().longValue();
+        maxDocumentRequestSize = Long.parseLong(ComponentUtil.getFessConfig().getIndexerDataMaxDocumentRequestSize());
+        maxDocumentCacheSize = ComponentUtil.getFessConfig().getIndexerDataMaxDocumentCacheSizeAsInteger();
     }
 
     /* (non-Javadoc)
@@ -106,20 +110,19 @@ public class IndexUpdateCallbackImpl implements IndexUpdateCallback {
 
         synchronized (docList) {
             docList.add(dataMap);
+            final long contentSize = indexingHelper.calculateDocumentSize(dataMap);
+            docList.addContentSize(contentSize);
+            final long processingTime = System.currentTimeMillis() - startTime;
+            docList.addProcessingTime(processingTime);
             if (logger.isDebugEnabled()) {
-                logger.debug("Added the document. The number of a document cache is {}.", docList.size());
+                logger.debug("Added the document({}, {}ms). The number of a document cache is {}.",
+                        MemoryUtil.byteCountToDisplaySize(contentSize), processingTime, docList.size());
             }
 
-            final Long contentLength = DocumentUtil.getValue(dataMap, fessConfig.getIndexFieldContentLength(), Long.class);
-            if (contentLength != null) {
-                docList.addContentSize(indexingHelper.calculateDocumentSize(dataMap, contentLength.longValue()));
-                if (docList.getContentSize() >= maxDocumentRequestSize) {
-                    indexingHelper.sendDocuments(fessEsClient, docList);
-                }
-            } else if (docList.size() >= fessConfig.getIndexerDataMaxDocumentCacheSizeAsInteger().intValue()) {
+            if (docList.getContentSize() >= maxDocumentRequestSize || docList.size() >= maxDocumentCacheSize) {
                 indexingHelper.sendDocuments(fessEsClient, docList);
             }
-            executeTime += System.currentTimeMillis() - startTime;
+            executeTime += processingTime;
         }
 
         documentSize.getAndIncrement();

src/main/java/org/codelibs/fess/helper/IndexingHelper.java

@@ -203,12 +203,8 @@ public class IndexingHelper {
 
     }
 
-    public long calculateDocumentSize(final Map<String, Object> dataMap, final long size) {
-        final long objSize = MemoryUtil.sizeOf(dataMap);
-        if (objSize > size) {
-            return objSize;
-        }
-        return size;
+    public long calculateDocumentSize(final Map<String, Object> dataMap) {
+        return MemoryUtil.sizeOf(dataMap);
     }
 
     public void setMaxRetryCount(final int maxRetryCount) {

src/main/java/org/codelibs/fess/indexer/IndexUpdater.java

@@ -308,7 +308,7 @@ public class IndexUpdater extends Thread {
 
     private void processAccessResults(final DocList docList, final List<EsAccessResult> accessResultList, final List<EsAccessResult> arList) {
         final FessConfig fessConfig = ComponentUtil.getFessConfig();
-        final long maxDocumentRequestSize = fessConfig.getIndexerWebfsMaxDocumentRequestSizeAsInteger().longValue();
+        final long maxDocumentRequestSize = Long.parseLong(fessConfig.getIndexerWebfsMaxDocumentRequestSize());
         for (final EsAccessResult accessResult : arList) {
             if (logger.isDebugEnabled()) {
                 logger.debug("Indexing {}", accessResult.getUrl());
@@ -355,20 +355,17 @@ public class IndexUpdater extends Thread {
                     updateDocument(map);
 
                     docList.add(map);
+                    final long contentSize = indexingHelper.calculateDocumentSize(map);
+                    docList.addContentSize(contentSize);
                     final long processingTime = System.currentTimeMillis() - startTime;
                     docList.addProcessingTime(processingTime);
                     if (logger.isDebugEnabled()) {
-                        logger.debug("Added the document({}, {}ms). " + "The number of a document cache is {}.",
-                                MemoryUtil.byteCountToDisplaySize(docList.getContentSize()), processingTime, docList.size());
+                        logger.debug("Added the document({}, {}ms). The number of a document cache is {}.",
+                                MemoryUtil.byteCountToDisplaySize(contentSize), processingTime, docList.size());
                     }
 
-                    if (accessResult.getContentLength() == null) {
+                    if (docList.getContentSize() >= maxDocumentRequestSize) {
                         indexingHelper.sendDocuments(fessEsClient, docList);
-                    } else {
-                        docList.addContentSize(accessResult.getContentLength().longValue());
-                        if (docList.getContentSize() >= maxDocumentRequestSize) {
-                            indexingHelper.sendDocuments(fessEsClient, docList);
-                        }
                     }
                     documentSize++;
                     if (logger.isDebugEnabled()) {

src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java

@@ -417,7 +417,7 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
     /** The key of the configuration. e.g. 1048576 */
     String INDEXER_WEBFS_MAX_DOCUMENT_REQUEST_SIZE = "indexer.webfs.max.document.request.size";
 
-    /** The key of the configuration. e.g. 5 */
+    /** The key of the configuration. e.g. 10000 */
     String INDEXER_DATA_MAX_DOCUMENT_CACHE_SIZE = "indexer.data.max.document.cache.size";
 
     /** The key of the configuration. e.g. 1048576 */
@@ -2809,14 +2809,14 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
 
     /**
      * Get the value for the key 'indexer.data.max.document.cache.size'. <br>
-     * The value is, e.g. 5 <br>
+     * The value is, e.g. 10000 <br>
      * @return The value of found property. (NotNull: if not found, exception but basically no way)
      */
     String getIndexerDataMaxDocumentCacheSize();
 
     /**
      * Get the value for the key 'indexer.data.max.document.cache.size' as {@link Integer}. <br>
-     * The value is, e.g. 5 <br>
+     * The value is, e.g. 10000 <br>
      * @return The value of found property. (NotNull: if not found, exception but basically no way)
      * @throws NumberFormatException When the property is not integer.
      */
@@ -9167,7 +9167,7 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
             defaultMap.put(FessConfig.INDEXER_WEBFS_UPDATE_INTERVAL, "10000");
             defaultMap.put(FessConfig.INDEXER_WEBFS_MAX_DOCUMENT_CACHE_SIZE, "10");
             defaultMap.put(FessConfig.INDEXER_WEBFS_MAX_DOCUMENT_REQUEST_SIZE, "1048576");
-            defaultMap.put(FessConfig.INDEXER_DATA_MAX_DOCUMENT_CACHE_SIZE, "5");
+            defaultMap.put(FessConfig.INDEXER_DATA_MAX_DOCUMENT_CACHE_SIZE, "10000");
             defaultMap.put(FessConfig.INDEXER_DATA_MAX_DOCUMENT_REQUEST_SIZE, "1048576");
             defaultMap.put(FessConfig.INDEXER_LANGUAGE_FIELDS, "content,important_content,title");
             defaultMap.put(FessConfig.INDEXER_LANGUAGE_DETECT_LENGTH, "1000");

src/main/resources/fess_config.properties

@@ -246,7 +246,7 @@ indexer.webfs.max.empty.list.count=3600
 indexer.webfs.update.interval=10000
 indexer.webfs.max.document.cache.size=10
 indexer.webfs.max.document.request.size=1048576
-indexer.data.max.document.cache.size=5
+indexer.data.max.document.cache.size=10000
 indexer.data.max.document.request.size=1048576
 indexer.language.fields=content,important_content,title
 indexer.language.detect.length=1000