fix #2830 Add batch_size parameter to search log and click log processing.

src/main/java/org/codelibs/fess/helper/SearchLogHelper.java

@@ -341,9 +341,17 @@ public class SearchLogHelper {
 
     protected void storeSearchLogList(final List<SearchLog> searchLogList) {
         final SearchLogBhv searchLogBhv = ComponentUtil.getComponent(SearchLogBhv.class);
-        searchLogBhv.batchUpdate(searchLogList, op -> {
-            op.setRefreshPolicy(Constants.TRUE);
-        });
+        final int batchSize = ComponentUtil.getFessConfig().getSearchlogProcessBatchSizeAsInteger();
+        final int totalSize = searchLogList.size();
+        for (int i = 0; i < totalSize; i += batchSize) {
+            final int end = Math.min(totalSize, i + batchSize);
+            if (logger.isDebugEnabled()) {
+                logger.debug("Sending {} search logs. ({}-{}/{})", end - i, i, end, totalSize);
+            }
+            searchLogBhv.batchUpdate(searchLogList.subList(i, end), op -> {
+                op.setRefreshPolicy(Constants.TRUE);
+            });
+        }
     }
 
     protected void processClickLogQueue(final Queue<ClickLog> queue) {
@@ -416,7 +424,15 @@ public class SearchLogHelper {
             final FessConfig fessConfig = ComponentUtil.getFessConfig();
             try {
                 final ClickLogBhv clickLogBhv = ComponentUtil.getComponent(ClickLogBhv.class);
-                clickLogBhv.batchInsert(clickLogList);
+                final int batchSize = fessConfig.getSearchlogProcessBatchSizeAsInteger();
+                final int totalSize = clickLogList.size();
+                for (int i = 0; i < totalSize; i += batchSize) {
+                    final int end = Math.min(totalSize, i + batchSize);
+                    if (logger.isDebugEnabled()) {
+                        logger.debug("Sending {} click logs. ({}-{}/{})", end - i, i, end, totalSize);
+                    }
+                    clickLogBhv.batchInsert(clickLogList.subList(i, end));
+                }
             } catch (final Exception e) {
                 logger.warn("Failed to insert: {}", clickLogList, e);
             }

src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java

@@ -1365,6 +1365,9 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
     /** The key of the configuration. e.g.  */
     String SEARCHLOG_REQUEST_HEADERS = "searchlog.request.headers";
 
+    /** The key of the configuration. e.g. 100 */
+    String SEARCHLOG_PROCESS_batch_size = "searchlog.process.batch_size";
+
     /** The key of the configuration. e.g. 100 */
     String THUMBNAIL_HTML_IMAGE_MIN_WIDTH = "thumbnail.html.image.min.width";
 
@@ -6367,6 +6370,21 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
      */
     Integer getSearchlogRequestHeadersAsInteger();
 
+    /**
+     * Get the value for the key 'searchlog.process.batch_size'. <br>
+     * The value is, e.g. 100 <br>
+     * @return The value of found property. (NotNull: if not found, exception but basically no way)
+     */
+    String getSearchlogProcessBatchSize();
+
+    /**
+     * Get the value for the key 'searchlog.process.batch_size' as {@link Integer}. <br>
+     * The value is, e.g. 100 <br>
+     * @return The value of found property. (NotNull: if not found, exception but basically no way)
+     * @throws NumberFormatException When the property is not integer.
+     */
+    Integer getSearchlogProcessBatchSizeAsInteger();
+
     /**
      * Get the value for the key 'thumbnail.html.image.min.width'. <br>
      * The value is, e.g. 100 <br>
@@ -10127,6 +10145,14 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
             return getAsInteger(FessConfig.SEARCHLOG_REQUEST_HEADERS);
         }
 
+        public String getSearchlogProcessBatchSize() {
+            return get(FessConfig.SEARCHLOG_PROCESS_batch_size);
+        }
+
+        public Integer getSearchlogProcessBatchSizeAsInteger() {
+            return getAsInteger(FessConfig.SEARCHLOG_PROCESS_batch_size);
+        }
+
         public String getThumbnailHtmlImageMinWidth() {
             return get(FessConfig.THUMBNAIL_HTML_IMAGE_MIN_WIDTH);
         }
@@ -11311,6 +11337,7 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
             defaultMap.put(FessConfig.PAGING_SEARCH_PAGE_MAX_SIZE, "100");
             defaultMap.put(FessConfig.SEARCHLOG_AGG_SHARD_SIZE, "-1");
             defaultMap.put(FessConfig.SEARCHLOG_REQUEST_HEADERS, "");
+            defaultMap.put(FessConfig.SEARCHLOG_PROCESS_batch_size, "100");
             defaultMap.put(FessConfig.THUMBNAIL_HTML_IMAGE_MIN_WIDTH, "100");
             defaultMap.put(FessConfig.THUMBNAIL_HTML_IMAGE_MIN_HEIGHT, "100");
             defaultMap.put(FessConfig.THUMBNAIL_HTML_IMAGE_MAX_ASPECT_RATIO, "3.0");

src/main/resources/fess_config.properties

@@ -720,6 +720,7 @@ paging.search.page.max.size=100
 
 searchlog.agg.shard.size=-1
 searchlog.request.headers=
+searchlog.process.batch_size=100
 
 thumbnail.html.image.min.width=100
 thumbnail.html.image.min.height=100