0ct0pu5/fess
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix #2830 Add batch_size parameter to search log and click log processing.

Browse source
This commit is contained in:
Shinsuke Sugaya 2024-07-22 11:07:22 +09:00
parent 9e78a39cc1
commit a7d00092fb
3 changed files with 48 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
src/main/java/org/codelibs/fess/helper/SearchLogHelper.java
View file

@ -341,9 +341,17 @@ public class SearchLogHelper {
protected void storeSearchLogList(final List<SearchLog> searchLogList) {
final SearchLogBhv searchLogBhv = ComponentUtil.getComponent(SearchLogBhv.class);
searchLogBhv.batchUpdate(searchLogList, op -> {
op.setRefreshPolicy(Constants.TRUE);
});
final int batchSize = ComponentUtil.getFessConfig().getSearchlogProcessBatchSizeAsInteger();
final int totalSize = searchLogList.size();
for (int i = 0; i < totalSize; i += batchSize) {
final int end = Math.min(totalSize, i + batchSize);
if (logger.isDebugEnabled()) {
logger.debug("Sending {} search logs. ({}-{}/{})", end - i, i, end, totalSize);
}
searchLogBhv.batchUpdate(searchLogList.subList(i, end), op -> {
op.setRefreshPolicy(Constants.TRUE);
});
}
}
protected void processClickLogQueue(final Queue<ClickLog> queue) {
@ -416,7 +424,15 @@ public class SearchLogHelper {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
try {
final ClickLogBhv clickLogBhv = ComponentUtil.getComponent(ClickLogBhv.class);
clickLogBhv.batchInsert(clickLogList);
final int batchSize = fessConfig.getSearchlogProcessBatchSizeAsInteger();
final int totalSize = clickLogList.size();
for (int i = 0; i < totalSize; i += batchSize) {
final int end = Math.min(totalSize, i + batchSize);
if (logger.isDebugEnabled()) {
logger.debug("Sending {} click logs. ({}-{}/{})", end - i, i, end, totalSize);
}
clickLogBhv.batchInsert(clickLogList.subList(i, end));
}
} catch (final Exception e) {
logger.warn("Failed to insert: {}", clickLogList, e);
}

27
src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java
View file

@ -1365,6 +1365,9 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
/** The key of the configuration. e.g. */
String SEARCHLOG_REQUEST_HEADERS = "searchlog.request.headers";
/** The key of the configuration. e.g. 100 */
String SEARCHLOG_PROCESS_batch_size = "searchlog.process.batch_size";
/** The key of the configuration. e.g. 100 */
String THUMBNAIL_HTML_IMAGE_MIN_WIDTH = "thumbnail.html.image.min.width";
@ -6367,6 +6370,21 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
*/
Integer getSearchlogRequestHeadersAsInteger();
/**
* Get the value for the key 'searchlog.process.batch_size'. <br>
* The value is, e.g. 100 <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getSearchlogProcessBatchSize();
/**
* Get the value for the key 'searchlog.process.batch_size' as {@link Integer}. <br>
* The value is, e.g. 100 <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
* @throws NumberFormatException When the property is not integer.
*/
Integer getSearchlogProcessBatchSizeAsInteger();
/**
* Get the value for the key 'thumbnail.html.image.min.width'. <br>
* The value is, e.g. 100 <br>
@ -10127,6 +10145,14 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
return getAsInteger(FessConfig.SEARCHLOG_REQUEST_HEADERS);
}
public String getSearchlogProcessBatchSize() {
return get(FessConfig.SEARCHLOG_PROCESS_batch_size);
}
public Integer getSearchlogProcessBatchSizeAsInteger() {
return getAsInteger(FessConfig.SEARCHLOG_PROCESS_batch_size);
}
public String getThumbnailHtmlImageMinWidth() {
return get(FessConfig.THUMBNAIL_HTML_IMAGE_MIN_WIDTH);
}
@ -11311,6 +11337,7 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
defaultMap.put(FessConfig.PAGING_SEARCH_PAGE_MAX_SIZE, "100");
defaultMap.put(FessConfig.SEARCHLOG_AGG_SHARD_SIZE, "-1");
defaultMap.put(FessConfig.SEARCHLOG_REQUEST_HEADERS, "");
defaultMap.put(FessConfig.SEARCHLOG_PROCESS_batch_size, "100");
defaultMap.put(FessConfig.THUMBNAIL_HTML_IMAGE_MIN_WIDTH, "100");
defaultMap.put(FessConfig.THUMBNAIL_HTML_IMAGE_MIN_HEIGHT, "100");
defaultMap.put(FessConfig.THUMBNAIL_HTML_IMAGE_MAX_ASPECT_RATIO, "3.0");

1
src/main/resources/fess_config.properties
View file

@ -720,6 +720,7 @@ paging.search.page.max.size=100
searchlog.agg.shard.size=-1
searchlog.request.headers=
searchlog.process.batch_size=100
thumbnail.html.image.min.width=100
thumbnail.html.image.min.height=100