improve log handling

dbflute_fess/freegen/elasticsearch/AbstractBehavior.vm

@@ -337,19 +337,24 @@ public abstract class AbstractBehavior<ENTITY extends Entity, CB extends Conditi
                 bulkBuilder.add((DeleteRequestBuilder) builder);
             }
         }
-        RequestOptionCall<BulkRequestBuilder> builderCall = bulkList.getCall();
+        final RequestOptionCall<BulkRequestBuilder> builderCall = bulkList.getCall();
         if (builderCall != null) {
             builderCall.callback(bulkBuilder);
         }
-        BulkResponse response = bulkBuilder.execute().actionGet();
 
-        List<Integer> resultList = new ArrayList<>();
-        for (BulkItemResponse itemResponse : response.getItems()) {
-            resultList.add(itemResponse.isFailed() ? 0 : 1);
+        final BulkResponse response = bulkBuilder.execute().actionGet();
+        final BulkItemResponse[] itemResponses = response.getItems();
+        if (itemResponses.length != entityList.size()) {
+            throw new IllegalStateException("Invalid response size: " + itemResponses.length + " != " + entityList.size());
         }
-        int[] results = new int[resultList.size()];
-        for (int i = 0; i < resultList.size(); i++) {
-            results[i] = resultList.get(i);
+        final int[] results = new int[itemResponses.length];
+        for (int i = 0; i < itemResponses.length; i++) {
+            final BulkItemResponse itemResponse = itemResponses[i];
+            final Entity entity = entityList.get(i);
+            if (entity instanceof AbstractEntity) {
+                ((AbstractEntity) entity).asDocMeta().id(itemResponse.getId());
+            }
+            results[i] = itemResponse.isFailed() ? 0 : 1;
         }
         return results;
     }

src/main/java/org/codelibs/fess/app/service/SearchService.java

@@ -7,6 +7,7 @@ import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.ExecutionException;
 import java.util.function.Consumer;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -22,6 +23,7 @@ import org.codelibs.fess.entity.SearchRenderData;
 import org.codelibs.fess.entity.SearchRequestParams;
 import org.codelibs.fess.es.client.FessEsClient;
 import org.codelibs.fess.es.client.FessEsClient.SearchConditionBuilder;
+import org.codelibs.fess.es.client.FessEsClientException;
 import org.codelibs.fess.es.exentity.SearchLog;
 import org.codelibs.fess.es.exentity.UserInfo;
 import org.codelibs.fess.helper.FieldHelper;
@@ -32,16 +34,15 @@ import org.codelibs.fess.helper.UserInfoHelper;
 import org.codelibs.fess.util.ComponentUtil;
 import org.codelibs.fess.util.QueryResponseList;
 import org.dbflute.optional.OptionalEntity;
+import org.elasticsearch.action.bulk.BulkRequestBuilder;
+import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.index.query.QueryBuilders;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 public class SearchService {
 
     // ===================================================================================
     //                                                                            Constant
     //
-    private static final Logger logger = LoggerFactory.getLogger(SearchService.class);
 
     protected static final Pattern FIELD_EXTRACTION_PATTERN = Pattern.compile("^([a-zA-Z0-9_]+):.*");
 
@@ -71,6 +72,8 @@ public class SearchService {
     //                                                                      ==============
 
     public void search(final HttpServletRequest request, final SearchRequestParams params, final SearchRenderData data) {
+        final long requestedTime = systemHelper.getCurrentTimeAsLong();
+
         final long startTime = System.currentTimeMillis();
         final boolean searchLogSupport =
                 Constants.TRUE.equals(crawlerProperties.getProperty(Constants.SEARCH_LOG_PROPERTY, Constants.TRUE));
@@ -136,7 +139,7 @@ public class SearchService {
 
         // search log
         if (searchLogSupport) {
-            storeSearchLog(request, query, pageStart, pageSize, queryResponseList);
+            storeSearchLog(request, requestedTime, query, pageStart, pageSize, queryResponseList);
         }
 
         queryResponseList.setExecTime(System.currentTimeMillis() - startTime);
@@ -163,11 +166,11 @@ public class SearchService {
         data.setPartialResults(queryResponseList.isPartialResults());
         data.setQueryTime(queryResponseList.getQueryTime());
         data.setSearchQuery(query);
+        data.setRequestedTime(requestedTime);
     }
 
-    protected void storeSearchLog(final HttpServletRequest request, final String query, final int pageStart, final int pageSize,
-            final QueryResponseList queryResponseList) {
-        final long now = systemHelper.getCurrentTimeAsLong();
+    protected void storeSearchLog(final HttpServletRequest request, final long requestedTime, final String query, final int pageStart,
+            final int pageSize, final QueryResponseList queryResponseList) {
 
         final SearchLogHelper searchLogHelper = ComponentUtil.getSearchLogHelper();
         final SearchLog searchLog = new SearchLog();
@@ -178,8 +181,8 @@ public class SearchService {
             if (StringUtil.isNotBlank(userCode)) {
                 final UserInfo userInfo = new UserInfo();
                 userInfo.setCode(userCode);
-                userInfo.setCreatedTime(now);
-                userInfo.setUpdatedTime(now);
+                userInfo.setCreatedTime(requestedTime);
+                userInfo.setUpdatedTime(requestedTime);
                 searchLog.setUserInfo(OptionalEntity.of(userInfo));
             }
         }
@@ -188,7 +191,7 @@ public class SearchService {
         searchLog.setResponseTime(Integer.valueOf((int) queryResponseList.getExecTime()));
         searchLog.setSearchWord(StringUtils.abbreviate(query, 1000));
         searchLog.setSearchQuery(StringUtils.abbreviate(queryResponseList.getSearchQuery(), 1000));
-        searchLog.setRequestedTime(now);
+        searchLog.setRequestedTime(requestedTime);
         searchLog.setQueryOffset(pageStart);
         searchLog.setQueryPageSize(pageSize);
 
@@ -321,4 +324,19 @@ public class SearchService {
     public boolean update(final String id, final String field, final Object value) {
         return fessEsClient.update(fieldHelper.docIndex, fieldHelper.docType, id, field, value);
     }
+
+    public boolean bulkUpdate(Consumer<BulkRequestBuilder> consumer) {
+        BulkRequestBuilder builder = fessEsClient.prepareBulk();
+        consumer.accept(builder);
+        try {
+            BulkResponse response = builder.execute().get();
+            if (response.hasFailures()) {
+                throw new FessEsClientException(response.buildFailureMessage());
+            } else {
+                return true;
+            }
+        } catch (InterruptedException | ExecutionException e) {
+            throw new FessEsClientException("Failed to update bulk data.", e);
+        }
+    }
 }

src/main/java/org/codelibs/fess/app/web/search/SearchAction.java

@@ -134,6 +134,7 @@ public class SearchAction extends FessSearchAction {
             try {
                 final WebRenderData renderData = new WebRenderData(data);
                 searchService.search(request, form, renderData);
+                form.rt = Long.toString(renderData.getRequestedTime());
                 // favorite or screenshot
                 if (favoriteSupport || screenShotManager != null) {
                     final String searchQuery = renderData.getSearchQuery();
@@ -157,7 +158,6 @@ public class SearchAction extends FessSearchAction {
                     messages.addErrorsResultSizeExceeded(GLOBAL);
                 }, () -> asHtml(path_ErrorJsp));
             }
-            form.rt = Long.toString(systemHelper.getCurrentTimeAsLong());
             data.register("displayQuery", getDisplayQuery(form, labelTypeHelper.getLabelTypeItemList()));
             data.register("pagingQuery", getPagingQuery(form));
         });

src/main/java/org/codelibs/fess/entity/SearchRenderData.java

@@ -39,6 +39,8 @@ public class SearchRenderData {
 
     private long queryTime;
 
+    private long requestedTime;
+
     public void setDocumentItems(final List<Map<String, Object>> documentItems) {
         this.documentItems = documentItems;
     }
@@ -103,6 +105,10 @@ public class SearchRenderData {
         this.searchQuery = searchQuery;
     }
 
+    public void setRequestedTime(final long requestedTime) {
+        this.requestedTime = requestedTime;
+    }
+
     public List<Map<String, Object>> getDocumentItems() {
         return documentItems;
     }
@@ -167,4 +173,8 @@ public class SearchRenderData {
         return queryTime;
     }
 
+    public long getRequestedTime() {
+        return requestedTime;
+    }
+
 }

src/main/java/org/codelibs/fess/es/bsbhv/AbstractBehavior.java

@@ -339,19 +339,24 @@ public abstract class AbstractBehavior<ENTITY extends Entity, CB extends Conditi
                 bulkBuilder.add((DeleteRequestBuilder) builder);
             }
         }
-        RequestOptionCall<BulkRequestBuilder> builderCall = bulkList.getCall();
+        final RequestOptionCall<BulkRequestBuilder> builderCall = bulkList.getCall();
         if (builderCall != null) {
             builderCall.callback(bulkBuilder);
         }
-        BulkResponse response = bulkBuilder.execute().actionGet();
 
-        List<Integer> resultList = new ArrayList<>();
-        for (BulkItemResponse itemResponse : response.getItems()) {
-            resultList.add(itemResponse.isFailed() ? 0 : 1);
+        final BulkResponse response = bulkBuilder.execute().actionGet();
+        final BulkItemResponse[] itemResponses = response.getItems();
+        if (itemResponses.length != entityList.size()) {
+            throw new IllegalStateException("Invalid response size: " + itemResponses.length + " != " + entityList.size());
         }
-        int[] results = new int[resultList.size()];
-        for (int i = 0; i < resultList.size(); i++) {
-            results[i] = resultList.get(i);
+        final int[] results = new int[itemResponses.length];
+        for (int i = 0; i < itemResponses.length; i++) {
+            final BulkItemResponse itemResponse = itemResponses[i];
+            final Entity entity = entityList.get(i);
+            if (entity instanceof AbstractEntity) {
+                ((AbstractEntity) entity).asDocMeta().id(itemResponse.getId());
+            }
+            results[i] = itemResponse.isFailed() ? 0 : 1;
         }
         return results;
     }

src/main/java/org/codelibs/fess/helper/SearchLogHelper.java

@@ -16,23 +16,48 @@
 
 package org.codelibs.fess.helper;
 
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Queue;
 import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.stream.Stream;
 
 import javax.annotation.PostConstruct;
 import javax.annotation.Resource;
 
+import org.codelibs.core.beans.util.BeanUtil;
 import org.codelibs.core.collection.LruHashMap;
+import org.codelibs.core.lang.StringUtil;
 import org.codelibs.core.misc.DynamicProperties;
+import org.codelibs.fess.Constants;
+import org.codelibs.fess.app.service.SearchService;
+import org.codelibs.fess.es.exbhv.ClickLogBhv;
+import org.codelibs.fess.es.exbhv.FavoriteLogBhv;
+import org.codelibs.fess.es.exbhv.SearchFieldLogBhv;
+import org.codelibs.fess.es.exbhv.SearchLogBhv;
+import org.codelibs.fess.es.exbhv.UserInfoBhv;
 import org.codelibs.fess.es.exentity.ClickLog;
+import org.codelibs.fess.es.exentity.SearchFieldLog;
 import org.codelibs.fess.es.exentity.SearchLog;
+import org.codelibs.fess.es.exentity.UserInfo;
+import org.codelibs.fess.util.ComponentUtil;
+import org.codelibs.fess.util.DocumentUtil;
+import org.elasticsearch.action.update.UpdateRequest;
+import org.lastaflute.di.core.SingletonLaContainer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
-public abstract class SearchLogHelper {
+public class SearchLogHelper {
+    private static final Logger logger = LoggerFactory.getLogger(SearchLogHelper.class);
 
     @Resource
     protected DynamicProperties crawlerProperties;
 
+    @Resource
+    protected FieldHelper fieldHelper;
+
     public long userCheckInterval = 5 * 60 * 1000;// 5 min
 
     public int userInfoCacheSize = 1000;
@@ -48,8 +73,6 @@ public abstract class SearchLogHelper {
         userInfoCache = new LruHashMap<String, Long>(userInfoCacheSize);
     }
 
-    public abstract void updateUserInfo(final String userCode);
-
     public void addSearchLog(final SearchLog searchLog) {
         searchLogQueue.add(searchLog);
     }
@@ -73,21 +96,185 @@ public abstract class SearchLogHelper {
     }
 
     public int getClickCount(final String url) {
-        // TODO
-        return 0;
+        final ClickLogBhv clickLogBhv = ComponentUtil.getComponent(ClickLogBhv.class);
+        return clickLogBhv.selectCount(cb -> {
+            cb.query().setUrl_Equal(url);
+        });
     }
 
     public long getFavoriteCount(final String url) {
-        // TODO
-        return 0;
+        final FavoriteLogBhv favoriteLogBhv = ComponentUtil.getComponent(FavoriteLogBhv.class);
+        return favoriteLogBhv.selectCount(cb -> {
+            cb.query().setUrl_Equal(url);
+        });
+    }
+
+    public void updateUserInfo(final String userCode) {
+        final long current = System.currentTimeMillis();
+        final Long time = userInfoCache.get(userCode);
+        if (time == null || current - time.longValue() > userCheckInterval) {
+
+            final UserInfoBhv userInfoBhv = ComponentUtil.getComponent(UserInfoBhv.class);
+
+            final List<UserInfo> list = userInfoBhv.selectList(cb -> {
+                cb.query().setCode_Equal(userCode);
+                cb.query().addOrderBy_UpdatedTime_Desc();
+            });
+
+            final UserInfo userInfo;
+            final long now = ComponentUtil.getSystemHelper().getCurrentTimeAsLong();
+            if (list.isEmpty()) {
+                userInfo = new UserInfo();
+                userInfo.setCode(userCode);
+                userInfo.setCreatedTime(now);
+            } else {
+                userInfo = list.get(0);
+            }
+            userInfo.setUpdatedTime(now);
+            new Thread(() -> {
+                userInfoBhv.insertOrUpdate(userInfo);
+            }).start();
+            userInfoCache.put(userCode, current);
+        }
+    }
+
+    protected void processSearchLogQueue(final Queue<SearchLog> queue) {
+        final String value = crawlerProperties.getProperty(Constants.PURGE_BY_BOTS_PROPERTY, StringUtil.EMPTY);
+        String[] botNames;
+        if (StringUtil.isBlank(value)) {
+            botNames = StringUtil.EMPTY_STRINGS;
+        } else {
+            botNames = value.split(",");
+        }
+
+        final List<SearchLog> searchLogList = new ArrayList<>();
+        final Map<String, UserInfo> userInfoMap = new HashMap<>();
+        queue.stream().forEach(searchLog -> {
+            final String userAgent = searchLog.getUserAgent();
+            boolean isBot = userAgent != null && Stream.of(botNames).anyMatch(botName -> userAgent.indexOf(botName) >= 0);
+            if (!isBot) {
+                searchLog.getUserInfo().ifPresent(userInfo -> {
+                    final String code = userInfo.getCode();
+                    final UserInfo oldUserInfo = userInfoMap.get(code);
+                    if (oldUserInfo != null) {
+                        userInfo.setCreatedTime(oldUserInfo.getCreatedTime());
+                    }
+                    userInfoMap.put(code, userInfo);
+                });
+                searchLogList.add(searchLog);
+            }
+        });
+
+        if (!userInfoMap.isEmpty()) {
+            final List<UserInfo> insertList = new ArrayList<>(userInfoMap.values());
+            final List<UserInfo> updateList = new ArrayList<>();
+            final UserInfoBhv userInfoBhv = SingletonLaContainer.getComponent(UserInfoBhv.class);
+            final List<UserInfo> list = userInfoBhv.selectList(cb -> {
+                cb.query().setCode_InScope(userInfoMap.keySet());
+            });
+            for (final UserInfo userInfo : list) {
+                final String code = userInfo.getCode();
+                final UserInfo entity = userInfoMap.get(code);
+                BeanUtil.copyBeanToBean(userInfo, entity, option -> option.include("id", "createdTime"));
+                updateList.add(entity);
+                insertList.remove(entity);
+            }
+            userInfoBhv.batchInsert(insertList);
+            userInfoBhv.batchUpdate(updateList);
+            searchLogList.stream().forEach(searchLog -> {
+                searchLog.getUserInfo().ifPresent(userInfo -> {
+                    final UserInfo entity = userInfoMap.get(userInfo.getCode());
+                    searchLog.setUserInfoId(entity.getId());
+                });
+            });
+        }
+
+        if (!searchLogList.isEmpty()) {
+            storeSearchLogList(searchLogList);
+        }
     }
 
-    protected abstract void processSearchLogQueue(Queue<SearchLog> queue);
+    private void storeSearchLogList(final List<SearchLog> searchLogList) {
+        final SearchLogBhv searchLogBhv = ComponentUtil.getComponent(SearchLogBhv.class);
+        final SearchFieldLogBhv searchFieldLogBhv = ComponentUtil.getComponent(SearchFieldLogBhv.class);
+        searchLogBhv.batchUpdate(searchLogList);
+        searchLogList.stream().forEach(searchLog -> {
+            final List<SearchFieldLog> fieldLogList = new ArrayList<>();
+            searchLog.getSearchFieldLogList().stream().forEach(fieldLog -> {
+                fieldLog.setSearchLogId(searchLog.getId());
+                fieldLogList.add(fieldLog);
+            });
+            searchFieldLogBhv.batchInsert(fieldLogList);
+        });
+    }
 
-    protected abstract void processClickLogQueue(Queue<ClickLog> queue);
+    protected void processClickLogQueue(final Queue<ClickLog> queue) {
+        final Map<String, Long> clickCountMap = new HashMap<>();
+        final List<ClickLog> clickLogList = new ArrayList<>();
+        for (final ClickLog clickLog : queue) {
+            try {
+                final SearchLogBhv searchLogBhv = SingletonLaContainer.getComponent(SearchLogBhv.class);
+                searchLogBhv.selectEntity(cb -> {
+                    cb.query().setRequestedTime_Equal(clickLog.getQueryRequestedTime());
+                    cb.query().setUserSessionId_Equal(clickLog.getUserSessionId());
+                }).ifPresent(entity -> {
+                    clickLog.setSearchLogId(entity.getId());
+                    clickLogList.add(clickLog);
+                }).orElse(() -> {
+                    logger.warn("Not Found for SearchLog: " + clickLog);
+                });
 
-    public boolean addfavoriteLog(final String userCode, final String favoriteUrl) {
-        // TODO Auto-generated method stub
-        return false;
+                final String docId = clickLog.getDocId();
+                Long countObj = clickCountMap.get(docId);
+                if (countObj == null) {
+                    final long clickCount = clickLog.getClickCount();
+                    countObj = Long.valueOf(clickCount + 1);
+                } else {
+                    countObj = countObj.longValue() + 1;
+                }
+                clickCountMap.put(docId, countObj);
+            } catch (final Exception e) {
+                logger.warn("Failed to process: " + clickLog, e);
+            }
+        }
+        if (!clickLogList.isEmpty()) {
+            try {
+                final ClickLogBhv clickLogBhv = SingletonLaContainer.getComponent(ClickLogBhv.class);
+                clickLogBhv.batchInsert(clickLogList);
+            } catch (final Exception e) {
+                logger.warn("Failed to insert: " + clickLogList, e);
+            }
+        }
+
+        if (!clickCountMap.isEmpty()) {
+            final SearchService searchService = ComponentUtil.getComponent(SearchService.class);
+            try {
+                final Map<String, String> docIdMap = new HashMap<>();
+                searchService
+                        .getDocumentListByDocIds(clickCountMap.keySet().toArray(new String[clickCountMap.size()]),
+                                new String[] { fieldHelper.idField, fieldHelper.docIdField })
+                        .stream()
+                        .forEach(
+                                doc -> {
+                                    docIdMap.put(DocumentUtil.getValue(doc, fieldHelper.docIdField, String.class),
+                                            DocumentUtil.getValue(doc, fieldHelper.idField, String.class));
+                                });
+                searchService.bulkUpdate(builder -> {
+                    clickCountMap
+                            .entrySet()
+                            .stream()
+                            .forEach(
+                                    entry -> {
+                                        String id = docIdMap.get(entry.getKey());
+                                        if (id != null) {
+                                            builder.add(new UpdateRequest(fieldHelper.docIndex, fieldHelper.docType, id).doc(
+                                                    fieldHelper.clickCountField, entry.getValue()));
+                                        }
+                                    });
+                });
+            } catch (final Exception e) {
+                logger.warn("Failed to update clickCounts", e);
+            }
+        }
     }
 }

src/main/java/org/codelibs/fess/helper/impl/SearchLogHelperImpl.java

@@ -1,214 +0,0 @@
-/*
- * Copyright 2009-2015 the CodeLibs Project and the Others.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
- * either express or implied. See the License for the specific language
- * governing permissions and limitations under the License.
- */
-
-package org.codelibs.fess.helper.impl;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Queue;
-
-import org.codelibs.core.beans.util.BeanUtil;
-import org.codelibs.core.lang.StringUtil;
-import org.codelibs.fess.Constants;
-import org.codelibs.fess.es.client.FessEsClient;
-import org.codelibs.fess.es.exbhv.ClickLogBhv;
-import org.codelibs.fess.es.exbhv.SearchLogBhv;
-import org.codelibs.fess.es.exbhv.UserInfoBhv;
-import org.codelibs.fess.es.exentity.ClickLog;
-import org.codelibs.fess.es.exentity.SearchLog;
-import org.codelibs.fess.es.exentity.UserInfo;
-import org.codelibs.fess.helper.FieldHelper;
-import org.codelibs.fess.helper.SearchLogHelper;
-import org.codelibs.fess.util.ComponentUtil;
-import org.lastaflute.di.core.SingletonLaContainer;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class SearchLogHelperImpl extends SearchLogHelper {
-    private static final Logger logger = LoggerFactory // NOPMD
-            .getLogger(SearchLogHelperImpl.class);
-
-    @Override
-    public void updateUserInfo(final String userCode) {
-        final long current = System.currentTimeMillis();
-        final Long time = userInfoCache.get(userCode);
-        if (time == null || current - time.longValue() > userCheckInterval) {
-
-            final UserInfoBhv userInfoBhv = ComponentUtil.getComponent(UserInfoBhv.class);
-
-            final List<UserInfo> list = userInfoBhv.selectList(cb -> {
-                cb.query().setCode_Equal(userCode);
-                cb.query().addOrderBy_UpdatedTime_Desc();
-            });
-
-            final UserInfo userInfo;
-            final long now = ComponentUtil.getSystemHelper().getCurrentTimeAsLong();
-            if (list.isEmpty()) {
-                userInfo = new UserInfo();
-                userInfo.setCode(userCode);
-                userInfo.setCreatedTime(now);
-            } else {
-                userInfo = list.get(0);
-            }
-            userInfo.setUpdatedTime(now);
-            new Thread(() -> {
-                userInfoBhv.insertOrUpdate(userInfo);
-            }).start();
-            userInfoCache.put(userCode, current);
-        }
-    }
-
-    @Override
-    protected void processSearchLogQueue(final Queue<SearchLog> queue) {
-        final List<SearchLog> searchLogList = new ArrayList<SearchLog>();
-        final String value = crawlerProperties.getProperty(Constants.PURGE_BY_BOTS_PROPERTY, StringUtil.EMPTY);
-        String[] botNames;
-        if (StringUtil.isBlank(value)) {
-            botNames = StringUtil.EMPTY_STRINGS;
-        } else {
-            botNames = value.split(",");
-        }
-
-        Constants.TRUE.equals(crawlerProperties.getProperty(Constants.SUGGEST_SEARCH_LOG_PROPERTY, Constants.TRUE));
-        final String dayForCleanupStr = crawlerProperties.getProperty(Constants.PURGE_SUGGEST_SEARCH_LOG_DAY_PROPERTY, "30");
-        try {
-            Integer.parseInt(dayForCleanupStr);
-        } catch (final NumberFormatException e) {}
-
-        final Map<String, UserInfo> userInfoMap = new HashMap<String, UserInfo>();
-        for (final SearchLog searchLog : queue) {
-            boolean add = true;
-            for (final String botName : botNames) {
-                if (searchLog.getUserAgent() != null && searchLog.getUserAgent().indexOf(botName) >= 0) {
-                    add = false;
-                    break;
-                }
-            }
-            if (add) {
-                final UserInfo userInfo = searchLog.getUserInfo().orElse(null);// TODO
-                if (userInfo != null) {
-                    final String code = userInfo.getCode();
-                    final UserInfo oldUserInfo = userInfoMap.get(code);
-                    if (oldUserInfo != null) {
-                        userInfo.setCreatedTime(oldUserInfo.getCreatedTime());
-                    }
-                    userInfoMap.put(code, userInfo);
-                }
-                searchLogList.add(searchLog);
-
-                // TODO
-                //                if (suggestAvailable && searchLog.getHitCount() > 0) {
-                //                    final List<SearchFieldLog> searchFieldLogList = searchLog.getSearchFieldLogList();
-                //                    for (final SearchFieldLog searchFieldLog : searchFieldLogList) {
-                //                        if ("solrQuery".equals(searchFieldLog.getName())) {
-                //                            suggestService.addSolrParams(searchFieldLog.getValue(), dayForCleanup);
-                //                            addedSuggest = true;
-                //                        }
-                //                    }
-                //                }
-            }
-        }
-        //        if (addedSuggest) {
-        //            suggestService.commit();
-        //        }
-
-        if (!userInfoMap.isEmpty()) {
-            final List<UserInfo> insertList = new ArrayList<UserInfo>(userInfoMap.values());
-            final List<UserInfo> updateList = new ArrayList<UserInfo>();
-            final UserInfoBhv userInfoBhv = SingletonLaContainer.getComponent(UserInfoBhv.class);
-            final List<UserInfo> list = userInfoBhv.selectList(cb -> {
-                cb.query().setCode_InScope(userInfoMap.keySet());
-            });
-            for (final UserInfo userInfo : list) {
-                final String code = userInfo.getCode();
-                final UserInfo entity = userInfoMap.get(code);
-                BeanUtil.copyBeanToBean(userInfo, entity, option -> option.include("id", "createdTime"));
-                updateList.add(entity);
-                insertList.remove(entity);
-            }
-            userInfoBhv.batchInsert(insertList);
-            userInfoBhv.batchUpdate(updateList);
-            for (final SearchLog searchLog : searchLogList) {
-                final UserInfo userInfo = searchLog.getUserInfo().orElse(null);//TODO
-                if (userInfo != null) {
-                    final UserInfo entity = userInfoMap.get(userInfo.getCode());
-                    searchLog.setUserInfoId(entity.getId());
-                }
-            }
-        }
-
-        if (!searchLogList.isEmpty()) {
-            final SearchLogBhv searchLogBhv = ComponentUtil.getComponent(SearchLogBhv.class);
-            searchLogBhv.batchUpdate(searchLogList);
-            // TODO SearchLogValue
-        }
-    }
-
-    @Override
-    protected void processClickLogQueue(final Queue<ClickLog> queue) {
-        final Map<String, Long> clickCountMap = new HashMap<String, Long>();
-        final List<ClickLog> clickLogList = new ArrayList<ClickLog>();
-        for (final ClickLog clickLog : queue) {
-            try {
-                final SearchLogBhv searchLogBhv = SingletonLaContainer.getComponent(SearchLogBhv.class);
-                final SearchLog entity = searchLogBhv.selectEntity(cb -> {
-                    cb.query().setRequestedTime_Equal(clickLog.getQueryRequestedTime());
-                    cb.query().setUserSessionId_Equal(clickLog.getUserSessionId());
-                }).orElse(null);//TODO
-                if (entity != null) {
-                    clickLog.setSearchLogId(entity.getId());
-                    clickLogList.add(clickLog);
-                } else {
-                    logger.warn("Not Found[ClickLog]: " + clickLog);
-                }
-
-                final String docId = clickLog.getDocId();
-                Long countObj = clickCountMap.get(docId);
-                final long clickCount = clickLog.getClickCount();
-                if (countObj == null) {
-                    countObj = Long.valueOf(clickCount);
-                } else {
-                    countObj = Math.max(countObj.longValue(), clickCount) + 1;
-                }
-                clickCountMap.put(docId, countObj);
-            } catch (final Exception e) {
-                logger.warn("Failed to process: " + clickLog, e);
-            }
-        }
-        if (!clickLogList.isEmpty()) {
-            try {
-                final ClickLogBhv clickLogBhv = SingletonLaContainer.getComponent(ClickLogBhv.class);
-                clickLogBhv.batchInsert(clickLogList);
-            } catch (final Exception e) {
-                logger.warn("Failed to insert: " + clickLogList, e);
-            }
-        }
-
-        final FessEsClient fessEsClient = ComponentUtil.getElasticsearchClient();
-        final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
-        for (final Map.Entry<String, Long> entry : clickCountMap.entrySet()) {
-            try {
-                // TODO buik update
-                fessEsClient.update(fieldHelper.docIndex, fieldHelper.docType, entry.getKey(), fieldHelper.clickCountField,
-                        entry.getValue() + 1);
-            } catch (final Exception e) {
-                logger.warn("Failed to update a clickCount(" + entry.getValue() + ") for " + entry.getKey(), e);
-            }
-        }
-    }
-}

src/main/resources/fess.xml

@@ -6,7 +6,7 @@
 	<include path="fess_ds.xml"/>
 	<include path="fess_es.xml"/>
 
-	<component name="searchLogHelper" class="org.codelibs.fess.helper.impl.SearchLogHelperImpl">
+	<component name="searchLogHelper" class="org.codelibs.fess.helper.SearchLogHelper">
 		<!-- 
 		<property name="userCheckInterval">5 * 60 * 1000</property>
 		<property name="userInfoCacheSize">1000</property>

src/main/resources/fess_indices/fess_log.json

@@ -1,7 +1,7 @@
 {
   "settings": {
     "index": {
-      "refresh_interval": "60s",
+      "refresh_interval": "1s",
       "number_of_shards": 10,
       "number_of_replicas": 0
     }