diff --git a/dbflute_fess/freegen/elasticsearch/AbstractBehavior.vm b/dbflute_fess/freegen/elasticsearch/AbstractBehavior.vm index 8bf7e2f15..fe13d41dd 100644 --- a/dbflute_fess/freegen/elasticsearch/AbstractBehavior.vm +++ b/dbflute_fess/freegen/elasticsearch/AbstractBehavior.vm @@ -337,19 +337,24 @@ public abstract class AbstractBehavior builderCall = bulkList.getCall(); + final RequestOptionCall builderCall = bulkList.getCall(); if (builderCall != null) { builderCall.callback(bulkBuilder); } - BulkResponse response = bulkBuilder.execute().actionGet(); - List resultList = new ArrayList<>(); - for (BulkItemResponse itemResponse : response.getItems()) { - resultList.add(itemResponse.isFailed() ? 0 : 1); + final BulkResponse response = bulkBuilder.execute().actionGet(); + final BulkItemResponse[] itemResponses = response.getItems(); + if (itemResponses.length != entityList.size()) { + throw new IllegalStateException("Invalid response size: " + itemResponses.length + " != " + entityList.size()); } - int[] results = new int[resultList.size()]; - for (int i = 0; i < resultList.size(); i++) { - results[i] = resultList.get(i); + final int[] results = new int[itemResponses.length]; + for (int i = 0; i < itemResponses.length; i++) { + final BulkItemResponse itemResponse = itemResponses[i]; + final Entity entity = entityList.get(i); + if (entity instanceof AbstractEntity) { + ((AbstractEntity) entity).asDocMeta().id(itemResponse.getId()); + } + results[i] = itemResponse.isFailed() ? 0 : 1; } return results; } diff --git a/src/main/java/org/codelibs/fess/app/service/SearchService.java b/src/main/java/org/codelibs/fess/app/service/SearchService.java index c8a409c95..306907b6b 100644 --- a/src/main/java/org/codelibs/fess/app/service/SearchService.java +++ b/src/main/java/org/codelibs/fess/app/service/SearchService.java @@ -7,6 +7,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Set; +import java.util.concurrent.ExecutionException; import java.util.function.Consumer; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -22,6 +23,7 @@ import org.codelibs.fess.entity.SearchRenderData; import org.codelibs.fess.entity.SearchRequestParams; import org.codelibs.fess.es.client.FessEsClient; import org.codelibs.fess.es.client.FessEsClient.SearchConditionBuilder; +import org.codelibs.fess.es.client.FessEsClientException; import org.codelibs.fess.es.exentity.SearchLog; import org.codelibs.fess.es.exentity.UserInfo; import org.codelibs.fess.helper.FieldHelper; @@ -32,16 +34,15 @@ import org.codelibs.fess.helper.UserInfoHelper; import org.codelibs.fess.util.ComponentUtil; import org.codelibs.fess.util.QueryResponseList; import org.dbflute.optional.OptionalEntity; +import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.index.query.QueryBuilders; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class SearchService { // =================================================================================== // Constant // - private static final Logger logger = LoggerFactory.getLogger(SearchService.class); protected static final Pattern FIELD_EXTRACTION_PATTERN = Pattern.compile("^([a-zA-Z0-9_]+):.*"); @@ -71,6 +72,8 @@ public class SearchService { // ============== public void search(final HttpServletRequest request, final SearchRequestParams params, final SearchRenderData data) { + final long requestedTime = systemHelper.getCurrentTimeAsLong(); + final long startTime = System.currentTimeMillis(); final boolean searchLogSupport = Constants.TRUE.equals(crawlerProperties.getProperty(Constants.SEARCH_LOG_PROPERTY, Constants.TRUE)); @@ -136,7 +139,7 @@ public class SearchService { // search log if (searchLogSupport) { - storeSearchLog(request, query, pageStart, pageSize, queryResponseList); + storeSearchLog(request, requestedTime, query, pageStart, pageSize, queryResponseList); } queryResponseList.setExecTime(System.currentTimeMillis() - startTime); @@ -163,11 +166,11 @@ public class SearchService { data.setPartialResults(queryResponseList.isPartialResults()); data.setQueryTime(queryResponseList.getQueryTime()); data.setSearchQuery(query); + data.setRequestedTime(requestedTime); } - protected void storeSearchLog(final HttpServletRequest request, final String query, final int pageStart, final int pageSize, - final QueryResponseList queryResponseList) { - final long now = systemHelper.getCurrentTimeAsLong(); + protected void storeSearchLog(final HttpServletRequest request, final long requestedTime, final String query, final int pageStart, + final int pageSize, final QueryResponseList queryResponseList) { final SearchLogHelper searchLogHelper = ComponentUtil.getSearchLogHelper(); final SearchLog searchLog = new SearchLog(); @@ -178,8 +181,8 @@ public class SearchService { if (StringUtil.isNotBlank(userCode)) { final UserInfo userInfo = new UserInfo(); userInfo.setCode(userCode); - userInfo.setCreatedTime(now); - userInfo.setUpdatedTime(now); + userInfo.setCreatedTime(requestedTime); + userInfo.setUpdatedTime(requestedTime); searchLog.setUserInfo(OptionalEntity.of(userInfo)); } } @@ -188,7 +191,7 @@ public class SearchService { searchLog.setResponseTime(Integer.valueOf((int) queryResponseList.getExecTime())); searchLog.setSearchWord(StringUtils.abbreviate(query, 1000)); searchLog.setSearchQuery(StringUtils.abbreviate(queryResponseList.getSearchQuery(), 1000)); - searchLog.setRequestedTime(now); + searchLog.setRequestedTime(requestedTime); searchLog.setQueryOffset(pageStart); searchLog.setQueryPageSize(pageSize); @@ -321,4 +324,19 @@ public class SearchService { public boolean update(final String id, final String field, final Object value) { return fessEsClient.update(fieldHelper.docIndex, fieldHelper.docType, id, field, value); } + + public boolean bulkUpdate(Consumer consumer) { + BulkRequestBuilder builder = fessEsClient.prepareBulk(); + consumer.accept(builder); + try { + BulkResponse response = builder.execute().get(); + if (response.hasFailures()) { + throw new FessEsClientException(response.buildFailureMessage()); + } else { + return true; + } + } catch (InterruptedException | ExecutionException e) { + throw new FessEsClientException("Failed to update bulk data.", e); + } + } } diff --git a/src/main/java/org/codelibs/fess/app/web/search/SearchAction.java b/src/main/java/org/codelibs/fess/app/web/search/SearchAction.java index d6c967578..01ea48b2f 100644 --- a/src/main/java/org/codelibs/fess/app/web/search/SearchAction.java +++ b/src/main/java/org/codelibs/fess/app/web/search/SearchAction.java @@ -134,6 +134,7 @@ public class SearchAction extends FessSearchAction { try { final WebRenderData renderData = new WebRenderData(data); searchService.search(request, form, renderData); + form.rt = Long.toString(renderData.getRequestedTime()); // favorite or screenshot if (favoriteSupport || screenShotManager != null) { final String searchQuery = renderData.getSearchQuery(); @@ -157,7 +158,6 @@ public class SearchAction extends FessSearchAction { messages.addErrorsResultSizeExceeded(GLOBAL); }, () -> asHtml(path_ErrorJsp)); } - form.rt = Long.toString(systemHelper.getCurrentTimeAsLong()); data.register("displayQuery", getDisplayQuery(form, labelTypeHelper.getLabelTypeItemList())); data.register("pagingQuery", getPagingQuery(form)); }); diff --git a/src/main/java/org/codelibs/fess/entity/SearchRenderData.java b/src/main/java/org/codelibs/fess/entity/SearchRenderData.java index addb3b8c4..df5d22152 100644 --- a/src/main/java/org/codelibs/fess/entity/SearchRenderData.java +++ b/src/main/java/org/codelibs/fess/entity/SearchRenderData.java @@ -39,6 +39,8 @@ public class SearchRenderData { private long queryTime; + private long requestedTime; + public void setDocumentItems(final List> documentItems) { this.documentItems = documentItems; } @@ -103,6 +105,10 @@ public class SearchRenderData { this.searchQuery = searchQuery; } + public void setRequestedTime(final long requestedTime) { + this.requestedTime = requestedTime; + } + public List> getDocumentItems() { return documentItems; } @@ -167,4 +173,8 @@ public class SearchRenderData { return queryTime; } + public long getRequestedTime() { + return requestedTime; + } + } diff --git a/src/main/java/org/codelibs/fess/es/bsbhv/AbstractBehavior.java b/src/main/java/org/codelibs/fess/es/bsbhv/AbstractBehavior.java index a6a517131..fdc8b985e 100644 --- a/src/main/java/org/codelibs/fess/es/bsbhv/AbstractBehavior.java +++ b/src/main/java/org/codelibs/fess/es/bsbhv/AbstractBehavior.java @@ -339,19 +339,24 @@ public abstract class AbstractBehavior builderCall = bulkList.getCall(); + final RequestOptionCall builderCall = bulkList.getCall(); if (builderCall != null) { builderCall.callback(bulkBuilder); } - BulkResponse response = bulkBuilder.execute().actionGet(); - List resultList = new ArrayList<>(); - for (BulkItemResponse itemResponse : response.getItems()) { - resultList.add(itemResponse.isFailed() ? 0 : 1); + final BulkResponse response = bulkBuilder.execute().actionGet(); + final BulkItemResponse[] itemResponses = response.getItems(); + if (itemResponses.length != entityList.size()) { + throw new IllegalStateException("Invalid response size: " + itemResponses.length + " != " + entityList.size()); } - int[] results = new int[resultList.size()]; - for (int i = 0; i < resultList.size(); i++) { - results[i] = resultList.get(i); + final int[] results = new int[itemResponses.length]; + for (int i = 0; i < itemResponses.length; i++) { + final BulkItemResponse itemResponse = itemResponses[i]; + final Entity entity = entityList.get(i); + if (entity instanceof AbstractEntity) { + ((AbstractEntity) entity).asDocMeta().id(itemResponse.getId()); + } + results[i] = itemResponse.isFailed() ? 0 : 1; } return results; } diff --git a/src/main/java/org/codelibs/fess/helper/SearchLogHelper.java b/src/main/java/org/codelibs/fess/helper/SearchLogHelper.java index c08a90747..e80358cc3 100644 --- a/src/main/java/org/codelibs/fess/helper/SearchLogHelper.java +++ b/src/main/java/org/codelibs/fess/helper/SearchLogHelper.java @@ -16,23 +16,48 @@ package org.codelibs.fess.helper; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Queue; import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.stream.Stream; import javax.annotation.PostConstruct; import javax.annotation.Resource; +import org.codelibs.core.beans.util.BeanUtil; import org.codelibs.core.collection.LruHashMap; +import org.codelibs.core.lang.StringUtil; import org.codelibs.core.misc.DynamicProperties; +import org.codelibs.fess.Constants; +import org.codelibs.fess.app.service.SearchService; +import org.codelibs.fess.es.exbhv.ClickLogBhv; +import org.codelibs.fess.es.exbhv.FavoriteLogBhv; +import org.codelibs.fess.es.exbhv.SearchFieldLogBhv; +import org.codelibs.fess.es.exbhv.SearchLogBhv; +import org.codelibs.fess.es.exbhv.UserInfoBhv; import org.codelibs.fess.es.exentity.ClickLog; +import org.codelibs.fess.es.exentity.SearchFieldLog; import org.codelibs.fess.es.exentity.SearchLog; +import org.codelibs.fess.es.exentity.UserInfo; +import org.codelibs.fess.util.ComponentUtil; +import org.codelibs.fess.util.DocumentUtil; +import org.elasticsearch.action.update.UpdateRequest; +import org.lastaflute.di.core.SingletonLaContainer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public abstract class SearchLogHelper { +public class SearchLogHelper { + private static final Logger logger = LoggerFactory.getLogger(SearchLogHelper.class); @Resource protected DynamicProperties crawlerProperties; + @Resource + protected FieldHelper fieldHelper; + public long userCheckInterval = 5 * 60 * 1000;// 5 min public int userInfoCacheSize = 1000; @@ -48,8 +73,6 @@ public abstract class SearchLogHelper { userInfoCache = new LruHashMap(userInfoCacheSize); } - public abstract void updateUserInfo(final String userCode); - public void addSearchLog(final SearchLog searchLog) { searchLogQueue.add(searchLog); } @@ -73,21 +96,185 @@ public abstract class SearchLogHelper { } public int getClickCount(final String url) { - // TODO - return 0; + final ClickLogBhv clickLogBhv = ComponentUtil.getComponent(ClickLogBhv.class); + return clickLogBhv.selectCount(cb -> { + cb.query().setUrl_Equal(url); + }); } public long getFavoriteCount(final String url) { - // TODO - return 0; + final FavoriteLogBhv favoriteLogBhv = ComponentUtil.getComponent(FavoriteLogBhv.class); + return favoriteLogBhv.selectCount(cb -> { + cb.query().setUrl_Equal(url); + }); } - protected abstract void processSearchLogQueue(Queue queue); + public void updateUserInfo(final String userCode) { + final long current = System.currentTimeMillis(); + final Long time = userInfoCache.get(userCode); + if (time == null || current - time.longValue() > userCheckInterval) { - protected abstract void processClickLogQueue(Queue queue); + final UserInfoBhv userInfoBhv = ComponentUtil.getComponent(UserInfoBhv.class); - public boolean addfavoriteLog(final String userCode, final String favoriteUrl) { - // TODO Auto-generated method stub - return false; + final List list = userInfoBhv.selectList(cb -> { + cb.query().setCode_Equal(userCode); + cb.query().addOrderBy_UpdatedTime_Desc(); + }); + + final UserInfo userInfo; + final long now = ComponentUtil.getSystemHelper().getCurrentTimeAsLong(); + if (list.isEmpty()) { + userInfo = new UserInfo(); + userInfo.setCode(userCode); + userInfo.setCreatedTime(now); + } else { + userInfo = list.get(0); + } + userInfo.setUpdatedTime(now); + new Thread(() -> { + userInfoBhv.insertOrUpdate(userInfo); + }).start(); + userInfoCache.put(userCode, current); + } + } + + protected void processSearchLogQueue(final Queue queue) { + final String value = crawlerProperties.getProperty(Constants.PURGE_BY_BOTS_PROPERTY, StringUtil.EMPTY); + String[] botNames; + if (StringUtil.isBlank(value)) { + botNames = StringUtil.EMPTY_STRINGS; + } else { + botNames = value.split(","); + } + + final List searchLogList = new ArrayList<>(); + final Map userInfoMap = new HashMap<>(); + queue.stream().forEach(searchLog -> { + final String userAgent = searchLog.getUserAgent(); + boolean isBot = userAgent != null && Stream.of(botNames).anyMatch(botName -> userAgent.indexOf(botName) >= 0); + if (!isBot) { + searchLog.getUserInfo().ifPresent(userInfo -> { + final String code = userInfo.getCode(); + final UserInfo oldUserInfo = userInfoMap.get(code); + if (oldUserInfo != null) { + userInfo.setCreatedTime(oldUserInfo.getCreatedTime()); + } + userInfoMap.put(code, userInfo); + }); + searchLogList.add(searchLog); + } + }); + + if (!userInfoMap.isEmpty()) { + final List insertList = new ArrayList<>(userInfoMap.values()); + final List updateList = new ArrayList<>(); + final UserInfoBhv userInfoBhv = SingletonLaContainer.getComponent(UserInfoBhv.class); + final List list = userInfoBhv.selectList(cb -> { + cb.query().setCode_InScope(userInfoMap.keySet()); + }); + for (final UserInfo userInfo : list) { + final String code = userInfo.getCode(); + final UserInfo entity = userInfoMap.get(code); + BeanUtil.copyBeanToBean(userInfo, entity, option -> option.include("id", "createdTime")); + updateList.add(entity); + insertList.remove(entity); + } + userInfoBhv.batchInsert(insertList); + userInfoBhv.batchUpdate(updateList); + searchLogList.stream().forEach(searchLog -> { + searchLog.getUserInfo().ifPresent(userInfo -> { + final UserInfo entity = userInfoMap.get(userInfo.getCode()); + searchLog.setUserInfoId(entity.getId()); + }); + }); + } + + if (!searchLogList.isEmpty()) { + storeSearchLogList(searchLogList); + } + } + + private void storeSearchLogList(final List searchLogList) { + final SearchLogBhv searchLogBhv = ComponentUtil.getComponent(SearchLogBhv.class); + final SearchFieldLogBhv searchFieldLogBhv = ComponentUtil.getComponent(SearchFieldLogBhv.class); + searchLogBhv.batchUpdate(searchLogList); + searchLogList.stream().forEach(searchLog -> { + final List fieldLogList = new ArrayList<>(); + searchLog.getSearchFieldLogList().stream().forEach(fieldLog -> { + fieldLog.setSearchLogId(searchLog.getId()); + fieldLogList.add(fieldLog); + }); + searchFieldLogBhv.batchInsert(fieldLogList); + }); + } + + protected void processClickLogQueue(final Queue queue) { + final Map clickCountMap = new HashMap<>(); + final List clickLogList = new ArrayList<>(); + for (final ClickLog clickLog : queue) { + try { + final SearchLogBhv searchLogBhv = SingletonLaContainer.getComponent(SearchLogBhv.class); + searchLogBhv.selectEntity(cb -> { + cb.query().setRequestedTime_Equal(clickLog.getQueryRequestedTime()); + cb.query().setUserSessionId_Equal(clickLog.getUserSessionId()); + }).ifPresent(entity -> { + clickLog.setSearchLogId(entity.getId()); + clickLogList.add(clickLog); + }).orElse(() -> { + logger.warn("Not Found for SearchLog: " + clickLog); + }); + + final String docId = clickLog.getDocId(); + Long countObj = clickCountMap.get(docId); + if (countObj == null) { + final long clickCount = clickLog.getClickCount(); + countObj = Long.valueOf(clickCount + 1); + } else { + countObj = countObj.longValue() + 1; + } + clickCountMap.put(docId, countObj); + } catch (final Exception e) { + logger.warn("Failed to process: " + clickLog, e); + } + } + if (!clickLogList.isEmpty()) { + try { + final ClickLogBhv clickLogBhv = SingletonLaContainer.getComponent(ClickLogBhv.class); + clickLogBhv.batchInsert(clickLogList); + } catch (final Exception e) { + logger.warn("Failed to insert: " + clickLogList, e); + } + } + + if (!clickCountMap.isEmpty()) { + final SearchService searchService = ComponentUtil.getComponent(SearchService.class); + try { + final Map docIdMap = new HashMap<>(); + searchService + .getDocumentListByDocIds(clickCountMap.keySet().toArray(new String[clickCountMap.size()]), + new String[] { fieldHelper.idField, fieldHelper.docIdField }) + .stream() + .forEach( + doc -> { + docIdMap.put(DocumentUtil.getValue(doc, fieldHelper.docIdField, String.class), + DocumentUtil.getValue(doc, fieldHelper.idField, String.class)); + }); + searchService.bulkUpdate(builder -> { + clickCountMap + .entrySet() + .stream() + .forEach( + entry -> { + String id = docIdMap.get(entry.getKey()); + if (id != null) { + builder.add(new UpdateRequest(fieldHelper.docIndex, fieldHelper.docType, id).doc( + fieldHelper.clickCountField, entry.getValue())); + } + }); + }); + } catch (final Exception e) { + logger.warn("Failed to update clickCounts", e); + } + } } } diff --git a/src/main/java/org/codelibs/fess/helper/impl/SearchLogHelperImpl.java b/src/main/java/org/codelibs/fess/helper/impl/SearchLogHelperImpl.java deleted file mode 100644 index 9e525ee15..000000000 --- a/src/main/java/org/codelibs/fess/helper/impl/SearchLogHelperImpl.java +++ /dev/null @@ -1,214 +0,0 @@ -/* - * Copyright 2009-2015 the CodeLibs Project and the Others. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied. See the License for the specific language - * governing permissions and limitations under the License. - */ - -package org.codelibs.fess.helper.impl; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Queue; - -import org.codelibs.core.beans.util.BeanUtil; -import org.codelibs.core.lang.StringUtil; -import org.codelibs.fess.Constants; -import org.codelibs.fess.es.client.FessEsClient; -import org.codelibs.fess.es.exbhv.ClickLogBhv; -import org.codelibs.fess.es.exbhv.SearchLogBhv; -import org.codelibs.fess.es.exbhv.UserInfoBhv; -import org.codelibs.fess.es.exentity.ClickLog; -import org.codelibs.fess.es.exentity.SearchLog; -import org.codelibs.fess.es.exentity.UserInfo; -import org.codelibs.fess.helper.FieldHelper; -import org.codelibs.fess.helper.SearchLogHelper; -import org.codelibs.fess.util.ComponentUtil; -import org.lastaflute.di.core.SingletonLaContainer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class SearchLogHelperImpl extends SearchLogHelper { - private static final Logger logger = LoggerFactory // NOPMD - .getLogger(SearchLogHelperImpl.class); - - @Override - public void updateUserInfo(final String userCode) { - final long current = System.currentTimeMillis(); - final Long time = userInfoCache.get(userCode); - if (time == null || current - time.longValue() > userCheckInterval) { - - final UserInfoBhv userInfoBhv = ComponentUtil.getComponent(UserInfoBhv.class); - - final List list = userInfoBhv.selectList(cb -> { - cb.query().setCode_Equal(userCode); - cb.query().addOrderBy_UpdatedTime_Desc(); - }); - - final UserInfo userInfo; - final long now = ComponentUtil.getSystemHelper().getCurrentTimeAsLong(); - if (list.isEmpty()) { - userInfo = new UserInfo(); - userInfo.setCode(userCode); - userInfo.setCreatedTime(now); - } else { - userInfo = list.get(0); - } - userInfo.setUpdatedTime(now); - new Thread(() -> { - userInfoBhv.insertOrUpdate(userInfo); - }).start(); - userInfoCache.put(userCode, current); - } - } - - @Override - protected void processSearchLogQueue(final Queue queue) { - final List searchLogList = new ArrayList(); - final String value = crawlerProperties.getProperty(Constants.PURGE_BY_BOTS_PROPERTY, StringUtil.EMPTY); - String[] botNames; - if (StringUtil.isBlank(value)) { - botNames = StringUtil.EMPTY_STRINGS; - } else { - botNames = value.split(","); - } - - Constants.TRUE.equals(crawlerProperties.getProperty(Constants.SUGGEST_SEARCH_LOG_PROPERTY, Constants.TRUE)); - final String dayForCleanupStr = crawlerProperties.getProperty(Constants.PURGE_SUGGEST_SEARCH_LOG_DAY_PROPERTY, "30"); - try { - Integer.parseInt(dayForCleanupStr); - } catch (final NumberFormatException e) {} - - final Map userInfoMap = new HashMap(); - for (final SearchLog searchLog : queue) { - boolean add = true; - for (final String botName : botNames) { - if (searchLog.getUserAgent() != null && searchLog.getUserAgent().indexOf(botName) >= 0) { - add = false; - break; - } - } - if (add) { - final UserInfo userInfo = searchLog.getUserInfo().orElse(null);// TODO - if (userInfo != null) { - final String code = userInfo.getCode(); - final UserInfo oldUserInfo = userInfoMap.get(code); - if (oldUserInfo != null) { - userInfo.setCreatedTime(oldUserInfo.getCreatedTime()); - } - userInfoMap.put(code, userInfo); - } - searchLogList.add(searchLog); - - // TODO - // if (suggestAvailable && searchLog.getHitCount() > 0) { - // final List searchFieldLogList = searchLog.getSearchFieldLogList(); - // for (final SearchFieldLog searchFieldLog : searchFieldLogList) { - // if ("solrQuery".equals(searchFieldLog.getName())) { - // suggestService.addSolrParams(searchFieldLog.getValue(), dayForCleanup); - // addedSuggest = true; - // } - // } - // } - } - } - // if (addedSuggest) { - // suggestService.commit(); - // } - - if (!userInfoMap.isEmpty()) { - final List insertList = new ArrayList(userInfoMap.values()); - final List updateList = new ArrayList(); - final UserInfoBhv userInfoBhv = SingletonLaContainer.getComponent(UserInfoBhv.class); - final List list = userInfoBhv.selectList(cb -> { - cb.query().setCode_InScope(userInfoMap.keySet()); - }); - for (final UserInfo userInfo : list) { - final String code = userInfo.getCode(); - final UserInfo entity = userInfoMap.get(code); - BeanUtil.copyBeanToBean(userInfo, entity, option -> option.include("id", "createdTime")); - updateList.add(entity); - insertList.remove(entity); - } - userInfoBhv.batchInsert(insertList); - userInfoBhv.batchUpdate(updateList); - for (final SearchLog searchLog : searchLogList) { - final UserInfo userInfo = searchLog.getUserInfo().orElse(null);//TODO - if (userInfo != null) { - final UserInfo entity = userInfoMap.get(userInfo.getCode()); - searchLog.setUserInfoId(entity.getId()); - } - } - } - - if (!searchLogList.isEmpty()) { - final SearchLogBhv searchLogBhv = ComponentUtil.getComponent(SearchLogBhv.class); - searchLogBhv.batchUpdate(searchLogList); - // TODO SearchLogValue - } - } - - @Override - protected void processClickLogQueue(final Queue queue) { - final Map clickCountMap = new HashMap(); - final List clickLogList = new ArrayList(); - for (final ClickLog clickLog : queue) { - try { - final SearchLogBhv searchLogBhv = SingletonLaContainer.getComponent(SearchLogBhv.class); - final SearchLog entity = searchLogBhv.selectEntity(cb -> { - cb.query().setRequestedTime_Equal(clickLog.getQueryRequestedTime()); - cb.query().setUserSessionId_Equal(clickLog.getUserSessionId()); - }).orElse(null);//TODO - if (entity != null) { - clickLog.setSearchLogId(entity.getId()); - clickLogList.add(clickLog); - } else { - logger.warn("Not Found[ClickLog]: " + clickLog); - } - - final String docId = clickLog.getDocId(); - Long countObj = clickCountMap.get(docId); - final long clickCount = clickLog.getClickCount(); - if (countObj == null) { - countObj = Long.valueOf(clickCount); - } else { - countObj = Math.max(countObj.longValue(), clickCount) + 1; - } - clickCountMap.put(docId, countObj); - } catch (final Exception e) { - logger.warn("Failed to process: " + clickLog, e); - } - } - if (!clickLogList.isEmpty()) { - try { - final ClickLogBhv clickLogBhv = SingletonLaContainer.getComponent(ClickLogBhv.class); - clickLogBhv.batchInsert(clickLogList); - } catch (final Exception e) { - logger.warn("Failed to insert: " + clickLogList, e); - } - } - - final FessEsClient fessEsClient = ComponentUtil.getElasticsearchClient(); - final FieldHelper fieldHelper = ComponentUtil.getFieldHelper(); - for (final Map.Entry entry : clickCountMap.entrySet()) { - try { - // TODO buik update - fessEsClient.update(fieldHelper.docIndex, fieldHelper.docType, entry.getKey(), fieldHelper.clickCountField, - entry.getValue() + 1); - } catch (final Exception e) { - logger.warn("Failed to update a clickCount(" + entry.getValue() + ") for " + entry.getKey(), e); - } - } - } -} diff --git a/src/main/resources/fess.xml b/src/main/resources/fess.xml index 186e40225..0bbf083c0 100644 --- a/src/main/resources/fess.xml +++ b/src/main/resources/fess.xml @@ -6,7 +6,7 @@ - +