0ct0pu5/fess
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

improve log handling

Browse source
This commit is contained in:
Shinsuke Sugaya 2015-10-22 22:04:25 +09:00
parent 2fc6a7d0e1
commit 5193d958cb
9 changed files with 266 additions and 255 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
dbflute_fess/freegen/elasticsearch/AbstractBehavior.vm
View file

@ -337,19 +337,24 @@ public abstract class AbstractBehavior<ENTITY extends Entity, CB extends Conditi
bulkBuilder.add((DeleteRequestBuilder) builder);
}
}
RequestOptionCall<BulkRequestBuilder> builderCall = bulkList.getCall();
final RequestOptionCall<BulkRequestBuilder> builderCall = bulkList.getCall();
if (builderCall != null) {
builderCall.callback(bulkBuilder);
}
BulkResponse response = bulkBuilder.execute().actionGet();
List<Integer> resultList = new ArrayList<>();
for (BulkItemResponse itemResponse : response.getItems()) {
resultList.add(itemResponse.isFailed() ? 0 : 1);
final BulkResponse response = bulkBuilder.execute().actionGet();
final BulkItemResponse[] itemResponses = response.getItems();
if (itemResponses.length != entityList.size()) {
throw new IllegalStateException("Invalid response size: " + itemResponses.length + " != " + entityList.size());
}
int[] results = new int[resultList.size()];
for (int i = 0; i < resultList.size(); i++) {
results[i] = resultList.get(i);
final int[] results = new int[itemResponses.length];
for (int i = 0; i < itemResponses.length; i++) {
final BulkItemResponse itemResponse = itemResponses[i];
final Entity entity = entityList.get(i);
if (entity instanceof AbstractEntity) {
((AbstractEntity) entity).asDocMeta().id(itemResponse.getId());
}
results[i] = itemResponse.isFailed() ? 0 : 1;
}
return results;
}

38
src/main/java/org/codelibs/fess/app/service/SearchService.java
View file

@ -7,6 +7,7 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.function.Consumer;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@ -22,6 +23,7 @@ import org.codelibs.fess.entity.SearchRenderData;
import org.codelibs.fess.entity.SearchRequestParams;
import org.codelibs.fess.es.client.FessEsClient;
import org.codelibs.fess.es.client.FessEsClient.SearchConditionBuilder;
import org.codelibs.fess.es.client.FessEsClientException;
import org.codelibs.fess.es.exentity.SearchLog;
import org.codelibs.fess.es.exentity.UserInfo;
import org.codelibs.fess.helper.FieldHelper;
@ -32,16 +34,15 @@ import org.codelibs.fess.helper.UserInfoHelper;
import org.codelibs.fess.util.ComponentUtil;
import org.codelibs.fess.util.QueryResponseList;
import org.dbflute.optional.OptionalEntity;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.index.query.QueryBuilders;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class SearchService {
// ===================================================================================
// Constant
//
private static final Logger logger = LoggerFactory.getLogger(SearchService.class);
protected static final Pattern FIELD_EXTRACTION_PATTERN = Pattern.compile("^([a-zA-Z0-9_]+):.*");
@ -71,6 +72,8 @@ public class SearchService {
// ==============
public void search(final HttpServletRequest request, final SearchRequestParams params, final SearchRenderData data) {
final long requestedTime = systemHelper.getCurrentTimeAsLong();
final long startTime = System.currentTimeMillis();
final boolean searchLogSupport =
Constants.TRUE.equals(crawlerProperties.getProperty(Constants.SEARCH_LOG_PROPERTY, Constants.TRUE));
@ -136,7 +139,7 @@ public class SearchService {
// search log
if (searchLogSupport) {
storeSearchLog(request, query, pageStart, pageSize, queryResponseList);
storeSearchLog(request, requestedTime, query, pageStart, pageSize, queryResponseList);
}
queryResponseList.setExecTime(System.currentTimeMillis() - startTime);
@ -163,11 +166,11 @@ public class SearchService {
data.setPartialResults(queryResponseList.isPartialResults());
data.setQueryTime(queryResponseList.getQueryTime());
data.setSearchQuery(query);
data.setRequestedTime(requestedTime);
}
protected void storeSearchLog(final HttpServletRequest request, final String query, final int pageStart, final int pageSize,
final QueryResponseList queryResponseList) {
final long now = systemHelper.getCurrentTimeAsLong();
protected void storeSearchLog(final HttpServletRequest request, final long requestedTime, final String query, final int pageStart,
final int pageSize, final QueryResponseList queryResponseList) {
final SearchLogHelper searchLogHelper = ComponentUtil.getSearchLogHelper();
final SearchLog searchLog = new SearchLog();
@ -178,8 +181,8 @@ public class SearchService {
if (StringUtil.isNotBlank(userCode)) {
final UserInfo userInfo = new UserInfo();
userInfo.setCode(userCode);
userInfo.setCreatedTime(now);
userInfo.setUpdatedTime(now);
userInfo.setCreatedTime(requestedTime);
userInfo.setUpdatedTime(requestedTime);
searchLog.setUserInfo(OptionalEntity.of(userInfo));
}
}
@ -188,7 +191,7 @@ public class SearchService {
searchLog.setResponseTime(Integer.valueOf((int) queryResponseList.getExecTime()));
searchLog.setSearchWord(StringUtils.abbreviate(query, 1000));
searchLog.setSearchQuery(StringUtils.abbreviate(queryResponseList.getSearchQuery(), 1000));
searchLog.setRequestedTime(now);
searchLog.setRequestedTime(requestedTime);
searchLog.setQueryOffset(pageStart);
searchLog.setQueryPageSize(pageSize);
@ -321,4 +324,19 @@ public class SearchService {
public boolean update(final String id, final String field, final Object value) {
return fessEsClient.update(fieldHelper.docIndex, fieldHelper.docType, id, field, value);
}
public boolean bulkUpdate(Consumer<BulkRequestBuilder> consumer) {
BulkRequestBuilder builder = fessEsClient.prepareBulk();
consumer.accept(builder);
try {
BulkResponse response = builder.execute().get();
if (response.hasFailures()) {
throw new FessEsClientException(response.buildFailureMessage());
} else {
return true;
}
} catch (InterruptedException | ExecutionException e) {
throw new FessEsClientException("Failed to update bulk data.", e);
}
}
}

2
src/main/java/org/codelibs/fess/app/web/search/SearchAction.java
View file

@ -134,6 +134,7 @@ public class SearchAction extends FessSearchAction {
try {
final WebRenderData renderData = new WebRenderData(data);
searchService.search(request, form, renderData);
form.rt = Long.toString(renderData.getRequestedTime());
// favorite or screenshot
if (favoriteSupport || screenShotManager != null) {
final String searchQuery = renderData.getSearchQuery();
@ -157,7 +158,6 @@ public class SearchAction extends FessSearchAction {
messages.addErrorsResultSizeExceeded(GLOBAL);
}, () -> asHtml(path_ErrorJsp));
}
form.rt = Long.toString(systemHelper.getCurrentTimeAsLong());
data.register("displayQuery", getDisplayQuery(form, labelTypeHelper.getLabelTypeItemList()));
data.register("pagingQuery", getPagingQuery(form));
});

10
src/main/java/org/codelibs/fess/entity/SearchRenderData.java
View file

@ -39,6 +39,8 @@ public class SearchRenderData {
private long queryTime;
private long requestedTime;
public void setDocumentItems(final List<Map<String, Object>> documentItems) {
this.documentItems = documentItems;
}
@ -103,6 +105,10 @@ public class SearchRenderData {
this.searchQuery = searchQuery;
}
public void setRequestedTime(final long requestedTime) {
this.requestedTime = requestedTime;
}
public List<Map<String, Object>> getDocumentItems() {
return documentItems;
}
@ -167,4 +173,8 @@ public class SearchRenderData {
return queryTime;
}
public long getRequestedTime() {
return requestedTime;
}
}

21
src/main/java/org/codelibs/fess/es/bsbhv/AbstractBehavior.java
View file

@ -339,19 +339,24 @@ public abstract class AbstractBehavior<ENTITY extends Entity, CB extends Conditi
bulkBuilder.add((DeleteRequestBuilder) builder);
}
}
RequestOptionCall<BulkRequestBuilder> builderCall = bulkList.getCall();
final RequestOptionCall<BulkRequestBuilder> builderCall = bulkList.getCall();
if (builderCall != null) {
builderCall.callback(bulkBuilder);
}
BulkResponse response = bulkBuilder.execute().actionGet();
List<Integer> resultList = new ArrayList<>();
for (BulkItemResponse itemResponse : response.getItems()) {
resultList.add(itemResponse.isFailed() ? 0 : 1);
final BulkResponse response = bulkBuilder.execute().actionGet();
final BulkItemResponse[] itemResponses = response.getItems();
if (itemResponses.length != entityList.size()) {
throw new IllegalStateException("Invalid response size: " + itemResponses.length + " != " + entityList.size());
}
int[] results = new int[resultList.size()];
for (int i = 0; i < resultList.size(); i++) {
results[i] = resultList.get(i);
final int[] results = new int[itemResponses.length];
for (int i = 0; i < itemResponses.length; i++) {
final BulkItemResponse itemResponse = itemResponses[i];
final Entity entity = entityList.get(i);
if (entity instanceof AbstractEntity) {
((AbstractEntity) entity).asDocMeta().id(itemResponse.getId());
}
results[i] = itemResponse.isFailed() ? 0 : 1;
}
return results;
}

211
src/main/java/org/codelibs/fess/helper/SearchLogHelper.java
View file

@ -16,23 +16,48 @@
package org.codelibs.fess.helper;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Queue;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.stream.Stream;
import javax.annotation.PostConstruct;
import javax.annotation.Resource;
import org.codelibs.core.beans.util.BeanUtil;
import org.codelibs.core.collection.LruHashMap;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.core.misc.DynamicProperties;
import org.codelibs.fess.Constants;
import org.codelibs.fess.app.service.SearchService;
import org.codelibs.fess.es.exbhv.ClickLogBhv;
import org.codelibs.fess.es.exbhv.FavoriteLogBhv;
import org.codelibs.fess.es.exbhv.SearchFieldLogBhv;
import org.codelibs.fess.es.exbhv.SearchLogBhv;
import org.codelibs.fess.es.exbhv.UserInfoBhv;
import org.codelibs.fess.es.exentity.ClickLog;
import org.codelibs.fess.es.exentity.SearchFieldLog;
import org.codelibs.fess.es.exentity.SearchLog;
import org.codelibs.fess.es.exentity.UserInfo;
import org.codelibs.fess.util.ComponentUtil;
import org.codelibs.fess.util.DocumentUtil;
import org.elasticsearch.action.update.UpdateRequest;
import org.lastaflute.di.core.SingletonLaContainer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public abstract class SearchLogHelper {
public class SearchLogHelper {
private static final Logger logger = LoggerFactory.getLogger(SearchLogHelper.class);
@Resource
protected DynamicProperties crawlerProperties;
@Resource
protected FieldHelper fieldHelper;
public long userCheckInterval = 5 * 60 * 1000;// 5 min
public int userInfoCacheSize = 1000;
@ -48,8 +73,6 @@ public abstract class SearchLogHelper {
userInfoCache = new LruHashMap<String, Long>(userInfoCacheSize);
}
public abstract void updateUserInfo(final String userCode);
public void addSearchLog(final SearchLog searchLog) {
searchLogQueue.add(searchLog);
}
@ -73,21 +96,185 @@ public abstract class SearchLogHelper {
}
public int getClickCount(final String url) {
// TODO
return 0;
final ClickLogBhv clickLogBhv = ComponentUtil.getComponent(ClickLogBhv.class);
return clickLogBhv.selectCount(cb -> {
cb.query().setUrl_Equal(url);
});
}
public long getFavoriteCount(final String url) {
// TODO
return 0;
final FavoriteLogBhv favoriteLogBhv = ComponentUtil.getComponent(FavoriteLogBhv.class);
return favoriteLogBhv.selectCount(cb -> {
cb.query().setUrl_Equal(url);
});
}
protected abstract void processSearchLogQueue(Queue<SearchLog> queue);
public void updateUserInfo(final String userCode) {
final long current = System.currentTimeMillis();
final Long time = userInfoCache.get(userCode);
if (time == null || current - time.longValue() > userCheckInterval) {
protected abstract void processClickLogQueue(Queue<ClickLog> queue);
final UserInfoBhv userInfoBhv = ComponentUtil.getComponent(UserInfoBhv.class);
public boolean addfavoriteLog(final String userCode, final String favoriteUrl) {
// TODO Auto-generated method stub
return false;
final List<UserInfo> list = userInfoBhv.selectList(cb -> {
cb.query().setCode_Equal(userCode);
cb.query().addOrderBy_UpdatedTime_Desc();
});
final UserInfo userInfo;
final long now = ComponentUtil.getSystemHelper().getCurrentTimeAsLong();
if (list.isEmpty()) {
userInfo = new UserInfo();
userInfo.setCode(userCode);
userInfo.setCreatedTime(now);
} else {
userInfo = list.get(0);
}
userInfo.setUpdatedTime(now);
new Thread(() -> {
userInfoBhv.insertOrUpdate(userInfo);
}).start();
userInfoCache.put(userCode, current);
}
}
protected void processSearchLogQueue(final Queue<SearchLog> queue) {
final String value = crawlerProperties.getProperty(Constants.PURGE_BY_BOTS_PROPERTY, StringUtil.EMPTY);
String[] botNames;
if (StringUtil.isBlank(value)) {
botNames = StringUtil.EMPTY_STRINGS;
} else {
botNames = value.split(",");
}
final List<SearchLog> searchLogList = new ArrayList<>();
final Map<String, UserInfo> userInfoMap = new HashMap<>();
queue.stream().forEach(searchLog -> {
final String userAgent = searchLog.getUserAgent();
boolean isBot = userAgent != null && Stream.of(botNames).anyMatch(botName -> userAgent.indexOf(botName) >= 0);
if (!isBot) {
searchLog.getUserInfo().ifPresent(userInfo -> {
final String code = userInfo.getCode();
final UserInfo oldUserInfo = userInfoMap.get(code);
if (oldUserInfo != null) {
userInfo.setCreatedTime(oldUserInfo.getCreatedTime());
}
userInfoMap.put(code, userInfo);
});
searchLogList.add(searchLog);
}
});
if (!userInfoMap.isEmpty()) {
final List<UserInfo> insertList = new ArrayList<>(userInfoMap.values());
final List<UserInfo> updateList = new ArrayList<>();
final UserInfoBhv userInfoBhv = SingletonLaContainer.getComponent(UserInfoBhv.class);
final List<UserInfo> list = userInfoBhv.selectList(cb -> {
cb.query().setCode_InScope(userInfoMap.keySet());
});
for (final UserInfo userInfo : list) {
final String code = userInfo.getCode();
final UserInfo entity = userInfoMap.get(code);
BeanUtil.copyBeanToBean(userInfo, entity, option -> option.include("id", "createdTime"));
updateList.add(entity);
insertList.remove(entity);
}
userInfoBhv.batchInsert(insertList);
userInfoBhv.batchUpdate(updateList);
searchLogList.stream().forEach(searchLog -> {
searchLog.getUserInfo().ifPresent(userInfo -> {
final UserInfo entity = userInfoMap.get(userInfo.getCode());
searchLog.setUserInfoId(entity.getId());
});
});
}
if (!searchLogList.isEmpty()) {
storeSearchLogList(searchLogList);
}
}
private void storeSearchLogList(final List<SearchLog> searchLogList) {
final SearchLogBhv searchLogBhv = ComponentUtil.getComponent(SearchLogBhv.class);
final SearchFieldLogBhv searchFieldLogBhv = ComponentUtil.getComponent(SearchFieldLogBhv.class);
searchLogBhv.batchUpdate(searchLogList);
searchLogList.stream().forEach(searchLog -> {
final List<SearchFieldLog> fieldLogList = new ArrayList<>();
searchLog.getSearchFieldLogList().stream().forEach(fieldLog -> {
fieldLog.setSearchLogId(searchLog.getId());
fieldLogList.add(fieldLog);
});
searchFieldLogBhv.batchInsert(fieldLogList);
});
}
protected void processClickLogQueue(final Queue<ClickLog> queue) {
final Map<String, Long> clickCountMap = new HashMap<>();
final List<ClickLog> clickLogList = new ArrayList<>();
for (final ClickLog clickLog : queue) {
try {
final SearchLogBhv searchLogBhv = SingletonLaContainer.getComponent(SearchLogBhv.class);
searchLogBhv.selectEntity(cb -> {
cb.query().setRequestedTime_Equal(clickLog.getQueryRequestedTime());
cb.query().setUserSessionId_Equal(clickLog.getUserSessionId());
}).ifPresent(entity -> {
clickLog.setSearchLogId(entity.getId());
clickLogList.add(clickLog);
}).orElse(() -> {
logger.warn("Not Found for SearchLog: " + clickLog);
});
final String docId = clickLog.getDocId();
Long countObj = clickCountMap.get(docId);
if (countObj == null) {
final long clickCount = clickLog.getClickCount();
countObj = Long.valueOf(clickCount + 1);
} else {
countObj = countObj.longValue() + 1;
}
clickCountMap.put(docId, countObj);
} catch (final Exception e) {
logger.warn("Failed to process: " + clickLog, e);
}
}
if (!clickLogList.isEmpty()) {
try {
final ClickLogBhv clickLogBhv = SingletonLaContainer.getComponent(ClickLogBhv.class);
clickLogBhv.batchInsert(clickLogList);
} catch (final Exception e) {
logger.warn("Failed to insert: " + clickLogList, e);
}
}
if (!clickCountMap.isEmpty()) {
final SearchService searchService = ComponentUtil.getComponent(SearchService.class);
try {
final Map<String, String> docIdMap = new HashMap<>();
searchService
.getDocumentListByDocIds(clickCountMap.keySet().toArray(new String[clickCountMap.size()]),
new String[] { fieldHelper.idField, fieldHelper.docIdField })
.stream()
.forEach(
doc -> {
docIdMap.put(DocumentUtil.getValue(doc, fieldHelper.docIdField, String.class),
DocumentUtil.getValue(doc, fieldHelper.idField, String.class));
});
searchService.bulkUpdate(builder -> {
clickCountMap
.entrySet()
.stream()
.forEach(
entry -> {
String id = docIdMap.get(entry.getKey());
if (id != null) {
builder.add(new UpdateRequest(fieldHelper.docIndex, fieldHelper.docType, id).doc(
fieldHelper.clickCountField, entry.getValue()));
}
});
});
} catch (final Exception e) {
logger.warn("Failed to update clickCounts", e);
}
}
}
}

214
src/main/java/org/codelibs/fess/helper/impl/SearchLogHelperImpl.java
View file

@ -1,214 +0,0 @@
/*
* Copyright 2009-2015 the CodeLibs Project and the Others.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
* either express or implied. See the License for the specific language
* governing permissions and limitations under the License.
*/
package org.codelibs.fess.helper.impl;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Queue;
import org.codelibs.core.beans.util.BeanUtil;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.fess.Constants;
import org.codelibs.fess.es.client.FessEsClient;
import org.codelibs.fess.es.exbhv.ClickLogBhv;
import org.codelibs.fess.es.exbhv.SearchLogBhv;
import org.codelibs.fess.es.exbhv.UserInfoBhv;
import org.codelibs.fess.es.exentity.ClickLog;
import org.codelibs.fess.es.exentity.SearchLog;
import org.codelibs.fess.es.exentity.UserInfo;
import org.codelibs.fess.helper.FieldHelper;
import org.codelibs.fess.helper.SearchLogHelper;
import org.codelibs.fess.util.ComponentUtil;
import org.lastaflute.di.core.SingletonLaContainer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class SearchLogHelperImpl extends SearchLogHelper {
private static final Logger logger = LoggerFactory // NOPMD
.getLogger(SearchLogHelperImpl.class);
@Override
public void updateUserInfo(final String userCode) {
final long current = System.currentTimeMillis();
final Long time = userInfoCache.get(userCode);
if (time == null || current - time.longValue() > userCheckInterval) {
final UserInfoBhv userInfoBhv = ComponentUtil.getComponent(UserInfoBhv.class);
final List<UserInfo> list = userInfoBhv.selectList(cb -> {
cb.query().setCode_Equal(userCode);
cb.query().addOrderBy_UpdatedTime_Desc();
});
final UserInfo userInfo;
final long now = ComponentUtil.getSystemHelper().getCurrentTimeAsLong();
if (list.isEmpty()) {
userInfo = new UserInfo();
userInfo.setCode(userCode);
userInfo.setCreatedTime(now);
} else {
userInfo = list.get(0);
}
userInfo.setUpdatedTime(now);
new Thread(() -> {
userInfoBhv.insertOrUpdate(userInfo);
}).start();
userInfoCache.put(userCode, current);
}
}
@Override
protected void processSearchLogQueue(final Queue<SearchLog> queue) {
final List<SearchLog> searchLogList = new ArrayList<SearchLog>();
final String value = crawlerProperties.getProperty(Constants.PURGE_BY_BOTS_PROPERTY, StringUtil.EMPTY);
String[] botNames;
if (StringUtil.isBlank(value)) {
botNames = StringUtil.EMPTY_STRINGS;
} else {
botNames = value.split(",");
}
Constants.TRUE.equals(crawlerProperties.getProperty(Constants.SUGGEST_SEARCH_LOG_PROPERTY, Constants.TRUE));
final String dayForCleanupStr = crawlerProperties.getProperty(Constants.PURGE_SUGGEST_SEARCH_LOG_DAY_PROPERTY, "30");
try {
Integer.parseInt(dayForCleanupStr);
} catch (final NumberFormatException e) {}
final Map<String, UserInfo> userInfoMap = new HashMap<String, UserInfo>();
for (final SearchLog searchLog : queue) {
boolean add = true;
for (final String botName : botNames) {
if (searchLog.getUserAgent() != null && searchLog.getUserAgent().indexOf(botName) >= 0) {
add = false;
break;
}
}
if (add) {
final UserInfo userInfo = searchLog.getUserInfo().orElse(null);// TODO
if (userInfo != null) {
final String code = userInfo.getCode();
final UserInfo oldUserInfo = userInfoMap.get(code);
if (oldUserInfo != null) {
userInfo.setCreatedTime(oldUserInfo.getCreatedTime());
}
userInfoMap.put(code, userInfo);
}
searchLogList.add(searchLog);
// TODO
// if (suggestAvailable && searchLog.getHitCount() > 0) {
// final List<SearchFieldLog> searchFieldLogList = searchLog.getSearchFieldLogList();
// for (final SearchFieldLog searchFieldLog : searchFieldLogList) {
// if ("solrQuery".equals(searchFieldLog.getName())) {
// suggestService.addSolrParams(searchFieldLog.getValue(), dayForCleanup);
// addedSuggest = true;
// }
// }
// }
}
}
// if (addedSuggest) {
// suggestService.commit();
// }
if (!userInfoMap.isEmpty()) {
final List<UserInfo> insertList = new ArrayList<UserInfo>(userInfoMap.values());
final List<UserInfo> updateList = new ArrayList<UserInfo>();
final UserInfoBhv userInfoBhv = SingletonLaContainer.getComponent(UserInfoBhv.class);
final List<UserInfo> list = userInfoBhv.selectList(cb -> {
cb.query().setCode_InScope(userInfoMap.keySet());
});
for (final UserInfo userInfo : list) {
final String code = userInfo.getCode();
final UserInfo entity = userInfoMap.get(code);
BeanUtil.copyBeanToBean(userInfo, entity, option -> option.include("id", "createdTime"));
updateList.add(entity);
insertList.remove(entity);
}
userInfoBhv.batchInsert(insertList);
userInfoBhv.batchUpdate(updateList);
for (final SearchLog searchLog : searchLogList) {
final UserInfo userInfo = searchLog.getUserInfo().orElse(null);//TODO
if (userInfo != null) {
final UserInfo entity = userInfoMap.get(userInfo.getCode());
searchLog.setUserInfoId(entity.getId());
}
}
}
if (!searchLogList.isEmpty()) {
final SearchLogBhv searchLogBhv = ComponentUtil.getComponent(SearchLogBhv.class);
searchLogBhv.batchUpdate(searchLogList);
// TODO SearchLogValue
}
}
@Override
protected void processClickLogQueue(final Queue<ClickLog> queue) {
final Map<String, Long> clickCountMap = new HashMap<String, Long>();
final List<ClickLog> clickLogList = new ArrayList<ClickLog>();
for (final ClickLog clickLog : queue) {
try {
final SearchLogBhv searchLogBhv = SingletonLaContainer.getComponent(SearchLogBhv.class);
final SearchLog entity = searchLogBhv.selectEntity(cb -> {
cb.query().setRequestedTime_Equal(clickLog.getQueryRequestedTime());
cb.query().setUserSessionId_Equal(clickLog.getUserSessionId());
}).orElse(null);//TODO
if (entity != null) {
clickLog.setSearchLogId(entity.getId());
clickLogList.add(clickLog);
} else {
logger.warn("Not Found[ClickLog]: " + clickLog);
}
final String docId = clickLog.getDocId();
Long countObj = clickCountMap.get(docId);
final long clickCount = clickLog.getClickCount();
if (countObj == null) {
countObj = Long.valueOf(clickCount);
} else {
countObj = Math.max(countObj.longValue(), clickCount) + 1;
}
clickCountMap.put(docId, countObj);
} catch (final Exception e) {
logger.warn("Failed to process: " + clickLog, e);
}
}
if (!clickLogList.isEmpty()) {
try {
final ClickLogBhv clickLogBhv = SingletonLaContainer.getComponent(ClickLogBhv.class);
clickLogBhv.batchInsert(clickLogList);
} catch (final Exception e) {
logger.warn("Failed to insert: " + clickLogList, e);
}
}
final FessEsClient fessEsClient = ComponentUtil.getElasticsearchClient();
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
for (final Map.Entry<String, Long> entry : clickCountMap.entrySet()) {
try {
// TODO buik update
fessEsClient.update(fieldHelper.docIndex, fieldHelper.docType, entry.getKey(), fieldHelper.clickCountField,
entry.getValue() + 1);
} catch (final Exception e) {
logger.warn("Failed to update a clickCount(" + entry.getValue() + ") for " + entry.getKey(), e);
}
}
}
}

2
src/main/resources/fess.xml
View file

@ -6,7 +6,7 @@
<include path="fess_ds.xml"/>
<include path="fess_es.xml"/>
<component name="searchLogHelper" class="org.codelibs.fess.helper.impl.SearchLogHelperImpl">
<component name="searchLogHelper" class="org.codelibs.fess.helper.SearchLogHelper">
<!--
<property name="userCheckInterval">5 * 60 * 1000</property>
<property name="userInfoCacheSize">1000</property>

2
src/main/resources/fess_indices/fess_log.json
View file

@ -1,7 +1,7 @@
{
"settings": {
"index": {
"refresh_interval": "60s",
"refresh_interval": "1s",
"number_of_shards": 10,
"number_of_replicas": 0
}