0ct0pu5/fess
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix #34

Browse source
This commit is contained in:
Shinsuke Sugaya 2013-11-16 15:52:04 +09:00
parent 840d1048b0
commit 714495baab
6 changed files with 237 additions and 192 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

32
src/main/java/jp/sf/fess/ds/impl/IndexUpdateCallbackImpl.java
View file

@ -23,19 +23,13 @@ import java.util.concurrent.atomic.AtomicLong;
import jp.sf.fess.Constants;
import jp.sf.fess.FessSystemException;
import jp.sf.fess.db.bsbhv.BsFavoriteLogBhv;
import jp.sf.fess.db.cbean.ClickLogCB;
import jp.sf.fess.db.exbhv.ClickLogBhv;
import jp.sf.fess.db.exbhv.FavoriteLogBhv;
import jp.sf.fess.db.exbhv.pmbean.FavoriteUrlCountPmb;
import jp.sf.fess.db.exentity.customize.FavoriteUrlCount;
import jp.sf.fess.ds.IndexUpdateCallback;
import jp.sf.fess.helper.CrawlingSessionHelper;
import jp.sf.fess.helper.SearchLogHelper;
import jp.sf.fess.helper.SystemHelper;
import org.apache.solr.common.SolrInputDocument;
import org.codelibs.solr.lib.SolrGroup;
import org.seasar.dbflute.cbean.ListResultBean;
import org.seasar.framework.container.SingletonS2Container;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -190,11 +184,9 @@ public class IndexUpdateCallbackImpl implements IndexUpdateCallback {
protected void addClickCountField(final SolrInputDocument doc,
final String url) {
final ClickLogBhv clickLogBhv = SingletonS2Container
.getComponent(ClickLogBhv.class);
final ClickLogCB cb = new ClickLogCB();
cb.query().setUrl_Equal(url);
final int count = clickLogBhv.selectCount(cb);
final SearchLogHelper searchLogHelper = SingletonS2Container
.getComponent(SearchLogHelper.class);
final int count = searchLogHelper.getClickCount(url);
doc.addField(clickCountField, count);
if (logger.isDebugEnabled()) {
logger.debug("Click Count: " + count + ", url: " + url);
@ -203,19 +195,9 @@ public class IndexUpdateCallbackImpl implements IndexUpdateCallback {
protected void addFavoriteCountField(final SolrInputDocument doc,
final String url) {
final FavoriteLogBhv favoriteLogBhv = SingletonS2Container
.getComponent(FavoriteLogBhv.class);
final FavoriteUrlCountPmb pmb = new FavoriteUrlCountPmb();
pmb.setUrl(url);
final String path = BsFavoriteLogBhv.PATH_selectFavoriteUrlCount;
final ListResultBean<FavoriteUrlCount> list = favoriteLogBhv
.outsideSql().selectList(path, pmb, FavoriteUrlCount.class);
long count = 0;
if (!list.isEmpty()) {
count = list.get(0).getCnt().longValue();
}
final SearchLogHelper searchLogHelper = SingletonS2Container
.getComponent(SearchLogHelper.class);
final long count = searchLogHelper.getFavoriteCount(url);
doc.addField(favoriteCountField, count);
if (logger.isDebugEnabled()) {
logger.debug("Favorite Count: " + count + ", url: " + url);

176
src/main/java/jp/sf/fess/helper/SearchLogHelper.java
View file

@ -16,58 +16,48 @@
package jp.sf.fess.helper;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Queue;
import java.util.concurrent.ConcurrentLinkedQueue;
import javax.annotation.Resource;
import jp.sf.fess.Constants;
import jp.sf.fess.db.cbean.SearchLogCB;
import jp.sf.fess.db.cbean.UserInfoCB;
import jp.sf.fess.db.bsbhv.BsFavoriteLogBhv;
import jp.sf.fess.db.cbean.ClickLogCB;
import jp.sf.fess.db.exbhv.ClickLogBhv;
import jp.sf.fess.db.exbhv.SearchFieldLogBhv;
import jp.sf.fess.db.exbhv.SearchLogBhv;
import jp.sf.fess.db.exbhv.UserInfoBhv;
import jp.sf.fess.db.exbhv.FavoriteLogBhv;
import jp.sf.fess.db.exbhv.pmbean.FavoriteUrlCountPmb;
import jp.sf.fess.db.exentity.ClickLog;
import jp.sf.fess.db.exentity.SearchLog;
import jp.sf.fess.db.exentity.UserInfo;
import jp.sf.fess.service.SearchLogService;
import jp.sf.fess.util.FessBeans;
import jp.sf.fess.db.exentity.customize.FavoriteUrlCount;
import org.codelibs.core.util.DynamicProperties;
import org.seasar.framework.util.StringUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.seasar.dbflute.cbean.ListResultBean;
import org.seasar.framework.container.SingletonS2Container;
import org.seasar.framework.container.annotation.tiger.InitMethod;
import org.seasar.robot.util.LruHashMap;
public class SearchLogHelper {
private static final Logger logger = LoggerFactory // NOPMD
.getLogger(SearchLogHelper.class);
@Resource
protected SearchLogService searchLogService;
@Resource
protected SearchLogBhv searchLogBhv;
@Resource
protected SearchFieldLogBhv searchFieldLogBhv;
@Resource
protected ClickLogBhv clickLogBhv;
@Resource
protected UserInfoBhv userInfoBhv;
public abstract class SearchLogHelper {
@Resource
protected DynamicProperties crawlerProperties;
private volatile Queue<SearchLog> searchLogQueue = new ConcurrentLinkedQueue<SearchLog>();
public long userCheckInterval = 5 * 60 * 1000;// 5 min
private volatile Queue<ClickLog> clickLogQueue = new ConcurrentLinkedQueue<ClickLog>();
public int userInfoCacheSize = 1000;
protected volatile Queue<SearchLog> searchLogQueue = new ConcurrentLinkedQueue<SearchLog>();
protected volatile Queue<ClickLog> clickLogQueue = new ConcurrentLinkedQueue<ClickLog>();
protected Map<String, Long> userInfoCache;
@InitMethod
public void init() {
userInfoCache = new LruHashMap<String, Long>(userInfoCacheSize);
}
public abstract void updateUserInfo(final String userCode);
public void addSearchLog(final SearchLog searchLog) {
searchLogQueue.add(searchLog);
@ -79,100 +69,44 @@ public class SearchLogHelper {
public void storeSearchLog() {
if (!searchLogQueue.isEmpty()) {
processSearchLogQueue();
final Queue<SearchLog> queue = searchLogQueue;
searchLogQueue = new ConcurrentLinkedQueue<SearchLog>();
processSearchLogQueue(queue);
}
if (!clickLogQueue.isEmpty()) {
processClickLogQueue();
final Queue<ClickLog> queue = clickLogQueue;
clickLogQueue = new ConcurrentLinkedQueue<ClickLog>();
processClickLogQueue(queue);
}
}
protected void processSearchLogQueue() {
final Queue<SearchLog> queue = searchLogQueue;
searchLogQueue = new ConcurrentLinkedQueue<SearchLog>();
final List<SearchLog> searchLogList = new ArrayList<SearchLog>();
final String value = crawlerProperties.getProperty(
Constants.PURGE_BY_BOTS_PROPERTY, Constants.EMPTY_STRING);
String[] botNames;
if (StringUtil.isBlank(value)) {
botNames = new String[0];
} else {
botNames = value.split(",");
}
final Map<String, UserInfo> userInfoMap = new HashMap<String, UserInfo>();
for (final SearchLog searchLog : queue) {
boolean add = true;
for (final String botName : botNames) {
if (searchLog.getUserAgent() != null
&& searchLog.getUserAgent().indexOf(botName) >= 0) {
add = false;
break;
}
}
if (add) {
final UserInfo userInfo = searchLog.getUserInfo();
if (userInfo != null) {
final String code = userInfo.getCode();
final UserInfo oldUserInfo = userInfoMap.get(code);
if (oldUserInfo != null) {
userInfo.setCreatedTime(oldUserInfo.getCreatedTime());
}
userInfoMap.put(code, userInfo);
}
searchLogList.add(searchLog);
}
}
if (!userInfoMap.isEmpty()) {
final List<UserInfo> insertList = new ArrayList<UserInfo>(
userInfoMap.values());
final List<UserInfo> updateList = new ArrayList<UserInfo>();
final UserInfoCB cb = new UserInfoCB();
cb.query().setCode_InScope(userInfoMap.keySet());
final List<UserInfo> list = userInfoBhv.selectList(cb);
for (final UserInfo userInfo : list) {
final String code = userInfo.getCode();
final UserInfo entity = userInfoMap.get(code);
FessBeans.copy(userInfo, entity).includes("id", "createdTime")
.execute();
updateList.add(entity);
insertList.remove(entity);
}
userInfoBhv.batchInsert(insertList);
userInfoBhv.batchUpdate(updateList);
for (final SearchLog searchLog : searchLogList) {
final UserInfo userInfo = searchLog.getUserInfo();
if (userInfo != null) {
final UserInfo entity = userInfoMap.get(userInfo.getCode());
searchLog.setUserId(entity.getId());
}
}
}
if (!searchLogList.isEmpty()) {
searchLogService.store(searchLogList);
}
public int getClickCount(final String url) {
final ClickLogBhv clickLogBhv = SingletonS2Container
.getComponent(ClickLogBhv.class);
final ClickLogCB cb = new ClickLogCB();
cb.query().setUrl_Equal(url);
final int count = clickLogBhv.selectCount(cb);
return count;
}
protected void processClickLogQueue() {
final Queue<ClickLog> queue = clickLogQueue;
clickLogQueue = new ConcurrentLinkedQueue<ClickLog>();
final List<ClickLog> clickLogList = new ArrayList<ClickLog>();
for (final ClickLog clickLog : queue) {
final SearchLogCB cb = new SearchLogCB();
cb.query().setRequestedTime_Equal(clickLog.getQueryRequestedTime());
cb.query().setUserSessionId_Equal(clickLog.getUserSessionId());
final SearchLog entity = searchLogBhv.selectEntity(cb);
if (entity != null) {
clickLog.setSearchId(entity.getId());
clickLogList.add(clickLog);
} else {
logger.warn("Not Found[ClickLog]: " + clickLog);
}
}
if (!clickLogList.isEmpty()) {
clickLogBhv.batchInsert(clickLogList);
public long getFavoriteCount(final String url) {
final FavoriteLogBhv favoriteLogBhv = SingletonS2Container
.getComponent(FavoriteLogBhv.class);
final FavoriteUrlCountPmb pmb = new FavoriteUrlCountPmb();
pmb.setUrl(url);
final String path = BsFavoriteLogBhv.PATH_selectFavoriteUrlCount;
final ListResultBean<FavoriteUrlCount> list = favoriteLogBhv
.outsideSql().selectList(path, pmb, FavoriteUrlCount.class);
long count = 0;
if (!list.isEmpty()) {
count = list.get(0).getCnt().longValue();
}
return count;
}
protected abstract void processSearchLogQueue(Queue<SearchLog> queue);
protected abstract void processClickLogQueue(Queue<ClickLog> queue);
}

33
src/main/java/jp/sf/fess/helper/impl/CookieUserInfoHelperImpl.java
View file

@ -16,7 +16,6 @@
package jp.sf.fess.helper.impl;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@ -28,11 +27,9 @@ import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpSession;
import jp.sf.fess.Constants;
import jp.sf.fess.db.exentity.UserInfo;
import jp.sf.fess.helper.SearchLogHelper;
import jp.sf.fess.helper.UserInfoHelper;
import jp.sf.fess.service.UserInfoService;
import org.seasar.framework.container.annotation.tiger.InitMethod;
import org.seasar.framework.util.StringUtil;
import org.seasar.robot.util.LruHashMap;
import org.seasar.struts.util.RequestUtil;
@ -41,9 +38,7 @@ import org.seasar.struts.util.ResponseUtil;
public class CookieUserInfoHelperImpl implements UserInfoHelper {
@Resource
protected UserInfoService userInfoService;
public int userInfoCacheSize = 1000;
protected SearchLogHelper searchLogHelper;
public int resultDocIdsCacheSize = 20;
@ -57,15 +52,6 @@ public class CookieUserInfoHelperImpl implements UserInfoHelper {
public Boolean cookieSecure;
public long userCheckInterval = 5 * 60 * 1000;// 5 min
private Map<String, Long> userInfoCache;
@InitMethod
public void init() {
userInfoCache = new LruHashMap<String, Long>(userInfoCacheSize);
}
/* (non-Javadoc)
* @see jp.sf.fess.helper.impl.UserInfoHelper#getUserCode()
*/
@ -98,20 +84,7 @@ public class CookieUserInfoHelperImpl implements UserInfoHelper {
}
protected void updateUserSessionId(final String userCode) {
final long current = System.currentTimeMillis();
final Long time = userInfoCache.get(userCode);
if (time == null || current - time.longValue() > userCheckInterval) {
UserInfo userInfo = userInfoService.getUserInfo(userCode);
if (userInfo == null) {
final Timestamp now = new Timestamp(current);
userInfo = new UserInfo();
userInfo.setCode(userCode);
userInfo.setCreatedTime(now);
userInfo.setUpdatedTime(now);
userInfoService.store(userInfo);
}
userInfoCache.put(userCode, current);
}
searchLogHelper.updateUserInfo(userCode);
final HttpServletRequest request = RequestUtil.getRequest();
request.setAttribute(Constants.USER_CODE, userCode);

157
src/main/java/jp/sf/fess/helper/impl/SearchLogHelperImpl.java Normal file
View file

@ -0,0 +1,157 @@
package jp.sf.fess.helper.impl;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Queue;
import javax.annotation.Resource;
import jp.sf.fess.Constants;
import jp.sf.fess.db.cbean.SearchLogCB;
import jp.sf.fess.db.cbean.UserInfoCB;
import jp.sf.fess.db.exbhv.ClickLogBhv;
import jp.sf.fess.db.exbhv.SearchFieldLogBhv;
import jp.sf.fess.db.exbhv.SearchLogBhv;
import jp.sf.fess.db.exbhv.UserInfoBhv;
import jp.sf.fess.db.exentity.ClickLog;
import jp.sf.fess.db.exentity.SearchLog;
import jp.sf.fess.db.exentity.UserInfo;
import jp.sf.fess.helper.SearchLogHelper;
import jp.sf.fess.service.SearchLogService;
import jp.sf.fess.service.UserInfoService;
import jp.sf.fess.util.FessBeans;
import org.seasar.framework.util.StringUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class SearchLogHelperImpl extends SearchLogHelper {
private static final Logger logger = LoggerFactory // NOPMD
.getLogger(SearchLogHelperImpl.class);
@Resource
protected UserInfoService userInfoService;
@Resource
protected SearchLogService searchLogService;
@Resource
protected SearchLogBhv searchLogBhv;
@Resource
protected SearchFieldLogBhv searchFieldLogBhv;
@Resource
protected ClickLogBhv clickLogBhv;
@Resource
protected UserInfoBhv userInfoBhv;
@Override
public void updateUserInfo(final String userCode) {
final long current = System.currentTimeMillis();
final Long time = userInfoCache.get(userCode);
if (time == null || current - time.longValue() > userCheckInterval) {
UserInfo userInfo = userInfoService.getUserInfo(userCode);
if (userInfo == null) {
final Timestamp now = new Timestamp(current);
userInfo = new UserInfo();
userInfo.setCode(userCode);
userInfo.setCreatedTime(now);
userInfo.setUpdatedTime(now);
userInfoService.store(userInfo);
}
userInfoCache.put(userCode, current);
}
}
@Override
protected void processSearchLogQueue(final Queue<SearchLog> queue) {
final List<SearchLog> searchLogList = new ArrayList<SearchLog>();
final String value = crawlerProperties.getProperty(
Constants.PURGE_BY_BOTS_PROPERTY, Constants.EMPTY_STRING);
String[] botNames;
if (StringUtil.isBlank(value)) {
botNames = new String[0];
} else {
botNames = value.split(",");
}
final Map<String, UserInfo> userInfoMap = new HashMap<String, UserInfo>();
for (final SearchLog searchLog : queue) {
boolean add = true;
for (final String botName : botNames) {
if (searchLog.getUserAgent() != null
&& searchLog.getUserAgent().indexOf(botName) >= 0) {
add = false;
break;
}
}
if (add) {
final UserInfo userInfo = searchLog.getUserInfo();
if (userInfo != null) {
final String code = userInfo.getCode();
final UserInfo oldUserInfo = userInfoMap.get(code);
if (oldUserInfo != null) {
userInfo.setCreatedTime(oldUserInfo.getCreatedTime());
}
userInfoMap.put(code, userInfo);
}
searchLogList.add(searchLog);
}
}
if (!userInfoMap.isEmpty()) {
final List<UserInfo> insertList = new ArrayList<UserInfo>(
userInfoMap.values());
final List<UserInfo> updateList = new ArrayList<UserInfo>();
final UserInfoCB cb = new UserInfoCB();
cb.query().setCode_InScope(userInfoMap.keySet());
final List<UserInfo> list = userInfoBhv.selectList(cb);
for (final UserInfo userInfo : list) {
final String code = userInfo.getCode();
final UserInfo entity = userInfoMap.get(code);
FessBeans.copy(userInfo, entity).includes("id", "createdTime")
.execute();
updateList.add(entity);
insertList.remove(entity);
}
userInfoBhv.batchInsert(insertList);
userInfoBhv.batchUpdate(updateList);
for (final SearchLog searchLog : searchLogList) {
final UserInfo userInfo = searchLog.getUserInfo();
if (userInfo != null) {
final UserInfo entity = userInfoMap.get(userInfo.getCode());
searchLog.setUserId(entity.getId());
}
}
}
if (!searchLogList.isEmpty()) {
searchLogService.store(searchLogList);
}
}
@Override
protected void processClickLogQueue(final Queue<ClickLog> queue) {
final List<ClickLog> clickLogList = new ArrayList<ClickLog>();
for (final ClickLog clickLog : queue) {
final SearchLogCB cb = new SearchLogCB();
cb.query().setRequestedTime_Equal(clickLog.getQueryRequestedTime());
cb.query().setUserSessionId_Equal(clickLog.getUserSessionId());
final SearchLog entity = searchLogBhv.selectEntity(cb);
if (entity != null) {
clickLog.setSearchId(entity.getId());
clickLogList.add(clickLog);
} else {
logger.warn("Not Found[ClickLog]: " + clickLog);
}
}
if (!clickLogList.isEmpty()) {
clickLogBhv.batchInsert(clickLogList);
}
}
}

25
src/main/java/jp/sf/fess/robot/FessS2RobotThread.java
View file

@ -28,14 +28,11 @@ import java.util.Set;
import jcifs.smb.ACE;
import jcifs.smb.SID;
import jp.sf.fess.Constants;
import jp.sf.fess.db.cbean.ClickLogCB;
import jp.sf.fess.db.cbean.FavoriteLogCB;
import jp.sf.fess.db.exbhv.ClickLogBhv;
import jp.sf.fess.db.exbhv.FavoriteLogBhv;
import jp.sf.fess.db.exentity.CrawlingConfig;
import jp.sf.fess.helper.CrawlingConfigHelper;
import jp.sf.fess.helper.CrawlingSessionHelper;
import jp.sf.fess.helper.SambaHelper;
import jp.sf.fess.helper.SearchLogHelper;
import org.apache.commons.io.IOUtils;
import org.apache.solr.client.solrj.SolrQuery;
@ -168,11 +165,10 @@ public class FessS2RobotThread extends S2RobotThread {
final Integer clickCount = (Integer) solrDocument
.get(clickCountField);
if (clickCount != null) {
final ClickLogBhv clickLogBhv = SingletonS2Container
.getComponent(ClickLogBhv.class);
final ClickLogCB cb = new ClickLogCB();
cb.query().setUrl_Equal(urlQueue.getUrl());
final int count = clickLogBhv.selectCount(cb);
final SearchLogHelper searchLogHelper = SingletonS2Container
.getComponent(SearchLogHelper.class);
final int count = searchLogHelper.getClickCount(urlQueue
.getUrl());
if (count != clickCount.intValue()) {
deleteSolrDocumentList(oldDocWithRoleList);
return true;
@ -182,12 +178,11 @@ public class FessS2RobotThread extends S2RobotThread {
final Integer favoriteCount = (Integer) solrDocument
.get(favoriteCountField);
if (favoriteCount != null) {
final FavoriteLogBhv favoriteLogBhv = SingletonS2Container
.getComponent(FavoriteLogBhv.class);
final FavoriteLogCB cb = new FavoriteLogCB();
cb.query().setUrl_Equal(urlQueue.getUrl());
final int count = favoriteLogBhv.selectCount(cb);
if (count != favoriteCount.intValue()) {
final SearchLogHelper searchLogHelper = SingletonS2Container
.getComponent(SearchLogHelper.class);
final long count = searchLogHelper
.getFavoriteCount(urlQueue.getUrl());
if (count != favoriteCount.longValue()) {
deleteSolrDocumentList(oldDocWithRoleList);
return true;
}

6
src/main/resources/app.dicon
View file

@ -186,7 +186,11 @@
</arg>
</initMethod>
</component>
<component name="searchLogHelper" class="jp.sf.fess.helper.SearchLogHelper">
<component name="searchLogHelper" class="jp.sf.fess.helper.impl.SearchLogHelperImpl">
<!--
<property name="userCheckInterval">5 * 60 * 1000</property>
<property name="userInfoCacheSize">1000</property>
-->
</component>
<component name="hotSearchWordHelper" class="jp.sf.fess.helper.HotSearchWordHelper">
<property name="excludedWordPattern">@java.util.regex.Pattern@compile(".*[a-zA-Z0-9_]+:.*")</property>