diff --git a/src/main/java/jp/sf/fess/ds/impl/IndexUpdateCallbackImpl.java b/src/main/java/jp/sf/fess/ds/impl/IndexUpdateCallbackImpl.java index ea277c6e7..4ef14c132 100644 --- a/src/main/java/jp/sf/fess/ds/impl/IndexUpdateCallbackImpl.java +++ b/src/main/java/jp/sf/fess/ds/impl/IndexUpdateCallbackImpl.java @@ -23,19 +23,13 @@ import java.util.concurrent.atomic.AtomicLong; import jp.sf.fess.Constants; import jp.sf.fess.FessSystemException; -import jp.sf.fess.db.bsbhv.BsFavoriteLogBhv; -import jp.sf.fess.db.cbean.ClickLogCB; -import jp.sf.fess.db.exbhv.ClickLogBhv; -import jp.sf.fess.db.exbhv.FavoriteLogBhv; -import jp.sf.fess.db.exbhv.pmbean.FavoriteUrlCountPmb; -import jp.sf.fess.db.exentity.customize.FavoriteUrlCount; import jp.sf.fess.ds.IndexUpdateCallback; import jp.sf.fess.helper.CrawlingSessionHelper; +import jp.sf.fess.helper.SearchLogHelper; import jp.sf.fess.helper.SystemHelper; import org.apache.solr.common.SolrInputDocument; import org.codelibs.solr.lib.SolrGroup; -import org.seasar.dbflute.cbean.ListResultBean; import org.seasar.framework.container.SingletonS2Container; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -190,11 +184,9 @@ public class IndexUpdateCallbackImpl implements IndexUpdateCallback { protected void addClickCountField(final SolrInputDocument doc, final String url) { - final ClickLogBhv clickLogBhv = SingletonS2Container - .getComponent(ClickLogBhv.class); - final ClickLogCB cb = new ClickLogCB(); - cb.query().setUrl_Equal(url); - final int count = clickLogBhv.selectCount(cb); + final SearchLogHelper searchLogHelper = SingletonS2Container + .getComponent(SearchLogHelper.class); + final int count = searchLogHelper.getClickCount(url); doc.addField(clickCountField, count); if (logger.isDebugEnabled()) { logger.debug("Click Count: " + count + ", url: " + url); @@ -203,19 +195,9 @@ public class IndexUpdateCallbackImpl implements IndexUpdateCallback { protected void addFavoriteCountField(final SolrInputDocument doc, final String url) { - final FavoriteLogBhv favoriteLogBhv = SingletonS2Container - .getComponent(FavoriteLogBhv.class); - final FavoriteUrlCountPmb pmb = new FavoriteUrlCountPmb(); - pmb.setUrl(url); - final String path = BsFavoriteLogBhv.PATH_selectFavoriteUrlCount; - final ListResultBean list = favoriteLogBhv - .outsideSql().selectList(path, pmb, FavoriteUrlCount.class); - - long count = 0; - if (!list.isEmpty()) { - count = list.get(0).getCnt().longValue(); - } - + final SearchLogHelper searchLogHelper = SingletonS2Container + .getComponent(SearchLogHelper.class); + final long count = searchLogHelper.getFavoriteCount(url); doc.addField(favoriteCountField, count); if (logger.isDebugEnabled()) { logger.debug("Favorite Count: " + count + ", url: " + url); diff --git a/src/main/java/jp/sf/fess/helper/SearchLogHelper.java b/src/main/java/jp/sf/fess/helper/SearchLogHelper.java index 3d9251762..0895f5a52 100644 --- a/src/main/java/jp/sf/fess/helper/SearchLogHelper.java +++ b/src/main/java/jp/sf/fess/helper/SearchLogHelper.java @@ -16,58 +16,48 @@ package jp.sf.fess.helper; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.Queue; import java.util.concurrent.ConcurrentLinkedQueue; import javax.annotation.Resource; -import jp.sf.fess.Constants; -import jp.sf.fess.db.cbean.SearchLogCB; -import jp.sf.fess.db.cbean.UserInfoCB; +import jp.sf.fess.db.bsbhv.BsFavoriteLogBhv; +import jp.sf.fess.db.cbean.ClickLogCB; import jp.sf.fess.db.exbhv.ClickLogBhv; -import jp.sf.fess.db.exbhv.SearchFieldLogBhv; -import jp.sf.fess.db.exbhv.SearchLogBhv; -import jp.sf.fess.db.exbhv.UserInfoBhv; +import jp.sf.fess.db.exbhv.FavoriteLogBhv; +import jp.sf.fess.db.exbhv.pmbean.FavoriteUrlCountPmb; import jp.sf.fess.db.exentity.ClickLog; import jp.sf.fess.db.exentity.SearchLog; -import jp.sf.fess.db.exentity.UserInfo; -import jp.sf.fess.service.SearchLogService; -import jp.sf.fess.util.FessBeans; +import jp.sf.fess.db.exentity.customize.FavoriteUrlCount; import org.codelibs.core.util.DynamicProperties; -import org.seasar.framework.util.StringUtil; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.seasar.dbflute.cbean.ListResultBean; +import org.seasar.framework.container.SingletonS2Container; +import org.seasar.framework.container.annotation.tiger.InitMethod; +import org.seasar.robot.util.LruHashMap; -public class SearchLogHelper { - private static final Logger logger = LoggerFactory // NOPMD - .getLogger(SearchLogHelper.class); - - @Resource - protected SearchLogService searchLogService; - - @Resource - protected SearchLogBhv searchLogBhv; - - @Resource - protected SearchFieldLogBhv searchFieldLogBhv; - - @Resource - protected ClickLogBhv clickLogBhv; - - @Resource - protected UserInfoBhv userInfoBhv; +public abstract class SearchLogHelper { @Resource protected DynamicProperties crawlerProperties; - private volatile Queue searchLogQueue = new ConcurrentLinkedQueue(); + public long userCheckInterval = 5 * 60 * 1000;// 5 min - private volatile Queue clickLogQueue = new ConcurrentLinkedQueue(); + public int userInfoCacheSize = 1000; + + protected volatile Queue searchLogQueue = new ConcurrentLinkedQueue(); + + protected volatile Queue clickLogQueue = new ConcurrentLinkedQueue(); + + protected Map userInfoCache; + + @InitMethod + public void init() { + userInfoCache = new LruHashMap(userInfoCacheSize); + } + + public abstract void updateUserInfo(final String userCode); public void addSearchLog(final SearchLog searchLog) { searchLogQueue.add(searchLog); @@ -79,100 +69,44 @@ public class SearchLogHelper { public void storeSearchLog() { if (!searchLogQueue.isEmpty()) { - processSearchLogQueue(); + final Queue queue = searchLogQueue; + searchLogQueue = new ConcurrentLinkedQueue(); + processSearchLogQueue(queue); } if (!clickLogQueue.isEmpty()) { - processClickLogQueue(); + final Queue queue = clickLogQueue; + clickLogQueue = new ConcurrentLinkedQueue(); + processClickLogQueue(queue); } } - protected void processSearchLogQueue() { - final Queue queue = searchLogQueue; - searchLogQueue = new ConcurrentLinkedQueue(); - final List searchLogList = new ArrayList(); - final String value = crawlerProperties.getProperty( - Constants.PURGE_BY_BOTS_PROPERTY, Constants.EMPTY_STRING); - String[] botNames; - if (StringUtil.isBlank(value)) { - botNames = new String[0]; - } else { - botNames = value.split(","); - } - - final Map userInfoMap = new HashMap(); - for (final SearchLog searchLog : queue) { - boolean add = true; - for (final String botName : botNames) { - if (searchLog.getUserAgent() != null - && searchLog.getUserAgent().indexOf(botName) >= 0) { - add = false; - break; - } - } - if (add) { - final UserInfo userInfo = searchLog.getUserInfo(); - if (userInfo != null) { - final String code = userInfo.getCode(); - final UserInfo oldUserInfo = userInfoMap.get(code); - if (oldUserInfo != null) { - userInfo.setCreatedTime(oldUserInfo.getCreatedTime()); - } - userInfoMap.put(code, userInfo); - } - searchLogList.add(searchLog); - } - } - - if (!userInfoMap.isEmpty()) { - final List insertList = new ArrayList( - userInfoMap.values()); - final List updateList = new ArrayList(); - final UserInfoCB cb = new UserInfoCB(); - cb.query().setCode_InScope(userInfoMap.keySet()); - final List list = userInfoBhv.selectList(cb); - for (final UserInfo userInfo : list) { - final String code = userInfo.getCode(); - final UserInfo entity = userInfoMap.get(code); - FessBeans.copy(userInfo, entity).includes("id", "createdTime") - .execute(); - updateList.add(entity); - insertList.remove(entity); - } - userInfoBhv.batchInsert(insertList); - userInfoBhv.batchUpdate(updateList); - for (final SearchLog searchLog : searchLogList) { - final UserInfo userInfo = searchLog.getUserInfo(); - if (userInfo != null) { - final UserInfo entity = userInfoMap.get(userInfo.getCode()); - searchLog.setUserId(entity.getId()); - } - } - } - - if (!searchLogList.isEmpty()) { - searchLogService.store(searchLogList); - } + public int getClickCount(final String url) { + final ClickLogBhv clickLogBhv = SingletonS2Container + .getComponent(ClickLogBhv.class); + final ClickLogCB cb = new ClickLogCB(); + cb.query().setUrl_Equal(url); + final int count = clickLogBhv.selectCount(cb); + return count; } - protected void processClickLogQueue() { - final Queue queue = clickLogQueue; - clickLogQueue = new ConcurrentLinkedQueue(); - final List clickLogList = new ArrayList(); - for (final ClickLog clickLog : queue) { - final SearchLogCB cb = new SearchLogCB(); - cb.query().setRequestedTime_Equal(clickLog.getQueryRequestedTime()); - cb.query().setUserSessionId_Equal(clickLog.getUserSessionId()); - final SearchLog entity = searchLogBhv.selectEntity(cb); - if (entity != null) { - clickLog.setSearchId(entity.getId()); - clickLogList.add(clickLog); - } else { - logger.warn("Not Found[ClickLog]: " + clickLog); - } - } - if (!clickLogList.isEmpty()) { - clickLogBhv.batchInsert(clickLogList); + public long getFavoriteCount(final String url) { + final FavoriteLogBhv favoriteLogBhv = SingletonS2Container + .getComponent(FavoriteLogBhv.class); + final FavoriteUrlCountPmb pmb = new FavoriteUrlCountPmb(); + pmb.setUrl(url); + final String path = BsFavoriteLogBhv.PATH_selectFavoriteUrlCount; + final ListResultBean list = favoriteLogBhv + .outsideSql().selectList(path, pmb, FavoriteUrlCount.class); + + long count = 0; + if (!list.isEmpty()) { + count = list.get(0).getCnt().longValue(); } + return count; } + + protected abstract void processSearchLogQueue(Queue queue); + + protected abstract void processClickLogQueue(Queue queue); } diff --git a/src/main/java/jp/sf/fess/helper/impl/CookieUserInfoHelperImpl.java b/src/main/java/jp/sf/fess/helper/impl/CookieUserInfoHelperImpl.java index 2d0a7421e..ca979a114 100644 --- a/src/main/java/jp/sf/fess/helper/impl/CookieUserInfoHelperImpl.java +++ b/src/main/java/jp/sf/fess/helper/impl/CookieUserInfoHelperImpl.java @@ -16,7 +16,6 @@ package jp.sf.fess.helper.impl; -import java.sql.Timestamp; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -28,11 +27,9 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpSession; import jp.sf.fess.Constants; -import jp.sf.fess.db.exentity.UserInfo; +import jp.sf.fess.helper.SearchLogHelper; import jp.sf.fess.helper.UserInfoHelper; -import jp.sf.fess.service.UserInfoService; -import org.seasar.framework.container.annotation.tiger.InitMethod; import org.seasar.framework.util.StringUtil; import org.seasar.robot.util.LruHashMap; import org.seasar.struts.util.RequestUtil; @@ -41,9 +38,7 @@ import org.seasar.struts.util.ResponseUtil; public class CookieUserInfoHelperImpl implements UserInfoHelper { @Resource - protected UserInfoService userInfoService; - - public int userInfoCacheSize = 1000; + protected SearchLogHelper searchLogHelper; public int resultDocIdsCacheSize = 20; @@ -57,15 +52,6 @@ public class CookieUserInfoHelperImpl implements UserInfoHelper { public Boolean cookieSecure; - public long userCheckInterval = 5 * 60 * 1000;// 5 min - - private Map userInfoCache; - - @InitMethod - public void init() { - userInfoCache = new LruHashMap(userInfoCacheSize); - } - /* (non-Javadoc) * @see jp.sf.fess.helper.impl.UserInfoHelper#getUserCode() */ @@ -98,20 +84,7 @@ public class CookieUserInfoHelperImpl implements UserInfoHelper { } protected void updateUserSessionId(final String userCode) { - final long current = System.currentTimeMillis(); - final Long time = userInfoCache.get(userCode); - if (time == null || current - time.longValue() > userCheckInterval) { - UserInfo userInfo = userInfoService.getUserInfo(userCode); - if (userInfo == null) { - final Timestamp now = new Timestamp(current); - userInfo = new UserInfo(); - userInfo.setCode(userCode); - userInfo.setCreatedTime(now); - userInfo.setUpdatedTime(now); - userInfoService.store(userInfo); - } - userInfoCache.put(userCode, current); - } + searchLogHelper.updateUserInfo(userCode); final HttpServletRequest request = RequestUtil.getRequest(); request.setAttribute(Constants.USER_CODE, userCode); diff --git a/src/main/java/jp/sf/fess/helper/impl/SearchLogHelperImpl.java b/src/main/java/jp/sf/fess/helper/impl/SearchLogHelperImpl.java new file mode 100644 index 000000000..98bca0f8e --- /dev/null +++ b/src/main/java/jp/sf/fess/helper/impl/SearchLogHelperImpl.java @@ -0,0 +1,157 @@ +package jp.sf.fess.helper.impl; + +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Queue; + +import javax.annotation.Resource; + +import jp.sf.fess.Constants; +import jp.sf.fess.db.cbean.SearchLogCB; +import jp.sf.fess.db.cbean.UserInfoCB; +import jp.sf.fess.db.exbhv.ClickLogBhv; +import jp.sf.fess.db.exbhv.SearchFieldLogBhv; +import jp.sf.fess.db.exbhv.SearchLogBhv; +import jp.sf.fess.db.exbhv.UserInfoBhv; +import jp.sf.fess.db.exentity.ClickLog; +import jp.sf.fess.db.exentity.SearchLog; +import jp.sf.fess.db.exentity.UserInfo; +import jp.sf.fess.helper.SearchLogHelper; +import jp.sf.fess.service.SearchLogService; +import jp.sf.fess.service.UserInfoService; +import jp.sf.fess.util.FessBeans; + +import org.seasar.framework.util.StringUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class SearchLogHelperImpl extends SearchLogHelper { + private static final Logger logger = LoggerFactory // NOPMD + .getLogger(SearchLogHelperImpl.class); + + @Resource + protected UserInfoService userInfoService; + + @Resource + protected SearchLogService searchLogService; + + @Resource + protected SearchLogBhv searchLogBhv; + + @Resource + protected SearchFieldLogBhv searchFieldLogBhv; + + @Resource + protected ClickLogBhv clickLogBhv; + + @Resource + protected UserInfoBhv userInfoBhv; + + @Override + public void updateUserInfo(final String userCode) { + final long current = System.currentTimeMillis(); + final Long time = userInfoCache.get(userCode); + if (time == null || current - time.longValue() > userCheckInterval) { + UserInfo userInfo = userInfoService.getUserInfo(userCode); + if (userInfo == null) { + final Timestamp now = new Timestamp(current); + userInfo = new UserInfo(); + userInfo.setCode(userCode); + userInfo.setCreatedTime(now); + userInfo.setUpdatedTime(now); + userInfoService.store(userInfo); + } + userInfoCache.put(userCode, current); + } + } + + @Override + protected void processSearchLogQueue(final Queue queue) { + final List searchLogList = new ArrayList(); + final String value = crawlerProperties.getProperty( + Constants.PURGE_BY_BOTS_PROPERTY, Constants.EMPTY_STRING); + String[] botNames; + if (StringUtil.isBlank(value)) { + botNames = new String[0]; + } else { + botNames = value.split(","); + } + + final Map userInfoMap = new HashMap(); + for (final SearchLog searchLog : queue) { + boolean add = true; + for (final String botName : botNames) { + if (searchLog.getUserAgent() != null + && searchLog.getUserAgent().indexOf(botName) >= 0) { + add = false; + break; + } + } + if (add) { + final UserInfo userInfo = searchLog.getUserInfo(); + if (userInfo != null) { + final String code = userInfo.getCode(); + final UserInfo oldUserInfo = userInfoMap.get(code); + if (oldUserInfo != null) { + userInfo.setCreatedTime(oldUserInfo.getCreatedTime()); + } + userInfoMap.put(code, userInfo); + } + searchLogList.add(searchLog); + } + } + + if (!userInfoMap.isEmpty()) { + final List insertList = new ArrayList( + userInfoMap.values()); + final List updateList = new ArrayList(); + final UserInfoCB cb = new UserInfoCB(); + cb.query().setCode_InScope(userInfoMap.keySet()); + final List list = userInfoBhv.selectList(cb); + for (final UserInfo userInfo : list) { + final String code = userInfo.getCode(); + final UserInfo entity = userInfoMap.get(code); + FessBeans.copy(userInfo, entity).includes("id", "createdTime") + .execute(); + updateList.add(entity); + insertList.remove(entity); + } + userInfoBhv.batchInsert(insertList); + userInfoBhv.batchUpdate(updateList); + for (final SearchLog searchLog : searchLogList) { + final UserInfo userInfo = searchLog.getUserInfo(); + if (userInfo != null) { + final UserInfo entity = userInfoMap.get(userInfo.getCode()); + searchLog.setUserId(entity.getId()); + } + } + } + + if (!searchLogList.isEmpty()) { + searchLogService.store(searchLogList); + } + } + + @Override + protected void processClickLogQueue(final Queue queue) { + final List clickLogList = new ArrayList(); + for (final ClickLog clickLog : queue) { + final SearchLogCB cb = new SearchLogCB(); + cb.query().setRequestedTime_Equal(clickLog.getQueryRequestedTime()); + cb.query().setUserSessionId_Equal(clickLog.getUserSessionId()); + final SearchLog entity = searchLogBhv.selectEntity(cb); + if (entity != null) { + clickLog.setSearchId(entity.getId()); + clickLogList.add(clickLog); + } else { + logger.warn("Not Found[ClickLog]: " + clickLog); + } + } + if (!clickLogList.isEmpty()) { + clickLogBhv.batchInsert(clickLogList); + } + } +} diff --git a/src/main/java/jp/sf/fess/robot/FessS2RobotThread.java b/src/main/java/jp/sf/fess/robot/FessS2RobotThread.java index 3e8cb7b37..039e31ff4 100644 --- a/src/main/java/jp/sf/fess/robot/FessS2RobotThread.java +++ b/src/main/java/jp/sf/fess/robot/FessS2RobotThread.java @@ -28,14 +28,11 @@ import java.util.Set; import jcifs.smb.ACE; import jcifs.smb.SID; import jp.sf.fess.Constants; -import jp.sf.fess.db.cbean.ClickLogCB; -import jp.sf.fess.db.cbean.FavoriteLogCB; -import jp.sf.fess.db.exbhv.ClickLogBhv; -import jp.sf.fess.db.exbhv.FavoriteLogBhv; import jp.sf.fess.db.exentity.CrawlingConfig; import jp.sf.fess.helper.CrawlingConfigHelper; import jp.sf.fess.helper.CrawlingSessionHelper; import jp.sf.fess.helper.SambaHelper; +import jp.sf.fess.helper.SearchLogHelper; import org.apache.commons.io.IOUtils; import org.apache.solr.client.solrj.SolrQuery; @@ -168,11 +165,10 @@ public class FessS2RobotThread extends S2RobotThread { final Integer clickCount = (Integer) solrDocument .get(clickCountField); if (clickCount != null) { - final ClickLogBhv clickLogBhv = SingletonS2Container - .getComponent(ClickLogBhv.class); - final ClickLogCB cb = new ClickLogCB(); - cb.query().setUrl_Equal(urlQueue.getUrl()); - final int count = clickLogBhv.selectCount(cb); + final SearchLogHelper searchLogHelper = SingletonS2Container + .getComponent(SearchLogHelper.class); + final int count = searchLogHelper.getClickCount(urlQueue + .getUrl()); if (count != clickCount.intValue()) { deleteSolrDocumentList(oldDocWithRoleList); return true; @@ -182,12 +178,11 @@ public class FessS2RobotThread extends S2RobotThread { final Integer favoriteCount = (Integer) solrDocument .get(favoriteCountField); if (favoriteCount != null) { - final FavoriteLogBhv favoriteLogBhv = SingletonS2Container - .getComponent(FavoriteLogBhv.class); - final FavoriteLogCB cb = new FavoriteLogCB(); - cb.query().setUrl_Equal(urlQueue.getUrl()); - final int count = favoriteLogBhv.selectCount(cb); - if (count != favoriteCount.intValue()) { + final SearchLogHelper searchLogHelper = SingletonS2Container + .getComponent(SearchLogHelper.class); + final long count = searchLogHelper + .getFavoriteCount(urlQueue.getUrl()); + if (count != favoriteCount.longValue()) { deleteSolrDocumentList(oldDocWithRoleList); return true; } diff --git a/src/main/resources/app.dicon b/src/main/resources/app.dicon index 158d1a7a2..db80436d5 100644 --- a/src/main/resources/app.dicon +++ b/src/main/resources/app.dicon @@ -186,7 +186,11 @@ - + + @java.util.regex.Pattern@compile(".*[a-zA-Z0-9_]+:.*")