0ct0pu5/fess
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix #38

Browse source
This commit is contained in:
Shinsuke Sugaya 2013-12-07 22:37:22 +09:00
parent 6fcb8fac05
commit 7dff6d385a
17 changed files with 67 additions and 74 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/main/java/jp/sf/fess/action/IndexAction.java
View file

@ -621,7 +621,7 @@ public class IndexAction {
return null;
}
if (!favoriteLogService.addUrl(userCode, favoriteUrl)) {
if (!favoriteLogService.addUrl(userCode, favoriteUrl, doc)) {
WebApiUtil.setError(4, "Failed to add url: " + favoriteUrl);
return null;
}

11
src/main/java/jp/sf/fess/ds/impl/AbstractDataStoreImpl.java
View file

@ -26,8 +26,8 @@ import jp.sf.fess.Constants;
import jp.sf.fess.db.exentity.DataCrawlingConfig;
import jp.sf.fess.ds.DataStore;
import jp.sf.fess.ds.IndexUpdateCallback;
import jp.sf.fess.helper.CrawlingConfigHelper;
import jp.sf.fess.helper.CrawlingSessionHelper;
import jp.sf.fess.helper.SystemHelper;
import jp.sf.fess.taglib.FessFunctions;
import org.seasar.framework.container.SingletonS2Container;
@ -61,8 +61,8 @@ public abstract class AbstractDataStoreImpl implements DataStore {
final CrawlingSessionHelper crawlingSessionHelper = SingletonS2Container
.getComponent("crawlingSessionHelper");
final Date documentExpires = crawlingSessionHelper.getDocumentExpires();
final CrawlingConfigHelper crawlingConfigHelper = SingletonS2Container
.getComponent("crawlingConfigHelper");
final SystemHelper systemHelper = SingletonS2Container
.getComponent("systemHelper");
initParamMap.putAll(configParamMap);
final Map<String, String> paramMap = initParamMap;
@ -73,12 +73,11 @@ public abstract class AbstractDataStoreImpl implements DataStore {
// cid
final String configId = config.getConfigId();
if (configId != null) {
defaultDataMap.put(crawlingConfigHelper.getConfigIdField(),
configId);
defaultDataMap.put(systemHelper.configIdField, configId);
}
// expires
if (documentExpires != null) {
defaultDataMap.put(crawlingSessionHelper.getExpiresField(),
defaultDataMap.put(systemHelper.expiresField,
FessFunctions.formatDate(documentExpires));
}
// segment

14
src/main/java/jp/sf/fess/ds/impl/IndexUpdateCallbackImpl.java
View file

@ -46,10 +46,6 @@ public class IndexUpdateCallbackImpl implements IndexUpdateCallback {
public boolean favoriteCountEnabled = true;
public String clickCountField = "clickCount_i";
public String favoriteCountField = "favoriteCount_i";
protected volatile AtomicLong documentSize = new AtomicLong(0);
protected volatile long commitPerCount = 0;
@ -137,7 +133,7 @@ public class IndexUpdateCallbackImpl implements IndexUpdateCallback {
if (!dataMap.containsKey(Constants.DOC_ID)) {
final SystemHelper systemHelper = SingletonS2Container
.getComponent(SystemHelper.class);
.getComponent("systemHelper");
doc.addField(Constants.DOC_ID, systemHelper.generateDocId(dataMap));
}
@ -186,8 +182,10 @@ public class IndexUpdateCallbackImpl implements IndexUpdateCallback {
final String url) {
final SearchLogHelper searchLogHelper = SingletonS2Container
.getComponent(SearchLogHelper.class);
final SystemHelper systemHelper = SingletonS2Container
.getComponent("systemHelper");
final int count = searchLogHelper.getClickCount(url);
doc.addField(clickCountField, count);
doc.addField(systemHelper.clickCountField, count);
if (logger.isDebugEnabled()) {
logger.debug("Click Count: " + count + ", url: " + url);
}
@ -197,8 +195,10 @@ public class IndexUpdateCallbackImpl implements IndexUpdateCallback {
final String url) {
final SearchLogHelper searchLogHelper = SingletonS2Container
.getComponent(SearchLogHelper.class);
final SystemHelper systemHelper = SingletonS2Container
.getComponent("systemHelper");
final long count = searchLogHelper.getFavoriteCount(url);
doc.addField(favoriteCountField, count);
doc.addField(systemHelper.favoriteCountField, count);
if (logger.isDebugEnabled()) {
logger.debug("Favorite Count: " + count + ", url: " + url);
}

9
src/main/java/jp/sf/fess/exec/Crawler.java
View file

@ -42,6 +42,7 @@ import jp.sf.fess.helper.DatabaseHelper;
import jp.sf.fess.helper.MailHelper;
import jp.sf.fess.helper.OverlappingHostHelper;
import jp.sf.fess.helper.PathMappingHelper;
import jp.sf.fess.helper.SystemHelper;
import jp.sf.fess.helper.WebFsIndexHelper;
import jp.sf.fess.screenshot.ScreenShotManager;
import jp.sf.fess.service.CrawlingSessionService;
@ -378,6 +379,8 @@ public class Crawler implements Serializable {
final CrawlingSessionHelper crawlingSessionHelper = SingletonS2Container
.getComponent("crawlingSessionHelper");
final SystemHelper systemHelper = SingletonS2Container
.getComponent("systemHelper");
boolean completed = false;
int exitCode = Constants.EXIT_OK;
@ -459,10 +462,8 @@ public class Crawler implements Serializable {
// clean up
try {
updateSolrGroup.deleteByQuery(crawlingSessionHelper
.getExpiresField()
+ ":[* TO "
+ FessFunctions.formatDate(new Date())
updateSolrGroup.deleteByQuery(systemHelper.expiresField
+ ":[* TO " + FessFunctions.formatDate(new Date())
+ "] NOT segment:" + options.sessionId);
} catch (final Exception e) {
if (logger.isWarnEnabled()) {

2
src/main/java/jp/sf/fess/filter/LoginInfoFilter.java
View file

@ -73,7 +73,7 @@ public class LoginInfoFilter implements Filter {
private void updateRoleList(final HttpServletRequest hRequest,
final LoginInfo loginInfo) {
final SystemHelper systemHelper = SingletonS2Container
.getComponent(SystemHelper.class);
.getComponent("systemHelper");
final List<String> authenticatedRoleList = systemHelper
.getAuthenticatedRoleList();
final List<String> roleList = new ArrayList<String>();

26
src/main/java/jp/sf/fess/helper/CrawlingConfigHelper.java
View file

@ -60,18 +60,6 @@ public class CrawlingConfigHelper implements Serializable {
protected int count = 1;
protected String configIdField = "cid_s_s";
protected String urlField = "url";
public String getUrlField() {
return urlField;
}
public void setUrlField(final String urlField) {
this.urlField = urlField;
}
public ConfigType getConfigType(final String configId) {
if (configId == null || configId.length() < 2) {
return null;
@ -143,19 +131,13 @@ public class CrawlingConfigHelper implements Serializable {
return crawlingConfigMap.get(sessionId);
}
public String getConfigIdField() {
return configIdField;
}
public void setConfigIdField(final String configIdField) {
this.configIdField = configIdField;
}
public void writeContent(final Map<String, Object> doc) {
if (logger.isDebugEnabled()) {
logger.debug("writing the content of: " + doc);
}
final Object configIdObj = doc.get(configIdField);
final SystemHelper systemHelper = SingletonS2Container
.getComponent("systemHelper");
final Object configIdObj = doc.get(systemHelper.configIdField);
if (configIdObj == null) {
throw new FessSystemException("Invalid configId: " + configIdObj);
}
@ -188,7 +170,7 @@ public class CrawlingConfigHelper implements Serializable {
if (config == null) {
throw new FessSystemException("No crawlingConfig: " + configIdObj);
}
final String url = (String) doc.get(urlField);
final String url = (String) doc.get(systemHelper.urlField);
final S2RobotClientFactory robotClientFactory = SingletonS2Container
.getComponent(S2RobotClientFactory.class);
config.initializeClientFactory(robotClientFactory);

10
src/main/java/jp/sf/fess/helper/CrawlingSessionHelper.java
View file

@ -57,8 +57,6 @@ public class CrawlingSessionHelper implements Serializable {
protected Date documentExpires;
protected String expiresField = "expires_dt";
protected CrawlingSessionService getCrawlingSessionService() {
return SingletonS2Container.getComponent(CrawlingSessionService.class);
}
@ -220,12 +218,4 @@ public class CrawlingSessionHelper implements Serializable {
return value.replace('"', ' ');
}
public String getExpiresField() {
return expiresField;
}
public void setExpiresField(final String expiresField) {
this.expiresField = expiresField;
}
}

12
src/main/java/jp/sf/fess/helper/SystemHelper.java
View file

@ -105,6 +105,18 @@ public class SystemHelper implements Serializable {
private final AtomicBoolean forceStop = new AtomicBoolean(false);
public String favoriteCountField = "favoriteCount_i";
public String clickCountField = "clickCount_i";
public String screenshotField = "screenshot_s_s";
public String configIdField = "cid_s_s";
public String expiresField = "expires_dt";
public String urlField = "url";
@InitMethod
public void init() {
final File[] files = ResourceUtil.getJarFiles(launcherFileNamePrefix);

2
src/main/java/jp/sf/fess/job/CrawlJob.java
View file

@ -210,7 +210,7 @@ public class CrawlJob {
final ServletContext servletContext = SingletonS2Container
.getComponent(ServletContext.class);
final SystemHelper systemHelper = SingletonS2Container
.getComponent(SystemHelper.class);
.getComponent("systemHelper");
final JobHelper jobHelper = SingletonS2Container
.getComponent(JobHelper.class);

2
src/main/java/jp/sf/fess/job/TriggeredJob.java
View file

@ -51,7 +51,7 @@ public class TriggeredJob implements Job {
public void execute(final ScheduledJob scheduledJob) {
final SystemHelper systemHelper = SingletonS2Container
.getComponent(SystemHelper.class);
.getComponent("systemHelper");
final JobHelper jobHelper = SingletonS2Container
.getComponent(JobHelper.class);
final JobLog jobLog = new JobLog(scheduledJob);

5
src/main/java/jp/sf/fess/robot/FessS2RobotThread.java
View file

@ -33,6 +33,7 @@ import jp.sf.fess.helper.CrawlingConfigHelper;
import jp.sf.fess.helper.CrawlingSessionHelper;
import jp.sf.fess.helper.SambaHelper;
import jp.sf.fess.helper.SearchLogHelper;
import jp.sf.fess.helper.SystemHelper;
import org.apache.commons.io.IOUtils;
import org.apache.solr.client.solrj.SolrQuery;
@ -81,12 +82,14 @@ public class FessS2RobotThread extends S2RobotThread {
.getComponent(CrawlingConfigHelper.class);
final CrawlingSessionHelper crawlingSessionHelper = SingletonS2Container
.getComponent(CrawlingSessionHelper.class);
final SystemHelper systemHelper = SingletonS2Container
.getComponent("systemHelper");
final SambaHelper sambaHelper = SingletonS2Container
.getComponent(SambaHelper.class);
final boolean useAclAsRole = crawlerProperties.getProperty(
Constants.USE_ACL_AS_ROLE, Constants.FALSE).equals(
Constants.TRUE);
final String expiresField = crawlingSessionHelper.getExpiresField();
final String expiresField = systemHelper.expiresField;
ResponseData responseData = null;
try {

13
src/main/java/jp/sf/fess/screenshot/ScreenShotManager.java
View file

@ -32,8 +32,10 @@ import javax.servlet.http.HttpSession;
import jp.sf.fess.Constants;
import jp.sf.fess.FessSystemException;
import jp.sf.fess.helper.SystemHelper;
import org.apache.commons.io.FileUtils;
import org.seasar.framework.container.SingletonS2Container;
import org.seasar.framework.container.annotation.tiger.InitMethod;
import org.seasar.framework.util.StringUtil;
import org.seasar.robot.util.LruHashMap;
@ -54,8 +56,6 @@ public class ScreenShotManager {
public int threadNum = 10;
public String solrFieldName = "screenshot_s_s";
public long shutdownTimeout = 5 * 60 * 1000; // 5min
public int screenShotPathCacheSize = 10;
@ -90,12 +90,14 @@ public class ScreenShotManager {
}
public void generate(final Map<String, Object> docMap) {
final SystemHelper systemHelper = SingletonS2Container
.getComponent("systemHelper");
for (final ScreenShotGenerator generator : generatorList) {
if (generator.isTarget(docMap)) {
final String segment = (String) docMap.get("segment");
final String url = (String) docMap.get("url");
final String path = segment + "/" + generator.getPath(docMap);
docMap.put(solrFieldName, path);
docMap.put(systemHelper.screenshotField, path);
executorService.execute(new GenerateTask(url, new File(baseDir,
path), generator));
break;
@ -105,11 +107,14 @@ public class ScreenShotManager {
public void storeRequest(final String queryId,
final List<Map<String, Object>> documentItems) {
final SystemHelper systemHelper = SingletonS2Container
.getComponent("systemHelper");
final Map<String, String> dataMap = new HashMap<String, String>(
documentItems.size());
for (final Map<String, Object> docMap : documentItems) {
final String url = (String) docMap.get("url");
final String screenShotPath = (String) docMap.get(solrFieldName);
final String screenShotPath = (String) docMap
.get(systemHelper.screenshotField);
if (StringUtil.isNotBlank(url)
&& StringUtil.isNotBlank(screenShotPath)) {
dataMap.put(url, screenShotPath);

2
src/main/java/jp/sf/fess/service/FailureUrlService.java
View file

@ -202,7 +202,7 @@ public class FailureUrlService extends BsFailureUrlService implements
private String getStackTrace(final Throwable t) {
final SystemHelper systemHelper = SingletonS2Container
.getComponent(SystemHelper.class);
.getComponent("systemHelper");
final StringWriter sw = new StringWriter();
final PrintWriter pw = new PrintWriter(sw, true);
t.printStackTrace(pw);

3
src/main/java/jp/sf/fess/service/FavoriteLogService.java
View file

@ -98,7 +98,8 @@ public class FavoriteLogService extends BsFavoriteLogService implements
}
public boolean addUrl(final String userCode, final String url) {
public boolean addUrl(final String userCode, final String url,
final Map<String, Object> doc) {
final UserInfoCB cb = new UserInfoCB();
cb.query().setCode_Equal(userCode);
final UserInfo userInfo = userInfoBhv.selectEntity(cb);

12
src/main/java/jp/sf/fess/solr/IndexUpdater.java
View file

@ -123,10 +123,6 @@ public class IndexUpdater extends Thread {
public boolean favoriteCountEnabled = true;
public String clickCountField = "clickCount_i";
public String favoriteCountField = "favoriteCount_i";
private final List<BoostDocumentRule> boostRuleList = new ArrayList<BoostDocumentRule>();
private final Map<String, Object> docValueMap = new HashMap<String, Object>();
@ -497,8 +493,8 @@ public class IndexUpdater extends Thread {
final ClickLogCB cb = new ClickLogCB();
cb.query().setUrl_Equal(url);
final int count = clickLogBhv.selectCount(cb);
doc.addField(clickCountField, count);
map.put(clickCountField, count);
doc.addField(systemHelper.clickCountField, count);
map.put(systemHelper.clickCountField, count);
if (logger.isDebugEnabled()) {
logger.debug("Click Count: " + count + ", url: " + url);
}
@ -520,8 +516,8 @@ public class IndexUpdater extends Thread {
count = list.get(0).getCnt().longValue();
}
doc.addField(favoriteCountField, count);
map.put(favoriteCountField, count);
doc.addField(systemHelper.favoriteCountField, count);
map.put(systemHelper.favoriteCountField, count);
if (logger.isDebugEnabled()) {
logger.debug("Favorite Count: " + count + ", url: " + url);
}

8
src/main/java/jp/sf/fess/transformer/AbstractFessFileTransformer.java
View file

@ -37,6 +37,7 @@ import jp.sf.fess.helper.CrawlingSessionHelper;
import jp.sf.fess.helper.LabelTypeHelper;
import jp.sf.fess.helper.PathMappingHelper;
import jp.sf.fess.helper.SambaHelper;
import jp.sf.fess.helper.SystemHelper;
import jp.sf.fess.taglib.FessFunctions;
import org.apache.commons.io.IOUtils;
@ -185,6 +186,8 @@ public abstract class AbstractFessFileTransformer extends
.getComponent("crawlingConfigHelper");
final CrawlingConfig crawlingConfig = crawlingConfigHelper
.get(responseData.getSessionId());
final SystemHelper systemHelper = SingletonS2Container
.getComponent("systemHelper");
String urlEncoding;
final UrlQueue urlQueue = CrawlingParameterUtil.getUrlQueue();
if (urlQueue != null && urlQueue.getEncoding() != null) {
@ -196,12 +199,11 @@ public abstract class AbstractFessFileTransformer extends
// cid
final String configId = crawlingConfig.getConfigId();
if (configId != null) {
putResultDataBody(dataMap, crawlingConfigHelper.getConfigIdField(),
configId);
putResultDataBody(dataMap, systemHelper.configIdField, configId);
}
// expires
if (documentExpires != null) {
putResultDataBody(dataMap, crawlingSessionHelper.getExpiresField(),
putResultDataBody(dataMap, systemHelper.expiresField,
FessFunctions.formatDate(documentExpires));
}
// segment

8
src/main/java/jp/sf/fess/transformer/FessXpathTransformer.java
View file

@ -43,6 +43,7 @@ import jp.sf.fess.helper.CrawlingSessionHelper;
import jp.sf.fess.helper.LabelTypeHelper;
import jp.sf.fess.helper.OverlappingHostHelper;
import jp.sf.fess.helper.PathMappingHelper;
import jp.sf.fess.helper.SystemHelper;
import jp.sf.fess.taglib.FessFunctions;
import org.apache.commons.io.IOUtils;
@ -220,6 +221,8 @@ public class FessXpathTransformer extends AbstractFessXpathTransformer {
.getComponent("crawlingConfigHelper");
final CrawlingConfig crawlingConfig = crawlingConfigHelper
.get(responseData.getSessionId());
final SystemHelper systemHelper = SingletonS2Container
.getComponent("systemHelper");
String urlEncoding;
final UrlQueue urlQueue = CrawlingParameterUtil.getUrlQueue();
if (urlQueue != null && urlQueue.getEncoding() != null) {
@ -231,12 +234,11 @@ public class FessXpathTransformer extends AbstractFessXpathTransformer {
// cid
final String configId = crawlingConfig.getConfigId();
if (configId != null) {
putResultDataBody(dataMap, crawlingConfigHelper.getConfigIdField(),
configId);
putResultDataBody(dataMap, systemHelper.configIdField, configId);
}
// expires
if (documentExpires != null) {
putResultDataBody(dataMap, crawlingSessionHelper.getExpiresField(),
putResultDataBody(dataMap, systemHelper.expiresField,
FessFunctions.formatDate(documentExpires));
}
// title