0ct0pu5/fess
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

#11 fixed FileListDataStore.

Browse source
This commit is contained in:
Shinsuke Sugaya 2013-10-05 14:08:38 +09:00
parent 88846fac4b
commit 3277dbee67
6 changed files with 60 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/main/java/jp/sf/fess/Constants.java
View file

@ -236,6 +236,8 @@ public class Constants extends CoreLibConstants {
public static final String DATA_INDEX_SIZE = "DataIndexSize";
public static final String SESSION_ID = "sessionId";
public static final String INDEXING_TARGET = "indexingTarget";
public static final String DIGEST_PREFIX = "...";

2
src/main/java/jp/sf/fess/ds/impl/AbstractDataStoreImpl.java
View file

@ -61,7 +61,7 @@ public abstract class AbstractDataStoreImpl implements DataStore {
// default values
final Map<String, Object> defaultDataMap = new HashMap<String, Object>();
// segment
defaultDataMap.put("segment", initParamMap.get("sessionId"));
defaultDataMap.put("segment", initParamMap.get(Constants.SESSION_ID));
// tstamp
defaultDataMap.put("tstamp", "NOW");
// boost

22
src/main/java/jp/sf/fess/ds/impl/FileListDataStoreImpl.java
View file

@ -101,6 +101,9 @@ public class FileListDataStoreImpl extends CsvDataStoreImpl {
public Map<String, String> parentEncodingMap = Collections
.synchronizedMap(new LruHashMap<String, String>(1000));
public String[] ignoreFieldNames = new String[] {
Constants.INDEXING_TARGET, Constants.SESSION_ID };
@Override
protected boolean isCsvFile(final File parentFile, final String filename) {
if (super.isCsvFile(parentFile, filename)) {
@ -233,9 +236,12 @@ public class FileListDataStoreImpl extends CsvDataStoreImpl {
headerName = paramMap.get(S2ROBOT_WEB_HEADER_PREFIX + count
+ ".name");
}
initParamMap.put(HcHttpClient.REQUERT_HEADERS_PROPERTY, rhList
.toArray(new org.seasar.robot.client.http.RequestHeader[rhList
.size()]));
if (!rhList.isEmpty()) {
initParamMap
.put(HcHttpClient.REQUERT_HEADERS_PROPERTY,
rhList.toArray(new org.seasar.robot.client.http.RequestHeader[rhList
.size()]));
}
// file auth
final String fileAuthStr = paramMap.get(S2ROBOT_FILE_AUTH);
@ -337,7 +343,7 @@ public class FileListDataStoreImpl extends CsvDataStoreImpl {
@Override
public boolean store(final Map<String, Object> dataMap) {
final Object eventType = dataMap.get(eventTypeField);
final Object eventType = dataMap.remove(eventTypeField);
if (createEventName.equals(eventType)
|| modifyEventName.equals(eventType)) {
@ -375,7 +381,8 @@ public class FileListDataStoreImpl extends CsvDataStoreImpl {
final ResponseData responseData = client.doGet(url);
responseData.setExecutionTime(System.currentTimeMillis()
- startTime);
responseData.setSessionId((String) dataMap.get("sessionId"));
responseData.setSessionId((String) dataMap
.get(Constants.SESSION_ID));
final RuleManager ruleManager = SingletonS2Container
.getComponent(RuleManager.class);
@ -406,6 +413,11 @@ public class FileListDataStoreImpl extends CsvDataStoreImpl {
}
}
// remove
for (final String fieldName : ignoreFieldNames) {
dataMap.remove(fieldName);
}
return indexUpdateCallback.store(dataMap);
} else {
logger.warn("The response processor is not DefaultResponseProcessor. responseProcessor: "

2
src/main/java/jp/sf/fess/helper/DataIndexHelper.java
View file

@ -129,7 +129,7 @@ public class DataIndexHelper implements Serializable {
dataCrawlingConfig);
sessionIdList.add(sid);
initParamMap.put("sessionId", sessionId);
initParamMap.put(Constants.SESSION_ID, sessionId);
initParamMap.put("crawlingSessionId", sid);
final DataCrawlingThread dataCrawlingThread = new DataCrawlingThread(

6
src/main/java/jp/sf/fess/helper/SystemHelper.java
View file

@ -294,6 +294,12 @@ public class SystemHelper implements Serializable {
logger.info("Crawler: Exit Code=" + exitValue
+ " - Crawler Process Output:\n" + it.getOutput());
}
if (exitValue != 0) {
throw new FessSystemException("Exit code is " + exitValue
+ "\nOutput:\n" + it.getOutput());
}
} catch (final FessSystemException e) {
throw e;
} catch (final InterruptedException e) {
logger.warn("Crawler Process interrupted.");
} catch (final Exception e) {

35
src/main/java/jp/sf/fess/job/CrawlJob.java
View file

@ -20,6 +20,7 @@ import java.text.SimpleDateFormat;
import java.util.Date;
import jp.sf.fess.Constants;
import jp.sf.fess.FessSystemException;
import jp.sf.fess.helper.CrawlingSessionHelper;
import jp.sf.fess.helper.SystemHelper;
import jp.sf.fess.job.JobExecutor.ShutdownListener;
@ -56,6 +57,35 @@ public class CrawlJob {
final String operation) {
final StringBuilder resultBuf = new StringBuilder();
resultBuf.append("Session Id: ").append(sessionId).append("\n");
resultBuf.append("Web Config Id:");
if (webConfigIds == null) {
resultBuf.append(" ALL\n");
} else {
for (final String id : webConfigIds) {
resultBuf.append(' ').append(id);
}
resultBuf.append('\n');
}
resultBuf.append("File Config Id:");
if (fileConfigIds == null) {
resultBuf.append(" ALL\n");
} else {
for (final String id : fileConfigIds) {
resultBuf.append(' ').append(id);
}
resultBuf.append('\n');
}
resultBuf.append("Data Config Id:");
if (dataConfigIds == null) {
resultBuf.append(" ALL\n");
} else {
for (final String id : dataConfigIds) {
resultBuf.append(' ').append(id);
}
resultBuf.append('\n');
}
if (jobExecutor != null) {
jobExecutor.addShutdownListener(new ShutdownListener() {
@Override
@ -74,9 +104,10 @@ public class CrawlJob {
SingletonS2Container.getComponent(SystemHelper.class)
.executeCrawler(sessionId, webConfigIds, fileConfigIds,
dataConfigIds, operation);
} catch (final FessSystemException e) {
throw e;
} catch (final Exception e) {
logger.error("Failed to execute a crawl job.", e);
resultBuf.append(e.getMessage());
throw new FessSystemException("Failed to execute a crawl job.", e);
}
return resultBuf.toString();