diff --git a/src/main/java/jp/sf/fess/Constants.java b/src/main/java/jp/sf/fess/Constants.java index 8dba8655e..ef1442e0d 100644 --- a/src/main/java/jp/sf/fess/Constants.java +++ b/src/main/java/jp/sf/fess/Constants.java @@ -236,6 +236,8 @@ public class Constants extends CoreLibConstants { public static final String DATA_INDEX_SIZE = "DataIndexSize"; + public static final String SESSION_ID = "sessionId"; + public static final String INDEXING_TARGET = "indexingTarget"; public static final String DIGEST_PREFIX = "..."; diff --git a/src/main/java/jp/sf/fess/ds/impl/AbstractDataStoreImpl.java b/src/main/java/jp/sf/fess/ds/impl/AbstractDataStoreImpl.java index f7480f441..56158ebc1 100644 --- a/src/main/java/jp/sf/fess/ds/impl/AbstractDataStoreImpl.java +++ b/src/main/java/jp/sf/fess/ds/impl/AbstractDataStoreImpl.java @@ -61,7 +61,7 @@ public abstract class AbstractDataStoreImpl implements DataStore { // default values final Map defaultDataMap = new HashMap(); // segment - defaultDataMap.put("segment", initParamMap.get("sessionId")); + defaultDataMap.put("segment", initParamMap.get(Constants.SESSION_ID)); // tstamp defaultDataMap.put("tstamp", "NOW"); // boost diff --git a/src/main/java/jp/sf/fess/ds/impl/FileListDataStoreImpl.java b/src/main/java/jp/sf/fess/ds/impl/FileListDataStoreImpl.java index 8c293ff83..5d3cc18eb 100644 --- a/src/main/java/jp/sf/fess/ds/impl/FileListDataStoreImpl.java +++ b/src/main/java/jp/sf/fess/ds/impl/FileListDataStoreImpl.java @@ -101,6 +101,9 @@ public class FileListDataStoreImpl extends CsvDataStoreImpl { public Map parentEncodingMap = Collections .synchronizedMap(new LruHashMap(1000)); + public String[] ignoreFieldNames = new String[] { + Constants.INDEXING_TARGET, Constants.SESSION_ID }; + @Override protected boolean isCsvFile(final File parentFile, final String filename) { if (super.isCsvFile(parentFile, filename)) { @@ -233,9 +236,12 @@ public class FileListDataStoreImpl extends CsvDataStoreImpl { headerName = paramMap.get(S2ROBOT_WEB_HEADER_PREFIX + count + ".name"); } - initParamMap.put(HcHttpClient.REQUERT_HEADERS_PROPERTY, rhList - .toArray(new org.seasar.robot.client.http.RequestHeader[rhList - .size()])); + if (!rhList.isEmpty()) { + initParamMap + .put(HcHttpClient.REQUERT_HEADERS_PROPERTY, + rhList.toArray(new org.seasar.robot.client.http.RequestHeader[rhList + .size()])); + } // file auth final String fileAuthStr = paramMap.get(S2ROBOT_FILE_AUTH); @@ -337,7 +343,7 @@ public class FileListDataStoreImpl extends CsvDataStoreImpl { @Override public boolean store(final Map dataMap) { - final Object eventType = dataMap.get(eventTypeField); + final Object eventType = dataMap.remove(eventTypeField); if (createEventName.equals(eventType) || modifyEventName.equals(eventType)) { @@ -375,7 +381,8 @@ public class FileListDataStoreImpl extends CsvDataStoreImpl { final ResponseData responseData = client.doGet(url); responseData.setExecutionTime(System.currentTimeMillis() - startTime); - responseData.setSessionId((String) dataMap.get("sessionId")); + responseData.setSessionId((String) dataMap + .get(Constants.SESSION_ID)); final RuleManager ruleManager = SingletonS2Container .getComponent(RuleManager.class); @@ -406,6 +413,11 @@ public class FileListDataStoreImpl extends CsvDataStoreImpl { } } + // remove + for (final String fieldName : ignoreFieldNames) { + dataMap.remove(fieldName); + } + return indexUpdateCallback.store(dataMap); } else { logger.warn("The response processor is not DefaultResponseProcessor. responseProcessor: " diff --git a/src/main/java/jp/sf/fess/helper/DataIndexHelper.java b/src/main/java/jp/sf/fess/helper/DataIndexHelper.java index adc35d070..728f081de 100644 --- a/src/main/java/jp/sf/fess/helper/DataIndexHelper.java +++ b/src/main/java/jp/sf/fess/helper/DataIndexHelper.java @@ -129,7 +129,7 @@ public class DataIndexHelper implements Serializable { dataCrawlingConfig); sessionIdList.add(sid); - initParamMap.put("sessionId", sessionId); + initParamMap.put(Constants.SESSION_ID, sessionId); initParamMap.put("crawlingSessionId", sid); final DataCrawlingThread dataCrawlingThread = new DataCrawlingThread( diff --git a/src/main/java/jp/sf/fess/helper/SystemHelper.java b/src/main/java/jp/sf/fess/helper/SystemHelper.java index c28315d89..cc29e543a 100644 --- a/src/main/java/jp/sf/fess/helper/SystemHelper.java +++ b/src/main/java/jp/sf/fess/helper/SystemHelper.java @@ -294,6 +294,12 @@ public class SystemHelper implements Serializable { logger.info("Crawler: Exit Code=" + exitValue + " - Crawler Process Output:\n" + it.getOutput()); } + if (exitValue != 0) { + throw new FessSystemException("Exit code is " + exitValue + + "\nOutput:\n" + it.getOutput()); + } + } catch (final FessSystemException e) { + throw e; } catch (final InterruptedException e) { logger.warn("Crawler Process interrupted."); } catch (final Exception e) { diff --git a/src/main/java/jp/sf/fess/job/CrawlJob.java b/src/main/java/jp/sf/fess/job/CrawlJob.java index d5f0998ab..f7f1ee16c 100644 --- a/src/main/java/jp/sf/fess/job/CrawlJob.java +++ b/src/main/java/jp/sf/fess/job/CrawlJob.java @@ -20,6 +20,7 @@ import java.text.SimpleDateFormat; import java.util.Date; import jp.sf.fess.Constants; +import jp.sf.fess.FessSystemException; import jp.sf.fess.helper.CrawlingSessionHelper; import jp.sf.fess.helper.SystemHelper; import jp.sf.fess.job.JobExecutor.ShutdownListener; @@ -56,6 +57,35 @@ public class CrawlJob { final String operation) { final StringBuilder resultBuf = new StringBuilder(); + resultBuf.append("Session Id: ").append(sessionId).append("\n"); + resultBuf.append("Web Config Id:"); + if (webConfigIds == null) { + resultBuf.append(" ALL\n"); + } else { + for (final String id : webConfigIds) { + resultBuf.append(' ').append(id); + } + resultBuf.append('\n'); + } + resultBuf.append("File Config Id:"); + if (fileConfigIds == null) { + resultBuf.append(" ALL\n"); + } else { + for (final String id : fileConfigIds) { + resultBuf.append(' ').append(id); + } + resultBuf.append('\n'); + } + resultBuf.append("Data Config Id:"); + if (dataConfigIds == null) { + resultBuf.append(" ALL\n"); + } else { + for (final String id : dataConfigIds) { + resultBuf.append(' ').append(id); + } + resultBuf.append('\n'); + } + if (jobExecutor != null) { jobExecutor.addShutdownListener(new ShutdownListener() { @Override @@ -74,9 +104,10 @@ public class CrawlJob { SingletonS2Container.getComponent(SystemHelper.class) .executeCrawler(sessionId, webConfigIds, fileConfigIds, dataConfigIds, operation); + } catch (final FessSystemException e) { + throw e; } catch (final Exception e) { - logger.error("Failed to execute a crawl job.", e); - resultBuf.append(e.getMessage()); + throw new FessSystemException("Failed to execute a crawl job.", e); } return resultBuf.toString();