diff --git a/src/main/java/org/codelibs/fess/ds/impl/CsvDataStoreImpl.java b/src/main/java/org/codelibs/fess/ds/impl/CsvDataStoreImpl.java index 0349ef28b..fa28216dd 100644 --- a/src/main/java/org/codelibs/fess/ds/impl/CsvDataStoreImpl.java +++ b/src/main/java/org/codelibs/fess/ds/impl/CsvDataStoreImpl.java @@ -20,6 +20,8 @@ import java.io.File; import java.io.FileInputStream; import java.io.InputStreamReader; import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; @@ -96,7 +98,8 @@ public class CsvDataStoreImpl extends AbstractDataStoreImpl { for (final String path : values) { final File dir = new File(path); if (dir.isDirectory()) { - StreamUtil.of(dir.listFiles()).filter(f -> isCsvFile(f.getParentFile(), f.getName())).forEach(f -> fileList.add(f)); + StreamUtil.of(dir.listFiles()).filter(f -> isCsvFile(f.getParentFile(), f.getName())) + .sorted((f1, f2) -> (int) (f1.lastModified() - f2.lastModified())).forEach(f -> fileList.add(f)); } else { logger.warn(path + " is not a directory."); } @@ -263,7 +266,7 @@ public class CsvDataStoreImpl extends AbstractDataStoreImpl { } final FailureUrlService failureUrlService = ComponentUtil.getComponent(FailureUrlService.class); failureUrlService.store(dataConfig, errorName, url, target); - } catch (final Exception e) { + } catch (final Exception | OutOfMemoryError e) { final String url = csvFile.getAbsolutePath() + ":" + csvReader.getLineNumber(); final FailureUrlService failureUrlService = ComponentUtil.getComponent(FailureUrlService.class); failureUrlService.store(dataConfig, e.getClass().getCanonicalName(), url, e); diff --git a/src/main/java/org/codelibs/fess/ds/impl/FileListIndexUpdateCallbackImpl.java b/src/main/java/org/codelibs/fess/ds/impl/FileListIndexUpdateCallbackImpl.java index 82cb3f974..8d73d1381 100644 --- a/src/main/java/org/codelibs/fess/ds/impl/FileListIndexUpdateCallbackImpl.java +++ b/src/main/java/org/codelibs/fess/ds/impl/FileListIndexUpdateCallbackImpl.java @@ -73,7 +73,8 @@ public class FileListIndexUpdateCallbackImpl implements IndexUpdateCallback { } logger.warn("unknown event: " + eventType + ", data: " + dataMap); - return false; + // don't stop crawling + return true; } protected String getParamValue(Map paramMap, String key, String defaultValue) { diff --git a/src/main/java/org/codelibs/fess/helper/DataIndexHelper.java b/src/main/java/org/codelibs/fess/helper/DataIndexHelper.java index 4dd7358c4..49edb1960 100644 --- a/src/main/java/org/codelibs/fess/helper/DataIndexHelper.java +++ b/src/main/java/org/codelibs/fess/helper/DataIndexHelper.java @@ -227,7 +227,7 @@ public class DataIndexHelper implements Serializable { } else { try { dataStore.store(dataConfig, indexUpdateCallback, initParamMap); - } catch (final Exception e) { + } catch (final Throwable e) { logger.error("Failed to process a data crawling: " + dataConfig.getName(), e); ComponentUtil.getComponent(FailureUrlService.class).store(dataConfig, e.getClass().getCanonicalName(), dataConfig.getConfigId() + ":" + dataConfig.getName(), e);