Merge pull request #467 from codelibs/10.0.x

merge from 10.0.x

src/main/java/org/codelibs/fess/ds/impl/CsvDataStoreImpl.java

@@ -20,6 +20,8 @@ import java.io.File;
 import java.io.FileInputStream;
 import java.io.InputStreamReader;
 import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
@@ -96,7 +98,8 @@ public class CsvDataStoreImpl extends AbstractDataStoreImpl {
             for (final String path : values) {
                 final File dir = new File(path);
                 if (dir.isDirectory()) {
-                    StreamUtil.of(dir.listFiles()).filter(f -> isCsvFile(f.getParentFile(), f.getName())).forEach(f -> fileList.add(f));
+                    StreamUtil.of(dir.listFiles()).filter(f -> isCsvFile(f.getParentFile(), f.getName()))
+                            .sorted((f1, f2) -> (int) (f1.lastModified() - f2.lastModified())).forEach(f -> fileList.add(f));
                 } else {
                     logger.warn(path + " is not a directory.");
                 }
@@ -263,7 +266,7 @@ public class CsvDataStoreImpl extends AbstractDataStoreImpl {
                     }
                     final FailureUrlService failureUrlService = ComponentUtil.getComponent(FailureUrlService.class);
                     failureUrlService.store(dataConfig, errorName, url, target);
-                } catch (final Exception e) {
+                } catch (final Exception | OutOfMemoryError e) {
                     final String url = csvFile.getAbsolutePath() + ":" + csvReader.getLineNumber();
                     final FailureUrlService failureUrlService = ComponentUtil.getComponent(FailureUrlService.class);
                     failureUrlService.store(dataConfig, e.getClass().getCanonicalName(), url, e);

src/main/java/org/codelibs/fess/ds/impl/FileListIndexUpdateCallbackImpl.java

@@ -73,7 +73,8 @@ public class FileListIndexUpdateCallbackImpl implements IndexUpdateCallback {
         }
 
         logger.warn("unknown event: " + eventType + ", data: " + dataMap);
-        return false;
+        // don't stop crawling
+        return true;
     }
 
     protected String getParamValue(Map<String, String> paramMap, String key, String defaultValue) {

src/main/java/org/codelibs/fess/helper/DataIndexHelper.java

@@ -227,7 +227,7 @@ public class DataIndexHelper implements Serializable {
             } else {
                 try {
                     dataStore.store(dataConfig, indexUpdateCallback, initParamMap);
-                } catch (final Exception e) {
+                } catch (final Throwable e) {
                     logger.error("Failed to process a data crawling: " + dataConfig.getName(), e);
                     ComponentUtil.getComponent(FailureUrlService.class).store(dataConfig, e.getClass().getCanonicalName(),
                             dataConfig.getConfigId() + ":" + dataConfig.getName(), e);