Merge branch 'master' into 10.3.x

src/main/java/org/codelibs/fess/ds/impl/CsvDataStoreImpl.java

@@ -225,6 +225,9 @@ public class CsvDataStoreImpl extends AbstractDataStoreImpl {
                     }
                 }
 
+                final Map<String, Object> crawlingContext = new HashMap<>();
+                crawlingContext.put("doc", dataMap);
+                resultMap.put("crawlingContext", crawlingContext);
                 for (final Map.Entry<String, String> entry : scriptMap.entrySet()) {
                     final Object convertValue = convertValue(entry.getValue(), resultMap);
                     if (convertValue != null) {

src/main/java/org/codelibs/fess/ds/impl/DatabaseDataStoreImpl.java

@@ -108,8 +108,11 @@ public class DatabaseDataStoreImpl extends AbstractDataStoreImpl {
             while (rs.next() && loop && alive) {
                 final Map<String, Object> dataMap = new HashMap<>();
                 dataMap.putAll(defaultDataMap);
+                final Map<String, Object> crawlingContext = new HashMap<>();
+                crawlingContext.put("doc", dataMap);
                 for (final Map.Entry<String, String> entry : scriptMap.entrySet()) {
-                    final Object convertValue = convertValue(config, entry.getValue(), rs, paramMap);
+                    final Object convertValue =
+                            convertValue(entry.getValue(), new ResultSetParamMap(config, crawlingContext, rs, paramMap));
                     if (convertValue != null) {
                         dataMap.put(entry.getKey(), convertValue);
                     }
@@ -189,16 +192,14 @@ public class DatabaseDataStoreImpl extends AbstractDataStoreImpl {
         }
     }
 
-    protected Object convertValue(final DataConfig config, final String template, final ResultSet rs, final Map<String, String> paramMap) {
-        return convertValue(template, new ResultSetParamMap(config, rs, paramMap));
-    }
-
     protected static class ResultSetParamMap implements Map<String, Object> {
         private final Map<String, Object> paramMap = new HashMap<>();
 
-        public ResultSetParamMap(final DataConfig config, final ResultSet resultSet, final Map<String, String> paramMap) {
+        public ResultSetParamMap(final DataConfig config, final Map<String, Object> crawlingContext, final ResultSet resultSet,
+                final Map<String, String> paramMap) {
             this.paramMap.putAll(paramMap);
             this.paramMap.put("crawlingConfig", config);
+            this.paramMap.put("crawlingContext", crawlingContext);
 
             try {
                 final ResultSetMetaData metaData = resultSet.getMetaData();

src/main/java/org/codelibs/fess/ds/impl/EsDataStoreImpl.java

@@ -170,6 +170,9 @@ public class EsDataStoreImpl extends AbstractDataStoreImpl {
                         }
                     }
 
+                    final Map<String, Object> crawlingContext = new HashMap<>();
+                    crawlingContext.put("doc", dataMap);
+                    resultMap.put("crawlingContext", crawlingContext);
                     for (final Map.Entry<String, String> entry : scriptMap.entrySet()) {
                         final Object convertValue = convertValue(entry.getValue(), resultMap);
                         if (convertValue != null) {

src/main/java/org/codelibs/fess/helper/CrawlingInfoHelper.java

@@ -125,11 +125,13 @@ public class CrawlingInfoHelper {
     }
 
     public Date getDocumentExpires(final CrawlingConfig config) {
-        final Integer timeToLive = config.getTimeToLive();
-        if (timeToLive != null) {
-            // timeToLive minutes
-            final long now = ComponentUtil.getSystemHelper().getCurrentTimeAsLong();
-            return new Date(now + timeToLive.longValue() * 1000 * 60);
+        if (config != null) {
+            final Integer timeToLive = config.getTimeToLive();
+            if (timeToLive != null) {
+                // timeToLive minutes
+                final long now = ComponentUtil.getSystemHelper().getCurrentTimeAsLong();
+                return new Date(now + timeToLive.longValue() * 1000 * 60);
+            }
         }
         return documentExpires != null ? new Date(documentExpires) : null;
     }

src/main/java/org/codelibs/fess/helper/DataIndexHelper.java

@@ -30,6 +30,7 @@ import org.codelibs.fess.app.service.FailureUrlService;
 import org.codelibs.fess.ds.DataStore;
 import org.codelibs.fess.ds.DataStoreFactory;
 import org.codelibs.fess.ds.IndexUpdateCallback;
+import org.codelibs.fess.es.client.FessEsClient;
 import org.codelibs.fess.es.config.exentity.DataConfig;
 import org.codelibs.fess.mylasta.direction.FessConfig;
 import org.codelibs.fess.util.ComponentUtil;
@@ -262,8 +263,11 @@ public class DataIndexHelper {
                                     .should(QueryBuilders.missingQuery(fessConfig.getIndexFieldExpires())))
                             .mustNot(QueryBuilders.termQuery(fessConfig.getIndexFieldSegment(), sessionId));
             try {
-                ComponentUtil.getFessEsClient().deleteByQuery(fessConfig.getIndexDocumentUpdateIndex(), fessConfig.getIndexDocumentType(),
-                        queryBuilder);
+                final FessEsClient fessEsClient = ComponentUtil.getFessEsClient();
+                final String index = fessConfig.getIndexDocumentUpdateIndex();
+                fessEsClient.admin().indices().prepareRefresh(index).execute().actionGet();
+                final int numOfDeleted = fessEsClient.deleteByQuery(index, fessConfig.getIndexDocumentType(), queryBuilder);
+                logger.info("Deleted {} old docs.", numOfDeleted);
             } catch (final Exception e) {
                 logger.error("Could not delete old docs at " + dataConfig, e);
             }

src/main/java/org/codelibs/fess/helper/DocumentHelper.java

@@ -104,8 +104,8 @@ public class DocumentHelper {
         }
     }
 
-    public Map<String, Object> processRequest(final CrawlingConfig crawlingConfig, final String sessionId, final String url) {
-        if (StringUtil.isBlank(sessionId)) {
+    public Map<String, Object> processRequest(final CrawlingConfig crawlingConfig, final String crawlingInfoId, final String url) {
+        if (StringUtil.isBlank(crawlingInfoId)) {
             throw new CrawlingAccessException("sessionId is null.");
         }
 
@@ -124,7 +124,7 @@ public class DocumentHelper {
                 throw new ChildUrlsException(childUrlList, "Redirected from " + url);
             }
             responseData.setExecutionTime(System.currentTimeMillis() - startTime);
-            responseData.setSessionId(sessionId);
+            responseData.setSessionId(crawlingInfoId);
 
             final RuleManager ruleManager = SingletonLaContainer.getComponent(RuleManager.class);
             final Rule rule = ruleManager.getRule(responseData);

src/main/java/org/codelibs/fess/util/ParameterUtil.java

@@ -16,6 +16,7 @@
 package org.codelibs.fess.util;
 
 import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.Map;
 
 import org.codelibs.core.lang.StringUtil;
@@ -42,7 +43,7 @@ public class ParameterUtil {
     }
 
     public static Map<String, String> parse(final String value) {
-        final Map<String, String> paramMap = new HashMap<>();
+        final Map<String, String> paramMap = new LinkedHashMap<>();
         if (value != null) {
             final String[] lines = value.split("[\r\n]");
             for (final String line : lines) {
@@ -74,13 +75,13 @@ public class ParameterUtil {
 
     public static Map<ConfigName, Map<String, String>> createConfigParameterMap(final String configParameters) {
         final Map<ConfigName, Map<String, String>> map = new HashMap<>();
-        final Map<String, String> configConfigMap = new HashMap<>();
-        final Map<String, String> clientConfigMap = new HashMap<>();
-        final Map<String, String> xpathConfigMap = new HashMap<>();
-        final Map<String, String> metaConfigMap = new HashMap<>();
-        final Map<String, String> valueConfigMap = new HashMap<>();
-        final Map<String, String> scriptConfigMap = new HashMap<>();
-        final Map<String, String> fieldConfigMap = new HashMap<>();
+        final Map<String, String> configConfigMap = new LinkedHashMap<>();
+        final Map<String, String> clientConfigMap = new LinkedHashMap<>();
+        final Map<String, String> xpathConfigMap = new LinkedHashMap<>();
+        final Map<String, String> metaConfigMap = new LinkedHashMap<>();
+        final Map<String, String> valueConfigMap = new LinkedHashMap<>();
+        final Map<String, String> scriptConfigMap = new LinkedHashMap<>();
+        final Map<String, String> fieldConfigMap = new LinkedHashMap<>();
         map.put(ConfigName.CONFIG, configConfigMap);
         map.put(ConfigName.CLIENT, clientConfigMap);
         map.put(ConfigName.XPATH, xpathConfigMap);