diff --git a/src/main/java/org/codelibs/fess/ds/impl/CsvDataStoreImpl.java b/src/main/java/org/codelibs/fess/ds/impl/CsvDataStoreImpl.java index cd80e923d..c8060a288 100644 --- a/src/main/java/org/codelibs/fess/ds/impl/CsvDataStoreImpl.java +++ b/src/main/java/org/codelibs/fess/ds/impl/CsvDataStoreImpl.java @@ -225,6 +225,9 @@ public class CsvDataStoreImpl extends AbstractDataStoreImpl { } } + final Map crawlingContext = new HashMap<>(); + crawlingContext.put("doc", dataMap); + resultMap.put("crawlingContext", crawlingContext); for (final Map.Entry entry : scriptMap.entrySet()) { final Object convertValue = convertValue(entry.getValue(), resultMap); if (convertValue != null) { diff --git a/src/main/java/org/codelibs/fess/ds/impl/DatabaseDataStoreImpl.java b/src/main/java/org/codelibs/fess/ds/impl/DatabaseDataStoreImpl.java index 1ba9173c2..61eadb550 100644 --- a/src/main/java/org/codelibs/fess/ds/impl/DatabaseDataStoreImpl.java +++ b/src/main/java/org/codelibs/fess/ds/impl/DatabaseDataStoreImpl.java @@ -108,8 +108,11 @@ public class DatabaseDataStoreImpl extends AbstractDataStoreImpl { while (rs.next() && loop && alive) { final Map dataMap = new HashMap<>(); dataMap.putAll(defaultDataMap); + final Map crawlingContext = new HashMap<>(); + crawlingContext.put("doc", dataMap); for (final Map.Entry entry : scriptMap.entrySet()) { - final Object convertValue = convertValue(config, entry.getValue(), rs, paramMap); + final Object convertValue = + convertValue(entry.getValue(), new ResultSetParamMap(config, crawlingContext, rs, paramMap)); if (convertValue != null) { dataMap.put(entry.getKey(), convertValue); } @@ -189,16 +192,14 @@ public class DatabaseDataStoreImpl extends AbstractDataStoreImpl { } } - protected Object convertValue(final DataConfig config, final String template, final ResultSet rs, final Map paramMap) { - return convertValue(template, new ResultSetParamMap(config, rs, paramMap)); - } - protected static class ResultSetParamMap implements Map { private final Map paramMap = new HashMap<>(); - public ResultSetParamMap(final DataConfig config, final ResultSet resultSet, final Map paramMap) { + public ResultSetParamMap(final DataConfig config, final Map crawlingContext, final ResultSet resultSet, + final Map paramMap) { this.paramMap.putAll(paramMap); this.paramMap.put("crawlingConfig", config); + this.paramMap.put("crawlingContext", crawlingContext); try { final ResultSetMetaData metaData = resultSet.getMetaData(); diff --git a/src/main/java/org/codelibs/fess/ds/impl/EsDataStoreImpl.java b/src/main/java/org/codelibs/fess/ds/impl/EsDataStoreImpl.java index 3f51fb331..216a36fc5 100644 --- a/src/main/java/org/codelibs/fess/ds/impl/EsDataStoreImpl.java +++ b/src/main/java/org/codelibs/fess/ds/impl/EsDataStoreImpl.java @@ -170,6 +170,9 @@ public class EsDataStoreImpl extends AbstractDataStoreImpl { } } + final Map crawlingContext = new HashMap<>(); + crawlingContext.put("doc", dataMap); + resultMap.put("crawlingContext", crawlingContext); for (final Map.Entry entry : scriptMap.entrySet()) { final Object convertValue = convertValue(entry.getValue(), resultMap); if (convertValue != null) { diff --git a/src/main/java/org/codelibs/fess/helper/CrawlingInfoHelper.java b/src/main/java/org/codelibs/fess/helper/CrawlingInfoHelper.java index c21d19984..04b03445c 100644 --- a/src/main/java/org/codelibs/fess/helper/CrawlingInfoHelper.java +++ b/src/main/java/org/codelibs/fess/helper/CrawlingInfoHelper.java @@ -125,11 +125,13 @@ public class CrawlingInfoHelper { } public Date getDocumentExpires(final CrawlingConfig config) { - final Integer timeToLive = config.getTimeToLive(); - if (timeToLive != null) { - // timeToLive minutes - final long now = ComponentUtil.getSystemHelper().getCurrentTimeAsLong(); - return new Date(now + timeToLive.longValue() * 1000 * 60); + if (config != null) { + final Integer timeToLive = config.getTimeToLive(); + if (timeToLive != null) { + // timeToLive minutes + final long now = ComponentUtil.getSystemHelper().getCurrentTimeAsLong(); + return new Date(now + timeToLive.longValue() * 1000 * 60); + } } return documentExpires != null ? new Date(documentExpires) : null; } diff --git a/src/main/java/org/codelibs/fess/helper/DataIndexHelper.java b/src/main/java/org/codelibs/fess/helper/DataIndexHelper.java index 92c1ba8cb..4d6efb97a 100644 --- a/src/main/java/org/codelibs/fess/helper/DataIndexHelper.java +++ b/src/main/java/org/codelibs/fess/helper/DataIndexHelper.java @@ -30,6 +30,7 @@ import org.codelibs.fess.app.service.FailureUrlService; import org.codelibs.fess.ds.DataStore; import org.codelibs.fess.ds.DataStoreFactory; import org.codelibs.fess.ds.IndexUpdateCallback; +import org.codelibs.fess.es.client.FessEsClient; import org.codelibs.fess.es.config.exentity.DataConfig; import org.codelibs.fess.mylasta.direction.FessConfig; import org.codelibs.fess.util.ComponentUtil; @@ -262,8 +263,11 @@ public class DataIndexHelper { .should(QueryBuilders.missingQuery(fessConfig.getIndexFieldExpires()))) .mustNot(QueryBuilders.termQuery(fessConfig.getIndexFieldSegment(), sessionId)); try { - ComponentUtil.getFessEsClient().deleteByQuery(fessConfig.getIndexDocumentUpdateIndex(), fessConfig.getIndexDocumentType(), - queryBuilder); + final FessEsClient fessEsClient = ComponentUtil.getFessEsClient(); + final String index = fessConfig.getIndexDocumentUpdateIndex(); + fessEsClient.admin().indices().prepareRefresh(index).execute().actionGet(); + final int numOfDeleted = fessEsClient.deleteByQuery(index, fessConfig.getIndexDocumentType(), queryBuilder); + logger.info("Deleted {} old docs.", numOfDeleted); } catch (final Exception e) { logger.error("Could not delete old docs at " + dataConfig, e); } diff --git a/src/main/java/org/codelibs/fess/helper/DocumentHelper.java b/src/main/java/org/codelibs/fess/helper/DocumentHelper.java index 049000996..183b0dfab 100644 --- a/src/main/java/org/codelibs/fess/helper/DocumentHelper.java +++ b/src/main/java/org/codelibs/fess/helper/DocumentHelper.java @@ -104,8 +104,8 @@ public class DocumentHelper { } } - public Map processRequest(final CrawlingConfig crawlingConfig, final String sessionId, final String url) { - if (StringUtil.isBlank(sessionId)) { + public Map processRequest(final CrawlingConfig crawlingConfig, final String crawlingInfoId, final String url) { + if (StringUtil.isBlank(crawlingInfoId)) { throw new CrawlingAccessException("sessionId is null."); } @@ -124,7 +124,7 @@ public class DocumentHelper { throw new ChildUrlsException(childUrlList, "Redirected from " + url); } responseData.setExecutionTime(System.currentTimeMillis() - startTime); - responseData.setSessionId(sessionId); + responseData.setSessionId(crawlingInfoId); final RuleManager ruleManager = SingletonLaContainer.getComponent(RuleManager.class); final Rule rule = ruleManager.getRule(responseData); diff --git a/src/main/java/org/codelibs/fess/util/ParameterUtil.java b/src/main/java/org/codelibs/fess/util/ParameterUtil.java index e8a2e2aea..06a4ca623 100644 --- a/src/main/java/org/codelibs/fess/util/ParameterUtil.java +++ b/src/main/java/org/codelibs/fess/util/ParameterUtil.java @@ -16,6 +16,7 @@ package org.codelibs.fess.util; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.Map; import org.codelibs.core.lang.StringUtil; @@ -42,7 +43,7 @@ public class ParameterUtil { } public static Map parse(final String value) { - final Map paramMap = new HashMap<>(); + final Map paramMap = new LinkedHashMap<>(); if (value != null) { final String[] lines = value.split("[\r\n]"); for (final String line : lines) { @@ -74,13 +75,13 @@ public class ParameterUtil { public static Map> createConfigParameterMap(final String configParameters) { final Map> map = new HashMap<>(); - final Map configConfigMap = new HashMap<>(); - final Map clientConfigMap = new HashMap<>(); - final Map xpathConfigMap = new HashMap<>(); - final Map metaConfigMap = new HashMap<>(); - final Map valueConfigMap = new HashMap<>(); - final Map scriptConfigMap = new HashMap<>(); - final Map fieldConfigMap = new HashMap<>(); + final Map configConfigMap = new LinkedHashMap<>(); + final Map clientConfigMap = new LinkedHashMap<>(); + final Map xpathConfigMap = new LinkedHashMap<>(); + final Map metaConfigMap = new LinkedHashMap<>(); + final Map valueConfigMap = new LinkedHashMap<>(); + final Map scriptConfigMap = new LinkedHashMap<>(); + final Map fieldConfigMap = new LinkedHashMap<>(); map.put(ConfigName.CONFIG, configConfigMap); map.put(ConfigName.CLIENT, clientConfigMap); map.put(ConfigName.XPATH, xpathConfigMap);