0ct0pu5/fess
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix #728 add crawlingContext

Browse source
This commit is contained in:
Shinsuke Sugaya 2016-10-06 11:39:16 +09:00
parent a1aab906d4
commit fd09edb344
7 changed files with 38 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
src/main/java/org/codelibs/fess/ds/impl/CsvDataStoreImpl.java
View file

@ -225,6 +225,9 @@ public class CsvDataStoreImpl extends AbstractDataStoreImpl {
}
}
final Map<String, Object> crawlingContext = new HashMap<>();
crawlingContext.put("doc", dataMap);
resultMap.put("crawlingContext", crawlingContext);
for (final Map.Entry<String, String> entry : scriptMap.entrySet()) {
final Object convertValue = convertValue(entry.getValue(), resultMap);
if (convertValue != null) {

13
src/main/java/org/codelibs/fess/ds/impl/DatabaseDataStoreImpl.java
View file

@ -108,8 +108,11 @@ public class DatabaseDataStoreImpl extends AbstractDataStoreImpl {
while (rs.next() && loop && alive) {
final Map<String, Object> dataMap = new HashMap<>();
dataMap.putAll(defaultDataMap);
final Map<String, Object> crawlingContext = new HashMap<>();
crawlingContext.put("doc", dataMap);
for (final Map.Entry<String, String> entry : scriptMap.entrySet()) {
final Object convertValue = convertValue(config, entry.getValue(), rs, paramMap);
final Object convertValue =
convertValue(entry.getValue(), new ResultSetParamMap(config, crawlingContext, rs, paramMap));
if (convertValue != null) {
dataMap.put(entry.getKey(), convertValue);
}
@ -189,16 +192,14 @@ public class DatabaseDataStoreImpl extends AbstractDataStoreImpl {
}
}
protected Object convertValue(final DataConfig config, final String template, final ResultSet rs, final Map<String, String> paramMap) {
return convertValue(template, new ResultSetParamMap(config, rs, paramMap));
}
protected static class ResultSetParamMap implements Map<String, Object> {
private final Map<String, Object> paramMap = new HashMap<>();
public ResultSetParamMap(final DataConfig config, final ResultSet resultSet, final Map<String, String> paramMap) {
public ResultSetParamMap(final DataConfig config, final Map<String, Object> crawlingContext, final ResultSet resultSet,
final Map<String, String> paramMap) {
this.paramMap.putAll(paramMap);
this.paramMap.put("crawlingConfig", config);
this.paramMap.put("crawlingContext", crawlingContext);
try {
final ResultSetMetaData metaData = resultSet.getMetaData();

3
src/main/java/org/codelibs/fess/ds/impl/EsDataStoreImpl.java
View file

@ -170,6 +170,9 @@ public class EsDataStoreImpl extends AbstractDataStoreImpl {
}
}
final Map<String, Object> crawlingContext = new HashMap<>();
crawlingContext.put("doc", dataMap);
resultMap.put("crawlingContext", crawlingContext);
for (final Map.Entry<String, String> entry : scriptMap.entrySet()) {
final Object convertValue = convertValue(entry.getValue(), resultMap);
if (convertValue != null) {

12
src/main/java/org/codelibs/fess/helper/CrawlingInfoHelper.java
View file

@ -125,11 +125,13 @@ public class CrawlingInfoHelper {
}
public Date getDocumentExpires(final CrawlingConfig config) {
final Integer timeToLive = config.getTimeToLive();
if (timeToLive != null) {
// timeToLive minutes
final long now = ComponentUtil.getSystemHelper().getCurrentTimeAsLong();
return new Date(now + timeToLive.longValue() * 1000 * 60);
if (config != null) {
final Integer timeToLive = config.getTimeToLive();
if (timeToLive != null) {
// timeToLive minutes
final long now = ComponentUtil.getSystemHelper().getCurrentTimeAsLong();
return new Date(now + timeToLive.longValue() * 1000 * 60);
}
}
return documentExpires != null ? new Date(documentExpires) : null;
}

8
src/main/java/org/codelibs/fess/helper/DataIndexHelper.java
View file

@ -30,6 +30,7 @@ import org.codelibs.fess.app.service.FailureUrlService;
import org.codelibs.fess.ds.DataStore;
import org.codelibs.fess.ds.DataStoreFactory;
import org.codelibs.fess.ds.IndexUpdateCallback;
import org.codelibs.fess.es.client.FessEsClient;
import org.codelibs.fess.es.config.exentity.DataConfig;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.util.ComponentUtil;
@ -262,8 +263,11 @@ public class DataIndexHelper {
.should(QueryBuilders.missingQuery(fessConfig.getIndexFieldExpires())))
.mustNot(QueryBuilders.termQuery(fessConfig.getIndexFieldSegment(), sessionId));
try {
ComponentUtil.getFessEsClient().deleteByQuery(fessConfig.getIndexDocumentUpdateIndex(), fessConfig.getIndexDocumentType(),
queryBuilder);
final FessEsClient fessEsClient = ComponentUtil.getFessEsClient();
final String index = fessConfig.getIndexDocumentUpdateIndex();
fessEsClient.admin().indices().prepareRefresh(index).execute().actionGet();
final int numOfDeleted = fessEsClient.deleteByQuery(index, fessConfig.getIndexDocumentType(), queryBuilder);
logger.info("Deleted {} old docs.", numOfDeleted);
} catch (final Exception e) {
logger.error("Could not delete old docs at " + dataConfig, e);
}

6
src/main/java/org/codelibs/fess/helper/DocumentHelper.java
View file

@ -104,8 +104,8 @@ public class DocumentHelper {
}
}
public Map<String, Object> processRequest(final CrawlingConfig crawlingConfig, final String sessionId, final String url) {
if (StringUtil.isBlank(sessionId)) {
public Map<String, Object> processRequest(final CrawlingConfig crawlingConfig, final String crawlingInfoId, final String url) {
if (StringUtil.isBlank(crawlingInfoId)) {
throw new CrawlingAccessException("sessionId is null.");
}
@ -124,7 +124,7 @@ public class DocumentHelper {
throw new ChildUrlsException(childUrlList, "Redirected from " + url);
}
responseData.setExecutionTime(System.currentTimeMillis() - startTime);
responseData.setSessionId(sessionId);
responseData.setSessionId(crawlingInfoId);
final RuleManager ruleManager = SingletonLaContainer.getComponent(RuleManager.class);
final Rule rule = ruleManager.getRule(responseData);

17
src/main/java/org/codelibs/fess/util/ParameterUtil.java
View file

@ -16,6 +16,7 @@
package org.codelibs.fess.util;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.Map;
import org.codelibs.core.lang.StringUtil;
@ -42,7 +43,7 @@ public class ParameterUtil {
}
public static Map<String, String> parse(final String value) {
final Map<String, String> paramMap = new HashMap<>();
final Map<String, String> paramMap = new LinkedHashMap<>();
if (value != null) {
final String[] lines = value.split("[\r\n]");
for (final String line : lines) {
@ -74,13 +75,13 @@ public class ParameterUtil {
public static Map<ConfigName, Map<String, String>> createConfigParameterMap(final String configParameters) {
final Map<ConfigName, Map<String, String>> map = new HashMap<>();
final Map<String, String> configConfigMap = new HashMap<>();
final Map<String, String> clientConfigMap = new HashMap<>();
final Map<String, String> xpathConfigMap = new HashMap<>();
final Map<String, String> metaConfigMap = new HashMap<>();
final Map<String, String> valueConfigMap = new HashMap<>();
final Map<String, String> scriptConfigMap = new HashMap<>();
final Map<String, String> fieldConfigMap = new HashMap<>();
final Map<String, String> configConfigMap = new LinkedHashMap<>();
final Map<String, String> clientConfigMap = new LinkedHashMap<>();
final Map<String, String> xpathConfigMap = new LinkedHashMap<>();
final Map<String, String> metaConfigMap = new LinkedHashMap<>();
final Map<String, String> valueConfigMap = new LinkedHashMap<>();
final Map<String, String> scriptConfigMap = new LinkedHashMap<>();
final Map<String, String> fieldConfigMap = new LinkedHashMap<>();
map.put(ConfigName.CONFIG, configConfigMap);
map.put(ConfigName.CLIENT, clientConfigMap);
map.put(ConfigName.XPATH, xpathConfigMap);