diff --git a/src/main/java/jp/sf/fess/db/exentity/CrawlingConfig.java b/src/main/java/jp/sf/fess/db/exentity/CrawlingConfig.java index 4208426b4..8d8c52b60 100644 --- a/src/main/java/jp/sf/fess/db/exentity/CrawlingConfig.java +++ b/src/main/java/jp/sf/fess/db/exentity/CrawlingConfig.java @@ -62,6 +62,6 @@ public interface CrawlingConfig { } public enum ConfigName { - CLIENT, XPATH, VALUE, SCRIPT, FIELD; + CLIENT, XPATH, META, VALUE, SCRIPT, FIELD; } } \ No newline at end of file diff --git a/src/main/java/jp/sf/fess/transformer/AbstractFessFileTransformer.java b/src/main/java/jp/sf/fess/transformer/AbstractFessFileTransformer.java index 4b4225d0c..f1ac27e1c 100644 --- a/src/main/java/jp/sf/fess/transformer/AbstractFessFileTransformer.java +++ b/src/main/java/jp/sf/fess/transformer/AbstractFessFileTransformer.java @@ -114,17 +114,22 @@ public abstract class AbstractFessFileTransformer extends final StringBuilder contentBuf = new StringBuilder(1000); final StringBuilder contentMetaBuf = new StringBuilder(1000); final Map dataMap = new HashMap(); + final Map metaDataMap = new HashMap<>(); try { final ExtractData extractData = extractor.getText(in, params); if (ignoreEmptyContent && StringUtil.isBlank(extractData.getContent())) { return null; } + if (logger.isDebugEnabled()) { + logger.debug("ExtractData: " + extractData); + } contentBuf.append(extractData.getContent()); // meta for (final String key : extractData.getKeySet()) { final String[] values = extractData.getValues(key); if (values != null) { + metaDataMap.put(key, values); if (contentMetaBuf.length() > 0) { contentMetaBuf.append(' '); } @@ -335,6 +340,16 @@ public abstract class AbstractFessFileTransformer extends // from config final Map scriptConfigMap = crawlingConfig .getConfigParameterMap(ConfigName.SCRIPT); + final Map metaConfigMap = crawlingConfig + .getConfigParameterMap(ConfigName.META); + for (final Map.Entry entry : metaConfigMap.entrySet()) { + final String key = entry.getKey(); + final String[] values = entry.getValue().split(","); + for (final String value : values) { + putResultDataWithTemplate(dataMap, key, metaDataMap.get(value), + scriptConfigMap.get(key)); + } + } final Map valueConfigMap = crawlingConfig .getConfigParameterMap(ConfigName.VALUE); for (final Map.Entry entry : valueConfigMap.entrySet()) { diff --git a/src/main/java/jp/sf/fess/transformer/AbstractFessXpathTransformer.java b/src/main/java/jp/sf/fess/transformer/AbstractFessXpathTransformer.java index 8172a4694..32996d839 100644 --- a/src/main/java/jp/sf/fess/transformer/AbstractFessXpathTransformer.java +++ b/src/main/java/jp/sf/fess/transformer/AbstractFessXpathTransformer.java @@ -17,6 +17,7 @@ package jp.sf.fess.transformer; import java.net.URLDecoder; +import java.util.Arrays; import java.util.HashMap; import java.util.Map; @@ -39,6 +40,8 @@ public abstract class AbstractFessXpathTransformer extends XpathTransformer { public boolean replaceSiteEncodingWhenEnglish = false; + public boolean appendResultData = true; + protected String getHost(final String u) { if (StringUtil.isBlank(u)) { return StringUtil.EMPTY; // empty @@ -115,23 +118,65 @@ public abstract class AbstractFessXpathTransformer extends XpathTransformer { protected void putResultDataBody(final Map dataMap, final String key, final Object value) { - dataMap.put(key, value); + if (dataMap.containsKey(key)) { + if (appendResultData) { + final Object oldValue = dataMap.get(key); + if (key.endsWith("_m")) { + final Object[] oldValues = (Object[]) oldValue; + if (value.getClass().isArray()) { + final Object[] newValues = (Object[]) value; + final Object[] values = Arrays.copyOf(oldValues, + oldValues.length + newValues.length); + for (int i = 0; i < newValues.length; i++) { + values[values.length - 1 + i] = newValues[i]; + } + dataMap.put(key, values); + } else { + final Object[] values = Arrays.copyOf(oldValues, + oldValues.length + 1); + values[values.length - 1] = value; + dataMap.put(key, values); + } + } else { + dataMap.put(key, oldValue + " " + value); + } + } else { + dataMap.put(key, value); + } + } else { + dataMap.put(key, value); + } } protected void putResultDataWithTemplate(final Map dataMap, - final String key, String value, final String template) { + final String key, final Object value, final String template) { + Object target = value; if (template != null) { - final Map paramMap = new HashMap<>(dataMap.size()); + final Map paramMap = new HashMap<>( + dataMap.size() + 1); paramMap.putAll(dataMap); - paramMap.put("value", value); - value = convertValue(template, paramMap); + paramMap.put("value", target); + target = evaluateValue(template, paramMap); } - if (value != null) { - putResultDataBody(dataMap, key, value); + if (key != null && target != null) { + if (target.getClass().isArray()) { + if (key.endsWith("_m")) { + putResultDataBody(dataMap, key, target); + } else { + putResultDataBody(dataMap, key, + Arrays.toString((Object[]) target)); + } + } else { + if (key.endsWith("_m")) { + putResultDataBody(dataMap, key, new Object[] { target }); + } else { + putResultDataBody(dataMap, key, target); + } + } } } - protected String convertValue(final String template, + protected String evaluateValue(final String template, final Map paramMap) { if (StringUtil.isEmpty(template)) { return StringUtil.EMPTY; diff --git a/src/main/java/jp/sf/fess/util/ParameterUtil.java b/src/main/java/jp/sf/fess/util/ParameterUtil.java index a8a45fbb6..d9cc5e236 100644 --- a/src/main/java/jp/sf/fess/util/ParameterUtil.java +++ b/src/main/java/jp/sf/fess/util/ParameterUtil.java @@ -27,6 +27,8 @@ import org.codelibs.core.util.StringUtil; public class ParameterUtil { protected static final String XPATH_PREFIX = "field.xpath."; + protected static final String META_PREFIX = "field.meta."; + protected static final String VALUE_PREFIX = "field.value."; protected static final String SCRIPT_PREFIX = "field.script."; @@ -79,11 +81,13 @@ public class ParameterUtil { final Map> map = new HashMap<>(); final Map clientConfigMap = new HashMap<>(); final Map xpathConfigMap = new HashMap<>(); + final Map metaConfigMap = new HashMap<>(); final Map valueConfigMap = new HashMap<>(); final Map scriptConfigMap = new HashMap<>(); final Map fieldConfigMap = new HashMap<>(); map.put(ConfigName.CLIENT, clientConfigMap); map.put(ConfigName.XPATH, xpathConfigMap); + map.put(ConfigName.META, metaConfigMap); map.put(ConfigName.VALUE, valueConfigMap); map.put(ConfigName.SCRIPT, scriptConfigMap); map.put(ConfigName.FIELD, fieldConfigMap); @@ -96,6 +100,9 @@ public class ParameterUtil { } else if (key.startsWith(XPATH_PREFIX)) { xpathConfigMap.put(key.substring(XPATH_PREFIX.length()), entry.getValue()); + } else if (key.startsWith(META_PREFIX)) { + metaConfigMap.put(key.substring(META_PREFIX.length()), + entry.getValue()); } else if (key.startsWith(VALUE_PREFIX)) { valueConfigMap.put(key.substring(VALUE_PREFIX.length()), entry.getValue());