0ct0pu5/fess
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix #126

Browse source
This commit is contained in:
Shinsuke Sugaya 2014-03-08 16:48:46 +09:00
parent 19f949a0b2
commit b7aaf2eaa2
4 changed files with 76 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/main/java/jp/sf/fess/db/exentity/CrawlingConfig.java
View file

@ -62,6 +62,6 @@ public interface CrawlingConfig {
}
public enum ConfigName {
CLIENT, XPATH, VALUE, SCRIPT, FIELD;
CLIENT, XPATH, META, VALUE, SCRIPT, FIELD;
}
}

15
src/main/java/jp/sf/fess/transformer/AbstractFessFileTransformer.java
View file

@ -114,17 +114,22 @@ public abstract class AbstractFessFileTransformer extends
final StringBuilder contentBuf = new StringBuilder(1000);
final StringBuilder contentMetaBuf = new StringBuilder(1000);
final Map<String, Object> dataMap = new HashMap<String, Object>();
final Map<String, Object> metaDataMap = new HashMap<>();
try {
final ExtractData extractData = extractor.getText(in, params);
if (ignoreEmptyContent
&& StringUtil.isBlank(extractData.getContent())) {
return null;
}
if (logger.isDebugEnabled()) {
logger.debug("ExtractData: " + extractData);
}
contentBuf.append(extractData.getContent());
// meta
for (final String key : extractData.getKeySet()) {
final String[] values = extractData.getValues(key);
if (values != null) {
metaDataMap.put(key, values);
if (contentMetaBuf.length() > 0) {
contentMetaBuf.append(' ');
}
@ -335,6 +340,16 @@ public abstract class AbstractFessFileTransformer extends
// from config
final Map<String, String> scriptConfigMap = crawlingConfig
.getConfigParameterMap(ConfigName.SCRIPT);
final Map<String, String> metaConfigMap = crawlingConfig
.getConfigParameterMap(ConfigName.META);
for (final Map.Entry<String, String> entry : metaConfigMap.entrySet()) {
final String key = entry.getKey();
final String[] values = entry.getValue().split(",");
for (final String value : values) {
putResultDataWithTemplate(dataMap, key, metaDataMap.get(value),
scriptConfigMap.get(key));
}
}
final Map<String, String> valueConfigMap = crawlingConfig
.getConfigParameterMap(ConfigName.VALUE);
for (final Map.Entry<String, String> entry : valueConfigMap.entrySet()) {

61
src/main/java/jp/sf/fess/transformer/AbstractFessXpathTransformer.java
View file

@ -17,6 +17,7 @@
package jp.sf.fess.transformer;
import java.net.URLDecoder;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
@ -39,6 +40,8 @@ public abstract class AbstractFessXpathTransformer extends XpathTransformer {
public boolean replaceSiteEncodingWhenEnglish = false;
public boolean appendResultData = true;
protected String getHost(final String u) {
if (StringUtil.isBlank(u)) {
return StringUtil.EMPTY; // empty
@ -115,23 +118,65 @@ public abstract class AbstractFessXpathTransformer extends XpathTransformer {
protected void putResultDataBody(final Map<String, Object> dataMap,
final String key, final Object value) {
dataMap.put(key, value);
if (dataMap.containsKey(key)) {
if (appendResultData) {
final Object oldValue = dataMap.get(key);
if (key.endsWith("_m")) {
final Object[] oldValues = (Object[]) oldValue;
if (value.getClass().isArray()) {
final Object[] newValues = (Object[]) value;
final Object[] values = Arrays.copyOf(oldValues,
oldValues.length + newValues.length);
for (int i = 0; i < newValues.length; i++) {
values[values.length - 1 + i] = newValues[i];
}
dataMap.put(key, values);
} else {
final Object[] values = Arrays.copyOf(oldValues,
oldValues.length + 1);
values[values.length - 1] = value;
dataMap.put(key, values);
}
} else {
dataMap.put(key, oldValue + " " + value);
}
} else {
dataMap.put(key, value);
}
} else {
dataMap.put(key, value);
}
}
protected void putResultDataWithTemplate(final Map<String, Object> dataMap,
final String key, String value, final String template) {
final String key, final Object value, final String template) {
Object target = value;
if (template != null) {
final Map<String, Object> paramMap = new HashMap<>(dataMap.size());
final Map<String, Object> paramMap = new HashMap<>(
dataMap.size() + 1);
paramMap.putAll(dataMap);
paramMap.put("value", value);
value = convertValue(template, paramMap);
paramMap.put("value", target);
target = evaluateValue(template, paramMap);
}
if (value != null) {
putResultDataBody(dataMap, key, value);
if (key != null && target != null) {
if (target.getClass().isArray()) {
if (key.endsWith("_m")) {
putResultDataBody(dataMap, key, target);
} else {
putResultDataBody(dataMap, key,
Arrays.toString((Object[]) target));
}
} else {
if (key.endsWith("_m")) {
putResultDataBody(dataMap, key, new Object[] { target });
} else {
putResultDataBody(dataMap, key, target);
}
}
}
}
protected String convertValue(final String template,
protected String evaluateValue(final String template,
final Map<String, Object> paramMap) {
if (StringUtil.isEmpty(template)) {
return StringUtil.EMPTY;

7
src/main/java/jp/sf/fess/util/ParameterUtil.java
View file

@ -27,6 +27,8 @@ import org.codelibs.core.util.StringUtil;
public class ParameterUtil {
protected static final String XPATH_PREFIX = "field.xpath.";
protected static final String META_PREFIX = "field.meta.";
protected static final String VALUE_PREFIX = "field.value.";
protected static final String SCRIPT_PREFIX = "field.script.";
@ -79,11 +81,13 @@ public class ParameterUtil {
final Map<ConfigName, Map<String, String>> map = new HashMap<>();
final Map<String, String> clientConfigMap = new HashMap<>();
final Map<String, String> xpathConfigMap = new HashMap<>();
final Map<String, String> metaConfigMap = new HashMap<>();
final Map<String, String> valueConfigMap = new HashMap<>();
final Map<String, String> scriptConfigMap = new HashMap<>();
final Map<String, String> fieldConfigMap = new HashMap<>();
map.put(ConfigName.CLIENT, clientConfigMap);
map.put(ConfigName.XPATH, xpathConfigMap);
map.put(ConfigName.META, metaConfigMap);
map.put(ConfigName.VALUE, valueConfigMap);
map.put(ConfigName.SCRIPT, scriptConfigMap);
map.put(ConfigName.FIELD, fieldConfigMap);
@ -96,6 +100,9 @@ public class ParameterUtil {
} else if (key.startsWith(XPATH_PREFIX)) {
xpathConfigMap.put(key.substring(XPATH_PREFIX.length()),
entry.getValue());
} else if (key.startsWith(META_PREFIX)) {
metaConfigMap.put(key.substring(META_PREFIX.length()),
entry.getValue());
} else if (key.startsWith(VALUE_PREFIX)) {
valueConfigMap.put(key.substring(VALUE_PREFIX.length()),
entry.getValue());