0ct0pu5/fess
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix #2003 add date type

Browse source
This commit is contained in:
Shinsuke Sugaya 2019-02-02 18:02:46 +09:00
parent b9f9372505
commit 5da01723e1
3 changed files with 35 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/main/java/org/codelibs/fess/Constants.java
View file

@ -398,6 +398,8 @@ public class Constants extends CoreLibConstants {
public static final String MAPPING_TYPE_DOUBLE = "double";
public static final String MAPPING_TYPE_DATE = "date";
public static final String PAGING_QUERY_LIST = "pagingQueryList";
public static final String REQUEST_LANGUAGES = "requestLanguages";

35
src/main/java/org/codelibs/fess/crawler/transformer/AbstractFessFileTransformer.java
View file

@ -19,6 +19,7 @@ import static org.codelibs.core.stream.StreamUtil.stream;
import java.io.InputStream;
import java.net.URLDecoder;
import java.time.temporal.TemporalAccessor;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
@ -31,7 +32,7 @@ import org.apache.tika.metadata.HttpHeaders;
import org.apache.tika.metadata.TikaMetadataKeys;
import org.codelibs.core.io.SerializeUtil;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.core.misc.Pair;
import org.codelibs.core.misc.Tuple3;
import org.codelibs.fess.Constants;
import org.codelibs.fess.crawler.entity.AccessResultData;
import org.codelibs.fess.crawler.entity.ExtractData;
@ -55,6 +56,7 @@ import org.codelibs.fess.helper.PermissionHelper;
import org.codelibs.fess.helper.SystemHelper;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.util.ComponentUtil;
import org.elasticsearch.common.joda.Joda;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -105,8 +107,10 @@ public abstract class AbstractFessFileTransformer extends AbstractTransformer im
getLogger().debug("ExtractData: " + extractData);
}
// meta
extractData.getKeySet().stream()//
.filter(k -> extractData.getValues(k) != null)//
extractData
.getKeySet()
.stream()
.filter(k -> extractData.getValues(k) != null)
.forEach(key -> {
final String[] values = extractData.getValues(key);
metaDataMap.put(key, values);
@ -122,23 +126,28 @@ public abstract class AbstractFessFileTransformer extends AbstractTransformer im
}
}
final Pair<String, String> mapping = fessConfig.getCrawlerMetadataNameMapping(key);
final Tuple3<String, String, String> mapping = fessConfig.getCrawlerMetadataNameMapping(key);
if (mapping != null) {
if (Constants.MAPPING_TYPE_ARRAY.equalsIgnoreCase(mapping.getSecond())) {
dataMap.put(mapping.getFirst(), values);
} else if (Constants.MAPPING_TYPE_STRING.equalsIgnoreCase(mapping.getSecond())) {
if (Constants.MAPPING_TYPE_ARRAY.equalsIgnoreCase(mapping.getValue2())) {
dataMap.put(mapping.getValue1(), values);
} else if (Constants.MAPPING_TYPE_STRING.equalsIgnoreCase(mapping.getValue2())) {
final String joinedValue = StringUtils.join(values, ' ');
dataMap.put(mapping.getFirst(), joinedValue.trim());
dataMap.put(mapping.getValue1(), joinedValue.trim());
} else if (values.length == 1) {
try {
if (Constants.MAPPING_TYPE_LONG.equalsIgnoreCase(mapping.getSecond())) {
dataMap.put(mapping.getFirst(), Long.parseLong(values[0]));
} else if (Constants.MAPPING_TYPE_DOUBLE.equalsIgnoreCase(mapping.getSecond())) {
dataMap.put(mapping.getFirst(), Double.parseDouble(values[0]));
if (Constants.MAPPING_TYPE_LONG.equalsIgnoreCase(mapping.getValue2())) {
dataMap.put(mapping.getValue1(), Long.parseLong(values[0]));
} else if (Constants.MAPPING_TYPE_DOUBLE.equalsIgnoreCase(mapping.getValue2())) {
dataMap.put(mapping.getValue1(), Double.parseDouble(values[0]));
} else if (Constants.MAPPING_TYPE_DATE.equalsIgnoreCase(mapping.getValue2())) {
final String format =
StringUtil.isNotBlank(mapping.getValue3()) ? mapping.getValue3() : "date_optional_time";
final TemporalAccessor dt = Joda.forPattern(format).parse(mapping.getValue2());
dataMap.put(mapping.getValue1(), Joda.forPattern("date_optional_time").format(dt));
} else {
logger.warn("Unknown mapping type: {}={}", key, mapping);
}
} catch (final NumberFormatException e) {
} catch (final Exception e) {
logger.warn("Failed to parse " + values[0], e);
}
}

18
src/main/java/org/codelibs/fess/mylasta/direction/FessProp.java
View file

@ -48,6 +48,7 @@ import org.codelibs.core.exception.ClassNotFoundRuntimeException;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.core.misc.Pair;
import org.codelibs.core.misc.Tuple3;
import org.codelibs.core.misc.Tuple4;
import org.codelibs.fess.Constants;
import org.codelibs.fess.exception.FessSystemException;
import org.codelibs.fess.helper.PermissionHelper;
@ -890,22 +891,25 @@ public interface FessProp {
String getCrawlerMetadataNameMapping();
default Pair<String, String> getCrawlerMetadataNameMapping(final String name) {
default Tuple3<String, String, String> getCrawlerMetadataNameMapping(final String name) {
@SuppressWarnings("unchecked")
Map<String, Pair<String, String>> params = (Map<String, Pair<String, String>>) propMap.get(CRAWLER_METADATA_NAME_MAPPING);
Map<String, Tuple3<String, String, String>> params =
(Map<String, Tuple3<String, String, String>>) propMap.get(CRAWLER_METADATA_NAME_MAPPING);
if (params == null) {
params = split(getCrawlerMetadataNameMapping(), "\n").get(stream -> stream.filter(StringUtil::isNotBlank).map(v -> {
final String[] values = v.split("=");
if (values.length == 2) {
final String[] subValues = values[1].split(":");
if (subValues.length == 2) {
return new Tuple3<>(values[0], subValues[0], subValues[1]);
final String[] subValues = values[1].split(":", 3);
if (subValues.length == 3) {
return new Tuple4<>(values[0], subValues[0], subValues[1], subValues[2]);
} else if (subValues.length == 2) {
return new Tuple4<>(values[0], subValues[0], subValues[1], StringUtil.EMPTY);
} else {
return new Tuple3<>(values[0], values[1], Constants.MAPPING_TYPE_ARRAY);
return new Tuple4<>(values[0], values[1], Constants.MAPPING_TYPE_ARRAY, StringUtil.EMPTY);
}
}
return null;
}).collect(Collectors.toMap(Tuple3::getValue1, d -> new Pair<>(d.getValue2(), d.getValue3()))));
}).collect(Collectors.toMap(Tuple4::getValue1, d -> new Tuple3<>(d.getValue2(), d.getValue3(), d.getValue4()))));
propMap.put(CRAWLER_METADATA_NAME_MAPPING, params);
}
return params.get(name);