0ct0pu5/fess
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into 10.3.x

Browse source
This commit is contained in:
Shinsuke Sugaya 2016-10-03 23:24:47 +09:00
commit 7142c332c3
9 changed files with 98 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/main/java/org/codelibs/fess/ds/impl/AbstractDataStoreImpl.java
View file

@ -107,7 +107,7 @@ public abstract class AbstractDataStoreImpl implements DataStore {
}
protected <T> Object convertValue(final String template, final Map<String, T> paramMap) {
protected Object convertValue(final String template, final Map<String, Object> paramMap) {
if (StringUtil.isEmpty(template)) {
return StringUtil.EMPTY;
}

5
src/main/java/org/codelibs/fess/ds/impl/CsvDataStoreImpl.java
View file

@ -190,10 +190,11 @@ public class CsvDataStoreImpl extends AbstractDataStoreImpl {
while ((list = csvReader.readValues()) != null && loop && alive) {
final Map<String, Object> dataMap = new HashMap<>();
dataMap.putAll(defaultDataMap);
final Map<String, String> resultMap = new LinkedHashMap<>();
final Map<String, Object> resultMap = new LinkedHashMap<>();
resultMap.putAll(paramMap);
resultMap.put("csvfile", csvFile.getAbsolutePath());
resultMap.put("csvfilename", csvFile.getName());
resultMap.put("crawlingConfig", dataConfig);
boolean foundValues = false;
for (int i = 0; i < list.size(); i++) {
String key = null;
@ -219,7 +220,7 @@ public class CsvDataStoreImpl extends AbstractDataStoreImpl {
}
if (logger.isDebugEnabled()) {
for (final Map.Entry<String, String> entry : resultMap.entrySet()) {
for (final Map.Entry<String, Object> entry : resultMap.entrySet()) {
logger.debug(entry.getKey() + "=" + entry.getValue());
}
}

25
src/main/java/org/codelibs/fess/ds/impl/DatabaseDataStoreImpl.java
View file

@ -109,7 +109,7 @@ public class DatabaseDataStoreImpl extends AbstractDataStoreImpl {
final Map<String, Object> dataMap = new HashMap<>();
dataMap.putAll(defaultDataMap);
for (final Map.Entry<String, String> entry : scriptMap.entrySet()) {
final Object convertValue = convertValue(entry.getValue(), rs, paramMap);
final Object convertValue = convertValue(config, entry.getValue(), rs, paramMap);
if (convertValue != null) {
dataMap.put(entry.getKey(), convertValue);
}
@ -189,15 +189,16 @@ public class DatabaseDataStoreImpl extends AbstractDataStoreImpl {
}
}
protected Object convertValue(final String template, final ResultSet rs, final Map<String, String> paramMap) {
return convertValue(template, new ResultSetParamMap(rs, paramMap));
protected Object convertValue(final DataConfig config, final String template, final ResultSet rs, final Map<String, String> paramMap) {
return convertValue(template, new ResultSetParamMap(config, rs, paramMap));
}
protected static class ResultSetParamMap implements Map<String, String> {
private final Map<String, String> paramMap = new HashMap<>();
protected static class ResultSetParamMap implements Map<String, Object> {
private final Map<String, Object> paramMap = new HashMap<>();
public ResultSetParamMap(final ResultSet resultSet, final Map<String, String> paramMap) {
public ResultSetParamMap(final DataConfig config, final ResultSet resultSet, final Map<String, String> paramMap) {
this.paramMap.putAll(paramMap);
this.paramMap.put("crawlingConfig", config);
try {
final ResultSetMetaData metaData = resultSet.getMetaData();
@ -233,12 +234,12 @@ public class DatabaseDataStoreImpl extends AbstractDataStoreImpl {
}
@Override
public Set<java.util.Map.Entry<String, String>> entrySet() {
public Set<java.util.Map.Entry<String, Object>> entrySet() {
return paramMap.entrySet();
}
@Override
public String get(final Object key) {
public Object get(final Object key) {
return paramMap.get(key);
}
@ -253,17 +254,17 @@ public class DatabaseDataStoreImpl extends AbstractDataStoreImpl {
}
@Override
public String put(final String key, final String value) {
public Object put(final String key, final Object value) {
return paramMap.put(key, value);
}
@Override
public void putAll(final Map<? extends String, ? extends String> m) {
public void putAll(final Map<? extends String, ? extends Object> m) {
paramMap.putAll(m);
}
@Override
public String remove(final Object key) {
public Object remove(final Object key) {
return paramMap.remove(key);
}
@ -273,7 +274,7 @@ public class DatabaseDataStoreImpl extends AbstractDataStoreImpl {
}
@Override
public Collection<String> values() {
public Collection<Object> values() {
return paramMap.values();
}

1
src/main/java/org/codelibs/fess/ds/impl/EsDataStoreImpl.java
View file

@ -162,6 +162,7 @@ public class EsDataStoreImpl extends AbstractDataStoreImpl {
resultMap.put("version", Long.valueOf(hit.getVersion()));
resultMap.put("hit", hit);
resultMap.put("source", hit.getSource());
resultMap.put("crawlingConfig", dataConfig);
if (logger.isDebugEnabled()) {
for (final Map.Entry<String, Object> entry : resultMap.entrySet()) {

72
src/main/java/org/codelibs/fess/helper/DocumentHelper.java
View file

@ -18,14 +18,32 @@ package org.codelibs.fess.helper;
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import org.apache.commons.lang3.StringUtils;
import org.codelibs.core.io.SerializeUtil;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.fess.crawler.builder.RequestDataBuilder;
import org.codelibs.fess.crawler.client.CrawlerClient;
import org.codelibs.fess.crawler.client.CrawlerClientFactory;
import org.codelibs.fess.crawler.entity.RequestData;
import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.entity.ResultData;
import org.codelibs.fess.crawler.exception.ChildUrlsException;
import org.codelibs.fess.crawler.exception.CrawlerSystemException;
import org.codelibs.fess.crawler.exception.CrawlingAccessException;
import org.codelibs.fess.crawler.processor.ResponseProcessor;
import org.codelibs.fess.crawler.processor.impl.DefaultResponseProcessor;
import org.codelibs.fess.crawler.rule.Rule;
import org.codelibs.fess.crawler.rule.RuleManager;
import org.codelibs.fess.crawler.transformer.Transformer;
import org.codelibs.fess.crawler.util.TextUtil;
import org.codelibs.fess.es.config.exentity.CrawlingConfig;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.util.ComponentUtil;
import org.lastaflute.di.core.SingletonLaContainer;
public class DocumentHelper {
public String getContent(final ResponseData responseData, final String content, final Map<String, Object> dataMap) {
@ -85,4 +103,58 @@ public class DocumentHelper {
return StringUtil.EMPTY; // empty
}
}
public Map<String, Object> processRequest(final CrawlingConfig crawlingConfig, final String sessionId, final String url) {
if (StringUtil.isBlank(sessionId)) {
throw new CrawlingAccessException("sessionId is null.");
}
final CrawlerClientFactory crawlerClientFactory = ComponentUtil.getCrawlerClientFactory();
crawlingConfig.initializeClientFactory(crawlerClientFactory);
final CrawlerClient client = crawlerClientFactory.getClient(url);
if (client == null) {
throw new CrawlingAccessException("CrawlerClient is null for " + url);
}
final long startTime = System.currentTimeMillis();
try (final ResponseData responseData = client.execute(RequestDataBuilder.newRequestData().get().url(url).build())) {
if (responseData.getRedirectLocation() != null) {
Set<RequestData> childUrlList = new HashSet<>();
childUrlList.add(RequestDataBuilder.newRequestData().get().url(responseData.getRedirectLocation()).build());
throw new ChildUrlsException(childUrlList, "Redirected from " + url);
}
responseData.setExecutionTime(System.currentTimeMillis() - startTime);
responseData.setSessionId(sessionId);
final RuleManager ruleManager = SingletonLaContainer.getComponent(RuleManager.class);
final Rule rule = ruleManager.getRule(responseData);
if (rule == null) {
throw new CrawlingAccessException("No url rule for " + url);
} else {
responseData.setRuleId(rule.getRuleId());
final ResponseProcessor responseProcessor = rule.getResponseProcessor();
if (responseProcessor instanceof DefaultResponseProcessor) {
final Transformer transformer = ((DefaultResponseProcessor) responseProcessor).getTransformer();
final ResultData resultData = transformer.transform(responseData);
final byte[] data = resultData.getData();
if (data != null) {
try {
@SuppressWarnings("unchecked")
final Map<String, Object> result = (Map<String, Object>) SerializeUtil.fromBinaryToObject(data);
return result;
} catch (final Exception e) {
throw new CrawlerSystemException("Could not create an instance from bytes.", e);
}
}
} else {
throw new CrawlingAccessException("The response processor is not DefaultResponseProcessor. responseProcessor: "
+ responseProcessor + ", url: " + url);
}
}
return null;
} catch (final Exception e) {
throw new CrawlingAccessException("Failed to parse " + url, e);
}
}
}

2
src/main/java/org/codelibs/fess/helper/SystemHelper.java
View file

@ -295,7 +295,7 @@ public class SystemHelper {
ComponentUtil.getPathMappingHelper().init();
ComponentUtil.getSuggestHelper().init();
ComponentUtil.getPopularWordHelper().init();
// TODO restart jobs
ComponentUtil.getJobManager().reboot();
}
}

2
src/main/java/org/codelibs/fess/job/impl/GroovyExecutor.java
View file

@ -20,14 +20,12 @@ import java.util.Map;
import org.codelibs.fess.job.JobExecutor;
import org.codelibs.fess.util.GroovyUtil;
import org.lastaflute.di.core.factory.SingletonLaContainerFactory;
public class GroovyExecutor extends JobExecutor {
@Override
public Object execute(final String script) {
final Map<String, Object> params = new HashMap<>();
params.put("container", SingletonLaContainerFactory.getContainer());
params.put("executor", this);
return GroovyUtil.evaluate(script, params);

8
src/main/java/org/codelibs/fess/util/GroovyUtil.java
View file

@ -15,8 +15,10 @@
*/
package org.codelibs.fess.util;
import java.util.HashMap;
import java.util.Map;
import org.lastaflute.di.core.factory.SingletonLaContainerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -31,8 +33,10 @@ public final class GroovyUtil {
// nothing
}
public static <T> Object evaluate(final String template, final Map<String, T> paramMap) {
final GroovyShell groovyShell = new GroovyShell(new Binding(paramMap));
public static Object evaluate(final String template, final Map<String, Object> paramMap) {
final Map<String, Object> bindingMap = new HashMap<>(paramMap);
bindingMap.put("container", SingletonLaContainerFactory.getContainer());
final GroovyShell groovyShell = new GroovyShell(new Binding(bindingMap));
try {
return groovyShell.evaluate(template);
} catch (final Exception e) {

2
src/test/java/org/codelibs/fess/ds/impl/DatabaseDataStoreImplTest.java
View file

@ -31,7 +31,7 @@ public class DatabaseDataStoreImplTest extends UnitFessTestCase {
public void test_convertValue() {
String value;
final Map<String, String> paramMap = new HashMap<String, String>();
final Map<String, Object> paramMap = new HashMap<>();
paramMap.put("param1", "PARAM1");
paramMap.put("param2", "PARAM2+");
paramMap.put("param3", "PARAM3*");