Merge branch 'master' into 10.3.x

src/main/java/org/codelibs/fess/ds/impl/AbstractDataStoreImpl.java

@@ -107,7 +107,7 @@ public abstract class AbstractDataStoreImpl implements DataStore {
 
     }
 
-    protected <T> Object convertValue(final String template, final Map<String, T> paramMap) {
+    protected Object convertValue(final String template, final Map<String, Object> paramMap) {
         if (StringUtil.isEmpty(template)) {
             return StringUtil.EMPTY;
         }

src/main/java/org/codelibs/fess/ds/impl/CsvDataStoreImpl.java

@@ -190,10 +190,11 @@ public class CsvDataStoreImpl extends AbstractDataStoreImpl {
             while ((list = csvReader.readValues()) != null && loop && alive) {
                 final Map<String, Object> dataMap = new HashMap<>();
                 dataMap.putAll(defaultDataMap);
-                final Map<String, String> resultMap = new LinkedHashMap<>();
+                final Map<String, Object> resultMap = new LinkedHashMap<>();
                 resultMap.putAll(paramMap);
                 resultMap.put("csvfile", csvFile.getAbsolutePath());
                 resultMap.put("csvfilename", csvFile.getName());
+                resultMap.put("crawlingConfig", dataConfig);
                 boolean foundValues = false;
                 for (int i = 0; i < list.size(); i++) {
                     String key = null;
@@ -219,7 +220,7 @@ public class CsvDataStoreImpl extends AbstractDataStoreImpl {
                 }
 
                 if (logger.isDebugEnabled()) {
-                    for (final Map.Entry<String, String> entry : resultMap.entrySet()) {
+                    for (final Map.Entry<String, Object> entry : resultMap.entrySet()) {
                         logger.debug(entry.getKey() + "=" + entry.getValue());
                     }
                 }

src/main/java/org/codelibs/fess/ds/impl/DatabaseDataStoreImpl.java

@@ -109,7 +109,7 @@ public class DatabaseDataStoreImpl extends AbstractDataStoreImpl {
                 final Map<String, Object> dataMap = new HashMap<>();
                 dataMap.putAll(defaultDataMap);
                 for (final Map.Entry<String, String> entry : scriptMap.entrySet()) {
-                    final Object convertValue = convertValue(entry.getValue(), rs, paramMap);
+                    final Object convertValue = convertValue(config, entry.getValue(), rs, paramMap);
                     if (convertValue != null) {
                         dataMap.put(entry.getKey(), convertValue);
                     }
@@ -189,15 +189,16 @@ public class DatabaseDataStoreImpl extends AbstractDataStoreImpl {
         }
     }
 
-    protected Object convertValue(final String template, final ResultSet rs, final Map<String, String> paramMap) {
-        return convertValue(template, new ResultSetParamMap(rs, paramMap));
+    protected Object convertValue(final DataConfig config, final String template, final ResultSet rs, final Map<String, String> paramMap) {
+        return convertValue(template, new ResultSetParamMap(config, rs, paramMap));
     }
 
-    protected static class ResultSetParamMap implements Map<String, String> {
-        private final Map<String, String> paramMap = new HashMap<>();
+    protected static class ResultSetParamMap implements Map<String, Object> {
+        private final Map<String, Object> paramMap = new HashMap<>();
 
-        public ResultSetParamMap(final ResultSet resultSet, final Map<String, String> paramMap) {
+        public ResultSetParamMap(final DataConfig config, final ResultSet resultSet, final Map<String, String> paramMap) {
             this.paramMap.putAll(paramMap);
+            this.paramMap.put("crawlingConfig", config);
 
             try {
                 final ResultSetMetaData metaData = resultSet.getMetaData();
@@ -233,12 +234,12 @@ public class DatabaseDataStoreImpl extends AbstractDataStoreImpl {
         }
 
         @Override
-        public Set<java.util.Map.Entry<String, String>> entrySet() {
+        public Set<java.util.Map.Entry<String, Object>> entrySet() {
             return paramMap.entrySet();
         }
 
         @Override
-        public String get(final Object key) {
+        public Object get(final Object key) {
             return paramMap.get(key);
         }
 
@@ -253,17 +254,17 @@ public class DatabaseDataStoreImpl extends AbstractDataStoreImpl {
         }
 
         @Override
-        public String put(final String key, final String value) {
+        public Object put(final String key, final Object value) {
             return paramMap.put(key, value);
         }
 
         @Override
-        public void putAll(final Map<? extends String, ? extends String> m) {
+        public void putAll(final Map<? extends String, ? extends Object> m) {
             paramMap.putAll(m);
         }
 
         @Override
-        public String remove(final Object key) {
+        public Object remove(final Object key) {
             return paramMap.remove(key);
         }
 
@@ -273,7 +274,7 @@ public class DatabaseDataStoreImpl extends AbstractDataStoreImpl {
         }
 
         @Override
-        public Collection<String> values() {
+        public Collection<Object> values() {
             return paramMap.values();
         }
 

src/main/java/org/codelibs/fess/ds/impl/EsDataStoreImpl.java

@@ -162,6 +162,7 @@ public class EsDataStoreImpl extends AbstractDataStoreImpl {
                     resultMap.put("version", Long.valueOf(hit.getVersion()));
                     resultMap.put("hit", hit);
                     resultMap.put("source", hit.getSource());
+                    resultMap.put("crawlingConfig", dataConfig);
 
                     if (logger.isDebugEnabled()) {
                         for (final Map.Entry<String, Object> entry : resultMap.entrySet()) {

src/main/java/org/codelibs/fess/helper/DocumentHelper.java

@@ -18,14 +18,32 @@ package org.codelibs.fess.helper;
 import java.io.IOException;
 import java.io.Reader;
 import java.io.StringReader;
+import java.util.HashSet;
 import java.util.Map;
+import java.util.Set;
 
 import org.apache.commons.lang3.StringUtils;
+import org.codelibs.core.io.SerializeUtil;
 import org.codelibs.core.lang.StringUtil;
+import org.codelibs.fess.crawler.builder.RequestDataBuilder;
+import org.codelibs.fess.crawler.client.CrawlerClient;
+import org.codelibs.fess.crawler.client.CrawlerClientFactory;
+import org.codelibs.fess.crawler.entity.RequestData;
 import org.codelibs.fess.crawler.entity.ResponseData;
+import org.codelibs.fess.crawler.entity.ResultData;
+import org.codelibs.fess.crawler.exception.ChildUrlsException;
+import org.codelibs.fess.crawler.exception.CrawlerSystemException;
+import org.codelibs.fess.crawler.exception.CrawlingAccessException;
+import org.codelibs.fess.crawler.processor.ResponseProcessor;
+import org.codelibs.fess.crawler.processor.impl.DefaultResponseProcessor;
+import org.codelibs.fess.crawler.rule.Rule;
+import org.codelibs.fess.crawler.rule.RuleManager;
+import org.codelibs.fess.crawler.transformer.Transformer;
 import org.codelibs.fess.crawler.util.TextUtil;
+import org.codelibs.fess.es.config.exentity.CrawlingConfig;
 import org.codelibs.fess.mylasta.direction.FessConfig;
 import org.codelibs.fess.util.ComponentUtil;
+import org.lastaflute.di.core.SingletonLaContainer;
 
 public class DocumentHelper {
     public String getContent(final ResponseData responseData, final String content, final Map<String, Object> dataMap) {
@@ -85,4 +103,58 @@ public class DocumentHelper {
             return StringUtil.EMPTY; // empty
         }
     }
+
+    public Map<String, Object> processRequest(final CrawlingConfig crawlingConfig, final String sessionId, final String url) {
+        if (StringUtil.isBlank(sessionId)) {
+            throw new CrawlingAccessException("sessionId is null.");
+        }
+
+        final CrawlerClientFactory crawlerClientFactory = ComponentUtil.getCrawlerClientFactory();
+        crawlingConfig.initializeClientFactory(crawlerClientFactory);
+        final CrawlerClient client = crawlerClientFactory.getClient(url);
+        if (client == null) {
+            throw new CrawlingAccessException("CrawlerClient is null for " + url);
+        }
+
+        final long startTime = System.currentTimeMillis();
+        try (final ResponseData responseData = client.execute(RequestDataBuilder.newRequestData().get().url(url).build())) {
+            if (responseData.getRedirectLocation() != null) {
+                Set<RequestData> childUrlList = new HashSet<>();
+                childUrlList.add(RequestDataBuilder.newRequestData().get().url(responseData.getRedirectLocation()).build());
+                throw new ChildUrlsException(childUrlList, "Redirected from " + url);
+            }
+            responseData.setExecutionTime(System.currentTimeMillis() - startTime);
+            responseData.setSessionId(sessionId);
+
+            final RuleManager ruleManager = SingletonLaContainer.getComponent(RuleManager.class);
+            final Rule rule = ruleManager.getRule(responseData);
+            if (rule == null) {
+                throw new CrawlingAccessException("No url rule for " + url);
+            } else {
+                responseData.setRuleId(rule.getRuleId());
+                final ResponseProcessor responseProcessor = rule.getResponseProcessor();
+                if (responseProcessor instanceof DefaultResponseProcessor) {
+                    final Transformer transformer = ((DefaultResponseProcessor) responseProcessor).getTransformer();
+                    final ResultData resultData = transformer.transform(responseData);
+                    final byte[] data = resultData.getData();
+                    if (data != null) {
+                        try {
+                            @SuppressWarnings("unchecked")
+                            final Map<String, Object> result = (Map<String, Object>) SerializeUtil.fromBinaryToObject(data);
+                            return result;
+                        } catch (final Exception e) {
+                            throw new CrawlerSystemException("Could not create an instance from bytes.", e);
+                        }
+                    }
+                } else {
+                    throw new CrawlingAccessException("The response processor is not DefaultResponseProcessor. responseProcessor: "
+                            + responseProcessor + ", url: " + url);
+                }
+            }
+            return null;
+        } catch (final Exception e) {
+            throw new CrawlingAccessException("Failed to parse " + url, e);
+        }
+    }
+
 }

src/main/java/org/codelibs/fess/helper/SystemHelper.java

@@ -295,7 +295,7 @@ public class SystemHelper {
         ComponentUtil.getPathMappingHelper().init();
         ComponentUtil.getSuggestHelper().init();
         ComponentUtil.getPopularWordHelper().init();
-        // TODO restart jobs
+        ComponentUtil.getJobManager().reboot();
     }
 
 }

src/main/java/org/codelibs/fess/job/impl/GroovyExecutor.java

@@ -20,14 +20,12 @@ import java.util.Map;
 
 import org.codelibs.fess.job.JobExecutor;
 import org.codelibs.fess.util.GroovyUtil;
-import org.lastaflute.di.core.factory.SingletonLaContainerFactory;
 
 public class GroovyExecutor extends JobExecutor {
 
     @Override
     public Object execute(final String script) {
         final Map<String, Object> params = new HashMap<>();
-        params.put("container", SingletonLaContainerFactory.getContainer());
         params.put("executor", this);
 
         return GroovyUtil.evaluate(script, params);

src/main/java/org/codelibs/fess/util/GroovyUtil.java

@@ -15,8 +15,10 @@
  */
 package org.codelibs.fess.util;
 
+import java.util.HashMap;
 import java.util.Map;
 
+import org.lastaflute.di.core.factory.SingletonLaContainerFactory;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -31,8 +33,10 @@ public final class GroovyUtil {
         // nothing
     }
 
-    public static <T> Object evaluate(final String template, final Map<String, T> paramMap) {
-        final GroovyShell groovyShell = new GroovyShell(new Binding(paramMap));
+    public static Object evaluate(final String template, final Map<String, Object> paramMap) {
+        final Map<String, Object> bindingMap = new HashMap<>(paramMap);
+        bindingMap.put("container", SingletonLaContainerFactory.getContainer());
+        final GroovyShell groovyShell = new GroovyShell(new Binding(bindingMap));
         try {
             return groovyShell.evaluate(template);
         } catch (final Exception e) {

src/test/java/org/codelibs/fess/ds/impl/DatabaseDataStoreImplTest.java

@@ -31,7 +31,7 @@ public class DatabaseDataStoreImplTest extends UnitFessTestCase {
 
     public void test_convertValue() {
         String value;
-        final Map<String, String> paramMap = new HashMap<String, String>();
+        final Map<String, Object> paramMap = new HashMap<>();
         paramMap.put("param1", "PARAM1");
         paramMap.put("param2", "PARAM2+");
         paramMap.put("param3", "PARAM3*");