diff --git a/src/main/java/org/codelibs/fess/crawler/transformer/FessXpathTransformer.java b/src/main/java/org/codelibs/fess/crawler/transformer/FessXpathTransformer.java index a3900d944..436c7ae03 100644 --- a/src/main/java/org/codelibs/fess/crawler/transformer/FessXpathTransformer.java +++ b/src/main/java/org/codelibs/fess/crawler/transformer/FessXpathTransformer.java @@ -185,9 +185,7 @@ public class FessXpathTransformer extends XpathTransformer implements FessTransf } protected void processMetaRobots(final ResponseData responseData, final ResultData resultData, final Document document) { - final CrawlingConfigHelper crawlingConfigHelper = ComponentUtil.getCrawlingConfigHelper(); - final CrawlingConfig crawlingConfig = crawlingConfigHelper.get(responseData.getSessionId()); - final Map configMap = crawlingConfig.getConfigParameterMap(ConfigName.CONFIG); + final Map configMap = getConfigPrameterMap(responseData, ConfigName.CONFIG); String ignore = configMap.get(IGNORE_META_ROBOTS); if (ignore == null) { if (fessConfig.isCrawlerIgnoreMetaRobots()) { @@ -233,6 +231,13 @@ public class FessXpathTransformer extends XpathTransformer implements FessTransf } + protected Map getConfigPrameterMap(final ResponseData responseData, final ConfigName config) { + final CrawlingConfigHelper crawlingConfigHelper = ComponentUtil.getCrawlingConfigHelper(); + final CrawlingConfig crawlingConfig = crawlingConfigHelper.get(responseData.getSessionId()); + final Map configMap = crawlingConfig.getConfigParameterMap(config); + return configMap; + } + protected boolean isValidUrl(final String urlStr) { if (StringUtil.isBlank(urlStr)) { return false; @@ -464,9 +469,7 @@ public class FessXpathTransformer extends XpathTransformer implements FessTransf } protected String getCanonicalUrl(final ResponseData responseData, final Document document) { - final CrawlingConfigHelper crawlingConfigHelper = ComponentUtil.getCrawlingConfigHelper(); - final CrawlingConfig crawlingConfig = crawlingConfigHelper.get(responseData.getSessionId()); - final Map configMap = crawlingConfig.getConfigParameterMap(ConfigName.CONFIG); + final Map configMap = getConfigPrameterMap(responseData, ConfigName.CONFIG); String xpath = configMap.get(HTML_CANONICAL_XPATH); if (xpath == null) { xpath = fessConfig.getCrawlerDocumentHtmlCanonicalXpath(); diff --git a/src/test/java/org/codelibs/fess/crawler/transformer/FessXpathTransformerTest.java b/src/test/java/org/codelibs/fess/crawler/transformer/FessXpathTransformerTest.java index 66f23c7ce..3a6366bb4 100644 --- a/src/test/java/org/codelibs/fess/crawler/transformer/FessXpathTransformerTest.java +++ b/src/test/java/org/codelibs/fess/crawler/transformer/FessXpathTransformerTest.java @@ -20,6 +20,7 @@ import java.io.StringWriter; import java.lang.reflect.Field; import java.net.URL; import java.util.ArrayList; +import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.List; @@ -41,6 +42,7 @@ import org.codelibs.fess.crawler.entity.ResultData; import org.codelibs.fess.crawler.exception.ChildUrlsException; import org.codelibs.fess.es.config.exentity.LabelType; import org.codelibs.fess.es.config.exentity.WebConfig; +import org.codelibs.fess.es.config.exentity.CrawlingConfig.ConfigName; import org.codelibs.fess.helper.CrawlingConfigHelper; import org.codelibs.fess.helper.CrawlingInfoHelper; import org.codelibs.fess.helper.DocumentHelper; @@ -269,7 +271,12 @@ public class FessXpathTransformerTest extends UnitFessTestCase { final String data = "foo"; final Document document = getDocument(data); - final FessXpathTransformer transformer = new FessXpathTransformer(); + final FessXpathTransformer transformer = new FessXpathTransformer() { + @Override + protected Map getConfigPrameterMap(final ResponseData responseData, final ConfigName config) { + return Collections.emptyMap(); + } + }; final ResponseData responseData = new ResponseData(); responseData.setUrl("http://example.com/"); @@ -282,7 +289,11 @@ public class FessXpathTransformerTest extends UnitFessTestCase { final String data = ""; final Document document = getDocument(data); - final FessXpathTransformer transformer = new FessXpathTransformer(); + final FessXpathTransformer transformer = new FessXpathTransformer() { + protected Map getConfigPrameterMap(final ResponseData responseData, final ConfigName config) { + return Collections.emptyMap(); + } + }; final ResponseData responseData = new ResponseData(); responseData.setUrl("http://example.com/"); @@ -301,7 +312,11 @@ public class FessXpathTransformerTest extends UnitFessTestCase { final String data = ""; final Document document = getDocument(data); - final FessXpathTransformer transformer = new FessXpathTransformer(); + final FessXpathTransformer transformer = new FessXpathTransformer() { + protected Map getConfigPrameterMap(final ResponseData responseData, final ConfigName config) { + return Collections.emptyMap(); + } + }; final ResponseData responseData = new ResponseData(); responseData.setUrl("http://example.com/"); @@ -340,7 +355,11 @@ public class FessXpathTransformerTest extends UnitFessTestCase { final String data = ""; final Document document = getDocument(data); - final FessXpathTransformer transformer = new FessXpathTransformer(); + final FessXpathTransformer transformer = new FessXpathTransformer() { + protected Map getConfigPrameterMap(final ResponseData responseData, final ConfigName config) { + return Collections.emptyMap(); + } + }; final ResponseData responseData = new ResponseData(); responseData.setUrl("http://example.com/"); @@ -428,7 +447,11 @@ public class FessXpathTransformerTest extends UnitFessTestCase { } public void test_canonicalXpath() throws Exception { - final FessXpathTransformer transformer = new FessXpathTransformer(); + final FessXpathTransformer transformer = new FessXpathTransformer() { + protected Map getConfigPrameterMap(final ResponseData responseData, final ConfigName config) { + return Collections.emptyMap(); + } + }; transformer.init(); final Map dataMap = new HashMap();