diff --git a/src/main/java/org/codelibs/fess/ds/AbstractDataStore.java b/src/main/java/org/codelibs/fess/ds/AbstractDataStore.java index 50d917621..656880e8f 100644 --- a/src/main/java/org/codelibs/fess/ds/AbstractDataStore.java +++ b/src/main/java/org/codelibs/fess/ds/AbstractDataStore.java @@ -28,6 +28,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.codelibs.core.lang.StringUtil; import org.codelibs.core.lang.ThreadUtil; +import org.codelibs.core.misc.Pair; import org.codelibs.fess.Constants; import org.codelibs.fess.ds.callback.IndexUpdateCallback; import org.codelibs.fess.es.config.exentity.DataConfig; @@ -58,12 +59,20 @@ public abstract class AbstractDataStore implements DataStore { @Override public void store(final DataConfig config, final IndexUpdateCallback callback, final Map initParamMap) { - final Map configParamMap = config.getHandlerParameterMap(); - final Map configScriptMap = config.getHandlerScriptMap(); final CrawlingInfoHelper crawlingInfoHelper = ComponentUtil.getCrawlingInfoHelper(); final SystemHelper systemHelper = ComponentUtil.getSystemHelper(); final Date documentExpires = crawlingInfoHelper.getDocumentExpires(config); final FessConfig fessConfig = ComponentUtil.getFessConfig(); + final Map paramEnvMap = systemHelper.getFilteredEnvMap(fessConfig.getCrawlerDataEnvParamKeyPattern()); + final Map configParamMap = config.getHandlerParameterMap().entrySet().stream().map(e -> { + final String key = e.getKey(); + String value = e.getValue(); + for (Map.Entry entry : paramEnvMap.entrySet()) { + value = value.replace("${" + entry.getKey() + "}", entry.getValue()); + } + return new Pair(key, value); + }).collect(Collectors.toMap(Pair::getFirst, Pair::getSecond)); + final Map configScriptMap = config.getHandlerScriptMap(); initParamMap.putAll(configParamMap); final Map paramMap = initParamMap; diff --git a/src/main/java/org/codelibs/fess/helper/SystemHelper.java b/src/main/java/org/codelibs/fess/helper/SystemHelper.java index b17e0ab41..c0f0cb201 100644 --- a/src/main/java/org/codelibs/fess/helper/SystemHelper.java +++ b/src/main/java/org/codelibs/fess/helper/SystemHelper.java @@ -36,6 +36,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Map.Entry; import java.util.Properties; import java.util.TimeZone; import java.util.UUID; @@ -44,6 +45,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.regex.Pattern; +import java.util.stream.Collectors; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -398,7 +400,7 @@ public class SystemHelper { } public String getHostname() { - final Map env = System.getenv(); + final Map env = getEnvMap(); if (env.containsKey("COMPUTERNAME")) { return env.get("COMPUTERNAME"); } else if (env.containsKey("HOSTNAME")) { @@ -586,6 +588,21 @@ public class SystemHelper { return systemCpuPercent; } + public Map getFilteredEnvMap(final String keyPattern) { + final Pattern pattern = Pattern.compile(keyPattern); + return getEnvMap().entrySet().stream().filter(e -> { + String key = e.getKey(); + if (StringUtil.isBlank(key)) { + return false; + } + return pattern.matcher(key).matches(); + }).collect(Collectors.toMap(Entry::getKey, Entry::getValue)); + } + + protected Map getEnvMap() { + return System.getenv(); + } + public String getVersion() { return version; } diff --git a/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java b/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java index 5ca2b2bc1..af3b3c4f7 100644 --- a/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java +++ b/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java @@ -325,6 +325,9 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction /** The key of the configuration. e.g. file,smb,smb1,ftp,storage */ String CRAWLER_FILE_PROTOCOLS = "crawler.file.protocols"; + /** The key of the configuration. e.g. ^FESS_ENV_ */ + String CRAWLER_DATA_ENV_PARAM_KEY_PATTERN = "crawler.data.env.param.key.pattern"; + /** The key of the configuration. e.g. false */ String CRAWLER_IGNORE_ROBOTS_TXT = "crawler.ignore.robots.txt"; @@ -2517,6 +2520,13 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction */ String getCrawlerFileProtocols(); + /** + * Get the value for the key 'crawler.data.env.param.key.pattern'.
+ * The value is, e.g. ^FESS_ENV_
+ * @return The value of found property. (NotNull: if not found, exception but basically no way) + */ + String getCrawlerDataEnvParamKeyPattern(); + /** * Get the value for the key 'crawler.ignore.robots.txt'.
* The value is, e.g. false
@@ -7395,6 +7405,10 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction return get(FessConfig.CRAWLER_FILE_PROTOCOLS); } + public String getCrawlerDataEnvParamKeyPattern() { + return get(FessConfig.CRAWLER_DATA_ENV_PARAM_KEY_PATTERN); + } + public String getCrawlerIgnoreRobotsTxt() { return get(FessConfig.CRAWLER_IGNORE_ROBOTS_TXT); } @@ -9850,6 +9864,7 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction defaultMap.put(FessConfig.CRAWLER_CRAWLING_DATA_ENCODING, "UTF-8"); defaultMap.put(FessConfig.CRAWLER_WEB_PROTOCOLS, "http,https"); defaultMap.put(FessConfig.CRAWLER_FILE_PROTOCOLS, "file,smb,smb1,ftp,storage"); + defaultMap.put(FessConfig.CRAWLER_DATA_ENV_PARAM_KEY_PATTERN, "^FESS_ENV_"); defaultMap.put(FessConfig.CRAWLER_IGNORE_ROBOTS_TXT, "false"); defaultMap.put(FessConfig.CRAWLER_IGNORE_ROBOTS_TAGS, "false"); defaultMap.put(FessConfig.CRAWLER_IGNORE_CONTENT_EXCEPTION, "true"); diff --git a/src/main/resources/fess_config.properties b/src/main/resources/fess_config.properties index f1926321d..49c3cbc93 100644 --- a/src/main/resources/fess_config.properties +++ b/src/main/resources/fess_config.properties @@ -205,6 +205,7 @@ crawler.document.fullstop.chars=u002eu06d4u2e3cu3002 crawler.crawling.data.encoding=UTF-8 crawler.web.protocols=http,https crawler.file.protocols=file,smb,smb1,ftp,storage +crawler.data.env.param.key.pattern=^FESS_ENV_.* crawler.ignore.robots.txt=false crawler.ignore.robots.tags=false crawler.ignore.content.exception=true diff --git a/src/test/java/org/codelibs/fess/helper/SystemHelperTest.java b/src/test/java/org/codelibs/fess/helper/SystemHelperTest.java index def04ed58..30fb7122d 100644 --- a/src/test/java/org/codelibs/fess/helper/SystemHelperTest.java +++ b/src/test/java/org/codelibs/fess/helper/SystemHelperTest.java @@ -15,6 +15,9 @@ */ package org.codelibs.fess.helper; +import java.util.HashMap; +import java.util.Map; + import org.codelibs.fess.mylasta.direction.FessConfig; import org.codelibs.fess.unit.UnitFessTestCase; import org.codelibs.fess.util.ComponentUtil; @@ -23,6 +26,8 @@ public class SystemHelperTest extends UnitFessTestCase { public SystemHelper systemHelper; + private Map envMap = new HashMap<>(); + @Override public void setUp() throws Exception { super.setUp(); @@ -34,7 +39,13 @@ public class SystemHelperTest extends UnitFessTestCase { @Override public void updateSystemProperties() { } + + @Override + protected Map getEnvMap() { + return envMap; + } }; + envMap.clear(); systemHelper.init(); } @@ -118,4 +129,22 @@ public class SystemHelperTest extends UnitFessTestCase { assertEquals("aaa", systemHelper.normalizeConfigPath("regexpCase:aaa")); assertEquals("(?i)aaa", systemHelper.normalizeConfigPath("regexpIgnoreCase:aaa")); } + + public void test_getFilteredEnvMap() { + Map filteredEnvMap = systemHelper.getFilteredEnvMap("^FESS_ENV.*"); + assertEquals(0, filteredEnvMap.size()); + + envMap.put("FESS_ENV_TEST", "123"); + filteredEnvMap = systemHelper.getFilteredEnvMap("^FESS_ENV.*"); + assertEquals(1, filteredEnvMap.size()); + assertEquals("123", filteredEnvMap.get("FESS_ENV_TEST")); + + filteredEnvMap = systemHelper.getFilteredEnvMap("^XFESS_ENV.*"); + assertEquals(0, filteredEnvMap.size()); + + envMap.put("", "123"); + filteredEnvMap = systemHelper.getFilteredEnvMap("^FESS_ENV.*"); + assertEquals(1, filteredEnvMap.size()); + assertEquals("123", filteredEnvMap.get("FESS_ENV_TEST")); + } }