fix #2554 use environment variables in parameters

src/main/java/org/codelibs/fess/ds/AbstractDataStore.java

@@ -28,6 +28,7 @@ import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.codelibs.core.lang.StringUtil;
 import org.codelibs.core.lang.ThreadUtil;
+import org.codelibs.core.misc.Pair;
 import org.codelibs.fess.Constants;
 import org.codelibs.fess.ds.callback.IndexUpdateCallback;
 import org.codelibs.fess.es.config.exentity.DataConfig;
@@ -58,12 +59,20 @@ public abstract class AbstractDataStore implements DataStore {
 
     @Override
     public void store(final DataConfig config, final IndexUpdateCallback callback, final Map<String, String> initParamMap) {
-        final Map<String, String> configParamMap = config.getHandlerParameterMap();
-        final Map<String, String> configScriptMap = config.getHandlerScriptMap();
         final CrawlingInfoHelper crawlingInfoHelper = ComponentUtil.getCrawlingInfoHelper();
         final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
         final Date documentExpires = crawlingInfoHelper.getDocumentExpires(config);
         final FessConfig fessConfig = ComponentUtil.getFessConfig();
+        final Map<String, String> paramEnvMap = systemHelper.getFilteredEnvMap(fessConfig.getCrawlerDataEnvParamKeyPattern());
+        final Map<String, String> configParamMap = config.getHandlerParameterMap().entrySet().stream().map(e -> {
+            final String key = e.getKey();
+            String value = e.getValue();
+            for (Map.Entry<String, String> entry : paramEnvMap.entrySet()) {
+                value = value.replace("${" + entry.getKey() + "}", entry.getValue());
+            }
+            return new Pair<String, String>(key, value);
+        }).collect(Collectors.toMap(Pair<String, String>::getFirst, Pair<String, String>::getSecond));
+        final Map<String, String> configScriptMap = config.getHandlerScriptMap();
 
         initParamMap.putAll(configParamMap);
         final Map<String, String> paramMap = initParamMap;

src/main/java/org/codelibs/fess/helper/SystemHelper.java

@@ -36,6 +36,7 @@ import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Map.Entry;
 import java.util.Properties;
 import java.util.TimeZone;
 import java.util.UUID;
@@ -44,6 +45,7 @@ import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.regex.Pattern;
+import java.util.stream.Collectors;
 
 import javax.annotation.PostConstruct;
 import javax.annotation.PreDestroy;
@@ -398,7 +400,7 @@ public class SystemHelper {
     }
 
     public String getHostname() {
-        final Map<String, String> env = System.getenv();
+        final Map<String, String> env = getEnvMap();
         if (env.containsKey("COMPUTERNAME")) {
             return env.get("COMPUTERNAME");
         } else if (env.containsKey("HOSTNAME")) {
@@ -586,6 +588,21 @@ public class SystemHelper {
         return systemCpuPercent;
     }
 
+    public Map<String, String> getFilteredEnvMap(final String keyPattern) {
+        final Pattern pattern = Pattern.compile(keyPattern);
+        return getEnvMap().entrySet().stream().filter(e -> {
+            String key = e.getKey();
+            if (StringUtil.isBlank(key)) {
+                return false;
+            }
+            return pattern.matcher(key).matches();
+        }).collect(Collectors.toMap(Entry<String, String>::getKey, Entry<String, String>::getValue));
+    }
+
+    protected Map<String, String> getEnvMap() {
+        return System.getenv();
+    }
+
     public String getVersion() {
         return version;
     }

src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java

@@ -325,6 +325,9 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
     /** The key of the configuration. e.g. file,smb,smb1,ftp,storage */
     String CRAWLER_FILE_PROTOCOLS = "crawler.file.protocols";
 
+    /** The key of the configuration. e.g. ^FESS_ENV_ */
+    String CRAWLER_DATA_ENV_PARAM_KEY_PATTERN = "crawler.data.env.param.key.pattern";
+
     /** The key of the configuration. e.g. false */
     String CRAWLER_IGNORE_ROBOTS_TXT = "crawler.ignore.robots.txt";
 
@@ -2517,6 +2520,13 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
      */
     String getCrawlerFileProtocols();
 
+    /**
+     * Get the value for the key 'crawler.data.env.param.key.pattern'. <br>
+     * The value is, e.g. ^FESS_ENV_ <br>
+     * @return The value of found property. (NotNull: if not found, exception but basically no way)
+     */
+    String getCrawlerDataEnvParamKeyPattern();
+
     /**
      * Get the value for the key 'crawler.ignore.robots.txt'. <br>
      * The value is, e.g. false <br>
@@ -7395,6 +7405,10 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
             return get(FessConfig.CRAWLER_FILE_PROTOCOLS);
         }
 
+        public String getCrawlerDataEnvParamKeyPattern() {
+            return get(FessConfig.CRAWLER_DATA_ENV_PARAM_KEY_PATTERN);
+        }
+
         public String getCrawlerIgnoreRobotsTxt() {
             return get(FessConfig.CRAWLER_IGNORE_ROBOTS_TXT);
         }
@@ -9850,6 +9864,7 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
             defaultMap.put(FessConfig.CRAWLER_CRAWLING_DATA_ENCODING, "UTF-8");
             defaultMap.put(FessConfig.CRAWLER_WEB_PROTOCOLS, "http,https");
             defaultMap.put(FessConfig.CRAWLER_FILE_PROTOCOLS, "file,smb,smb1,ftp,storage");
+            defaultMap.put(FessConfig.CRAWLER_DATA_ENV_PARAM_KEY_PATTERN, "^FESS_ENV_");
             defaultMap.put(FessConfig.CRAWLER_IGNORE_ROBOTS_TXT, "false");
             defaultMap.put(FessConfig.CRAWLER_IGNORE_ROBOTS_TAGS, "false");
             defaultMap.put(FessConfig.CRAWLER_IGNORE_CONTENT_EXCEPTION, "true");

src/main/resources/fess_config.properties

@@ -205,6 +205,7 @@ crawler.document.fullstop.chars=u002eu06d4u2e3cu3002
 crawler.crawling.data.encoding=UTF-8
 crawler.web.protocols=http,https
 crawler.file.protocols=file,smb,smb1,ftp,storage
+crawler.data.env.param.key.pattern=^FESS_ENV_.*
 crawler.ignore.robots.txt=false
 crawler.ignore.robots.tags=false
 crawler.ignore.content.exception=true

src/test/java/org/codelibs/fess/helper/SystemHelperTest.java

@@ -15,6 +15,9 @@
  */
 package org.codelibs.fess.helper;
 
+import java.util.HashMap;
+import java.util.Map;
+
 import org.codelibs.fess.mylasta.direction.FessConfig;
 import org.codelibs.fess.unit.UnitFessTestCase;
 import org.codelibs.fess.util.ComponentUtil;
@@ -23,6 +26,8 @@ public class SystemHelperTest extends UnitFessTestCase {
 
     public SystemHelper systemHelper;
 
+    private Map<String, String> envMap = new HashMap<>();
+
     @Override
     public void setUp() throws Exception {
         super.setUp();
@@ -34,7 +39,13 @@ public class SystemHelperTest extends UnitFessTestCase {
             @Override
             public void updateSystemProperties() {
             }
+
+            @Override
+            protected Map<String, String> getEnvMap() {
+                return envMap;
+            }
         };
+        envMap.clear();
         systemHelper.init();
     }
 
@@ -118,4 +129,22 @@ public class SystemHelperTest extends UnitFessTestCase {
         assertEquals("aaa", systemHelper.normalizeConfigPath("regexpCase:aaa"));
         assertEquals("(?i)aaa", systemHelper.normalizeConfigPath("regexpIgnoreCase:aaa"));
     }
+
+    public void test_getFilteredEnvMap() {
+        Map<String, String> filteredEnvMap = systemHelper.getFilteredEnvMap("^FESS_ENV.*");
+        assertEquals(0, filteredEnvMap.size());
+
+        envMap.put("FESS_ENV_TEST", "123");
+        filteredEnvMap = systemHelper.getFilteredEnvMap("^FESS_ENV.*");
+        assertEquals(1, filteredEnvMap.size());
+        assertEquals("123", filteredEnvMap.get("FESS_ENV_TEST"));
+
+        filteredEnvMap = systemHelper.getFilteredEnvMap("^XFESS_ENV.*");
+        assertEquals(0, filteredEnvMap.size());
+
+        envMap.put("", "123");
+        filteredEnvMap = systemHelper.getFilteredEnvMap("^FESS_ENV.*");
+        assertEquals(1, filteredEnvMap.size());
+        assertEquals("123", filteredEnvMap.get("FESS_ENV_TEST"));
+    }
 }