0ct0pu5/fess
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix #2554 use environment variables in parameters

Browse source
This commit is contained in:
Shinsuke Sugaya 2021-04-02 22:03:11 +09:00
parent af41d1fdd6
commit cd41184064
5 changed files with 74 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
src/main/java/org/codelibs/fess/ds/AbstractDataStore.java
View file

@ -28,6 +28,7 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.core.lang.ThreadUtil;
import org.codelibs.core.misc.Pair;
import org.codelibs.fess.Constants;
import org.codelibs.fess.ds.callback.IndexUpdateCallback;
import org.codelibs.fess.es.config.exentity.DataConfig;
@ -58,12 +59,20 @@ public abstract class AbstractDataStore implements DataStore {
@Override
public void store(final DataConfig config, final IndexUpdateCallback callback, final Map<String, String> initParamMap) {
final Map<String, String> configParamMap = config.getHandlerParameterMap();
final Map<String, String> configScriptMap = config.getHandlerScriptMap();
final CrawlingInfoHelper crawlingInfoHelper = ComponentUtil.getCrawlingInfoHelper();
final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
final Date documentExpires = crawlingInfoHelper.getDocumentExpires(config);
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final Map<String, String> paramEnvMap = systemHelper.getFilteredEnvMap(fessConfig.getCrawlerDataEnvParamKeyPattern());
final Map<String, String> configParamMap = config.getHandlerParameterMap().entrySet().stream().map(e -> {
final String key = e.getKey();
String value = e.getValue();
for (Map.Entry<String, String> entry : paramEnvMap.entrySet()) {
value = value.replace("${" + entry.getKey() + "}", entry.getValue());
}
return new Pair<String, String>(key, value);
}).collect(Collectors.toMap(Pair<String, String>::getFirst, Pair<String, String>::getSecond));
final Map<String, String> configScriptMap = config.getHandlerScriptMap();
initParamMap.putAll(configParamMap);
final Map<String, String> paramMap = initParamMap;

19
src/main/java/org/codelibs/fess/helper/SystemHelper.java
View file

@ -36,6 +36,7 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
import java.util.TimeZone;
import java.util.UUID;
@ -44,6 +45,7 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
@ -398,7 +400,7 @@ public class SystemHelper {
}
public String getHostname() {
final Map<String, String> env = System.getenv();
final Map<String, String> env = getEnvMap();
if (env.containsKey("COMPUTERNAME")) {
return env.get("COMPUTERNAME");
} else if (env.containsKey("HOSTNAME")) {
@ -586,6 +588,21 @@ public class SystemHelper {
return systemCpuPercent;
}
public Map<String, String> getFilteredEnvMap(final String keyPattern) {
final Pattern pattern = Pattern.compile(keyPattern);
return getEnvMap().entrySet().stream().filter(e -> {
String key = e.getKey();
if (StringUtil.isBlank(key)) {
return false;
}
return pattern.matcher(key).matches();
}).collect(Collectors.toMap(Entry<String, String>::getKey, Entry<String, String>::getValue));
}
protected Map<String, String> getEnvMap() {
return System.getenv();
}
public String getVersion() {
return version;
}

15
src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java
View file

@ -325,6 +325,9 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
/** The key of the configuration. e.g. file,smb,smb1,ftp,storage */
String CRAWLER_FILE_PROTOCOLS = "crawler.file.protocols";
/** The key of the configuration. e.g. ^FESS_ENV_ */
String CRAWLER_DATA_ENV_PARAM_KEY_PATTERN = "crawler.data.env.param.key.pattern";
/** The key of the configuration. e.g. false */
String CRAWLER_IGNORE_ROBOTS_TXT = "crawler.ignore.robots.txt";
@ -2517,6 +2520,13 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
*/
String getCrawlerFileProtocols();
/**
* Get the value for the key 'crawler.data.env.param.key.pattern'. <br>
* The value is, e.g. ^FESS_ENV_ <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getCrawlerDataEnvParamKeyPattern();
/**
* Get the value for the key 'crawler.ignore.robots.txt'. <br>
* The value is, e.g. false <br>
@ -7395,6 +7405,10 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
return get(FessConfig.CRAWLER_FILE_PROTOCOLS);
}
public String getCrawlerDataEnvParamKeyPattern() {
return get(FessConfig.CRAWLER_DATA_ENV_PARAM_KEY_PATTERN);
}
public String getCrawlerIgnoreRobotsTxt() {
return get(FessConfig.CRAWLER_IGNORE_ROBOTS_TXT);
}
@ -9850,6 +9864,7 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
defaultMap.put(FessConfig.CRAWLER_CRAWLING_DATA_ENCODING, "UTF-8");
defaultMap.put(FessConfig.CRAWLER_WEB_PROTOCOLS, "http,https");
defaultMap.put(FessConfig.CRAWLER_FILE_PROTOCOLS, "file,smb,smb1,ftp,storage");
defaultMap.put(FessConfig.CRAWLER_DATA_ENV_PARAM_KEY_PATTERN, "^FESS_ENV_");
defaultMap.put(FessConfig.CRAWLER_IGNORE_ROBOTS_TXT, "false");
defaultMap.put(FessConfig.CRAWLER_IGNORE_ROBOTS_TAGS, "false");
defaultMap.put(FessConfig.CRAWLER_IGNORE_CONTENT_EXCEPTION, "true");

1
src/main/resources/fess_config.properties
View file

@ -205,6 +205,7 @@ crawler.document.fullstop.chars=u002eu06d4u2e3cu3002
crawler.crawling.data.encoding=UTF-8
crawler.web.protocols=http,https
crawler.file.protocols=file,smb,smb1,ftp,storage
crawler.data.env.param.key.pattern=^FESS_ENV_.*
crawler.ignore.robots.txt=false
crawler.ignore.robots.tags=false
crawler.ignore.content.exception=true

29
src/test/java/org/codelibs/fess/helper/SystemHelperTest.java
View file

@ -15,6 +15,9 @@
*/
package org.codelibs.fess.helper;
import java.util.HashMap;
import java.util.Map;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.unit.UnitFessTestCase;
import org.codelibs.fess.util.ComponentUtil;
@ -23,6 +26,8 @@ public class SystemHelperTest extends UnitFessTestCase {
public SystemHelper systemHelper;
private Map<String, String> envMap = new HashMap<>();
@Override
public void setUp() throws Exception {
super.setUp();
@ -34,7 +39,13 @@ public class SystemHelperTest extends UnitFessTestCase {
@Override
public void updateSystemProperties() {
}
@Override
protected Map<String, String> getEnvMap() {
return envMap;
}
};
envMap.clear();
systemHelper.init();
}
@ -118,4 +129,22 @@ public class SystemHelperTest extends UnitFessTestCase {
assertEquals("aaa", systemHelper.normalizeConfigPath("regexpCase:aaa"));
assertEquals("(?i)aaa", systemHelper.normalizeConfigPath("regexpIgnoreCase:aaa"));
}
public void test_getFilteredEnvMap() {
Map<String, String> filteredEnvMap = systemHelper.getFilteredEnvMap("^FESS_ENV.*");
assertEquals(0, filteredEnvMap.size());
envMap.put("FESS_ENV_TEST", "123");
filteredEnvMap = systemHelper.getFilteredEnvMap("^FESS_ENV.*");
assertEquals(1, filteredEnvMap.size());
assertEquals("123", filteredEnvMap.get("FESS_ENV_TEST"));
filteredEnvMap = systemHelper.getFilteredEnvMap("^XFESS_ENV.*");
assertEquals(0, filteredEnvMap.size());
envMap.put("", "123");
filteredEnvMap = systemHelper.getFilteredEnvMap("^FESS_ENV.*");
assertEquals(1, filteredEnvMap.size());
assertEquals("123", filteredEnvMap.get("FESS_ENV_TEST"));
}
}