0ct0pu5/fess
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix #2841 Enabled support for multiple Fess web applications on a single OpenSearch instance.

Browse source
This commit is contained in:
Shinsuke Sugaya 2024-08-22 21:43:18 +09:00
parent ec6385766e
commit 327d2eb160
6 changed files with 84 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/main/java/org/codelibs/fess/Constants.java
View file

@ -87,6 +87,8 @@ public class Constants extends CoreLibConstants {
public static final String CRAWLING_USER_AGENT_SUFFIX = "; +http://fess.codelibs.org/bot.html)";
public static final String DOCUMENT_INDEX_SUFFIX_PATTERN = "yyyyMMddHHmmssSSS";
// fess properties
public static final String USER_INFO_PROPERTY = "user.info";

2
src/main/java/org/codelibs/fess/app/web/admin/maintenance/AdminMaintenanceAction.java
View file

@ -300,7 +300,7 @@ public class AdminMaintenanceAction extends FessAdminAction {
final String autoExpandReplicas) {
final String docIndex = "fess";
final String fromIndex = fessConfig.getIndexDocumentUpdateIndex();
final String toIndex = docIndex + "." + new SimpleDateFormat("yyyyMMddHHmm").format(new Date());
final String toIndex = docIndex + "." + new SimpleDateFormat(Constants.DOCUMENT_INDEX_SUFFIX_PATTERN).format(new Date());
if (searchEngineClient.createIndex(docIndex, toIndex, numberOfShards, autoExpandReplicas, resetDictionaries)) {
searchEngineClient.admin().cluster().prepareHealth(toIndex).setWaitForYellowStatus().execute(ActionListener.wrap(response -> {
searchEngineClient.addMapping(docIndex, "doc", toIndex);

61
src/main/java/org/codelibs/fess/es/client/SearchEngineClient.java
View file

@ -184,6 +184,8 @@ public class SearchEngineClient implements Client {
private static final Logger logger = LogManager.getLogger(SearchEngineClient.class);
private static final String DOC_INDEX = "fess";
private static final String LOG_INDEX_PREFIX = "fess_log";
private static final String USER_INDEX_PREFIX = "fess_user";
@ -268,6 +270,18 @@ public class SearchEngineClient implements Client {
}
final FessConfig fessConfig = ComponentUtil.getFessConfig();
if (StringUtil.isNotBlank(fessConfig.getIndexDictionaryPrefix())) {
String dictionaryPath = System.getProperty("fess.dictionary.path", StringUtil.EMPTY);
if (StringUtil.isBlank(dictionaryPath)) {
System.setProperty("fess.dictionary.path", fessConfig.getIndexDictionaryPrefix() + "/");
} else {
if (!dictionaryPath.endsWith("/")) {
dictionaryPath = dictionaryPath + "/";
}
System.setProperty("fess.dictionary.path", dictionaryPath + fessConfig.getIndexDictionaryPrefix() + "/");
}
}
String httpAddress = SystemUtil.getSearchEngineHttpAddress();
if (StringUtil.isBlank(httpAddress) && (runner == null)) {
switch (fessConfig.getFesenType()) {
@ -322,7 +336,7 @@ public class SearchEngineClient implements Client {
final String configIndex = values[0];
final String configType = values[1];
final boolean isFessIndex = "fess".equals(configIndex);
final boolean isFessIndex = DOC_INDEX.equals(configIndex);
final String indexName;
if (isFessIndex) {
final boolean exists = existsIndex(fessConfig.getIndexDocumentUpdateIndex());
@ -349,10 +363,10 @@ public class SearchEngineClient implements Client {
indexName = configIndex.replaceFirst(Pattern.quote(CONFIG_INDEX_PREFIX), name);
} else if (configIndex.startsWith(USER_INDEX_PREFIX)) {
final String name = fessConfig.getIndexUserIndex();
indexName = configIndex.replaceFirst(Pattern.quote(CONFIG_INDEX_PREFIX), name);
indexName = configIndex.replaceFirst(Pattern.quote(USER_INDEX_PREFIX), name);
} else if (configIndex.startsWith(LOG_INDEX_PREFIX)) {
final String name = fessConfig.getIndexLogIndex();
indexName = configIndex.replaceFirst(Pattern.quote(CONFIG_INDEX_PREFIX), name);
indexName = configIndex.replaceFirst(Pattern.quote(LOG_INDEX_PREFIX), name);
} else {
throw new FessSystemException("Unknown config index: " + configIndex);
}
@ -553,7 +567,7 @@ public class SearchEngineClient implements Client {
final String mappingFile = getResourcePath(indexConfigPath, fessConfig.getFesenType(), "/" + index + "/" + docType + ".json");
try {
source = FileUtil.readUTF8(mappingFile);
if ("fess".equals(index)) {
if (DOC_INDEX.equals(index)) {
for (final UnaryOperator<String> rule : docMappingRewriteRuleList) {
source = rule.apply(source);
}
@ -624,7 +638,23 @@ public class SearchEngineClient implements Client {
final File aliasConfigDir = ResourceUtil.getResourceAsFile(aliasConfigDirPath);
if (aliasConfigDir.isDirectory()) {
stream(aliasConfigDir.listFiles((dir, name) -> name.endsWith(".json"))).of(stream -> stream.forEach(f -> {
final String aliasName = f.getName().replaceFirst(".json$", "");
String aliasName = f.getName().replaceFirst(".json$", "");
if (index.equals(DOC_INDEX)) {
if ("fess.search".equals(aliasName)) {
aliasName = fessConfig.getIndexDocumentSearchIndex();
} else if ("fess.update".equals(aliasName)) {
aliasName = fessConfig.getIndexDocumentUpdateIndex();
}
} else if (index.startsWith(CONFIG_INDEX_PREFIX)) {
final String name = fessConfig.getIndexConfigIndex();
aliasName = aliasName.replaceFirst(Pattern.quote(CONFIG_INDEX_PREFIX), name);
} else if (index.startsWith(USER_INDEX_PREFIX)) {
final String name = fessConfig.getIndexUserIndex();
aliasName = aliasName.replaceFirst(Pattern.quote(USER_INDEX_PREFIX), name);
} else if (index.startsWith(LOG_INDEX_PREFIX)) {
final String name = fessConfig.getIndexLogIndex();
aliasName = aliasName.replaceFirst(Pattern.quote(LOG_INDEX_PREFIX), name);
}
String source = FileUtil.readUTF8(f);
if ("{}".equals(source.trim())) {
source = null;
@ -646,13 +676,20 @@ public class SearchEngineClient implements Client {
}
protected void sendConfigFiles(final String index) {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
configListMap.getOrDefault(index, Collections.emptyList()).forEach(path -> {
String source = null;
final String filePath = indexConfigPath + "/" + index + "/" + path;
final String dictionaryPath;
if (StringUtil.isNotBlank(fessConfig.getIndexDictionaryPrefix())) {
dictionaryPath = fessConfig.getIndexDictionaryPrefix() + "/" + path;
} else {
dictionaryPath = path;
}
try {
source = FileUtil.readUTF8(filePath);
try (CurlResponse response =
ComponentUtil.getCurlHelper().post("/_configsync/file").param("path", path).body(source).execute()) {
ComponentUtil.getCurlHelper().post("/_configsync/file").param("path", dictionaryPath).body(source).execute()) {
if (response.getHttpStatusCode() == 200) {
logger.info("Register {} to {}", path, index);
} else if (response.getContentException() != null) {
@ -703,14 +740,20 @@ public class SearchEngineClient implements Client {
}
protected String generateNewIndexName(final String configIndex) {
return configIndex + "." + new SimpleDateFormat("yyyyMMdd").format(new Date());
return configIndex + "." + new SimpleDateFormat(Constants.DOCUMENT_INDEX_SUFFIX_PATTERN).format(new Date());
}
protected void insertBulkData(final FessConfig fessConfig, final String configIndex, final String dataPath) {
try {
final BulkRequestBuilder builder = client.prepareBulk();
final ObjectMapper mapper = new ObjectMapper();
Arrays.stream(FileUtil.readUTF8(dataPath).split("\n")).reduce((prev, line) -> {
final String userIndex = fessConfig.getIndexUserIndex() + ".user";
Arrays.stream(FileUtil.readUTF8(dataPath).split("\n")).map(line -> {
return line//
.replace("\"_index\":\"fess_config.", "\"_index\":\"" + fessConfig.getIndexConfigIndex() + ".")//
.replace("\"_index\":\"fess_user.", "\"_index\":\"" + fessConfig.getIndexUserIndex() + ".")//
.replace("\"_index\":\"fess_log.", "\"_index\":\"" + fessConfig.getIndexLogIndex() + ".");
}).reduce((prev, line) -> {
try {
if (StringUtil.isBlank(prev)) {
final Map<String, Map<String, String>> result =
@ -728,7 +771,7 @@ public class SearchEngineClient implements Client {
});
if (result.containsKey("index")) {
String source = line;
if ("fess_user.user".equals(configIndex)) {
if (userIndex.equals(configIndex)) {
source = source.replace("${fess.index.initial_password}", ComponentUtil.getComponent(FessLoginAssist.class)
.encryptPassword(fessConfig.getIndexUserInitialPassword()));
}

1
src/main/java/org/codelibs/fess/helper/SuggestHelper.java
View file

@ -108,6 +108,7 @@ public class SuggestHelper {
settingsBuilder.indexTimeout(fessConfig.getIndexIndexTimeout());
settingsBuilder.indicesTimeout(fessConfig.getIndexIndicesTimeout());
settingsBuilder.searchTimeout(fessConfig.getIndexSearchTimeout());
settingsBuilder.setSettingsIndexName(fessConfig.getIndexDocumentSuggestIndex() + "_suggest");
suggester = Suggester.builder().settings(settingsBuilder).build(searchEngineClient, fessConfig.getIndexDocumentSuggestIndex());
if (ComponentUtil.hasPopularWordHelper()) {
popularWordHelper = ComponentUtil.getPopularWordHelper();

27
src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java
View file

@ -718,6 +718,9 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
/** The key of the configuration. e.g. fess_log */
String INDEX_LOG_INDEX = "index.log.index";
/** The key of the configuration. e.g. */
String INDEX_DICTIONARY_PREFIX = "index.dictionary.prefix";
/** The key of the configuration. e.g. lang,role,label,anchor,virtual_host */
String INDEX_ADMIN_ARRAY_FIELDS = "index.admin.array.fields";
@ -4067,6 +4070,21 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
*/
String getIndexLogIndex();
/**
* Get the value for the key 'index.dictionary.prefix'. <br>
* The value is, e.g. <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getIndexDictionaryPrefix();
/**
* Get the value for the key 'index.dictionary.prefix' as {@link Integer}. <br>
* The value is, e.g. <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
* @throws NumberFormatException When the property is not integer.
*/
Integer getIndexDictionaryPrefixAsInteger();
/**
* Get the value for the key 'index.admin.array.fields'. <br>
* The value is, e.g. lang,role,label,anchor,virtual_host <br>
@ -8983,6 +9001,14 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
return get(FessConfig.INDEX_LOG_INDEX);
}
public String getIndexDictionaryPrefix() {
return get(FessConfig.INDEX_DICTIONARY_PREFIX);
}
public Integer getIndexDictionaryPrefixAsInteger() {
return getAsInteger(FessConfig.INDEX_DICTIONARY_PREFIX);
}
public String getIndexAdminArrayFields() {
return get(FessConfig.INDEX_ADMIN_ARRAY_FIELDS);
}
@ -11178,6 +11204,7 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
defaultMap.put(FessConfig.INDEX_CONFIG_INDEX, "fess_config");
defaultMap.put(FessConfig.INDEX_USER_INDEX, "fess_user");
defaultMap.put(FessConfig.INDEX_LOG_INDEX, "fess_log");
defaultMap.put(FessConfig.INDEX_DICTIONARY_PREFIX, "");
defaultMap.put(FessConfig.INDEX_ADMIN_ARRAY_FIELDS, "lang,role,label,anchor,virtual_host");
defaultMap.put(FessConfig.INDEX_ADMIN_DATE_FIELDS, "expires,created,timestamp,last_modified");
defaultMap.put(FessConfig.INDEX_ADMIN_INTEGER_FIELDS, "");

1
src/main/resources/fess_config.properties
View file

@ -354,6 +354,7 @@ index.document.crawler.filter.number_of_replicas=1
index.config.index=fess_config
index.user.index=fess_user
index.log.index=fess_log
index.dictionary.prefix=
# doc management
index.admin.array.fields=lang,role,label,anchor,virtual_host