0ct0pu5/fess
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix #2624 copy data from old indices

Browse source
This commit is contained in:
Shinsuke Sugaya 2022-02-10 12:15:13 +09:00
parent e75c5b5703
commit 0a51cfb8fc
4 changed files with 125 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/main/java/org/codelibs/fess/app/web/admin/maintenance/AdminMaintenanceAction.java
View file

@ -302,7 +302,7 @@ public class AdminMaintenanceAction extends FessAdminAction {
if (searchEngineClient.createIndex(docIndex, toIndex, numberOfShards, autoExpandReplicas, resetDictionaries)) {
searchEngineClient.admin().cluster().prepareHealth(toIndex).setWaitForYellowStatus().execute(ActionListener.wrap(response -> {
searchEngineClient.addMapping(docIndex, "doc", toIndex);
if (searchEngineClient.reindex(fromIndex, toIndex, replaceAliases)
if (searchEngineClient.copyDocIndex(fromIndex, toIndex, replaceAliases)
&& (replaceAliases && !searchEngineClient.updateAlias(toIndex))) {
logger.warn("Failed to update aliases for {} and {}", fromIndex, toIndex);
}

91
src/main/java/org/codelibs/fess/app/web/admin/upgrade/AdminUpgradeAction.java
View file

@ -30,6 +30,7 @@ import org.codelibs.fess.es.config.exbhv.LabelTypeBhv;
import org.codelibs.fess.es.config.exbhv.RoleTypeBhv;
import org.codelibs.fess.es.config.exbhv.WebConfigBhv;
import org.codelibs.fess.es.user.exbhv.RoleBhv;
import org.codelibs.fess.util.ComponentUtil;
import org.codelibs.fess.util.UpgradeUtil;
import org.lastaflute.web.Execute;
import org.lastaflute.web.response.HtmlResponse;
@ -76,6 +77,8 @@ public class AdminUpgradeAction extends FessAdminAction {
private static final String VERSION_13_15 = "13.15";
private static final String VERSION_13_16 = "13.16";
// ===================================================================================
// Attribute
//
@ -158,6 +161,7 @@ public class AdminUpgradeAction extends FessAdminAction {
upgradeFrom13_13();
upgradeFrom13_14();
upgradeFrom13_15();
upgradeFrom13_16();
upgradeFromAll();
saveInfo(messages -> messages.addSuccessStartedDataUpdate(GLOBAL));
@ -184,6 +188,7 @@ public class AdminUpgradeAction extends FessAdminAction {
upgradeFrom13_13();
upgradeFrom13_14();
upgradeFrom13_15();
upgradeFrom13_16();
upgradeFromAll();
saveInfo(messages -> messages.addSuccessStartedDataUpdate(GLOBAL));
@ -209,6 +214,7 @@ public class AdminUpgradeAction extends FessAdminAction {
upgradeFrom13_13();
upgradeFrom13_14();
upgradeFrom13_15();
upgradeFrom13_16();
upgradeFromAll();
saveInfo(messages -> messages.addSuccessStartedDataUpdate(GLOBAL));
@ -233,6 +239,7 @@ public class AdminUpgradeAction extends FessAdminAction {
upgradeFrom13_13();
upgradeFrom13_14();
upgradeFrom13_15();
upgradeFrom13_16();
upgradeFromAll();
saveInfo(messages -> messages.addSuccessStartedDataUpdate(GLOBAL));
@ -256,6 +263,7 @@ public class AdminUpgradeAction extends FessAdminAction {
upgradeFrom13_13();
upgradeFrom13_14();
upgradeFrom13_15();
upgradeFrom13_16();
upgradeFromAll();
saveInfo(messages -> messages.addSuccessStartedDataUpdate(GLOBAL));
@ -278,6 +286,7 @@ public class AdminUpgradeAction extends FessAdminAction {
upgradeFrom13_13();
upgradeFrom13_14();
upgradeFrom13_15();
upgradeFrom13_16();
upgradeFromAll();
saveInfo(messages -> messages.addSuccessStartedDataUpdate(GLOBAL));
@ -299,6 +308,7 @@ public class AdminUpgradeAction extends FessAdminAction {
upgradeFrom13_13();
upgradeFrom13_14();
upgradeFrom13_15();
upgradeFrom13_16();
upgradeFromAll();
saveInfo(messages -> messages.addSuccessStartedDataUpdate(GLOBAL));
@ -319,6 +329,7 @@ public class AdminUpgradeAction extends FessAdminAction {
upgradeFrom13_13();
upgradeFrom13_14();
upgradeFrom13_15();
upgradeFrom13_16();
upgradeFromAll();
saveInfo(messages -> messages.addSuccessStartedDataUpdate(GLOBAL));
@ -338,6 +349,7 @@ public class AdminUpgradeAction extends FessAdminAction {
upgradeFrom13_13();
upgradeFrom13_14();
upgradeFrom13_15();
upgradeFrom13_16();
upgradeFromAll();
saveInfo(messages -> messages.addSuccessStartedDataUpdate(GLOBAL));
@ -356,6 +368,7 @@ public class AdminUpgradeAction extends FessAdminAction {
upgradeFrom13_13();
upgradeFrom13_14();
upgradeFrom13_15();
upgradeFrom13_16();
upgradeFromAll();
saveInfo(messages -> messages.addSuccessStartedDataUpdate(GLOBAL));
@ -373,6 +386,7 @@ public class AdminUpgradeAction extends FessAdminAction {
upgradeFrom13_13();
upgradeFrom13_14();
upgradeFrom13_15();
upgradeFrom13_16();
upgradeFromAll();
saveInfo(messages -> messages.addSuccessStartedDataUpdate(GLOBAL));
@ -389,6 +403,7 @@ public class AdminUpgradeAction extends FessAdminAction {
upgradeFrom13_13();
upgradeFrom13_14();
upgradeFrom13_15();
upgradeFrom13_16();
upgradeFromAll();
saveInfo(messages -> messages.addSuccessStartedDataUpdate(GLOBAL));
@ -404,6 +419,7 @@ public class AdminUpgradeAction extends FessAdminAction {
upgradeFrom13_13();
upgradeFrom13_14();
upgradeFrom13_15();
upgradeFrom13_16();
upgradeFromAll();
saveInfo(messages -> messages.addSuccessStartedDataUpdate(GLOBAL));
@ -418,6 +434,7 @@ public class AdminUpgradeAction extends FessAdminAction {
upgradeFrom13_13();
upgradeFrom13_14();
upgradeFrom13_15();
upgradeFrom13_16();
upgradeFromAll();
saveInfo(messages -> messages.addSuccessStartedDataUpdate(GLOBAL));
@ -431,6 +448,7 @@ public class AdminUpgradeAction extends FessAdminAction {
try {
upgradeFrom13_14();
upgradeFrom13_15();
upgradeFrom13_16();
upgradeFromAll();
saveInfo(messages -> messages.addSuccessStartedDataUpdate(GLOBAL));
@ -443,6 +461,7 @@ public class AdminUpgradeAction extends FessAdminAction {
} else if (VERSION_13_15.equals(form.targetVersion)) {
try {
upgradeFrom13_15();
upgradeFrom13_16();
upgradeFromAll();
saveInfo(messages -> messages.addSuccessStartedDataUpdate(GLOBAL));
@ -452,6 +471,18 @@ public class AdminUpgradeAction extends FessAdminAction {
logger.warn("Failed to upgrade data.", e);
saveError(messages -> messages.addErrorsFailedToUpgradeFrom(GLOBAL, VERSION_13_15, e.getLocalizedMessage()));
}
} else if (VERSION_13_16.equals(form.targetVersion)) {
try {
upgradeFrom13_16();
upgradeFromAll();
saveInfo(messages -> messages.addSuccessStartedDataUpdate(GLOBAL));
systemHelper.reloadConfiguration();
} catch (final Exception e) {
logger.warn("Failed to upgrade data.", e);
saveError(messages -> messages.addErrorsFailedToUpgradeFrom(GLOBAL, VERSION_13_16, e.getLocalizedMessage()));
}
} else {
saveError(messages -> messages.addErrorsUnknownVersionForUpgrade(GLOBAL));
}
@ -523,6 +554,66 @@ public class AdminUpgradeAction extends FessAdminAction {
// nothing
}
private void upgradeFrom13_16() {
final String[] configIndices = { //
"fess_config.access_token", //
"fess_config.bad_word", //
"fess_config.boost_document_rule", //
"fess_config.crawling_info", //
"fess_config.crawling_info_param", //
"fess_config.data_config", //
"fess_config.duplicate_host", //
"fess_config.elevate_word", //
"fess_config.elevate_word_to_label", //
"fess_config.failure_url", //
"fess_config.file_authentication", //
"fess_config.file_config", //
"fess_config.job_log", //
"fess_config.key_match", //
"fess_config.label_type", //
"fess_config.path_mapping", //
"fess_config.related_content", //
"fess_config.related_query", //
"fess_config.request_header", //
"fess_config.role_type", //
"fess_config.scheduled_job", //
"fess_config.thumbnail_queue", //
"fess_config.web_authentication", //
"fess_config.web_config", //
"fess_user.group", //
"fess_user.role", //
"fess_user.user", //
"configsync", //
};
final String[] crawlerIndices = { ".crawler.data", //
".crawler.filter", //
".crawler.queue", //
};
final SearchEngineClient client = ComponentUtil.getSearchEngineClient();
for (final String index : configIndices) {
final String oldIndex = "." + index;
if (client.existsIndex(oldIndex)) {
logger.info("Copying from {} to {}", oldIndex, index);
if (!client.reindex(oldIndex, index, false)) {
logger.warn("Failed to copy from {} to {}", oldIndex, index);
}
} else if (logger.isDebugEnabled()) {
logger.debug("{} does not exist.", oldIndex);
}
}
for (final String index : crawlerIndices) {
if (client.existsIndex(index)) {
if (client.deleteIndex(index)) {
logger.warn("Deleted {}.", index);
} else {
logger.warn("Failed to delete {}.", index);
}
} else if (logger.isDebugEnabled()) {
logger.debug("{} does not exist.", index);
}
}
}
private void upgradeFromAll() {
// nothing
}

33
src/main/java/org/codelibs/fess/es/client/SearchEngineClient.java
View file

@ -356,18 +356,35 @@ public class SearchEngineClient implements Client {
client.admin().indices().prepareExists(indexName).execute().actionGet(fessConfig.getIndexSearchTimeout());
exists = response.isExists();
} catch (final Exception e) {
// ignore
logger.debug("Failed to check {} index status.", indexName, e);
}
return exists;
}
public boolean reindex(final String fromIndex, final String toIndex, final boolean waitForCompletion) {
public boolean copyDocIndex(final String fromIndex, final String toIndex, final boolean waitForCompletion) {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final String source = fessConfig.getIndexReindexBody()//
.replace("__SOURCE_INDEX__", fromIndex)//
.replace("__SIZE__", fessConfig.getIndexReindexSize())//
.replace("__DEST_INDEX__", toIndex)//
.replace("__SCRIPT_SOURCE__", ComponentUtil.getLanguageHelper().getReindexScriptSource());
return reindex(fromIndex, toIndex, source, waitForCompletion);
}
public boolean reindex(final String fromIndex, final String toIndex, final boolean waitForCompletion) {
final String template = """
{"source":{"index":"__SOURCE_INDEX__","size":__SIZE__},"dest":{"index":"__DEST_INDEX__"}}
""";
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final String source = template //
.replace("__SOURCE_INDEX__", fromIndex)//
.replace("__SIZE__", fessConfig.getIndexReindexSize())//
.replace("__DEST_INDEX__", toIndex);
return reindex(fromIndex, toIndex, source, waitForCompletion);
}
protected boolean reindex(final String fromIndex, final String toIndex, final String source, final boolean waitForCompletion) {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final String refresh = StringUtil.isNotBlank(fessConfig.getIndexReindexRefresh()) ? fessConfig.getIndexReindexRefresh() : null;
final String requestsPerSecond = getReindexRequestsPerSecound(fessConfig);
final String scroll = StringUtil.isNotBlank(fessConfig.getIndexReindexScroll()) ? fessConfig.getIndexReindexScroll() : null;
@ -443,6 +460,18 @@ public class SearchEngineClient implements Client {
return false;
}
public boolean deleteIndex(final String indexName) {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
try {
final AcknowledgedResponse response =
client.admin().indices().prepareDelete(indexName).execute().actionGet(fessConfig.getIndexIndicesTimeout());
return response.isAcknowledged();
} catch (Exception e) {
logger.debug("Failed to delete {}.", indexName, e);
}
return false;
}
protected String readIndexSetting(final String fesenType, final String indexConfigFile, final String numberOfShards,
final String autoExpandReplicas) {
final FessConfig fessConfig = ComponentUtil.getFessConfig();

3
src/main/webapp/WEB-INF/view/admin/upgrade/admin_upgrade.jsp
View file

@ -65,7 +65,8 @@
<la:option value="13.12">13.12</la:option>
<la:option value="13.13">13.13</la:option>
<la:option value="13.14">13.14</la:option>
<la:option value="13.14">13.15</la:option>
<la:option value="13.15">13.15</la:option>
<la:option value="13.16">13.16</la:option>
</la:select>
</div>
</div>