diff --git a/src/main/java/org/codelibs/fess/app/web/admin/upgrade/AdminUpgradeAction.java b/src/main/java/org/codelibs/fess/app/web/admin/upgrade/AdminUpgradeAction.java index 599d8e18b..832dcab77 100644 --- a/src/main/java/org/codelibs/fess/app/web/admin/upgrade/AdminUpgradeAction.java +++ b/src/main/java/org/codelibs/fess/app/web/admin/upgrade/AdminUpgradeAction.java @@ -17,7 +17,6 @@ package org.codelibs.fess.app.web.admin.upgrade; import java.text.SimpleDateFormat; import java.util.Date; -import java.util.function.Consumer; import javax.annotation.Resource; @@ -36,13 +35,8 @@ import org.codelibs.fess.es.config.exbhv.RoleTypeBhv; import org.codelibs.fess.es.config.exbhv.WebConfigBhv; import org.codelibs.fess.es.config.exbhv.WebConfigToRoleBhv; import org.codelibs.fess.es.user.exbhv.RoleBhv; -import org.codelibs.fess.mylasta.direction.FessConfig; +import org.codelibs.fess.util.UpgradeUtil; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse; -import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse; -import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse; -import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetaData; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.IndicesAdminClient; import org.lastaflute.web.Execute; @@ -62,12 +56,11 @@ public class AdminUpgradeAction extends FessAdminAction { private static final String VERSION_11_1 = "11.1"; + private static final String VERSION_11_2 = "11.2"; + // =================================================================================== // Attribute // - @Resource - protected FessConfig fessConfig; - @Resource protected RoleBhv roleBhv; @@ -152,6 +145,8 @@ public class AdminUpgradeAction extends FessAdminAction { if (VERSION_11_0.equals(form.targetVersion)) { try { upgradeFrom11_0(); + upgradeFrom11_1(); + upgradeFrom11_2(); upgradeFromAll(); saveInfo(messages -> messages.addSuccessStartedDataUpdate(GLOBAL)); @@ -163,8 +158,8 @@ public class AdminUpgradeAction extends FessAdminAction { } } else if (VERSION_11_1.equals(form.targetVersion)) { try { - upgradeFrom11_0(); upgradeFrom11_1(); + upgradeFrom11_2(); upgradeFromAll(); saveInfo(messages -> messages.addSuccessStartedDataUpdate(GLOBAL)); @@ -174,6 +169,18 @@ public class AdminUpgradeAction extends FessAdminAction { logger.warn("Failed to upgrade data.", e); saveError(messages -> messages.addErrorsFailedToUpgradeFrom(GLOBAL, VERSION_11_1, e.getLocalizedMessage())); } + } else if (VERSION_11_2.equals(form.targetVersion)) { + try { + upgradeFrom11_2(); + upgradeFromAll(); + + saveInfo(messages -> messages.addSuccessStartedDataUpdate(GLOBAL)); + + systemHelper.reloadConfiguration(); + } catch (final Exception e) { + logger.warn("Failed to upgrade data.", e); + saveError(messages -> messages.addErrorsFailedToUpgradeFrom(GLOBAL, VERSION_11_2, e.getLocalizedMessage())); + } } else { saveError(messages -> messages.addErrorsUnknownVersionForUpgrade(GLOBAL)); } @@ -188,17 +195,43 @@ public class AdminUpgradeAction extends FessAdminAction { final String configIndex = ".fess_config"; // update mapping - addFieldMapping(indicesClient, configIndex, "thumbnail_queue", "thumbnail_id", + UpgradeUtil.addFieldMapping(indicesClient, configIndex, "thumbnail_queue", "thumbnail_id", "{\"properties\":{\"thumbnail_id\":{\"type\":\"keyword\"}}}"); } + private void upgradeFrom11_2() { + final IndicesAdminClient indicesClient = fessEsClient.admin().indices(); + final String configIndex = ".fess_config"; + final String searchLogIndex = "fess_log"; + + // update mapping + UpgradeUtil.addMapping(indicesClient, configIndex, "related_content", "fess_indices/.fess_config"); + UpgradeUtil.addMapping(indicesClient, configIndex, "related_query", "fess_indices/.fess_config"); + + // update mapping + if (UpgradeUtil.addFieldMapping(indicesClient, searchLogIndex, "search_log", "virtualHost", + "{\"properties\":{\"virtualHost\":{\"type\":\"keyword\"}}}")) { + UpgradeUtil.putMapping(indicesClient, searchLogIndex, "{\"dynamic_templates\": [" + + "{\"search_fields\": {\"path_match\": \"searchField.*\",\"mapping\": {\"type\": \"keyword\"}}}"// + + "]}"); + } + UpgradeUtil.addFieldMapping(indicesClient, configIndex, "web_config", "virtualHost", + "{\"properties\":{\"virtualHost\":{\"type\":\"keyword\"}}}"); + UpgradeUtil.addFieldMapping(indicesClient, configIndex, "data_config", "virtualHost", + "{\"properties\":{\"virtualHost\":{\"type\":\"keyword\"}}}"); + UpgradeUtil.addFieldMapping(indicesClient, configIndex, "file_config", "virtualHost", + "{\"properties\":{\"virtualHost\":{\"type\":\"keyword\"}}}"); + UpgradeUtil.addFieldMapping(indicesClient, configIndex, "key_match", "virtualHost", + "{\"properties\":{\"virtualHost\":{\"type\":\"keyword\"}}}"); + } + private void upgradeFromAll() { final IndicesAdminClient indicesClient = fessEsClient.admin().indices(); final String crawlerIndex = fessConfig.getIndexDocumentCrawlerIndex(); // .crawler - if (existsIndex(indicesClient, crawlerIndex, IndicesOptions.fromOptions(false, true, true, true))) { - deleteIndex(indicesClient, crawlerIndex, response -> {}); + if (UpgradeUtil.existsIndex(indicesClient, crawlerIndex, IndicesOptions.fromOptions(false, true, true, true))) { + UpgradeUtil.deleteIndex(indicesClient, crawlerIndex, response -> {}); } } @@ -222,48 +255,4 @@ public class AdminUpgradeAction extends FessAdminAction { return false; } - private void addFieldMapping(final IndicesAdminClient indicesClient, final String index, final String type, final String field, - final String source) { - final GetFieldMappingsResponse gfmResponse = - indicesClient.prepareGetFieldMappings(index).addTypes(type).setFields(field).execute().actionGet(); - final FieldMappingMetaData fieldMappings = gfmResponse.fieldMappings(index, type, field); - if (fieldMappings == null || fieldMappings.isNull()) { - try { - final PutMappingResponse pmResponse = - indicesClient.preparePutMapping(index).setType(type).setSource(source).execute().actionGet(); - if (!pmResponse.isAcknowledged()) { - logger.warn("Failed to add " + field + " to " + index + "/" + type); - } - } catch (final Exception e) { - logger.warn("Failed to add " + field + " to " + index + "/" + type, e); - } - } - } - - private boolean existsIndex(final IndicesAdminClient indicesClient, final String index, final IndicesOptions options) { - try { - final IndicesExistsResponse response = - indicesClient.prepareExists(index).setIndicesOptions(options).execute().actionGet(fessConfig.getIndexSearchTimeout()); - return response.isExists(); - } catch (final Exception e) { - // ignore - } - return false; - } - - private void deleteIndex(final IndicesAdminClient indicesClient, final String index, final Consumer comsumer) { - indicesClient.prepareDelete(index).execute(new ActionListener() { - - @Override - public void onResponse(final DeleteIndexResponse response) { - logger.info("Deleted " + index + " index."); - comsumer.accept(response); - } - - @Override - public void onFailure(final Exception e) { - logger.warn("Failed to delete " + index + " index.", e); - } - }); - } } \ No newline at end of file diff --git a/src/main/java/org/codelibs/fess/util/UpgradeUtil.java b/src/main/java/org/codelibs/fess/util/UpgradeUtil.java new file mode 100644 index 000000000..d5d8d231d --- /dev/null +++ b/src/main/java/org/codelibs/fess/util/UpgradeUtil.java @@ -0,0 +1,203 @@ +/* + * Copyright 2012-2017 CodeLibs Project and the Others. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ +package org.codelibs.fess.util; + +import java.io.File; +import java.util.function.Consumer; + +import org.codelibs.core.exception.ResourceNotFoundRuntimeException; +import org.codelibs.core.io.FileUtil; +import org.codelibs.elasticsearch.runner.net.Curl; +import org.codelibs.elasticsearch.runner.net.CurlResponse; +import org.codelibs.fess.mylasta.direction.FessConfig; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse; +import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse; +import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse; +import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetaData; +import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; +import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.client.Client; +import org.elasticsearch.client.IndicesAdminClient; +import org.elasticsearch.cluster.metadata.MappingMetaData; +import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.common.xcontent.XContentType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public final class UpgradeUtil { + private static final Logger logger = LoggerFactory.getLogger(UpgradeUtil.class); + + private UpgradeUtil() { + } + + public static boolean uploadResource(final String indexConfigPath, final String indexName, final String path) { + final String filePath = indexConfigPath + "/" + indexName + "/" + path; + try { + final String source = FileUtil.readUTF8(filePath); + try (CurlResponse response = + Curl.post(org.codelibs.fess.util.ResourceUtil.getElasticsearchHttpUrl() + "/_configsync/file").param("path", path) + .body(source).execute()) { + if (response.getHttpStatusCode() == 200) { + logger.info("Register " + path + " to " + indexName); + return true; + } else { + logger.warn("Invalid request for " + path); + } + } + } catch (final Exception e) { + logger.warn("Failed to register " + filePath, e); + } + return false; + } + + public static boolean createAlias(final IndicesAdminClient indicesClient, final String indexConfigPath, final String indexName, + final String aliasName) { + final FessConfig fessConfig = ComponentUtil.getFessConfig(); + final String aliasConfigPath = indexConfigPath + "/" + indexName + "/alias/" + aliasName + ".json"; + try { + final File aliasConfigFile = org.codelibs.core.io.ResourceUtil.getResourceAsFile(aliasConfigPath); + if (aliasConfigFile.exists()) { + final String source = FileUtil.readUTF8(aliasConfigFile); + final IndicesAliasesResponse response = + indicesClient.prepareAliases().addAlias(indexName, aliasName, source).execute() + .actionGet(fessConfig.getIndexIndicesTimeout()); + if (response.isAcknowledged()) { + logger.info("Created " + aliasName + " alias for " + indexName); + return true; + } else if (logger.isDebugEnabled()) { + logger.debug("Failed to create " + aliasName + " alias for " + indexName); + } + } + } catch (final ResourceNotFoundRuntimeException e) { + // ignore + } catch (final Exception e) { + logger.warn(aliasConfigPath + " is not found.", e); + } + return false; + } + + public static boolean addMapping(final IndicesAdminClient indicesClient, final String index, final String type, + final String indexResourcePath) { + final FessConfig fessConfig = ComponentUtil.getFessConfig(); + final GetMappingsResponse getMappingsResponse = + indicesClient.prepareGetMappings(index).execute().actionGet(fessConfig.getIndexIndicesTimeout()); + final ImmutableOpenMap indexMappings = getMappingsResponse.mappings().get(index); + if (indexMappings == null || !indexMappings.containsKey(type)) { + String source = null; + final String mappingFile = indexResourcePath + "/" + type + ".json"; + try { + source = FileUtil.readUTF8(mappingFile); + } catch (final Exception e) { + logger.warn(mappingFile + " is not found.", e); + } + try { + final PutMappingResponse putMappingResponse = + indicesClient.preparePutMapping(index).setType(type).setSource(source, XContentType.JSON).execute() + .actionGet(fessConfig.getIndexIndicesTimeout()); + if (putMappingResponse.isAcknowledged()) { + logger.info("Created " + index + "/" + type + " mapping."); + return true; + } else { + logger.warn("Failed to create " + index + "/" + type + " mapping."); + } + // TODO bulk + } catch (final Exception e) { + logger.warn("Failed to create " + index + "/" + type + " mapping.", e); + } + } + return false; + } + + public static boolean addFieldMapping(final IndicesAdminClient indicesClient, final String index, final String type, + final String field, final String source) { + final GetFieldMappingsResponse gfmResponse = + indicesClient.prepareGetFieldMappings(index).addTypes(type).setFields(field).execute().actionGet(); + final FieldMappingMetaData fieldMappings = gfmResponse.fieldMappings(index, type, field); + if (fieldMappings == null || fieldMappings.isNull()) { + try { + final PutMappingResponse pmResponse = + indicesClient.preparePutMapping(index).setType(type).setSource(source, XContentType.JSON).execute().actionGet(); + if (!pmResponse.isAcknowledged()) { + logger.warn("Failed to add " + field + " to " + index + "/" + type); + } else { + return true; + } + } catch (final Exception e) { + logger.warn("Failed to add " + field + " to " + index + "/" + type, e); + } + } + return false; + } + + public static boolean putMapping(final IndicesAdminClient indicesClient, final String index, final String source) { + try { + final PutMappingResponse pmResponse = + indicesClient.preparePutMapping(index).setSource(source, XContentType.JSON).execute().actionGet(); + if (!pmResponse.isAcknowledged()) { + logger.warn("Failed to update " + index + " settings."); + } else { + return true; + } + } catch (final Exception e) { + logger.warn("Failed to update " + index + " settings.", e); + } + + return false; + } + + public static boolean addData(final Client fessEsClient, final String index, final String type, final String id, final String source) { + try { + final IndexRequest indexRequest = new IndexRequest(index, type, id).source(source, XContentType.JSON); + fessEsClient.index(indexRequest).actionGet(); + return true; + } catch (final Exception e) { + logger.warn("Failed to add " + id + " to " + index + "/" + type, e); + } + return false; + } + + public static boolean existsIndex(final IndicesAdminClient indicesClient, final String index, final IndicesOptions options) { + final FessConfig fessConfig = ComponentUtil.getFessConfig(); + try { + final IndicesExistsResponse response = + indicesClient.prepareExists(index).setIndicesOptions(options).execute().actionGet(fessConfig.getIndexSearchTimeout()); + return response.isExists(); + } catch (final Exception e) { + // ignore + } + return false; + } + + public static void deleteIndex(final IndicesAdminClient indicesClient, final String index, final Consumer comsumer) { + indicesClient.prepareDelete(index).execute(new ActionListener() { + + @Override + public void onResponse(final DeleteIndexResponse response) { + logger.info("Deleted " + index + " index."); + comsumer.accept(response); + } + + @Override + public void onFailure(final Exception e) { + logger.warn("Failed to delete " + index + " index.", e); + } + }); + } +} diff --git a/src/main/webapp/WEB-INF/view/admin/upgrade/admin_upgrade.jsp b/src/main/webapp/WEB-INF/view/admin/upgrade/admin_upgrade.jsp index 750b9f693..7d6f5305b 100644 --- a/src/main/webapp/WEB-INF/view/admin/upgrade/admin_upgrade.jsp +++ b/src/main/webapp/WEB-INF/view/admin/upgrade/admin_upgrade.jsp @@ -46,6 +46,7 @@ 11.0 11.1 + 11.2