0ct0pu5/fess
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix #1185 update upgrade page

Browse source
This commit is contained in:
Shinsuke Sugaya 2017-07-27 21:54:30 +09:00
parent c079ca8210
commit acde55fe09
3 changed files with 251 additions and 58 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

105
src/main/java/org/codelibs/fess/app/web/admin/upgrade/AdminUpgradeAction.java
View file

@ -17,7 +17,6 @@ package org.codelibs.fess.app.web.admin.upgrade;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.function.Consumer;
import javax.annotation.Resource;
@ -36,13 +35,8 @@ import org.codelibs.fess.es.config.exbhv.RoleTypeBhv;
import org.codelibs.fess.es.config.exbhv.WebConfigBhv;
import org.codelibs.fess.es.config.exbhv.WebConfigToRoleBhv;
import org.codelibs.fess.es.user.exbhv.RoleBhv;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.util.UpgradeUtil;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse;
import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetaData;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
import org.elasticsearch.action.support.IndicesOptions;
import org.elasticsearch.client.IndicesAdminClient;
import org.lastaflute.web.Execute;
@ -62,12 +56,11 @@ public class AdminUpgradeAction extends FessAdminAction {
private static final String VERSION_11_1 = "11.1";
private static final String VERSION_11_2 = "11.2";
// ===================================================================================
// Attribute
//
@Resource
protected FessConfig fessConfig;
@Resource
protected RoleBhv roleBhv;
@ -152,6 +145,8 @@ public class AdminUpgradeAction extends FessAdminAction {
if (VERSION_11_0.equals(form.targetVersion)) {
try {
upgradeFrom11_0();
upgradeFrom11_1();
upgradeFrom11_2();
upgradeFromAll();
saveInfo(messages -> messages.addSuccessStartedDataUpdate(GLOBAL));
@ -163,8 +158,8 @@ public class AdminUpgradeAction extends FessAdminAction {
}
} else if (VERSION_11_1.equals(form.targetVersion)) {
try {
upgradeFrom11_0();
upgradeFrom11_1();
upgradeFrom11_2();
upgradeFromAll();
saveInfo(messages -> messages.addSuccessStartedDataUpdate(GLOBAL));
@ -174,6 +169,18 @@ public class AdminUpgradeAction extends FessAdminAction {
logger.warn("Failed to upgrade data.", e);
saveError(messages -> messages.addErrorsFailedToUpgradeFrom(GLOBAL, VERSION_11_1, e.getLocalizedMessage()));
}
} else if (VERSION_11_2.equals(form.targetVersion)) {
try {
upgradeFrom11_2();
upgradeFromAll();
saveInfo(messages -> messages.addSuccessStartedDataUpdate(GLOBAL));
systemHelper.reloadConfiguration();
} catch (final Exception e) {
logger.warn("Failed to upgrade data.", e);
saveError(messages -> messages.addErrorsFailedToUpgradeFrom(GLOBAL, VERSION_11_2, e.getLocalizedMessage()));
}
} else {
saveError(messages -> messages.addErrorsUnknownVersionForUpgrade(GLOBAL));
}
@ -188,17 +195,43 @@ public class AdminUpgradeAction extends FessAdminAction {
final String configIndex = ".fess_config";
// update mapping
addFieldMapping(indicesClient, configIndex, "thumbnail_queue", "thumbnail_id",
UpgradeUtil.addFieldMapping(indicesClient, configIndex, "thumbnail_queue", "thumbnail_id",
"{\"properties\":{\"thumbnail_id\":{\"type\":\"keyword\"}}}");
}
private void upgradeFrom11_2() {
final IndicesAdminClient indicesClient = fessEsClient.admin().indices();
final String configIndex = ".fess_config";
final String searchLogIndex = "fess_log";
// update mapping
UpgradeUtil.addMapping(indicesClient, configIndex, "related_content", "fess_indices/.fess_config");
UpgradeUtil.addMapping(indicesClient, configIndex, "related_query", "fess_indices/.fess_config");
// update mapping
if (UpgradeUtil.addFieldMapping(indicesClient, searchLogIndex, "search_log", "virtualHost",
"{\"properties\":{\"virtualHost\":{\"type\":\"keyword\"}}}")) {
UpgradeUtil.putMapping(indicesClient, searchLogIndex, "{\"dynamic_templates\": ["
+ "{\"search_fields\": {\"path_match\": \"searchField.*\",\"mapping\": {\"type\": \"keyword\"}}}"//
+ "]}");
}
UpgradeUtil.addFieldMapping(indicesClient, configIndex, "web_config", "virtualHost",
"{\"properties\":{\"virtualHost\":{\"type\":\"keyword\"}}}");
UpgradeUtil.addFieldMapping(indicesClient, configIndex, "data_config", "virtualHost",
"{\"properties\":{\"virtualHost\":{\"type\":\"keyword\"}}}");
UpgradeUtil.addFieldMapping(indicesClient, configIndex, "file_config", "virtualHost",
"{\"properties\":{\"virtualHost\":{\"type\":\"keyword\"}}}");
UpgradeUtil.addFieldMapping(indicesClient, configIndex, "key_match", "virtualHost",
"{\"properties\":{\"virtualHost\":{\"type\":\"keyword\"}}}");
}
private void upgradeFromAll() {
final IndicesAdminClient indicesClient = fessEsClient.admin().indices();
final String crawlerIndex = fessConfig.getIndexDocumentCrawlerIndex();
// .crawler
if (existsIndex(indicesClient, crawlerIndex, IndicesOptions.fromOptions(false, true, true, true))) {
deleteIndex(indicesClient, crawlerIndex, response -> {});
if (UpgradeUtil.existsIndex(indicesClient, crawlerIndex, IndicesOptions.fromOptions(false, true, true, true))) {
UpgradeUtil.deleteIndex(indicesClient, crawlerIndex, response -> {});
}
}
@ -222,48 +255,4 @@ public class AdminUpgradeAction extends FessAdminAction {
return false;
}
private void addFieldMapping(final IndicesAdminClient indicesClient, final String index, final String type, final String field,
final String source) {
final GetFieldMappingsResponse gfmResponse =
indicesClient.prepareGetFieldMappings(index).addTypes(type).setFields(field).execute().actionGet();
final FieldMappingMetaData fieldMappings = gfmResponse.fieldMappings(index, type, field);
if (fieldMappings == null || fieldMappings.isNull()) {
try {
final PutMappingResponse pmResponse =
indicesClient.preparePutMapping(index).setType(type).setSource(source).execute().actionGet();
if (!pmResponse.isAcknowledged()) {
logger.warn("Failed to add " + field + " to " + index + "/" + type);
}
} catch (final Exception e) {
logger.warn("Failed to add " + field + " to " + index + "/" + type, e);
}
}
}
private boolean existsIndex(final IndicesAdminClient indicesClient, final String index, final IndicesOptions options) {
try {
final IndicesExistsResponse response =
indicesClient.prepareExists(index).setIndicesOptions(options).execute().actionGet(fessConfig.getIndexSearchTimeout());
return response.isExists();
} catch (final Exception e) {
// ignore
}
return false;
}
private void deleteIndex(final IndicesAdminClient indicesClient, final String index, final Consumer<DeleteIndexResponse> comsumer) {
indicesClient.prepareDelete(index).execute(new ActionListener<DeleteIndexResponse>() {
@Override
public void onResponse(final DeleteIndexResponse response) {
logger.info("Deleted " + index + " index.");
comsumer.accept(response);
}
@Override
public void onFailure(final Exception e) {
logger.warn("Failed to delete " + index + " index.", e);
}
});
}
}

203
src/main/java/org/codelibs/fess/util/UpgradeUtil.java Normal file
View file

@ -0,0 +1,203 @@
/*
* Copyright 2012-2017 CodeLibs Project and the Others.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
* either express or implied. See the License for the specific language
* governing permissions and limitations under the License.
*/
package org.codelibs.fess.util;
import java.io.File;
import java.util.function.Consumer;
import org.codelibs.core.exception.ResourceNotFoundRuntimeException;
import org.codelibs.core.io.FileUtil;
import org.codelibs.elasticsearch.runner.net.Curl;
import org.codelibs.elasticsearch.runner.net.CurlResponse;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse;
import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetaData;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.support.IndicesOptions;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.IndicesAdminClient;
import org.elasticsearch.cluster.metadata.MappingMetaData;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.xcontent.XContentType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public final class UpgradeUtil {
private static final Logger logger = LoggerFactory.getLogger(UpgradeUtil.class);
private UpgradeUtil() {
}
public static boolean uploadResource(final String indexConfigPath, final String indexName, final String path) {
final String filePath = indexConfigPath + "/" + indexName + "/" + path;
try {
final String source = FileUtil.readUTF8(filePath);
try (CurlResponse response =
Curl.post(org.codelibs.fess.util.ResourceUtil.getElasticsearchHttpUrl() + "/_configsync/file").param("path", path)
.body(source).execute()) {
if (response.getHttpStatusCode() == 200) {
logger.info("Register " + path + " to " + indexName);
return true;
} else {
logger.warn("Invalid request for " + path);
}
}
} catch (final Exception e) {
logger.warn("Failed to register " + filePath, e);
}
return false;
}
public static boolean createAlias(final IndicesAdminClient indicesClient, final String indexConfigPath, final String indexName,
final String aliasName) {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final String aliasConfigPath = indexConfigPath + "/" + indexName + "/alias/" + aliasName + ".json";
try {
final File aliasConfigFile = org.codelibs.core.io.ResourceUtil.getResourceAsFile(aliasConfigPath);
if (aliasConfigFile.exists()) {
final String source = FileUtil.readUTF8(aliasConfigFile);
final IndicesAliasesResponse response =
indicesClient.prepareAliases().addAlias(indexName, aliasName, source).execute()
.actionGet(fessConfig.getIndexIndicesTimeout());
if (response.isAcknowledged()) {
logger.info("Created " + aliasName + " alias for " + indexName);
return true;
} else if (logger.isDebugEnabled()) {
logger.debug("Failed to create " + aliasName + " alias for " + indexName);
}
}
} catch (final ResourceNotFoundRuntimeException e) {
// ignore
} catch (final Exception e) {
logger.warn(aliasConfigPath + " is not found.", e);
}
return false;
}
public static boolean addMapping(final IndicesAdminClient indicesClient, final String index, final String type,
final String indexResourcePath) {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final GetMappingsResponse getMappingsResponse =
indicesClient.prepareGetMappings(index).execute().actionGet(fessConfig.getIndexIndicesTimeout());
final ImmutableOpenMap<String, MappingMetaData> indexMappings = getMappingsResponse.mappings().get(index);
if (indexMappings == null || !indexMappings.containsKey(type)) {
String source = null;
final String mappingFile = indexResourcePath + "/" + type + ".json";
try {
source = FileUtil.readUTF8(mappingFile);
} catch (final Exception e) {
logger.warn(mappingFile + " is not found.", e);
}
try {
final PutMappingResponse putMappingResponse =
indicesClient.preparePutMapping(index).setType(type).setSource(source, XContentType.JSON).execute()
.actionGet(fessConfig.getIndexIndicesTimeout());
if (putMappingResponse.isAcknowledged()) {
logger.info("Created " + index + "/" + type + " mapping.");
return true;
} else {
logger.warn("Failed to create " + index + "/" + type + " mapping.");
}
// TODO bulk
} catch (final Exception e) {
logger.warn("Failed to create " + index + "/" + type + " mapping.", e);
}
}
return false;
}
public static boolean addFieldMapping(final IndicesAdminClient indicesClient, final String index, final String type,
final String field, final String source) {
final GetFieldMappingsResponse gfmResponse =
indicesClient.prepareGetFieldMappings(index).addTypes(type).setFields(field).execute().actionGet();
final FieldMappingMetaData fieldMappings = gfmResponse.fieldMappings(index, type, field);
if (fieldMappings == null || fieldMappings.isNull()) {
try {
final PutMappingResponse pmResponse =
indicesClient.preparePutMapping(index).setType(type).setSource(source, XContentType.JSON).execute().actionGet();
if (!pmResponse.isAcknowledged()) {
logger.warn("Failed to add " + field + " to " + index + "/" + type);
} else {
return true;
}
} catch (final Exception e) {
logger.warn("Failed to add " + field + " to " + index + "/" + type, e);
}
}
return false;
}
public static boolean putMapping(final IndicesAdminClient indicesClient, final String index, final String source) {
try {
final PutMappingResponse pmResponse =
indicesClient.preparePutMapping(index).setSource(source, XContentType.JSON).execute().actionGet();
if (!pmResponse.isAcknowledged()) {
logger.warn("Failed to update " + index + " settings.");
} else {
return true;
}
} catch (final Exception e) {
logger.warn("Failed to update " + index + " settings.", e);
}
return false;
}
public static boolean addData(final Client fessEsClient, final String index, final String type, final String id, final String source) {
try {
final IndexRequest indexRequest = new IndexRequest(index, type, id).source(source, XContentType.JSON);
fessEsClient.index(indexRequest).actionGet();
return true;
} catch (final Exception e) {
logger.warn("Failed to add " + id + " to " + index + "/" + type, e);
}
return false;
}
public static boolean existsIndex(final IndicesAdminClient indicesClient, final String index, final IndicesOptions options) {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
try {
final IndicesExistsResponse response =
indicesClient.prepareExists(index).setIndicesOptions(options).execute().actionGet(fessConfig.getIndexSearchTimeout());
return response.isExists();
} catch (final Exception e) {
// ignore
}
return false;
}
public static void deleteIndex(final IndicesAdminClient indicesClient, final String index, final Consumer<DeleteIndexResponse> comsumer) {
indicesClient.prepareDelete(index).execute(new ActionListener<DeleteIndexResponse>() {
@Override
public void onResponse(final DeleteIndexResponse response) {
logger.info("Deleted " + index + " index.");
comsumer.accept(response);
}
@Override
public void onFailure(final Exception e) {
logger.warn("Failed to delete " + index + " index.", e);
}
});
}
}

1
src/main/webapp/WEB-INF/view/admin/upgrade/admin_upgrade.jsp
View file

@ -46,6 +46,7 @@
<la:select styleId="targetVersion" property="targetVersion" styleClass="form-control">
<la:option value="11.0">11.0</la:option>
<la:option value="11.1">11.1</la:option>
<la:option value="11.2">11.2</la:option>
</la:select>
</div>
</div>