0ct0pu5/fess
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add Admin API: /api/admin/storage #2333 (#2337)

Browse source
This commit is contained in:
Koki Igarashi 2019-12-19 05:20:39 +09:00 • committed by Shinsuke Sugaya
parent bcb5612f06
commit f05d1c360e
6 changed files with 324 additions and 53 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

130
src/main/java/org/codelibs/fess/app/web/admin/storage/AdminStorageAction.java
View file

@ -34,14 +34,17 @@ import org.codelibs.fess.exception.StorageException;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.util.ComponentUtil;
import org.codelibs.fess.util.RenderDataUtil;
import org.dbflute.optional.OptionalThing;
import org.lastaflute.web.Execute;
import org.lastaflute.web.response.ActionResponse;
import org.lastaflute.web.response.HtmlResponse;
import org.lastaflute.web.ruts.multipart.MultipartFormFile;
import org.lastaflute.web.ruts.process.ActionRuntime;
import io.minio.MinioClient;
import io.minio.Result;
import io.minio.messages.Item;
import org.lastaflute.web.servlet.request.stream.WrittenStreamOut;
/**
* @author shinsuke
@ -67,39 +70,36 @@ public class AdminStorageAction extends FessAdminAction {
// public HtmlResponse create() {
// }
@Execute
public ActionResponse list(final OptionalThing<String> id) {
saveToken();
if (id.isPresent() && id.get() != null) {
return asListHtml(decodePath(id.get()));
}
return redirect(getClass());
}
@Execute
public HtmlResponse upload(final ItemForm form) {
validate(form, messages -> {}, () -> asListHtml(form.path));
if (form.uploadFile == null) {
throwValidationError(messages -> messages.addErrorsStorageNoUploadFile(GLOBAL), () -> asListHtml(form.path));
}
logger.debug("form.path = {}", form.path);
verifyToken(() -> asListHtml(form.path));
final String objectName = getObjectName(form.path, form.uploadFile.getFileName());
try (final InputStream in = form.uploadFile.getInputStream()) {
final MinioClient minioClient = createClient(fessConfig);
minioClient.putObject(fessConfig.getStorageBucket(), objectName, in, (long) form.uploadFile.getFileSize(), null, null,
"application/octet-stream");
} catch (final Exception e) {
try {
uploadObject(getObjectName(form.path, form.uploadFile.getFileName()), form.uploadFile);
} catch (final StorageException e) {
if (logger.isDebugEnabled()) {
logger.debug("Failed to upload {}", objectName, e);
logger.debug("Failed to upload {}", form.uploadFile.getFileName(), e);
}
throwValidationError(messages -> messages.addErrorsStorageFileUploadFailure(GLOBAL, e.getLocalizedMessage()),
() -> asListHtml(form.path));
throwValidationError(messages -> messages.addErrorsStorageFileUploadFailure(GLOBAL, form.uploadFile.getFileName()),
() -> asListHtml(encodeId(form.path)));
}
saveInfo(messages -> messages.addSuccessUploadFileToStorage(GLOBAL, form.uploadFile.getFileName()));
if (StringUtil.isEmpty(form.path)) {
return redirect(getClass());
}
return redirectWith(getClass(), moreUrl("list/" + encodeId(form.path)));
}
@Execute
public ActionResponse list(final String id) {
saveToken();
return asListHtml(decodePath(id));
}
@Execute
public ActionResponse download(final String id) {
final String[] values = decodeId(id);
@ -108,13 +108,13 @@ public class AdminStorageAction extends FessAdminAction {
}
return asStream(values[1]).contentTypeOctetStream().stream(
out -> {
try (InputStream in = createClient(fessConfig).getObject(fessConfig.getStorageBucket(), values[0] + values[1])) {
out.write(in);
} catch (final Exception e) {
try {
downloadObject(getObjectName(values[0], values[1]), out);
} catch (final StorageException e) {
if (logger.isDebugEnabled()) {
logger.debug("Failed to access {}", fessConfig.getStorageEndpoint(), e);
logger.debug("Failed to download {}", values[1], e);
}
throwValidationError(messages -> messages.addErrorsStorageAccessError(GLOBAL, e.getLocalizedMessage()),
throwValidationError(messages -> messages.addErrorsStorageFileDownloadFailure(GLOBAL, values[1]),
() -> asListHtml(encodeId(values[0])));
}
});
@ -126,20 +126,14 @@ public class AdminStorageAction extends FessAdminAction {
if (StringUtil.isEmpty(values[1])) {
throwValidationError(messages -> messages.addErrorsStorageFileNotFound(GLOBAL), () -> asListHtml(encodeId(values[0])));
}
logger.debug("values[0] = {}, values[1] = {}", values[0], values[1]);
final String objectName = getObjectName(values[0], values[1]);
try {
final MinioClient minioClient = createClient(fessConfig);
minioClient.removeObject(fessConfig.getStorageBucket(), objectName);
} catch (final Exception e) {
deleteObject(objectName);
} catch (final StorageException e) {
logger.debug("Failed to delete {}", values[1], e);
throwValidationError(messages -> messages.addErrorsFailedToDeleteFile(GLOBAL, e.getLocalizedMessage()),
() -> asListHtml(encodeId(values[0])));
throwValidationError(messages -> messages.addErrorsFailedToDeleteFile(GLOBAL, values[1]), () -> asListHtml(encodeId(values[0])));
}
saveInfo(messages -> messages.addSuccessDeleteFile(GLOBAL, values[1]));
if (StringUtil.isEmpty(values[0])) {
return redirect(getClass());
}
return redirectWith(getClass(), moreUrl("list/" + encodeId(values[0])));
}
@ -152,6 +146,44 @@ public class AdminStorageAction extends FessAdminAction {
return redirectWith(getClass(), moreUrl("list/" + encodeId(getObjectName(form.path, form.name))));
}
public static void uploadObject(final String objectName, final MultipartFormFile uploadFile) {
try (final InputStream in = uploadFile.getInputStream()) {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final MinioClient minioClient = createClient(fessConfig);
minioClient.putObject(fessConfig.getStorageBucket(), objectName, in, (long) uploadFile.getFileSize(), null, null,
"application/octet-stream");
} catch (final Exception e) {
throw new StorageException("Failed to upload " + objectName, e);
}
}
public static void downloadObject(final String objectName, final WrittenStreamOut out) {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
try (InputStream in = createClient(fessConfig).getObject(fessConfig.getStorageBucket(), objectName)) {
out.write(in);
} catch (final Exception e) {
throw new StorageException("Failed to download " + objectName, e);
}
}
public static void deleteObject(final String objectName) {
try {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final MinioClient minioClient = createClient(fessConfig);
minioClient.removeObject(fessConfig.getStorageBucket(), objectName);
} catch (final Exception e) {
throw new StorageException("Failed to delete " + objectName, e);
}
}
protected static MinioClient createClient(final FessConfig fessConfig) {
try {
return new MinioClient(fessConfig.getStorageEndpoint(), fessConfig.getStorageAccessKey(), fessConfig.getStorageSecretKey());
} catch (final Exception e) {
throw new StorageException("Failed to create MinioClient: " + fessConfig.getStorageEndpoint(), e);
}
}
public static List<Map<String, Object>> getFileItems(final String prefix) {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final ArrayList<Map<String, Object>> list = new ArrayList<>();
@ -162,7 +194,7 @@ public class AdminStorageAction extends FessAdminAction {
final Map<String, Object> map = new HashMap<>();
final Item item = result.get();
final String objectName = item.objectName();
map.put("id", URLEncoder.encode(objectName, Constants.UTF_8_CHARSET));
map.put("id", encodeId(objectName));
map.put("name", getName(objectName));
map.put("hashCode", item.hashCode());
map.put("size", item.objectSize());
@ -191,16 +223,7 @@ public class AdminStorageAction extends FessAdminAction {
return values[values.length - 1];
}
protected static MinioClient createClient(final FessConfig fessConfig) {
try {
return new MinioClient(fessConfig.getStorageEndpoint(), fessConfig.getStorageAccessKey(), fessConfig.getStorageSecretKey());
} catch (final Exception e) {
throw new StorageException("Failed to create MinioClient: " + fessConfig.getStorageEndpoint(), e);
}
}
protected static String decodePath(final String id) {
public static String decodePath(final String id) {
final String[] values = decodeId(id);
if (StringUtil.isEmpty(values[0]) && StringUtil.isEmpty(values[1])) {
return StringUtil.EMPTY;
@ -211,8 +234,8 @@ public class AdminStorageAction extends FessAdminAction {
}
}
protected static String[] decodeId(final String id) {
final String value = URLDecoder.decode(id, Constants.UTF_8_CHARSET);
public static String[] decodeId(final String id) {
final String value = urlDecode(urlDecode(id));
final String[] values = split(value, "/").get(stream -> stream.filter(StringUtil::isNotEmpty).toArray(n -> new String[n]));
if (values.length == 0) {
// invalid?
@ -269,19 +292,26 @@ public class AdminStorageAction extends FessAdminAction {
return StringUtil.isEmpty(path) ? StringUtil.EMPTY : path + "/";
}
protected static String getObjectName(final String path, final String name) {
public static String getObjectName(final String path, final String name) {
return getPathPrefix(path) + name;
}
protected static String urlEncode(final String str) {
if (str == null) {
return null;
return StringUtil.EMPTY;
}
return URLEncoder.encode(str, Constants.UTF_8_CHARSET);
}
protected static String encodeId(final String str) {
return urlEncode(urlEncode(str));
protected static String urlDecode(final String str) {
if (str == null) {
return StringUtil.EMPTY;
}
return URLDecoder.decode(str, Constants.UTF_8_CHARSET);
}
protected static String encodeId(final String objectName) {
return urlEncode(urlEncode(objectName));
}
private HtmlResponse asListHtml(final String prefix) {

15
src/main/java/org/codelibs/fess/app/web/api/ApiResult.java
View file

@ -361,4 +361,19 @@ public class ApiResult {
return new ApiResult(this);
}
}
public static class ApiStorageResponse extends ApiResponse {
protected List<Map<String, Object>> items;
public ApiStorageResponse items(final List<Map<String, Object>> items) {
this.items = items;
return this;
}
@Override
public ApiResult result() {
return new ApiResult(this);
}
}
}

113
src/main/java/org/codelibs/fess/app/web/api/admin/storage/ApiAdminStorageAction.java Normal file
View file

@ -0,0 +1,113 @@
/*
* Copyright 2012-2019 CodeLibs Project and the Others.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
* either express or implied. See the License for the specific language
* governing permissions and limitations under the License.
*/
package org.codelibs.fess.app.web.api.admin.storage;
import static org.codelibs.fess.app.web.admin.storage.AdminStorageAction.decodeId;
import static org.codelibs.fess.app.web.admin.storage.AdminStorageAction.decodePath;
import static org.codelibs.fess.app.web.admin.storage.AdminStorageAction.getFileItems;
import static org.codelibs.fess.app.web.admin.storage.AdminStorageAction.deleteObject;
import static org.codelibs.fess.app.web.admin.storage.AdminStorageAction.downloadObject;
import static org.codelibs.fess.app.web.admin.storage.AdminStorageAction.getObjectName;
import static org.codelibs.fess.app.web.admin.storage.AdminStorageAction.uploadObject;
import java.util.List;
import java.util.Map;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.fess.app.web.api.ApiResult;
import org.codelibs.fess.app.web.api.admin.FessApiAdminAction;
import org.codelibs.fess.exception.ResultOffsetExceededException;
import org.codelibs.fess.exception.StorageException;
import org.dbflute.optional.OptionalThing;
import org.lastaflute.web.Execute;
import org.lastaflute.web.response.JsonResponse;
import org.lastaflute.web.response.StreamResponse;
public class ApiAdminStorageAction extends FessApiAdminAction {
private static final Logger logger = LogManager.getLogger(ApiAdminStorageAction.class);
// GET /api/admin/storage/list/{id}
// POST /api/admin/storage/list/{id}
@Execute
public JsonResponse<ApiResult> list(final OptionalThing<String> id) {
final List<Map<String, Object>> list = getFileItems(id.isPresent() ? decodePath(id.get()) : null);
try {
return asJson(new ApiResult.ApiStorageResponse().items(list).status(ApiResult.Status.OK).result());
} catch (final ResultOffsetExceededException e) {
if (logger.isDebugEnabled()) {
logger.debug(e.getMessage(), e);
}
throwValidationErrorApi(messages -> messages.addErrorsResultSizeExceeded(GLOBAL));
}
return null;
}
// GET /api/admin/storage/download/{id}/
@Execute
public StreamResponse get$download(final String id) {
final String[] values = decodeId(id);
if (StringUtil.isEmpty(values[1])) {
throwValidationErrorApi(messages -> messages.addErrorsStorageFileNotFound(GLOBAL));
}
return asStream(values[1]).contentTypeOctetStream().stream(out -> {
try {
downloadObject(getObjectName(values[0], values[1]), out);
} catch (final StorageException e) {
throwValidationErrorApi(messages -> messages.addErrorsStorageFileDownloadFailure(GLOBAL, values[1]));
}
});
}
// DELETE /api/admin/storage/delete/{id}/
@Execute
public JsonResponse<ApiResult> delete$delete(final String id) {
final String[] values = decodeId(id);
if (StringUtil.isEmpty(values[1])) {
throwValidationErrorApi(messages -> messages.addErrorsStorageAccessError(GLOBAL, "id is invalid"));
}
final String objectName = getObjectName(values[0], values[1]);
try {
deleteObject(objectName);
saveInfo(messages -> messages.addSuccessDeleteFile(GLOBAL, values[1]));
return asJson(new ApiResult.ApiResponse().status(ApiResult.Status.OK).result());
} catch (final StorageException e) {
throwValidationErrorApi(messages -> messages.addErrorsFailedToDeleteFile(GLOBAL, values[1]));
}
return null;
}
// POST /api/admin/storage/upload/{pathId}/
@Execute
public JsonResponse<ApiResult> post$upload(final String pathId, final UploadForm form) {
validateApi(form, messages -> {});
if (form.uploadFile == null) {
throwValidationErrorApi(messages -> messages.addErrorsStorageNoUploadFile(GLOBAL));
}
try {
uploadObject(getObjectName(decodeId(pathId)[0], form.uploadFile.getFileName()), form.uploadFile);
saveInfo(messages -> messages.addSuccessUploadFileToStorage(GLOBAL, form.uploadFile.getFileName()));
return asJson(new ApiResult.ApiResponse().status(ApiResult.Status.OK).result());
} catch (final StorageException e) {
throwValidationErrorApi(messages -> messages.addErrorsStorageFileUploadFailure(GLOBAL, form.uploadFile.getFileName()));
}
return null;
}
}

27
src/main/java/org/codelibs/fess/app/web/api/admin/storage/UploadForm.java Normal file
View file

@ -0,0 +1,27 @@
/*
* Copyright 2012-2019 CodeLibs Project and the Others.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
* either express or implied. See the License for the specific language
* governing permissions and limitations under the License.
*/
package org.codelibs.fess.app.web.api.admin.storage;
import org.codelibs.fess.app.web.admin.storage.ItemForm;
import org.lastaflute.web.ruts.multipart.MultipartFormFile;
import org.lastaflute.web.validation.Required;
public class UploadForm extends ItemForm {
@Required
public MultipartFormFile uploadFile;
}

6
src/main/webapp/WEB-INF/view/admin/storage/admin_storage.jsp
View file

@ -151,7 +151,7 @@
</c:if>
<c:if test="${data.directory.booleanValue()}">
<tr
data-href="${contextPath}/admin/storage/list/${f:u(data.id)}/">
data-href="${contextPath}/admin/storage/list/${f:h(data.id)}/">
<td>
<em class="fa fa-folder-open" style="color:#F7C502;"></em>
${f:h(data.name)}
@ -162,7 +162,7 @@
<td>
<c:if test="${not data.directory}">
<a class="btn btn-primary btn-xs" role="button" name="download" data-toggle="modal"
href="${contextPath}/admin/storage/download/${f:u(data.id)}/" download="${f:u(data.name)}"
href="${contextPath}/admin/storage/download/${f:h(data.id)}/" download="${f:u(data.name)}"
value="<la:message key="labels.design_download_button" />"
>
<em class="fa fa-download"></em>
@ -197,7 +197,7 @@
<button type="button" class="btn btn-outline pull-left" data-dismiss="modal">
<la:message key="labels.crud_button_cancel" />
</button>
<la:form action="${contextPath}/admin/storage/delete/${f:u(data.id)}/" styleClass="form-horizontal">
<la:form action="${contextPath}/admin/storage/delete/${f:h(data.id)}/" styleClass="form-horizontal">
<button type="submit" class="btn btn-outline btn-danger" name="delete"
value="<la:message key="labels.crud_button_delete" />"
>

86
src/test/java/org/codelibs/fess/it/admin/StorageTests.java Normal file
View file

@ -0,0 +1,86 @@
/*
* Copyright 2012-2019 CodeLibs Project and the Others.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
* either express or implied. See the License for the specific language
* governing permissions and limitations under the License.
*/
package org.codelibs.fess.it.admin;
import static org.hamcrest.Matchers.equalTo;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import org.codelibs.fess.it.CrudTestBase;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
@Tag("it")
public class StorageTests extends CrudTestBase {
private static final String NAME_PREFIX = "storageTests_";
private static final String API_PATH = "/api/admin/storage";
private static final String LIST_ENDPOINT_SUFFIX = "list";
private static final String ITEM_ENDPOINT_SUFFIX = "";
private static final String KEY_PROPERTY = "";
@Override
protected String getNamePrefix() {
return NAME_PREFIX;
}
@Override
protected String getApiPath() {
return API_PATH;
}
@Override
protected String getKeyProperty() {
return KEY_PROPERTY;
}
@Override
protected String getListEndpointSuffix() {
return LIST_ENDPOINT_SUFFIX;
}
@Override
protected String getItemEndpointSuffix() {
return ITEM_ENDPOINT_SUFFIX;
}
@Override
protected Map<String, Object> createTestParam(int id) {
final Map<String, Object> requestBody = new HashMap<>();
return requestBody;
}
@Override
protected Map<String, Object> getUpdateMap() {
final Map<String, Object> updateMap = new HashMap<>();
return updateMap;
}
@AfterEach
protected void tearDown() {
// do nothing
}
@Test
void testList_ok() {
checkGetMethod(Collections.emptyMap(), getListEndpointSuffix() + "/").then().body("response.status", equalTo(0));
}
}