diff --git a/src/main/java/org/codelibs/fess/app/web/admin/backup/AdminBackupAction.java b/src/main/java/org/codelibs/fess/app/web/admin/backup/AdminBackupAction.java index 08211e870..ad2726ec9 100644 --- a/src/main/java/org/codelibs/fess/app/web/admin/backup/AdminBackupAction.java +++ b/src/main/java/org/codelibs/fess/app/web/admin/backup/AdminBackupAction.java @@ -17,24 +17,32 @@ package org.codelibs.fess.app.web.admin.backup; import static org.codelibs.core.stream.StreamUtil.stream; +import java.io.BufferedWriter; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; import javax.annotation.Resource; +import javax.servlet.http.HttpServletResponse; +import org.codelibs.core.CoreLibConstants; import org.codelibs.core.exception.IORuntimeException; import org.codelibs.core.io.CopyUtil; import org.codelibs.core.lang.StringUtil; import org.codelibs.elasticsearch.runner.net.Curl; import org.codelibs.elasticsearch.runner.net.CurlResponse; import org.codelibs.fess.app.web.base.FessAdminAction; +import org.codelibs.fess.es.log.exbhv.SearchLogBhv; import org.codelibs.fess.util.ComponentUtil; import org.codelibs.fess.util.RenderDataUtil; import org.codelibs.fess.util.ResourceUtil; @@ -46,6 +54,10 @@ import org.lastaflute.web.ruts.process.ActionRuntime; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.healthmarketscience.jackcess.RuntimeIOException; +import com.orangesignal.csv.CsvConfig; +import com.orangesignal.csv.CsvWriter; + /** * @author shinsuke */ @@ -53,9 +65,16 @@ public class AdminBackupAction extends FessAdminAction { private static final Logger logger = LoggerFactory.getLogger(AdminBackupAction.class); + private static final String CSV_EXTENTION = ".csv"; + + private static final DateTimeFormatter ISO_8601_FORMATTER = DateTimeFormatter.ofPattern(CoreLibConstants.DATE_FORMAT_ISO_8601_EXTEND); + @Resource private AsyncManager asyncManager; + @Resource + private HttpServletResponse response; + @Override protected void setupHtmlData(final ActionRuntime runtime) { super.setupHtmlData(runtime); @@ -104,7 +123,7 @@ public class AdminBackupAction extends FessAdminAction { @Execute public ActionResponse download(final String id) { - if (stream(fessConfig.getIndexBackupTargetsAsArray()).get(stream -> stream.anyMatch(s -> s.equals(id)))) { + if (stream(fessConfig.getIndexBackupAllTargets()).get(stream -> stream.anyMatch(s -> s.equals(id)))) { if (id.equals("system.properties")) { return asStream(id).contentTypeOctetStream().stream(out -> { try (final ByteArrayOutputStream baos = new ByteArrayOutputStream()) { @@ -114,6 +133,17 @@ public class AdminBackupAction extends FessAdminAction { } } }); + } else if (id.endsWith(CSV_EXTENTION)) { + String name = id.substring(0, id.length() - CSV_EXTENTION.length()); + if ("search_log".equals(name)) { + writeSearchLogCsvResponse(id); + return redirect(getClass()); // no-op + } else if ("user_info".equals(id)) { + // TODO user_info=createdAt,updatedAt + } + // TODO search_field_log=name,searchLogId,value + // TODO favorite_log=queryId,userInfoId,docId,url,createdAt + // TODO click_log=queryId,userSessionId,docId,url,order,queryRequestedAt,requestedAt } else { final String index; final String filename; @@ -140,8 +170,70 @@ public class AdminBackupAction extends FessAdminAction { return redirect(getClass()); // no-op } + private void writeSearchLogCsvResponse(final String id) { + writeCsvResponseHeader(id); + final CsvConfig cfg = new CsvConfig(',', '"', '"'); + cfg.setEscapeDisabled(false); + cfg.setQuoteDisabled(false); + try (final CsvWriter writer = + new CsvWriter(new BufferedWriter(new OutputStreamWriter(response.getOutputStream(), fessConfig.getCsvFileEncoding())), cfg)) { + SearchLogBhv searchLogBhv = ComponentUtil.getComponent(SearchLogBhv.class); + searchLogBhv.selectCursor(cb -> { + cb.query().matchAll(); + cb.query().addOrderBy_RequestedAt_Asc(); + }, entity -> { + List list = new ArrayList<>(); + addToList(entity.getQueryId(), list); + addToList(entity.getUserInfoId(), list); + addToList(entity.getUserSessionId(), list); + addToList(entity.getUser(), list); + addToList(entity.getSearchWord(), list); + addToList(entity.getHitCount(), list); + addToList(entity.getQueryPageSize(), list); + addToList(entity.getQueryOffset(), list); + addToList(entity.getReferer(), list); + addToList(entity.getLanguages(), list); + addToList(entity.getRoles(), list); + addToList(entity.getUserAgent(), list); + addToList(entity.getClientIp(), list); + addToList(entity.getAccessType(), list); + addToList(entity.getQueryTime(), list); + addToList(entity.getResponseTime(), list); + addToList(entity.getRequestedAt(), list); + try { + writer.writeValues(list); + } catch (IOException e) { + throw new RuntimeIOException(e); + } + }); + writer.flush(); + } catch (final Exception e) { + logger.warn("Failed to write " + id + " to response.", e); + } + } + + private void addToList(final Object value, final List list) { + if (value == null) { + list.add(StringUtil.EMPTY); + } else if (value instanceof LocalDateTime) { + list.add(((LocalDateTime) value).format(ISO_8601_FORMATTER)); + } else if (value instanceof String[]) { + String.join(",", (String[]) value); + } else { + list.add(value.toString()); + } + } + + private void writeCsvResponseHeader(final String id) { + response.setContentType("application/octet-stream"); + response.addHeader("Content-Disposition", "attachment; filename=\"" + id + "\""); + response.addHeader("Pragma", "no-cache"); + response.addHeader("Cache-Control", "no-cache"); + response.addHeader("Expires", "Thu, 01 Dec 1994 16:00:00 GMT"); + } + private List> getBackupItems() { - return stream(fessConfig.getIndexBackupTargetsAsArray()).get(stream -> stream.filter(StringUtil::isNotBlank).map(name -> { + return stream(fessConfig.getIndexBackupAllTargets()).get(stream -> stream.map(name -> { final Map map = new HashMap<>(); map.put("id", name); map.put("name", name); diff --git a/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java b/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java index 125dd9297..8dd1e076e 100644 --- a/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java +++ b/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java @@ -576,6 +576,9 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction /** The key of the configuration. e.g. .fess_basic_config.bulk,.fess_config.bulk,.fess_user.bulk,system.properties */ String INDEX_BACKUP_TARGETS = "index.backup.targets"; + /** The key of the configuration. e.g. click_log.csv,favorite_log.csv,search_log.csv,search_field_log.csv,user_info.csv */ + String INDEX_BACKUP_LOG_TARGETS = "index.backup.log.targets"; + /** The key of the configuration. e.g. 4000 */ String FORM_ADMIN_MAX_INPUT_SIZE = "form.admin.max.input.size"; @@ -2848,6 +2851,13 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction */ String getIndexBackupTargets(); + /** + * Get the value for the key 'index.backup.log.targets'.
+ * The value is, e.g. click_log.csv,favorite_log.csv,search_log.csv,search_field_log.csv,user_info.csv
+ * @return The value of found property. (NotNull: if not found, exception but basically no way) + */ + String getIndexBackupLogTargets(); + /** * Get the value for the key 'form.admin.max.input.size'.
* The value is, e.g. 4000
@@ -5693,6 +5703,10 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction return get(FessConfig.INDEX_BACKUP_TARGETS); } + public String getIndexBackupLogTargets() { + return get(FessConfig.INDEX_BACKUP_LOG_TARGETS); + } + public String getFormAdminMaxInputSize() { return get(FessConfig.FORM_ADMIN_MAX_INPUT_SIZE); } diff --git a/src/main/java/org/codelibs/fess/mylasta/direction/FessProp.java b/src/main/java/org/codelibs/fess/mylasta/direction/FessProp.java index 0ceb34d06..f0a72e116 100644 --- a/src/main/java/org/codelibs/fess/mylasta/direction/FessProp.java +++ b/src/main/java/org/codelibs/fess/mylasta/direction/FessProp.java @@ -673,8 +673,11 @@ public interface FessProp { String getIndexBackupTargets(); - public default String[] getIndexBackupTargetsAsArray() { - return getIndexBackupTargets().split(","); + String getIndexBackupLogTargets(); + + public default String[] getIndexBackupAllTargets() { + return split(getIndexBackupTargets() + "," + getIndexBackupLogTargets(), ",").get( + stream -> stream.filter(StringUtil::isNotBlank).map(s -> s.trim()).toArray(n -> new String[n])); } String getJobSystemJobIds(); diff --git a/src/main/resources/fess_config.properties b/src/main/resources/fess_config.properties index d56db3ec1..f70c7a87b 100644 --- a/src/main/resources/fess_config.properties +++ b/src/main/resources/fess_config.properties @@ -291,6 +291,7 @@ smb.available.sid.types=1,2 # backup index.backup.targets=.fess_basic_config.bulk,.fess_config.bulk,.fess_user.bulk,system.properties +index.backup.log.targets=click_log.csv,favorite_log.csv,search_log.csv,search_field_log.csv,user_info.csv # ======================================================================================== # Web