#995 download search logs as csv file

src/main/java/org/codelibs/fess/app/web/admin/backup/AdminBackupAction.java

@@ -17,24 +17,32 @@ package org.codelibs.fess.app.web.admin.backup;
 
 import static org.codelibs.core.stream.StreamUtil.stream;
 
+import java.io.BufferedWriter;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
 
 import javax.annotation.Resource;
+import javax.servlet.http.HttpServletResponse;
 
+import org.codelibs.core.CoreLibConstants;
 import org.codelibs.core.exception.IORuntimeException;
 import org.codelibs.core.io.CopyUtil;
 import org.codelibs.core.lang.StringUtil;
 import org.codelibs.elasticsearch.runner.net.Curl;
 import org.codelibs.elasticsearch.runner.net.CurlResponse;
 import org.codelibs.fess.app.web.base.FessAdminAction;
+import org.codelibs.fess.es.log.exbhv.SearchLogBhv;
 import org.codelibs.fess.util.ComponentUtil;
 import org.codelibs.fess.util.RenderDataUtil;
 import org.codelibs.fess.util.ResourceUtil;
@@ -46,6 +54,10 @@ import org.lastaflute.web.ruts.process.ActionRuntime;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.healthmarketscience.jackcess.RuntimeIOException;
+import com.orangesignal.csv.CsvConfig;
+import com.orangesignal.csv.CsvWriter;
+
 /**
  * @author shinsuke
  */
@@ -53,9 +65,16 @@ public class AdminBackupAction extends FessAdminAction {
 
     private static final Logger logger = LoggerFactory.getLogger(AdminBackupAction.class);
 
+    private static final String CSV_EXTENTION = ".csv";
+
+    private static final DateTimeFormatter ISO_8601_FORMATTER = DateTimeFormatter.ofPattern(CoreLibConstants.DATE_FORMAT_ISO_8601_EXTEND);
+
     @Resource
     private AsyncManager asyncManager;
 
+    @Resource
+    private HttpServletResponse response;
+
     @Override
     protected void setupHtmlData(final ActionRuntime runtime) {
         super.setupHtmlData(runtime);
@@ -104,7 +123,7 @@ public class AdminBackupAction extends FessAdminAction {
 
     @Execute
     public ActionResponse download(final String id) {
-        if (stream(fessConfig.getIndexBackupTargetsAsArray()).get(stream -> stream.anyMatch(s -> s.equals(id)))) {
+        if (stream(fessConfig.getIndexBackupAllTargets()).get(stream -> stream.anyMatch(s -> s.equals(id)))) {
             if (id.equals("system.properties")) {
                 return asStream(id).contentTypeOctetStream().stream(out -> {
                     try (final ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
@@ -114,6 +133,17 @@ public class AdminBackupAction extends FessAdminAction {
                         }
                     }
                 });
+            } else if (id.endsWith(CSV_EXTENTION)) {
+                String name = id.substring(0, id.length() - CSV_EXTENTION.length());
+                if ("search_log".equals(name)) {
+                    writeSearchLogCsvResponse(id);
+                    return redirect(getClass()); // no-op
+                } else if ("user_info".equals(id)) {
+                    // TODO user_info=createdAt,updatedAt
+                }
+                // TODO search_field_log=name,searchLogId,value
+                // TODO favorite_log=queryId,userInfoId,docId,url,createdAt
+                // TODO click_log=queryId,userSessionId,docId,url,order,queryRequestedAt,requestedAt
             } else {
                 final String index;
                 final String filename;
@@ -140,8 +170,70 @@ public class AdminBackupAction extends FessAdminAction {
         return redirect(getClass()); // no-op
     }
 
+    private void writeSearchLogCsvResponse(final String id) {
+        writeCsvResponseHeader(id);
+        final CsvConfig cfg = new CsvConfig(',', '"', '"');
+        cfg.setEscapeDisabled(false);
+        cfg.setQuoteDisabled(false);
+        try (final CsvWriter writer =
+                new CsvWriter(new BufferedWriter(new OutputStreamWriter(response.getOutputStream(), fessConfig.getCsvFileEncoding())), cfg)) {
+            SearchLogBhv searchLogBhv = ComponentUtil.getComponent(SearchLogBhv.class);
+            searchLogBhv.selectCursor(cb -> {
+                cb.query().matchAll();
+                cb.query().addOrderBy_RequestedAt_Asc();
+            }, entity -> {
+                List<String> list = new ArrayList<>();
+                addToList(entity.getQueryId(), list);
+                addToList(entity.getUserInfoId(), list);
+                addToList(entity.getUserSessionId(), list);
+                addToList(entity.getUser(), list);
+                addToList(entity.getSearchWord(), list);
+                addToList(entity.getHitCount(), list);
+                addToList(entity.getQueryPageSize(), list);
+                addToList(entity.getQueryOffset(), list);
+                addToList(entity.getReferer(), list);
+                addToList(entity.getLanguages(), list);
+                addToList(entity.getRoles(), list);
+                addToList(entity.getUserAgent(), list);
+                addToList(entity.getClientIp(), list);
+                addToList(entity.getAccessType(), list);
+                addToList(entity.getQueryTime(), list);
+                addToList(entity.getResponseTime(), list);
+                addToList(entity.getRequestedAt(), list);
+                try {
+                    writer.writeValues(list);
+                } catch (IOException e) {
+                    throw new RuntimeIOException(e);
+                }
+            });
+            writer.flush();
+        } catch (final Exception e) {
+            logger.warn("Failed to write " + id + " to response.", e);
+        }
+    }
+
+    private void addToList(final Object value, final List<String> list) {
+        if (value == null) {
+            list.add(StringUtil.EMPTY);
+        } else if (value instanceof LocalDateTime) {
+            list.add(((LocalDateTime) value).format(ISO_8601_FORMATTER));
+        } else if (value instanceof String[]) {
+            String.join(",", (String[]) value);
+        } else {
+            list.add(value.toString());
+        }
+    }
+
+    private void writeCsvResponseHeader(final String id) {
+        response.setContentType("application/octet-stream");
+        response.addHeader("Content-Disposition", "attachment; filename=\"" + id + "\"");
+        response.addHeader("Pragma", "no-cache");
+        response.addHeader("Cache-Control", "no-cache");
+        response.addHeader("Expires", "Thu, 01 Dec 1994 16:00:00 GMT");
+    }
+
     private List<Map<String, String>> getBackupItems() {
-        return stream(fessConfig.getIndexBackupTargetsAsArray()).get(stream -> stream.filter(StringUtil::isNotBlank).map(name -> {
+        return stream(fessConfig.getIndexBackupAllTargets()).get(stream -> stream.map(name -> {
             final Map<String, String> map = new HashMap<>();
             map.put("id", name);
             map.put("name", name);

src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java

@@ -576,6 +576,9 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
     /** The key of the configuration. e.g. .fess_basic_config.bulk,.fess_config.bulk,.fess_user.bulk,system.properties */
     String INDEX_BACKUP_TARGETS = "index.backup.targets";
 
+    /** The key of the configuration. e.g. click_log.csv,favorite_log.csv,search_log.csv,search_field_log.csv,user_info.csv */
+    String INDEX_BACKUP_LOG_TARGETS = "index.backup.log.targets";
+
     /** The key of the configuration. e.g. 4000 */
     String FORM_ADMIN_MAX_INPUT_SIZE = "form.admin.max.input.size";
 
@@ -2848,6 +2851,13 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
      */
     String getIndexBackupTargets();
 
+    /**
+     * Get the value for the key 'index.backup.log.targets'. <br>
+     * The value is, e.g. click_log.csv,favorite_log.csv,search_log.csv,search_field_log.csv,user_info.csv <br>
+     * @return The value of found property. (NotNull: if not found, exception but basically no way)
+     */
+    String getIndexBackupLogTargets();
+
     /**
      * Get the value for the key 'form.admin.max.input.size'. <br>
      * The value is, e.g. 4000 <br>
@@ -5693,6 +5703,10 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
             return get(FessConfig.INDEX_BACKUP_TARGETS);
         }
 
+        public String getIndexBackupLogTargets() {
+            return get(FessConfig.INDEX_BACKUP_LOG_TARGETS);
+        }
+
         public String getFormAdminMaxInputSize() {
             return get(FessConfig.FORM_ADMIN_MAX_INPUT_SIZE);
         }

src/main/java/org/codelibs/fess/mylasta/direction/FessProp.java

@@ -673,8 +673,11 @@ public interface FessProp {
 
     String getIndexBackupTargets();
 
-    public default String[] getIndexBackupTargetsAsArray() {
-        return getIndexBackupTargets().split(",");
+    String getIndexBackupLogTargets();
+
+    public default String[] getIndexBackupAllTargets() {
+        return split(getIndexBackupTargets() + "," + getIndexBackupLogTargets(), ",").get(
+                stream -> stream.filter(StringUtil::isNotBlank).map(s -> s.trim()).toArray(n -> new String[n]));
     }
 
     String getJobSystemJobIds();

src/main/resources/fess_config.properties

@@ -291,6 +291,7 @@ smb.available.sid.types=1,2
 
 # backup
 index.backup.targets=.fess_basic_config.bulk,.fess_config.bulk,.fess_user.bulk,system.properties
+index.backup.log.targets=click_log.csv,favorite_log.csv,search_log.csv,search_field_log.csv,user_info.csv
 
 # ========================================================================================
 #                                                                                     Web