0ct0pu5/fess
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

#995 download search logs as csv file

Browse source
This commit is contained in:
Shinsuke Sugaya 2017-04-14 00:10:54 +09:00
parent f673402aac
commit 78ea946df6
4 changed files with 114 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

96
src/main/java/org/codelibs/fess/app/web/admin/backup/AdminBackupAction.java
View file

@ -17,24 +17,32 @@ package org.codelibs.fess.app.web.admin.backup;
import static org.codelibs.core.stream.StreamUtil.stream;
import java.io.BufferedWriter;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import javax.annotation.Resource;
import javax.servlet.http.HttpServletResponse;
import org.codelibs.core.CoreLibConstants;
import org.codelibs.core.exception.IORuntimeException;
import org.codelibs.core.io.CopyUtil;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.elasticsearch.runner.net.Curl;
import org.codelibs.elasticsearch.runner.net.CurlResponse;
import org.codelibs.fess.app.web.base.FessAdminAction;
import org.codelibs.fess.es.log.exbhv.SearchLogBhv;
import org.codelibs.fess.util.ComponentUtil;
import org.codelibs.fess.util.RenderDataUtil;
import org.codelibs.fess.util.ResourceUtil;
@ -46,6 +54,10 @@ import org.lastaflute.web.ruts.process.ActionRuntime;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.healthmarketscience.jackcess.RuntimeIOException;
import com.orangesignal.csv.CsvConfig;
import com.orangesignal.csv.CsvWriter;
/**
* @author shinsuke
*/
@ -53,9 +65,16 @@ public class AdminBackupAction extends FessAdminAction {
private static final Logger logger = LoggerFactory.getLogger(AdminBackupAction.class);
private static final String CSV_EXTENTION = ".csv";
private static final DateTimeFormatter ISO_8601_FORMATTER = DateTimeFormatter.ofPattern(CoreLibConstants.DATE_FORMAT_ISO_8601_EXTEND);
@Resource
private AsyncManager asyncManager;
@Resource
private HttpServletResponse response;
@Override
protected void setupHtmlData(final ActionRuntime runtime) {
super.setupHtmlData(runtime);
@ -104,7 +123,7 @@ public class AdminBackupAction extends FessAdminAction {
@Execute
public ActionResponse download(final String id) {
if (stream(fessConfig.getIndexBackupTargetsAsArray()).get(stream -> stream.anyMatch(s -> s.equals(id)))) {
if (stream(fessConfig.getIndexBackupAllTargets()).get(stream -> stream.anyMatch(s -> s.equals(id)))) {
if (id.equals("system.properties")) {
return asStream(id).contentTypeOctetStream().stream(out -> {
try (final ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
@ -114,6 +133,17 @@ public class AdminBackupAction extends FessAdminAction {
}
}
});
} else if (id.endsWith(CSV_EXTENTION)) {
String name = id.substring(0, id.length() - CSV_EXTENTION.length());
if ("search_log".equals(name)) {
writeSearchLogCsvResponse(id);
return redirect(getClass()); // no-op
} else if ("user_info".equals(id)) {
// TODO user_info=createdAt,updatedAt
}
// TODO search_field_log=name,searchLogId,value
// TODO favorite_log=queryId,userInfoId,docId,url,createdAt
// TODO click_log=queryId,userSessionId,docId,url,order,queryRequestedAt,requestedAt
} else {
final String index;
final String filename;
@ -140,8 +170,70 @@ public class AdminBackupAction extends FessAdminAction {
return redirect(getClass()); // no-op
}
private void writeSearchLogCsvResponse(final String id) {
writeCsvResponseHeader(id);
final CsvConfig cfg = new CsvConfig(',', '"', '"');
cfg.setEscapeDisabled(false);
cfg.setQuoteDisabled(false);
try (final CsvWriter writer =
new CsvWriter(new BufferedWriter(new OutputStreamWriter(response.getOutputStream(), fessConfig.getCsvFileEncoding())), cfg)) {
SearchLogBhv searchLogBhv = ComponentUtil.getComponent(SearchLogBhv.class);
searchLogBhv.selectCursor(cb -> {
cb.query().matchAll();
cb.query().addOrderBy_RequestedAt_Asc();
}, entity -> {
List<String> list = new ArrayList<>();
addToList(entity.getQueryId(), list);
addToList(entity.getUserInfoId(), list);
addToList(entity.getUserSessionId(), list);
addToList(entity.getUser(), list);
addToList(entity.getSearchWord(), list);
addToList(entity.getHitCount(), list);
addToList(entity.getQueryPageSize(), list);
addToList(entity.getQueryOffset(), list);
addToList(entity.getReferer(), list);
addToList(entity.getLanguages(), list);
addToList(entity.getRoles(), list);
addToList(entity.getUserAgent(), list);
addToList(entity.getClientIp(), list);
addToList(entity.getAccessType(), list);
addToList(entity.getQueryTime(), list);
addToList(entity.getResponseTime(), list);
addToList(entity.getRequestedAt(), list);
try {
writer.writeValues(list);
} catch (IOException e) {
throw new RuntimeIOException(e);
}
});
writer.flush();
} catch (final Exception e) {
logger.warn("Failed to write " + id + " to response.", e);
}
}
private void addToList(final Object value, final List<String> list) {
if (value == null) {
list.add(StringUtil.EMPTY);
} else if (value instanceof LocalDateTime) {
list.add(((LocalDateTime) value).format(ISO_8601_FORMATTER));
} else if (value instanceof String[]) {
String.join(",", (String[]) value);
} else {
list.add(value.toString());
}
}
private void writeCsvResponseHeader(final String id) {
response.setContentType("application/octet-stream");
response.addHeader("Content-Disposition", "attachment; filename=\"" + id + "\"");
response.addHeader("Pragma", "no-cache");
response.addHeader("Cache-Control", "no-cache");
response.addHeader("Expires", "Thu, 01 Dec 1994 16:00:00 GMT");
}
private List<Map<String, String>> getBackupItems() {
return stream(fessConfig.getIndexBackupTargetsAsArray()).get(stream -> stream.filter(StringUtil::isNotBlank).map(name -> {
return stream(fessConfig.getIndexBackupAllTargets()).get(stream -> stream.map(name -> {
final Map<String, String> map = new HashMap<>();
map.put("id", name);
map.put("name", name);

14
src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java
View file

@ -576,6 +576,9 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
/** The key of the configuration. e.g. .fess_basic_config.bulk,.fess_config.bulk,.fess_user.bulk,system.properties */
String INDEX_BACKUP_TARGETS = "index.backup.targets";
/** The key of the configuration. e.g. click_log.csv,favorite_log.csv,search_log.csv,search_field_log.csv,user_info.csv */
String INDEX_BACKUP_LOG_TARGETS = "index.backup.log.targets";
/** The key of the configuration. e.g. 4000 */
String FORM_ADMIN_MAX_INPUT_SIZE = "form.admin.max.input.size";
@ -2848,6 +2851,13 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
*/
String getIndexBackupTargets();
/**
* Get the value for the key 'index.backup.log.targets'. <br>
* The value is, e.g. click_log.csv,favorite_log.csv,search_log.csv,search_field_log.csv,user_info.csv <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getIndexBackupLogTargets();
/**
* Get the value for the key 'form.admin.max.input.size'. <br>
* The value is, e.g. 4000 <br>
@ -5693,6 +5703,10 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
return get(FessConfig.INDEX_BACKUP_TARGETS);
}
public String getIndexBackupLogTargets() {
return get(FessConfig.INDEX_BACKUP_LOG_TARGETS);
}
public String getFormAdminMaxInputSize() {
return get(FessConfig.FORM_ADMIN_MAX_INPUT_SIZE);
}

7
src/main/java/org/codelibs/fess/mylasta/direction/FessProp.java
View file

@ -673,8 +673,11 @@ public interface FessProp {
String getIndexBackupTargets();
public default String[] getIndexBackupTargetsAsArray() {
return getIndexBackupTargets().split(",");
String getIndexBackupLogTargets();
public default String[] getIndexBackupAllTargets() {
return split(getIndexBackupTargets() + "," + getIndexBackupLogTargets(), ",").get(
stream -> stream.filter(StringUtil::isNotBlank).map(s -> s.trim()).toArray(n -> new String[n]));
}
String getJobSystemJobIds();

1
src/main/resources/fess_config.properties
View file

@ -291,6 +291,7 @@ smb.available.sid.types=1,2
# backup
index.backup.targets=.fess_basic_config.bulk,.fess_config.bulk,.fess_user.bulk,system.properties
index.backup.log.targets=click_log.csv,favorite_log.csv,search_log.csv,search_field_log.csv,user_info.csv
# ========================================================================================
# Web