fix #1252 ndjson format support

This commit is contained in:
Shinsuke Sugaya 2017-08-31 12:44:18 +09:00
parent 97a19d96b6
commit 3ca467ae99
6 changed files with 260 additions and 7 deletions

View file

@ -24,9 +24,12 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -35,11 +38,14 @@ import java.util.stream.Collectors;
import javax.annotation.Resource;
import org.apache.commons.text.StringEscapeUtils;
import org.codelibs.core.exception.IORuntimeException;
import org.codelibs.core.io.CopyUtil;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.core.misc.Pair;
import org.codelibs.elasticsearch.runner.net.Curl;
import org.codelibs.elasticsearch.runner.net.CurlResponse;
import org.codelibs.fess.Constants;
import org.codelibs.fess.app.web.base.FessAdminAction;
import org.codelibs.fess.es.log.exbhv.ClickLogBhv;
import org.codelibs.fess.es.log.exbhv.FavoriteLogBhv;
@ -71,6 +77,8 @@ public class AdminBackupAction extends FessAdminAction {
public static final String CSV_EXTENTION = ".csv";
public static final String NDJSON_EXTENTION = ".ndjson";
private static final DateTimeFormatter ISO_8601_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSS");
@Resource
@ -136,6 +144,17 @@ public class AdminBackupAction extends FessAdminAction {
}
}
});
} else if (id.endsWith(NDJSON_EXTENTION)) {
final String name = id.substring(0, id.length() - NDJSON_EXTENTION.length());
if ("search_log".equals(name)) {
return writeNdjsonResponse(id, getSearchLogNdjsonWriteCall());
} else if ("user_info".equals(name)) {
return writeNdjsonResponse(id, getUserInfoNdjsonWriteCall());
} else if ("click_log".equals(name)) {
return writeNdjsonResponse(id, getClickLogNdjsonWriteCall());
} else if ("favorite_log".equals(name)) {
return writeNdjsonResponse(id, getFavoriteLogNdjsonWriteCall());
}
} else if (id.endsWith(CSV_EXTENTION)) {
final String name = id.substring(0, id.length() - CSV_EXTENTION.length());
if ("search_log".equals(name)) {
@ -173,6 +192,184 @@ public class AdminBackupAction extends FessAdminAction {
return redirect(getClass()); // no-op
}
private StreamResponse writeNdjsonResponse(final String id, final Consumer<Writer> writeCall) {
return asStream(id)//
.header("Pragma", "no-cache")//
.header("Cache-Control", "no-cache")//
.header("Expires", "Thu, 01 Dec 1994 16:00:00 GMT")//
.header("Content-Type", "application/x-ndjson")//
.stream(out -> {
try (final Writer writer = new BufferedWriter(new OutputStreamWriter(out.stream(), Constants.CHARSET_UTF_8))) {
writeCall.accept(writer);
writer.flush();
} catch (final Exception e) {
logger.warn("Failed to write " + id + " to response.", e);
}
});
}
private static StringBuilder appendJson(String field, Object value, StringBuilder buf) {
buf.append('"').append(StringEscapeUtils.escapeJson(field)).append('"').append(':');
if (value == null) {
buf.append("null");
} else if (value instanceof LocalDateTime) {
final String format =
((LocalDateTime) value).atZone(ZoneId.systemDefault()).withZoneSameInstant(ZoneId.of("UTC")).format(ISO_8601_FORMATTER);
buf.append('"').append(StringEscapeUtils.escapeJson(format)).append('"');
} else if (value instanceof String[]) {
final String json =
Arrays.stream((String[]) value).map(s -> "\"" + StringEscapeUtils.escapeJson(s) + "\"")
.collect(Collectors.joining(","));
buf.append('[').append(json).append(']');
} else if (value instanceof List) {
final String json =
((List<?>) value).stream().map(s -> "\"" + StringEscapeUtils.escapeJson(s.toString()) + "\"")
.collect(Collectors.joining(","));
buf.append('[').append(json).append(']');
} else if (value instanceof Map) {
buf.append('{');
final String json = ((Map<?, ?>) value).entrySet().stream().map(e -> {
StringBuilder tempBuf = new StringBuilder();
appendJson(e.getKey().toString(), e.getValue(), tempBuf);
return tempBuf.toString();
}).collect(Collectors.joining(","));
buf.append(json);
buf.append('}');
} else if (value instanceof Long || value instanceof Integer) {
buf.append(((Number) value).longValue());
} else if (value instanceof Number) {
buf.append(((Number) value).doubleValue());
} else {
buf.append('"').append(StringEscapeUtils.escapeJson(value.toString())).append('"');
}
return buf;
}
public static Consumer<Writer> getSearchLogNdjsonWriteCall() {
return writer -> {
final SearchLogBhv bhv = ComponentUtil.getComponent(SearchLogBhv.class);
bhv.selectCursor(
cb -> {
cb.query().matchAll();
cb.query().addOrderBy_RequestedAt_Asc();
},
entity -> {
StringBuilder buf = new StringBuilder();
buf.append('{');
appendJson("id", entity.getId(), buf).append(',');
appendJson("query-id", entity.getQueryId(), buf).append(',');
appendJson("user-info-id", entity.getUserInfoId(), buf).append(',');
appendJson("user-session-id", entity.getUserSessionId(), buf).append(',');
appendJson("user", entity.getUser(), buf).append(',');
appendJson("search-word", entity.getSearchWord(), buf).append(',');
appendJson("hit-count", entity.getHitCount(), buf).append(',');
appendJson("query-page-size", entity.getQueryPageSize(), buf).append(',');
appendJson("query-offset", entity.getQueryOffset(), buf).append(',');
appendJson("referer", entity.getReferer(), buf).append(',');
appendJson("languages", entity.getLanguages(), buf).append(',');
appendJson("roles", entity.getRoles(), buf).append(',');
appendJson("user-agent", entity.getUserAgent(), buf).append(',');
appendJson("client-ip", entity.getClientIp(), buf).append(',');
appendJson("access-type", entity.getAccessType(), buf).append(',');
appendJson("query-time", entity.getQueryTime(), buf).append(',');
appendJson("response-time", entity.getResponseTime(), buf).append(',');
appendJson("requested-at", entity.getRequestedAt(), buf).append(',');
final Map<String, List<String>> searchFieldMap =
entity.getSearchFieldLogList()
.stream()
.collect(
Collectors.groupingBy(Pair::getFirst,
Collectors.mapping(Pair::getSecond, Collectors.toList())));
appendJson("search-field", searchFieldMap, buf);
buf.append('}');
buf.append('\n');
try {
writer.write(buf.toString());
} catch (final IOException e) {
throw new RuntimeIOException(e);
}
});
};
}
public static Consumer<Writer> getUserInfoNdjsonWriteCall() {
return writer -> {
final UserInfoBhv bhv = ComponentUtil.getComponent(UserInfoBhv.class);
bhv.selectCursor(cb -> {
cb.query().matchAll();
cb.query().addOrderBy_CreatedAt_Asc();
}, entity -> {
StringBuilder buf = new StringBuilder();
buf.append('{');
appendJson("id", entity.getId(), buf).append(',');
appendJson("created-at", entity.getCreatedAt(), buf).append(',');
appendJson("updated-at", entity.getUpdatedAt(), buf);
buf.append('}');
buf.append('\n');
try {
writer.write(buf.toString());
} catch (final IOException e) {
throw new RuntimeIOException(e);
}
});
};
}
public static Consumer<Writer> getFavoriteLogNdjsonWriteCall() {
return writer -> {
final FavoriteLogBhv bhv = ComponentUtil.getComponent(FavoriteLogBhv.class);
bhv.selectCursor(cb -> {
cb.query().matchAll();
cb.query().addOrderBy_CreatedAt_Asc();
}, entity -> {
StringBuilder buf = new StringBuilder();
buf.append('{');
appendJson("id", entity.getId(), buf).append(',');
appendJson("created-at", entity.getCreatedAt(), buf).append(',');
appendJson("query-id", entity.getQueryId(), buf).append(',');
appendJson("user-info-id", entity.getUserInfoId(), buf).append(',');
appendJson("doc-id", entity.getDocId(), buf).append(',');
appendJson("url", entity.getUrl(), buf);
buf.append('}');
buf.append('\n');
try {
writer.write(buf.toString());
} catch (final IOException e) {
throw new RuntimeIOException(e);
}
});
};
}
public static Consumer<Writer> getClickLogNdjsonWriteCall() {
return writer -> {
final ClickLogBhv bhv = ComponentUtil.getComponent(ClickLogBhv.class);
bhv.selectCursor(cb -> {
cb.query().matchAll();
cb.query().addOrderBy_RequestedAt_Asc();
}, entity -> {
StringBuilder buf = new StringBuilder();
buf.append('{');
appendJson("id", entity.getId(), buf).append(',');
appendJson("query-id", entity.getQueryId(), buf).append(',');
appendJson("user-session-id", entity.getUserSessionId(), buf).append(',');
appendJson("doc-id", entity.getDocId(), buf).append(',');
appendJson("url", entity.getUrl(), buf).append(',');
appendJson("order", entity.getOrder(), buf).append(',');
appendJson("query-requested-at", entity.getQueryRequestedAt(), buf).append(',');
appendJson("requested-at", entity.getRequestedAt(), buf);
buf.append('}');
buf.append('\n');
try {
writer.write(buf.toString());
} catch (final IOException e) {
throw new RuntimeIOException(e);
}
});
};
}
@Deprecated
private StreamResponse writeCsvResponse(final String id, final Consumer<CsvWriter> writeCall) {
return asStream(id)
.contentTypeOctetStream()
@ -193,6 +390,7 @@ public class AdminBackupAction extends FessAdminAction {
});
}
@Deprecated
public static Consumer<CsvWriter> getSearchLogCsvWriteCall() {
return writer -> {
final SearchLogBhv bhv = ComponentUtil.getComponent(SearchLogBhv.class);
@ -231,6 +429,7 @@ public class AdminBackupAction extends FessAdminAction {
};
}
@Deprecated
public static Consumer<CsvWriter> getUserInfoCsvWriteCall() {
return writer -> {
final UserInfoBhv bhv = ComponentUtil.getComponent(UserInfoBhv.class);
@ -250,6 +449,7 @@ public class AdminBackupAction extends FessAdminAction {
};
}
@Deprecated
public static Consumer<CsvWriter> getFavoriteLogCsvWriteCall() {
return writer -> {
final FavoriteLogBhv bhv = ComponentUtil.getComponent(FavoriteLogBhv.class);
@ -272,6 +472,7 @@ public class AdminBackupAction extends FessAdminAction {
};
}
@Deprecated
public static Consumer<CsvWriter> getClickLogCsvWriteCall() {
return writer -> {
final ClickLogBhv bhv = ComponentUtil.getComponent(ClickLogBhv.class);

View file

@ -17,23 +17,30 @@ package org.codelibs.fess.app.web.api.admin.backup;
import static org.codelibs.core.stream.StreamUtil.stream;
import static org.codelibs.fess.app.web.admin.backup.AdminBackupAction.CSV_EXTENTION;
import static org.codelibs.fess.app.web.admin.backup.AdminBackupAction.NDJSON_EXTENTION;
import static org.codelibs.fess.app.web.admin.backup.AdminBackupAction.getBackupItems;
import static org.codelibs.fess.app.web.admin.backup.AdminBackupAction.getClickLogCsvWriteCall;
import static org.codelibs.fess.app.web.admin.backup.AdminBackupAction.getClickLogNdjsonWriteCall;
import static org.codelibs.fess.app.web.admin.backup.AdminBackupAction.getFavoriteLogCsvWriteCall;
import static org.codelibs.fess.app.web.admin.backup.AdminBackupAction.getFavoriteLogNdjsonWriteCall;
import static org.codelibs.fess.app.web.admin.backup.AdminBackupAction.getSearchLogCsvWriteCall;
import static org.codelibs.fess.app.web.admin.backup.AdminBackupAction.getSearchLogNdjsonWriteCall;
import static org.codelibs.fess.app.web.admin.backup.AdminBackupAction.getUserInfoCsvWriteCall;
import static org.codelibs.fess.app.web.admin.backup.AdminBackupAction.getUserInfoNdjsonWriteCall;
import java.io.BufferedWriter;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.InputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.util.List;
import java.util.Map;
import java.util.function.Consumer;
import org.codelibs.elasticsearch.runner.net.Curl;
import org.codelibs.elasticsearch.runner.net.CurlResponse;
import org.codelibs.fess.Constants;
import org.codelibs.fess.app.web.api.ApiResult;
import org.codelibs.fess.app.web.api.ApiResult.ApiBackupFilesResponse;
import org.codelibs.fess.app.web.api.admin.FessApiAdminAction;
@ -73,6 +80,17 @@ public class ApiAdminBackupAction extends FessApiAdminAction {
}
}
});
} else if (id.endsWith(NDJSON_EXTENTION)) {
final String name = id.substring(0, id.length() - NDJSON_EXTENTION.length());
if ("search_log".equals(name)) {
return writeNdjsonResponse(id, getSearchLogNdjsonWriteCall());
} else if ("user_info".equals(name)) {
return writeNdjsonResponse(id, getUserInfoNdjsonWriteCall());
} else if ("click_log".equals(name)) {
return writeNdjsonResponse(id, getClickLogNdjsonWriteCall());
} else if ("favorite_log".equals(name)) {
return writeNdjsonResponse(id, getFavoriteLogNdjsonWriteCall());
}
} else if (id.endsWith(CSV_EXTENTION)) {
final String name = id.substring(0, id.length() - CSV_EXTENTION.length());
if ("search_log".equals(name)) {
@ -109,12 +127,31 @@ public class ApiAdminBackupAction extends FessApiAdminAction {
return StreamResponse.asEmptyBody(); // no-op
}
private StreamResponse writeNdjsonResponse(final String id, final Consumer<Writer> writeCall) {
return asStream(id)//
.header("Pragma", "no-cache")//
.header("Cache-Control", "no-cache")//
.header("Expires", "Thu, 01 Dec 1994 16:00:00 GMT")//
.header("Content-Type", "application/x-ndjson")//
.stream(out -> {
try (final Writer writer = new BufferedWriter(new OutputStreamWriter(out.stream(), Constants.CHARSET_UTF_8))) {
writeCall.accept(writer);
writer.flush();
}
});
}
private StreamResponse writeCsvResponse(final String id, final Consumer<CsvWriter> writeCall) {
return asStream(id)
//
.contentTypeOctetStream()
//
.header("Pragma", "no-cache")
//
.header("Cache-Control", "no-cache")
//
.header("Expires", "Thu, 01 Dec 1994 16:00:00 GMT")
//
.stream(out -> {
final CsvConfig cfg = new CsvConfig(',', '"', '"');
cfg.setEscapeDisabled(false);

View file

@ -20,6 +20,7 @@ import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
import java.util.List;
import java.util.Map;
import org.codelibs.core.misc.Pair;
@ -56,14 +57,28 @@ public class SearchLogBhv extends BsSearchLogBhv {
return DfTypeUtil.toLocalDateTime(value);
}
@SuppressWarnings("unchecked")
@Override
protected <RESULT extends SearchLog> RESULT createEntity(final Map<String, Object> source, final Class<? extends RESULT> entityType) {
try {
final RESULT result = super.createEntity(source, entityType);
final Object searchFieldObj = source.get("searchField");
if (searchFieldObj instanceof Map) {
((Map<String, String>) searchFieldObj).entrySet().stream()
.forEach(e -> result.getSearchFieldLogList().add(new Pair(e.getKey(), e.getValue())));
((Map<String, ?>) searchFieldObj).entrySet().stream().forEach(e -> {
if (e.getValue() instanceof String[]) {
String[] values = (String[]) e.getValue();
for (final String v : values) {
result.getSearchFieldLogList().add(new Pair<>(e.getKey(), v));
}
} else if (e.getValue() instanceof List) {
List<String> values = (List<String>) e.getValue();
for (final String v : values) {
result.getSearchFieldLogList().add(new Pair<>(e.getKey(), v));
}
} else if (e.getValue() != null) {
result.getSearchFieldLogList().add(new Pair<>(e.getKey(), e.getValue().toString()));
}
});
}
return result;
} catch (Exception e) {

View file

@ -100,7 +100,7 @@ public class SearchLog extends BsSearchLog {
if (fields != null) {
sourceMap.putAll(fields);
}
final Map<String, List<Object>> searchFieldMap =
final Map<String, List<String>> searchFieldMap =
searchFieldLogList.stream().collect(
Collectors.groupingBy(Pair::getFirst, Collectors.mapping(Pair::getSecond, Collectors.toList())));
sourceMap.put("searchField", searchFieldMap);

View file

@ -626,7 +626,7 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
/** The key of the configuration. e.g. .fess_basic_config.bulk,.fess_config.bulk,.fess_user.bulk,system.properties */
String INDEX_BACKUP_TARGETS = "index.backup.targets";
/** The key of the configuration. e.g. click_log.csv,favorite_log.csv,search_log.csv,user_info.csv */
/** The key of the configuration. e.g. click_log.ndjson,favorite_log.ndjson,search_log.ndjson,user_info.ndjson */
String INDEX_BACKUP_LOG_TARGETS = "index.backup.log.targets";
/** The key of the configuration. e.g. 4000 */
@ -3112,7 +3112,7 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
/**
* Get the value for the key 'index.backup.log.targets'. <br>
* The value is, e.g. click_log.csv,favorite_log.csv,search_log.csv,user_info.csv <br>
* The value is, e.g. click_log.ndjson,favorite_log.ndjson,search_log.ndjson,user_info.ndjson <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getIndexBackupLogTargets();
@ -7623,7 +7623,7 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
defaultMap.put(FessConfig.SMB_ROLE_FROM_FILE, "true");
defaultMap.put(FessConfig.SMB_AVAILABLE_SID_TYPES, "1,2");
defaultMap.put(FessConfig.INDEX_BACKUP_TARGETS, ".fess_basic_config.bulk,.fess_config.bulk,.fess_user.bulk,system.properties");
defaultMap.put(FessConfig.INDEX_BACKUP_LOG_TARGETS, "click_log.csv,favorite_log.csv,search_log.csv,user_info.csv");
defaultMap.put(FessConfig.INDEX_BACKUP_LOG_TARGETS, "click_log.ndjson,favorite_log.ndjson,search_log.ndjson,user_info.ndjson");
defaultMap.put(FessConfig.FORM_ADMIN_MAX_INPUT_SIZE, "4000");
defaultMap.put(FessConfig.AUTHENTICATION_ADMIN_USERS, "admin");
defaultMap.put(FessConfig.AUTHENTICATION_ADMIN_ROLES, "admin");

View file

@ -317,7 +317,7 @@ smb.available.sid.types=1,2
# backup
index.backup.targets=.fess_basic_config.bulk,.fess_config.bulk,.fess_user.bulk,system.properties
index.backup.log.targets=click_log.csv,favorite_log.csv,search_log.csv,user_info.csv
index.backup.log.targets=click_log.ndjson,favorite_log.ndjson,search_log.ndjson,user_info.ndjson
# ========================================================================================
# Web