0ct0pu5/fess
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix #2496 add searchlog.log

Browse source
This commit is contained in:
Shinsuke Sugaya 2020-11-11 06:54:23 +09:00
parent 1864a45475
commit 040a95da72
10 changed files with 192 additions and 19 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
src/main/java/org/codelibs/fess/app/service/FavoriteLogService.java
View file

@ -28,6 +28,7 @@ import org.codelibs.fess.es.log.exentity.FavoriteLog;
import org.codelibs.fess.es.log.exentity.UserInfo;
import org.codelibs.fess.helper.SystemHelper;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.util.ComponentUtil;
import org.dbflute.cbean.result.ListResultBean;
public class FavoriteLogService {
@ -48,6 +49,10 @@ public class FavoriteLogService {
final FavoriteLog favoriteLog = new FavoriteLog();
favoriteLogLambda.accept(userInfo, favoriteLog);
favoriteLogBhv.insert(favoriteLog);
if (fessConfig.isLoggingSearchUseLogfile()) {
ComponentUtil.getSearchLogHelper().writeSearchLogEvent(favoriteLog);
;
}
return true;
}).orElse(false);
}

28
src/main/java/org/codelibs/fess/entity/SearchLogEvent.java Normal file
View file

@ -0,0 +1,28 @@
/*
* Copyright 2012-2020 CodeLibs Project and the Others.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
* either express or implied. See the License for the specific language
* governing permissions and limitations under the License.
*/
package org.codelibs.fess.entity;
import java.util.Map;
public interface SearchLogEvent {
String getId();
Long getVersionNo();
Map<String, Object> toSource();
String getEventType();
}

8
src/main/java/org/codelibs/fess/es/log/exentity/ClickLog.java
View file

@ -21,12 +21,13 @@ import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Map;
import org.codelibs.fess.entity.SearchLogEvent;
import org.codelibs.fess.es.log.bsentity.BsClickLog;
/**
* @author FreeGen
*/
public class ClickLog extends BsClickLog {
public class ClickLog extends BsClickLog implements SearchLogEvent {
private static final long serialVersionUID = 1L;
@ -81,4 +82,9 @@ public class ClickLog extends BsClickLog {
return "ClickLog [queryRequestedAt=" + queryRequestedAt + ", requestedAt=" + requestedAt + ", queryId=" + queryId + ", docId="
+ docId + ", userSessionId=" + userSessionId + ", url=" + url + ", order=" + order + ", docMeta=" + docMeta + "]";
}
@Override
public String getEventType() {
return "click";
}
}

8
src/main/java/org/codelibs/fess/es/log/exentity/FavoriteLog.java
View file

@ -21,12 +21,13 @@ import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Map;
import org.codelibs.fess.entity.SearchLogEvent;
import org.codelibs.fess.es.log.bsentity.BsFavoriteLog;
/**
* @author FreeGen
*/
public class FavoriteLog extends BsFavoriteLog {
public class FavoriteLog extends BsFavoriteLog implements SearchLogEvent {
private static final long serialVersionUID = 1L;
@ -85,4 +86,9 @@ public class FavoriteLog extends BsFavoriteLog {
return "FavoriteLog [createdAt=" + createdAt + ", url=" + url + ", docId=" + docId + ", queryId=" + queryId + ", userInfoId="
+ userInfoId + ", docMeta=" + docMeta + "]";
}
@Override
public String getEventType() {
return "favorite";
}
}

7
src/main/java/org/codelibs/fess/es/log/exentity/SearchLog.java
View file

@ -28,6 +28,7 @@ import java.util.stream.Collectors;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.core.misc.Pair;
import org.codelibs.fess.Constants;
import org.codelibs.fess.entity.SearchLogEvent;
import org.codelibs.fess.es.log.bsentity.BsSearchLog;
import org.codelibs.fess.es.log.exbhv.UserInfoBhv;
import org.codelibs.fess.util.ComponentUtil;
@ -36,7 +37,7 @@ import org.dbflute.optional.OptionalEntity;
/**
* @author FreeGen
*/
public class SearchLog extends BsSearchLog {
public class SearchLog extends BsSearchLog implements SearchLogEvent {
private static final long serialVersionUID = 1L;
@ -147,4 +148,8 @@ public class SearchLog extends BsSearchLog {
+ userSessionId + ", virtualHost=" + virtualHost + ", documents=" + documentList + "]";
}
@Override
public String getEventType() {
return "log";
}
}

8
src/main/java/org/codelibs/fess/es/log/exentity/UserInfo.java
View file

@ -21,12 +21,13 @@ import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Map;
import org.codelibs.fess.entity.SearchLogEvent;
import org.codelibs.fess.es.log.bsentity.BsUserInfo;
/**
* @author FreeGen
*/
public class UserInfo extends BsUserInfo {
public class UserInfo extends BsUserInfo implements SearchLogEvent {
private static final long serialVersionUID = 1L;
@ -84,4 +85,9 @@ public class UserInfo extends BsUserInfo {
public String toString() {
return "UserInfo [createdAt=" + createdAt + ", updatedAt=" + updatedAt + ", docMeta=" + docMeta + "]";
}
@Override
public String getEventType() {
return "user";
}
}

104
src/main/java/org/codelibs/fess/helper/SearchLogHelper.java
View file

@ -27,6 +27,7 @@ import java.util.Queue;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import javax.annotation.PostConstruct;
import javax.servlet.http.HttpServletRequest;
@ -37,6 +38,7 @@ import org.apache.logging.log4j.Logger;
import org.codelibs.core.concurrent.CommonPoolUtil;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.fess.Constants;
import org.codelibs.fess.entity.SearchLogEvent;
import org.codelibs.fess.entity.SearchRequestParams;
import org.codelibs.fess.entity.SearchRequestParams.SearchRequestType;
import org.codelibs.fess.es.log.exbhv.ClickLogBhv;
@ -57,6 +59,9 @@ import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.script.Script;
import org.lastaflute.web.util.LaRequestUtil;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.CaseFormat;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
@ -74,6 +79,12 @@ public class SearchLogHelper {
protected LoadingCache<String, UserInfo> userInfoCache;
protected String loggerName = "fess.log.searchlog";
protected Logger searchLogLogger = null;
protected ObjectMapper objectMapper = new ObjectMapper();
@PostConstruct
public void init() {
if (logger.isDebugEnabled()) {
@ -88,6 +99,7 @@ public class SearchLogHelper {
return storeUserInfo(key);
}
});
searchLogLogger = LogManager.getLogger(loggerName);
}
public void addSearchLog(final SearchRequestParams params, final LocalDateTime requestedTime, final String queryId, final String query,
@ -267,7 +279,27 @@ public class SearchLogHelper {
}
});
processUserInfoLog(searchLogList, userInfoMap);
processSearchLog(searchLogList);
}
private void processSearchLog(final List<SearchLog> searchLogList) {
if (!searchLogList.isEmpty()) {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
storeSearchLogList(searchLogList);
if (fessConfig.isSuggestSearchLog()) {
final SuggestHelper suggestHelper = ComponentUtil.getSuggestHelper();
suggestHelper.indexFromSearchLog(searchLogList);
}
if (fessConfig.isLoggingSearchUseLogfile()) {
searchLogList.forEach(this::writeSearchLogEvent);
}
}
}
protected void processUserInfoLog(final List<SearchLog> searchLogList, final Map<String, UserInfo> userInfoMap) {
if (!userInfoMap.isEmpty()) {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final List<UserInfo> insertList = new ArrayList<>(userInfoMap.values());
final List<UserInfo> updateList = new ArrayList<>();
final UserInfoBhv userInfoBhv = ComponentUtil.getComponent(UserInfoBhv.class);
@ -289,13 +321,9 @@ public class SearchLogHelper {
searchLog.setUserInfoId(userInfo.getId());
});
});
}
if (!searchLogList.isEmpty()) {
storeSearchLogList(searchLogList);
if (fessConfig.isSuggestSearchLog()) {
final SuggestHelper suggestHelper = ComponentUtil.getSuggestHelper();
suggestHelper.indexFromSearchLog(searchLogList);
if (fessConfig.isLoggingSearchUseLogfile()) {
insertList.forEach(this::writeSearchLogEvent);
updateList.forEach(this::writeSearchLogEvent);
}
}
}
@ -332,15 +360,12 @@ public class SearchLogHelper {
logger.warn("Failed to process: {}", clickLog, e);
}
}
if (!clickLogList.isEmpty()) {
try {
final ClickLogBhv clickLogBhv = ComponentUtil.getComponent(ClickLogBhv.class);
clickLogBhv.batchInsert(clickLogList);
} catch (final Exception e) {
logger.warn("Failed to insert: {}", clickLogList, e);
}
}
processClickLog(clickLogList);
updateClickFieldInIndex(clickCountMap);
}
protected void updateClickFieldInIndex(final Map<String, Integer> clickCountMap) {
if (!clickCountMap.isEmpty()) {
final SearchHelper searchHelper = ComponentUtil.getSearchHelper();
final FessConfig fessConfig = ComponentUtil.getFessConfig();
@ -375,6 +400,51 @@ public class SearchLogHelper {
}
}
protected void processClickLog(final List<ClickLog> clickLogList) {
if (!clickLogList.isEmpty()) {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
try {
final ClickLogBhv clickLogBhv = ComponentUtil.getComponent(ClickLogBhv.class);
clickLogBhv.batchInsert(clickLogList);
} catch (final Exception e) {
logger.warn("Failed to insert: {}", clickLogList, e);
}
if (fessConfig.isLoggingSearchUseLogfile()) {
clickLogList.forEach(this::writeSearchLogEvent);
}
}
}
public void writeSearchLogEvent(SearchLogEvent event) {
try {
final Map<String, Object> source = toSource(event);
searchLogLogger.info(objectMapper.writeValueAsString(source));
} catch (JsonProcessingException e) {
logger.warn("Failed to write {}", event, e);
}
}
protected Map<String, Object> toSource(SearchLogEvent searchLogEvent) {
final Map<String, Object> source = toLowerHyphen(searchLogEvent.toSource());
source.put("_id", searchLogEvent.getId());
// source.put("version_no", searchLogEvent.getVersionNo());
source.put("event_type", searchLogEvent.getEventType());
return source;
}
protected Map<String, Object> toLowerHyphen(Map<String, Object> source) {
return source.entrySet().stream()
.collect(Collectors.toMap(e -> CaseFormat.UPPER_CAMEL.to(CaseFormat.LOWER_UNDERSCORE, e.getKey()), e -> {
final Object value = e.getValue();
if (value instanceof Map) {
@SuppressWarnings("unchecked")
final Map<String, Object> mapValue = (Map<String, Object>) value;
return toLowerHyphen(mapValue);
}
return e.getValue();
}));
}
public void setUserCheckInterval(final long userCheckInterval) {
this.userCheckInterval = userCheckInterval;
}
@ -382,4 +452,8 @@ public class SearchLogHelper {
public void setUserInfoCacheSize(final int userInfoCacheSize) {
this.userInfoCacheSize = userInfoCacheSize;
}
public void setLoggerName(String loggerName) {
this.loggerName = loggerName;
}
}

26
src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java
View file

@ -983,6 +983,9 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
/** The key of the configuration. e.g. filetype,created,click_count,title,doc_id,url,score,site,filename,host,digest,boost,mimetype,favorite_count,_id,lang,last_modified,content_length,timestamp */
String LOGGING_SEARCH_DOCS_FIELDS = "logging.search.docs.fields";
/** The key of the configuration. e.g. true */
String LOGGING_SEARCH_USE_LOGFILE = "logging.search.use.logfile";
/** The key of the configuration. e.g. org.codelibs,org.dbflute,org.lastaflute */
String LOGGING_APP_PACKAGES = "logging.app.packages";
@ -4471,6 +4474,20 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
*/
String getLoggingSearchDocsFields();
/**
* Get the value for the key 'logging.search.use.logfile'. <br>
* The value is, e.g. true <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getLoggingSearchUseLogfile();
/**
* Is the property for the key 'logging.search.use.logfile' true? <br>
* The value is, e.g. true <br>
* @return The determination, true or false. (if not found, exception but basically no way)
*/
boolean isLoggingSearchUseLogfile();
/**
* Get the value for the key 'logging.app.packages'. <br>
* The value is, e.g. org.codelibs,org.dbflute,org.lastaflute <br>
@ -8033,6 +8050,14 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
return get(FessConfig.LOGGING_SEARCH_DOCS_FIELDS);
}
public String getLoggingSearchUseLogfile() {
return get(FessConfig.LOGGING_SEARCH_USE_LOGFILE);
}
public boolean isLoggingSearchUseLogfile() {
return is(FessConfig.LOGGING_SEARCH_USE_LOGFILE);
}
public String getLoggingAppPackages() {
return get(FessConfig.LOGGING_APP_PACKAGES);
}
@ -9447,6 +9472,7 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
defaultMap.put(FessConfig.LOGGING_SEARCH_DOCS_ENABLED, "true");
defaultMap.put(FessConfig.LOGGING_SEARCH_DOCS_FIELDS,
"filetype,created,click_count,title,doc_id,url,score,site,filename,host,digest,boost,mimetype,favorite_count,_id,lang,last_modified,content_length,timestamp");
defaultMap.put(FessConfig.LOGGING_SEARCH_USE_LOGFILE, "true");
defaultMap.put(FessConfig.LOGGING_APP_PACKAGES, "org.codelibs,org.dbflute,org.lastaflute");
defaultMap.put(FessConfig.FORM_ADMIN_MAX_INPUT_SIZE, "4000");
defaultMap.put(FessConfig.FORM_ADMIN_LABEL_IN_CONFIG_ENABLED, "false");

1
src/main/resources/fess_config.properties
View file

@ -554,6 +554,7 @@ index.backup.log.targets=click_log.ndjson,favorite_log.ndjson,search_log.ndjson,
# logging
logging.search.docs.enabled=true
logging.search.docs.fields=filetype,created,click_count,title,doc_id,url,score,site,filename,host,digest,boost,mimetype,favorite_count,_id,lang,last_modified,content_length,timestamp
logging.search.use.logfile=false
logging.app.packages=org.codelibs,org.dbflute,org.lastaflute
# ========================================================================================

16
src/main/resources/log4j2.xml
View file

@ -10,6 +10,7 @@
<Property name="backup.date.suffix" value="_%d{yyyyMMdd}" />
<Property name="backup.max.history" value="90" />
<Property name="audit.log.pattern" value="%msg%n" />
<Property name="searchlog.log.pattern" value="%msg%n" />
</Properties>
<Appenders>
@ -52,6 +53,18 @@
<DefaultRolloverStrategy fileIndex="max" min="1"
max="${backup.max.history}" compressionLevel="9" />
</RollingFile>
<RollingFile name="SearchLogFile" fileName="${log.file.basedir}/searchlog.log"
filePattern="${log.file.basedir}/searchlog${backup.date.suffix}-%i.log.gz">
<PatternLayout>
<Pattern>${searchlog.log.pattern}</Pattern>
</PatternLayout>
<Policies>
<TimeBasedTriggeringPolicy />
<SizeBasedTriggeringPolicy size="100 MB" />
</Policies>
<DefaultRolloverStrategy fileIndex="max" min="1"
max="${backup.max.history}" compressionLevel="9" />
</RollingFile>
</Appenders>
<Loggers>
@ -71,6 +84,9 @@
<AppenderRef ref="Console" />
<AppenderRef ref="AuditFile" />
</Logger>
<Logger name="fess.log.searchlog" additivity="false" level="info">
<AppenderRef ref="SearchLogFile" />
</Logger>
<Root level="${root.log.level}">
<AppenderRef ref="Console" />
<AppenderRef ref="AppFile" />