diff --git a/src/main/java/org/codelibs/fess/app/service/FavoriteLogService.java b/src/main/java/org/codelibs/fess/app/service/FavoriteLogService.java index dd4b692ce..87b1c089d 100644 --- a/src/main/java/org/codelibs/fess/app/service/FavoriteLogService.java +++ b/src/main/java/org/codelibs/fess/app/service/FavoriteLogService.java @@ -28,6 +28,7 @@ import org.codelibs.fess.es.log.exentity.FavoriteLog; import org.codelibs.fess.es.log.exentity.UserInfo; import org.codelibs.fess.helper.SystemHelper; import org.codelibs.fess.mylasta.direction.FessConfig; +import org.codelibs.fess.util.ComponentUtil; import org.dbflute.cbean.result.ListResultBean; public class FavoriteLogService { @@ -48,6 +49,10 @@ public class FavoriteLogService { final FavoriteLog favoriteLog = new FavoriteLog(); favoriteLogLambda.accept(userInfo, favoriteLog); favoriteLogBhv.insert(favoriteLog); + if (fessConfig.isLoggingSearchUseLogfile()) { + ComponentUtil.getSearchLogHelper().writeSearchLogEvent(favoriteLog); + ; + } return true; }).orElse(false); } diff --git a/src/main/java/org/codelibs/fess/entity/SearchLogEvent.java b/src/main/java/org/codelibs/fess/entity/SearchLogEvent.java new file mode 100644 index 000000000..9029b1391 --- /dev/null +++ b/src/main/java/org/codelibs/fess/entity/SearchLogEvent.java @@ -0,0 +1,28 @@ +/* + * Copyright 2012-2020 CodeLibs Project and the Others. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ +package org.codelibs.fess.entity; + +import java.util.Map; + +public interface SearchLogEvent { + String getId(); + + Long getVersionNo(); + + Map toSource(); + + String getEventType(); +} diff --git a/src/main/java/org/codelibs/fess/es/log/exentity/ClickLog.java b/src/main/java/org/codelibs/fess/es/log/exentity/ClickLog.java index 7c1a6f439..01c0819bc 100644 --- a/src/main/java/org/codelibs/fess/es/log/exentity/ClickLog.java +++ b/src/main/java/org/codelibs/fess/es/log/exentity/ClickLog.java @@ -21,12 +21,13 @@ import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.util.Map; +import org.codelibs.fess.entity.SearchLogEvent; import org.codelibs.fess.es.log.bsentity.BsClickLog; /** * @author FreeGen */ -public class ClickLog extends BsClickLog { +public class ClickLog extends BsClickLog implements SearchLogEvent { private static final long serialVersionUID = 1L; @@ -81,4 +82,9 @@ public class ClickLog extends BsClickLog { return "ClickLog [queryRequestedAt=" + queryRequestedAt + ", requestedAt=" + requestedAt + ", queryId=" + queryId + ", docId=" + docId + ", userSessionId=" + userSessionId + ", url=" + url + ", order=" + order + ", docMeta=" + docMeta + "]"; } + + @Override + public String getEventType() { + return "click"; + } } diff --git a/src/main/java/org/codelibs/fess/es/log/exentity/FavoriteLog.java b/src/main/java/org/codelibs/fess/es/log/exentity/FavoriteLog.java index c0e9f78df..e87c32aa9 100644 --- a/src/main/java/org/codelibs/fess/es/log/exentity/FavoriteLog.java +++ b/src/main/java/org/codelibs/fess/es/log/exentity/FavoriteLog.java @@ -21,12 +21,13 @@ import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.util.Map; +import org.codelibs.fess.entity.SearchLogEvent; import org.codelibs.fess.es.log.bsentity.BsFavoriteLog; /** * @author FreeGen */ -public class FavoriteLog extends BsFavoriteLog { +public class FavoriteLog extends BsFavoriteLog implements SearchLogEvent { private static final long serialVersionUID = 1L; @@ -85,4 +86,9 @@ public class FavoriteLog extends BsFavoriteLog { return "FavoriteLog [createdAt=" + createdAt + ", url=" + url + ", docId=" + docId + ", queryId=" + queryId + ", userInfoId=" + userInfoId + ", docMeta=" + docMeta + "]"; } + + @Override + public String getEventType() { + return "favorite"; + } } diff --git a/src/main/java/org/codelibs/fess/es/log/exentity/SearchLog.java b/src/main/java/org/codelibs/fess/es/log/exentity/SearchLog.java index c28e91695..c830df49d 100644 --- a/src/main/java/org/codelibs/fess/es/log/exentity/SearchLog.java +++ b/src/main/java/org/codelibs/fess/es/log/exentity/SearchLog.java @@ -28,6 +28,7 @@ import java.util.stream.Collectors; import org.codelibs.core.lang.StringUtil; import org.codelibs.core.misc.Pair; import org.codelibs.fess.Constants; +import org.codelibs.fess.entity.SearchLogEvent; import org.codelibs.fess.es.log.bsentity.BsSearchLog; import org.codelibs.fess.es.log.exbhv.UserInfoBhv; import org.codelibs.fess.util.ComponentUtil; @@ -36,7 +37,7 @@ import org.dbflute.optional.OptionalEntity; /** * @author FreeGen */ -public class SearchLog extends BsSearchLog { +public class SearchLog extends BsSearchLog implements SearchLogEvent { private static final long serialVersionUID = 1L; @@ -147,4 +148,8 @@ public class SearchLog extends BsSearchLog { + userSessionId + ", virtualHost=" + virtualHost + ", documents=" + documentList + "]"; } + @Override + public String getEventType() { + return "log"; + } } diff --git a/src/main/java/org/codelibs/fess/es/log/exentity/UserInfo.java b/src/main/java/org/codelibs/fess/es/log/exentity/UserInfo.java index fa8c17648..8635cfa87 100644 --- a/src/main/java/org/codelibs/fess/es/log/exentity/UserInfo.java +++ b/src/main/java/org/codelibs/fess/es/log/exentity/UserInfo.java @@ -21,12 +21,13 @@ import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.util.Map; +import org.codelibs.fess.entity.SearchLogEvent; import org.codelibs.fess.es.log.bsentity.BsUserInfo; /** * @author FreeGen */ -public class UserInfo extends BsUserInfo { +public class UserInfo extends BsUserInfo implements SearchLogEvent { private static final long serialVersionUID = 1L; @@ -84,4 +85,9 @@ public class UserInfo extends BsUserInfo { public String toString() { return "UserInfo [createdAt=" + createdAt + ", updatedAt=" + updatedAt + ", docMeta=" + docMeta + "]"; } + + @Override + public String getEventType() { + return "user"; + } } diff --git a/src/main/java/org/codelibs/fess/helper/SearchLogHelper.java b/src/main/java/org/codelibs/fess/helper/SearchLogHelper.java index 0235fc9f3..fb445550a 100644 --- a/src/main/java/org/codelibs/fess/helper/SearchLogHelper.java +++ b/src/main/java/org/codelibs/fess/helper/SearchLogHelper.java @@ -27,6 +27,7 @@ import java.util.Queue; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import javax.annotation.PostConstruct; import javax.servlet.http.HttpServletRequest; @@ -37,6 +38,7 @@ import org.apache.logging.log4j.Logger; import org.codelibs.core.concurrent.CommonPoolUtil; import org.codelibs.core.lang.StringUtil; import org.codelibs.fess.Constants; +import org.codelibs.fess.entity.SearchLogEvent; import org.codelibs.fess.entity.SearchRequestParams; import org.codelibs.fess.entity.SearchRequestParams.SearchRequestType; import org.codelibs.fess.es.log.exbhv.ClickLogBhv; @@ -57,6 +59,9 @@ import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.script.Script; import org.lastaflute.web.util.LaRequestUtil; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.CaseFormat; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; @@ -74,6 +79,12 @@ public class SearchLogHelper { protected LoadingCache userInfoCache; + protected String loggerName = "fess.log.searchlog"; + + protected Logger searchLogLogger = null; + + protected ObjectMapper objectMapper = new ObjectMapper(); + @PostConstruct public void init() { if (logger.isDebugEnabled()) { @@ -88,6 +99,7 @@ public class SearchLogHelper { return storeUserInfo(key); } }); + searchLogLogger = LogManager.getLogger(loggerName); } public void addSearchLog(final SearchRequestParams params, final LocalDateTime requestedTime, final String queryId, final String query, @@ -267,7 +279,27 @@ public class SearchLogHelper { } }); + processUserInfoLog(searchLogList, userInfoMap); + processSearchLog(searchLogList); + } + + private void processSearchLog(final List searchLogList) { + if (!searchLogList.isEmpty()) { + final FessConfig fessConfig = ComponentUtil.getFessConfig(); + storeSearchLogList(searchLogList); + if (fessConfig.isSuggestSearchLog()) { + final SuggestHelper suggestHelper = ComponentUtil.getSuggestHelper(); + suggestHelper.indexFromSearchLog(searchLogList); + } + if (fessConfig.isLoggingSearchUseLogfile()) { + searchLogList.forEach(this::writeSearchLogEvent); + } + } + } + + protected void processUserInfoLog(final List searchLogList, final Map userInfoMap) { if (!userInfoMap.isEmpty()) { + final FessConfig fessConfig = ComponentUtil.getFessConfig(); final List insertList = new ArrayList<>(userInfoMap.values()); final List updateList = new ArrayList<>(); final UserInfoBhv userInfoBhv = ComponentUtil.getComponent(UserInfoBhv.class); @@ -289,13 +321,9 @@ public class SearchLogHelper { searchLog.setUserInfoId(userInfo.getId()); }); }); - } - - if (!searchLogList.isEmpty()) { - storeSearchLogList(searchLogList); - if (fessConfig.isSuggestSearchLog()) { - final SuggestHelper suggestHelper = ComponentUtil.getSuggestHelper(); - suggestHelper.indexFromSearchLog(searchLogList); + if (fessConfig.isLoggingSearchUseLogfile()) { + insertList.forEach(this::writeSearchLogEvent); + updateList.forEach(this::writeSearchLogEvent); } } } @@ -332,15 +360,12 @@ public class SearchLogHelper { logger.warn("Failed to process: {}", clickLog, e); } } - if (!clickLogList.isEmpty()) { - try { - final ClickLogBhv clickLogBhv = ComponentUtil.getComponent(ClickLogBhv.class); - clickLogBhv.batchInsert(clickLogList); - } catch (final Exception e) { - logger.warn("Failed to insert: {}", clickLogList, e); - } - } + processClickLog(clickLogList); + updateClickFieldInIndex(clickCountMap); + } + + protected void updateClickFieldInIndex(final Map clickCountMap) { if (!clickCountMap.isEmpty()) { final SearchHelper searchHelper = ComponentUtil.getSearchHelper(); final FessConfig fessConfig = ComponentUtil.getFessConfig(); @@ -375,6 +400,51 @@ public class SearchLogHelper { } } + protected void processClickLog(final List clickLogList) { + if (!clickLogList.isEmpty()) { + final FessConfig fessConfig = ComponentUtil.getFessConfig(); + try { + final ClickLogBhv clickLogBhv = ComponentUtil.getComponent(ClickLogBhv.class); + clickLogBhv.batchInsert(clickLogList); + } catch (final Exception e) { + logger.warn("Failed to insert: {}", clickLogList, e); + } + if (fessConfig.isLoggingSearchUseLogfile()) { + clickLogList.forEach(this::writeSearchLogEvent); + } + } + } + + public void writeSearchLogEvent(SearchLogEvent event) { + try { + final Map source = toSource(event); + searchLogLogger.info(objectMapper.writeValueAsString(source)); + } catch (JsonProcessingException e) { + logger.warn("Failed to write {}", event, e); + } + } + + protected Map toSource(SearchLogEvent searchLogEvent) { + final Map source = toLowerHyphen(searchLogEvent.toSource()); + source.put("_id", searchLogEvent.getId()); + // source.put("version_no", searchLogEvent.getVersionNo()); + source.put("event_type", searchLogEvent.getEventType()); + return source; + } + + protected Map toLowerHyphen(Map source) { + return source.entrySet().stream() + .collect(Collectors.toMap(e -> CaseFormat.UPPER_CAMEL.to(CaseFormat.LOWER_UNDERSCORE, e.getKey()), e -> { + final Object value = e.getValue(); + if (value instanceof Map) { + @SuppressWarnings("unchecked") + final Map mapValue = (Map) value; + return toLowerHyphen(mapValue); + } + return e.getValue(); + })); + } + public void setUserCheckInterval(final long userCheckInterval) { this.userCheckInterval = userCheckInterval; } @@ -382,4 +452,8 @@ public class SearchLogHelper { public void setUserInfoCacheSize(final int userInfoCacheSize) { this.userInfoCacheSize = userInfoCacheSize; } + + public void setLoggerName(String loggerName) { + this.loggerName = loggerName; + } } diff --git a/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java b/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java index 4b5a5a89e..eb48de9a4 100644 --- a/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java +++ b/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java @@ -983,6 +983,9 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction /** The key of the configuration. e.g. filetype,created,click_count,title,doc_id,url,score,site,filename,host,digest,boost,mimetype,favorite_count,_id,lang,last_modified,content_length,timestamp */ String LOGGING_SEARCH_DOCS_FIELDS = "logging.search.docs.fields"; + /** The key of the configuration. e.g. true */ + String LOGGING_SEARCH_USE_LOGFILE = "logging.search.use.logfile"; + /** The key of the configuration. e.g. org.codelibs,org.dbflute,org.lastaflute */ String LOGGING_APP_PACKAGES = "logging.app.packages"; @@ -4471,6 +4474,20 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction */ String getLoggingSearchDocsFields(); + /** + * Get the value for the key 'logging.search.use.logfile'.
+ * The value is, e.g. true
+ * @return The value of found property. (NotNull: if not found, exception but basically no way) + */ + String getLoggingSearchUseLogfile(); + + /** + * Is the property for the key 'logging.search.use.logfile' true?
+ * The value is, e.g. true
+ * @return The determination, true or false. (if not found, exception but basically no way) + */ + boolean isLoggingSearchUseLogfile(); + /** * Get the value for the key 'logging.app.packages'.
* The value is, e.g. org.codelibs,org.dbflute,org.lastaflute
@@ -8033,6 +8050,14 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction return get(FessConfig.LOGGING_SEARCH_DOCS_FIELDS); } + public String getLoggingSearchUseLogfile() { + return get(FessConfig.LOGGING_SEARCH_USE_LOGFILE); + } + + public boolean isLoggingSearchUseLogfile() { + return is(FessConfig.LOGGING_SEARCH_USE_LOGFILE); + } + public String getLoggingAppPackages() { return get(FessConfig.LOGGING_APP_PACKAGES); } @@ -9447,6 +9472,7 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction defaultMap.put(FessConfig.LOGGING_SEARCH_DOCS_ENABLED, "true"); defaultMap.put(FessConfig.LOGGING_SEARCH_DOCS_FIELDS, "filetype,created,click_count,title,doc_id,url,score,site,filename,host,digest,boost,mimetype,favorite_count,_id,lang,last_modified,content_length,timestamp"); + defaultMap.put(FessConfig.LOGGING_SEARCH_USE_LOGFILE, "true"); defaultMap.put(FessConfig.LOGGING_APP_PACKAGES, "org.codelibs,org.dbflute,org.lastaflute"); defaultMap.put(FessConfig.FORM_ADMIN_MAX_INPUT_SIZE, "4000"); defaultMap.put(FessConfig.FORM_ADMIN_LABEL_IN_CONFIG_ENABLED, "false"); diff --git a/src/main/resources/fess_config.properties b/src/main/resources/fess_config.properties index b3162a112..820c55070 100644 --- a/src/main/resources/fess_config.properties +++ b/src/main/resources/fess_config.properties @@ -554,6 +554,7 @@ index.backup.log.targets=click_log.ndjson,favorite_log.ndjson,search_log.ndjson, # logging logging.search.docs.enabled=true logging.search.docs.fields=filetype,created,click_count,title,doc_id,url,score,site,filename,host,digest,boost,mimetype,favorite_count,_id,lang,last_modified,content_length,timestamp +logging.search.use.logfile=false logging.app.packages=org.codelibs,org.dbflute,org.lastaflute # ======================================================================================== diff --git a/src/main/resources/log4j2.xml b/src/main/resources/log4j2.xml index 60c3ae64e..e56ff516e 100644 --- a/src/main/resources/log4j2.xml +++ b/src/main/resources/log4j2.xml @@ -10,6 +10,7 @@ + @@ -52,6 +53,18 @@ + + + ${searchlog.log.pattern} + + + + + + + @@ -71,6 +84,9 @@ + + +