fix #2496 add searchlog.log

src/main/java/org/codelibs/fess/app/service/FavoriteLogService.java

@@ -28,6 +28,7 @@ import org.codelibs.fess.es.log.exentity.FavoriteLog;
 import org.codelibs.fess.es.log.exentity.UserInfo;
 import org.codelibs.fess.helper.SystemHelper;
 import org.codelibs.fess.mylasta.direction.FessConfig;
+import org.codelibs.fess.util.ComponentUtil;
 import org.dbflute.cbean.result.ListResultBean;
 
 public class FavoriteLogService {
@@ -48,6 +49,10 @@ public class FavoriteLogService {
             final FavoriteLog favoriteLog = new FavoriteLog();
             favoriteLogLambda.accept(userInfo, favoriteLog);
             favoriteLogBhv.insert(favoriteLog);
+            if (fessConfig.isLoggingSearchUseLogfile()) {
+                ComponentUtil.getSearchLogHelper().writeSearchLogEvent(favoriteLog);
+                ;
+            }
             return true;
         }).orElse(false);
     }

src/main/java/org/codelibs/fess/entity/SearchLogEvent.java

@@ -0,0 +1,28 @@
+/*
+ * Copyright 2012-2020 CodeLibs Project and the Others.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+ * either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+package org.codelibs.fess.entity;
+
+import java.util.Map;
+
+public interface SearchLogEvent {
+    String getId();
+
+    Long getVersionNo();
+
+    Map<String, Object> toSource();
+
+    String getEventType();
+}

src/main/java/org/codelibs/fess/es/log/exentity/ClickLog.java

@@ -21,12 +21,13 @@ import java.time.ZonedDateTime;
 import java.time.format.DateTimeFormatter;
 import java.util.Map;
 
+import org.codelibs.fess.entity.SearchLogEvent;
 import org.codelibs.fess.es.log.bsentity.BsClickLog;
 
 /**
  * @author FreeGen
  */
-public class ClickLog extends BsClickLog {
+public class ClickLog extends BsClickLog implements SearchLogEvent {
 
     private static final long serialVersionUID = 1L;
 
@@ -81,4 +82,9 @@ public class ClickLog extends BsClickLog {
         return "ClickLog [queryRequestedAt=" + queryRequestedAt + ", requestedAt=" + requestedAt + ", queryId=" + queryId + ", docId="
                 + docId + ", userSessionId=" + userSessionId + ", url=" + url + ", order=" + order + ", docMeta=" + docMeta + "]";
     }
+
+    @Override
+    public String getEventType() {
+        return "click";
+    }
 }

src/main/java/org/codelibs/fess/es/log/exentity/FavoriteLog.java

@@ -21,12 +21,13 @@ import java.time.ZonedDateTime;
 import java.time.format.DateTimeFormatter;
 import java.util.Map;
 
+import org.codelibs.fess.entity.SearchLogEvent;
 import org.codelibs.fess.es.log.bsentity.BsFavoriteLog;
 
 /**
  * @author FreeGen
  */
-public class FavoriteLog extends BsFavoriteLog {
+public class FavoriteLog extends BsFavoriteLog implements SearchLogEvent {
 
     private static final long serialVersionUID = 1L;
 
@@ -85,4 +86,9 @@ public class FavoriteLog extends BsFavoriteLog {
         return "FavoriteLog [createdAt=" + createdAt + ", url=" + url + ", docId=" + docId + ", queryId=" + queryId + ", userInfoId="
                 + userInfoId + ", docMeta=" + docMeta + "]";
     }
+
+    @Override
+    public String getEventType() {
+        return "favorite";
+    }
 }

src/main/java/org/codelibs/fess/es/log/exentity/SearchLog.java

@@ -28,6 +28,7 @@ import java.util.stream.Collectors;
 import org.codelibs.core.lang.StringUtil;
 import org.codelibs.core.misc.Pair;
 import org.codelibs.fess.Constants;
+import org.codelibs.fess.entity.SearchLogEvent;
 import org.codelibs.fess.es.log.bsentity.BsSearchLog;
 import org.codelibs.fess.es.log.exbhv.UserInfoBhv;
 import org.codelibs.fess.util.ComponentUtil;
@@ -36,7 +37,7 @@ import org.dbflute.optional.OptionalEntity;
 /**
  * @author FreeGen
  */
-public class SearchLog extends BsSearchLog {
+public class SearchLog extends BsSearchLog implements SearchLogEvent {
 
     private static final long serialVersionUID = 1L;
 
@@ -147,4 +148,8 @@ public class SearchLog extends BsSearchLog {
                 + userSessionId + ", virtualHost=" + virtualHost + ", documents=" + documentList + "]";
     }
 
+    @Override
+    public String getEventType() {
+        return "log";
+    }
 }

src/main/java/org/codelibs/fess/es/log/exentity/UserInfo.java

@@ -21,12 +21,13 @@ import java.time.ZonedDateTime;
 import java.time.format.DateTimeFormatter;
 import java.util.Map;
 
+import org.codelibs.fess.entity.SearchLogEvent;
 import org.codelibs.fess.es.log.bsentity.BsUserInfo;
 
 /**
  * @author FreeGen
  */
-public class UserInfo extends BsUserInfo {
+public class UserInfo extends BsUserInfo implements SearchLogEvent {
 
     private static final long serialVersionUID = 1L;
 
@@ -84,4 +85,9 @@ public class UserInfo extends BsUserInfo {
     public String toString() {
         return "UserInfo [createdAt=" + createdAt + ", updatedAt=" + updatedAt + ", docMeta=" + docMeta + "]";
     }
+
+    @Override
+    public String getEventType() {
+        return "user";
+    }
 }

src/main/java/org/codelibs/fess/helper/SearchLogHelper.java

@@ -27,6 +27,7 @@ import java.util.Queue;
 import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
 
 import javax.annotation.PostConstruct;
 import javax.servlet.http.HttpServletRequest;
@@ -37,6 +38,7 @@ import org.apache.logging.log4j.Logger;
 import org.codelibs.core.concurrent.CommonPoolUtil;
 import org.codelibs.core.lang.StringUtil;
 import org.codelibs.fess.Constants;
+import org.codelibs.fess.entity.SearchLogEvent;
 import org.codelibs.fess.entity.SearchRequestParams;
 import org.codelibs.fess.entity.SearchRequestParams.SearchRequestType;
 import org.codelibs.fess.es.log.exbhv.ClickLogBhv;
@@ -57,6 +59,9 @@ import org.elasticsearch.action.update.UpdateRequest;
 import org.elasticsearch.script.Script;
 import org.lastaflute.web.util.LaRequestUtil;
 
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.CaseFormat;
 import com.google.common.cache.CacheBuilder;
 import com.google.common.cache.CacheLoader;
 import com.google.common.cache.LoadingCache;
@@ -74,6 +79,12 @@ public class SearchLogHelper {
 
     protected LoadingCache<String, UserInfo> userInfoCache;
 
+    protected String loggerName = "fess.log.searchlog";
+
+    protected Logger searchLogLogger = null;
+
+    protected ObjectMapper objectMapper = new ObjectMapper();
+
     @PostConstruct
     public void init() {
         if (logger.isDebugEnabled()) {
@@ -88,6 +99,7 @@ public class SearchLogHelper {
                         return storeUserInfo(key);
                     }
                 });
+        searchLogLogger = LogManager.getLogger(loggerName);
     }
 
     public void addSearchLog(final SearchRequestParams params, final LocalDateTime requestedTime, final String queryId, final String query,
@@ -267,7 +279,27 @@ public class SearchLogHelper {
             }
         });
 
+        processUserInfoLog(searchLogList, userInfoMap);
+        processSearchLog(searchLogList);
+    }
+
+    private void processSearchLog(final List<SearchLog> searchLogList) {
+        if (!searchLogList.isEmpty()) {
+            final FessConfig fessConfig = ComponentUtil.getFessConfig();
+            storeSearchLogList(searchLogList);
+            if (fessConfig.isSuggestSearchLog()) {
+                final SuggestHelper suggestHelper = ComponentUtil.getSuggestHelper();
+                suggestHelper.indexFromSearchLog(searchLogList);
+            }
+            if (fessConfig.isLoggingSearchUseLogfile()) {
+                searchLogList.forEach(this::writeSearchLogEvent);
+            }
+        }
+    }
+
+    protected void processUserInfoLog(final List<SearchLog> searchLogList, final Map<String, UserInfo> userInfoMap) {
         if (!userInfoMap.isEmpty()) {
+            final FessConfig fessConfig = ComponentUtil.getFessConfig();
             final List<UserInfo> insertList = new ArrayList<>(userInfoMap.values());
             final List<UserInfo> updateList = new ArrayList<>();
             final UserInfoBhv userInfoBhv = ComponentUtil.getComponent(UserInfoBhv.class);
@@ -289,13 +321,9 @@ public class SearchLogHelper {
                     searchLog.setUserInfoId(userInfo.getId());
                 });
             });
-        }
-
-        if (!searchLogList.isEmpty()) {
-            storeSearchLogList(searchLogList);
-            if (fessConfig.isSuggestSearchLog()) {
-                final SuggestHelper suggestHelper = ComponentUtil.getSuggestHelper();
-                suggestHelper.indexFromSearchLog(searchLogList);
+            if (fessConfig.isLoggingSearchUseLogfile()) {
+                insertList.forEach(this::writeSearchLogEvent);
+                updateList.forEach(this::writeSearchLogEvent);
             }
         }
     }
@@ -332,15 +360,12 @@ public class SearchLogHelper {
                 logger.warn("Failed to process: {}", clickLog, e);
             }
         }
-        if (!clickLogList.isEmpty()) {
-            try {
-                final ClickLogBhv clickLogBhv = ComponentUtil.getComponent(ClickLogBhv.class);
-                clickLogBhv.batchInsert(clickLogList);
-            } catch (final Exception e) {
-                logger.warn("Failed to insert: {}", clickLogList, e);
-            }
-        }
+        processClickLog(clickLogList);
 
+        updateClickFieldInIndex(clickCountMap);
+    }
+
+    protected void updateClickFieldInIndex(final Map<String, Integer> clickCountMap) {
         if (!clickCountMap.isEmpty()) {
             final SearchHelper searchHelper = ComponentUtil.getSearchHelper();
             final FessConfig fessConfig = ComponentUtil.getFessConfig();
@@ -375,6 +400,51 @@ public class SearchLogHelper {
         }
     }
 
+    protected void processClickLog(final List<ClickLog> clickLogList) {
+        if (!clickLogList.isEmpty()) {
+            final FessConfig fessConfig = ComponentUtil.getFessConfig();
+            try {
+                final ClickLogBhv clickLogBhv = ComponentUtil.getComponent(ClickLogBhv.class);
+                clickLogBhv.batchInsert(clickLogList);
+            } catch (final Exception e) {
+                logger.warn("Failed to insert: {}", clickLogList, e);
+            }
+            if (fessConfig.isLoggingSearchUseLogfile()) {
+                clickLogList.forEach(this::writeSearchLogEvent);
+            }
+        }
+    }
+
+    public void writeSearchLogEvent(SearchLogEvent event) {
+        try {
+            final Map<String, Object> source = toSource(event);
+            searchLogLogger.info(objectMapper.writeValueAsString(source));
+        } catch (JsonProcessingException e) {
+            logger.warn("Failed to write {}", event, e);
+        }
+    }
+
+    protected Map<String, Object> toSource(SearchLogEvent searchLogEvent) {
+        final Map<String, Object> source = toLowerHyphen(searchLogEvent.toSource());
+        source.put("_id", searchLogEvent.getId());
+        // source.put("version_no", searchLogEvent.getVersionNo());
+        source.put("event_type", searchLogEvent.getEventType());
+        return source;
+    }
+
+    protected Map<String, Object> toLowerHyphen(Map<String, Object> source) {
+        return source.entrySet().stream()
+                .collect(Collectors.toMap(e -> CaseFormat.UPPER_CAMEL.to(CaseFormat.LOWER_UNDERSCORE, e.getKey()), e -> {
+                    final Object value = e.getValue();
+                    if (value instanceof Map) {
+                        @SuppressWarnings("unchecked")
+                        final Map<String, Object> mapValue = (Map<String, Object>) value;
+                        return toLowerHyphen(mapValue);
+                    }
+                    return e.getValue();
+                }));
+    }
+
     public void setUserCheckInterval(final long userCheckInterval) {
         this.userCheckInterval = userCheckInterval;
     }
@@ -382,4 +452,8 @@ public class SearchLogHelper {
     public void setUserInfoCacheSize(final int userInfoCacheSize) {
         this.userInfoCacheSize = userInfoCacheSize;
     }
+
+    public void setLoggerName(String loggerName) {
+        this.loggerName = loggerName;
+    }
 }

src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java

@@ -983,6 +983,9 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
     /** The key of the configuration. e.g. filetype,created,click_count,title,doc_id,url,score,site,filename,host,digest,boost,mimetype,favorite_count,_id,lang,last_modified,content_length,timestamp */
     String LOGGING_SEARCH_DOCS_FIELDS = "logging.search.docs.fields";
 
+    /** The key of the configuration. e.g. true */
+    String LOGGING_SEARCH_USE_LOGFILE = "logging.search.use.logfile";
+
     /** The key of the configuration. e.g. org.codelibs,org.dbflute,org.lastaflute */
     String LOGGING_APP_PACKAGES = "logging.app.packages";
 
@@ -4471,6 +4474,20 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
      */
     String getLoggingSearchDocsFields();
 
+    /**
+     * Get the value for the key 'logging.search.use.logfile'. <br>
+     * The value is, e.g. true <br>
+     * @return The value of found property. (NotNull: if not found, exception but basically no way)
+     */
+    String getLoggingSearchUseLogfile();
+
+    /**
+     * Is the property for the key 'logging.search.use.logfile' true? <br>
+     * The value is, e.g. true <br>
+     * @return The determination, true or false. (if not found, exception but basically no way)
+     */
+    boolean isLoggingSearchUseLogfile();
+
     /**
      * Get the value for the key 'logging.app.packages'. <br>
      * The value is, e.g. org.codelibs,org.dbflute,org.lastaflute <br>
@@ -8033,6 +8050,14 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
             return get(FessConfig.LOGGING_SEARCH_DOCS_FIELDS);
         }
 
+        public String getLoggingSearchUseLogfile() {
+            return get(FessConfig.LOGGING_SEARCH_USE_LOGFILE);
+        }
+
+        public boolean isLoggingSearchUseLogfile() {
+            return is(FessConfig.LOGGING_SEARCH_USE_LOGFILE);
+        }
+
         public String getLoggingAppPackages() {
             return get(FessConfig.LOGGING_APP_PACKAGES);
         }
@@ -9447,6 +9472,7 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
             defaultMap.put(FessConfig.LOGGING_SEARCH_DOCS_ENABLED, "true");
             defaultMap.put(FessConfig.LOGGING_SEARCH_DOCS_FIELDS,
                     "filetype,created,click_count,title,doc_id,url,score,site,filename,host,digest,boost,mimetype,favorite_count,_id,lang,last_modified,content_length,timestamp");
+            defaultMap.put(FessConfig.LOGGING_SEARCH_USE_LOGFILE, "true");
             defaultMap.put(FessConfig.LOGGING_APP_PACKAGES, "org.codelibs,org.dbflute,org.lastaflute");
             defaultMap.put(FessConfig.FORM_ADMIN_MAX_INPUT_SIZE, "4000");
             defaultMap.put(FessConfig.FORM_ADMIN_LABEL_IN_CONFIG_ENABLED, "false");

src/main/resources/fess_config.properties

@@ -554,6 +554,7 @@ index.backup.log.targets=click_log.ndjson,favorite_log.ndjson,search_log.ndjson,
 # logging
 logging.search.docs.enabled=true
 logging.search.docs.fields=filetype,created,click_count,title,doc_id,url,score,site,filename,host,digest,boost,mimetype,favorite_count,_id,lang,last_modified,content_length,timestamp
+logging.search.use.logfile=false
 logging.app.packages=org.codelibs,org.dbflute,org.lastaflute
 
 # ========================================================================================

src/main/resources/log4j2.xml

@@ -10,6 +10,7 @@
 		<Property name="backup.date.suffix" value="_%d{yyyyMMdd}" />
 		<Property name="backup.max.history" value="90" />
 		<Property name="audit.log.pattern" value="%msg%n" />
+		<Property name="searchlog.log.pattern" value="%msg%n" />
 	</Properties>
 
 	<Appenders>
@@ -52,6 +53,18 @@
 			<DefaultRolloverStrategy fileIndex="max" min="1"
 				max="${backup.max.history}" compressionLevel="9" />
 		</RollingFile>
+		<RollingFile name="SearchLogFile" fileName="${log.file.basedir}/searchlog.log"
+			filePattern="${log.file.basedir}/searchlog${backup.date.suffix}-%i.log.gz">
+			<PatternLayout>
+				<Pattern>${searchlog.log.pattern}</Pattern>
+			</PatternLayout>
+			<Policies>
+				<TimeBasedTriggeringPolicy />
+				<SizeBasedTriggeringPolicy size="100 MB" />
+			</Policies>
+			<DefaultRolloverStrategy fileIndex="max" min="1"
+				max="${backup.max.history}" compressionLevel="9" />
+		</RollingFile>
 	</Appenders>
 
 	<Loggers>
@@ -71,6 +84,9 @@
 			<AppenderRef ref="Console" />
 			<AppenderRef ref="AuditFile" />
 		</Logger>
+		<Logger name="fess.log.searchlog" additivity="false" level="info">
+			<AppenderRef ref="SearchLogFile" />
+		</Logger>
 		<Root level="${root.log.level}">
 			<AppenderRef ref="Console" />
 			<AppenderRef ref="AppFile" />