fix #1589 add query.additional.scroll.response.fields

src/main/java/org/codelibs/fess/api/json/JsonApiManager.java

@@ -138,34 +138,32 @@ public class JsonApiManager extends BaseJsonApiManager {
         final JsonRequestParams params = new JsonRequestParams(request, fessConfig);
         try {
             response.setContentType("application/x-ndjson; charset=UTF-8");
-            final long count =
-                    searchService.scrollSearch(params, doc -> {
-                        buf.setLength(0);
-                        buf.append('{');
-                        boolean first2 = true;
-                        for (final Map.Entry<String, Object> entry : doc.entrySet()) {
-                            final String name = entry.getKey();
-                            if (StringUtil.isNotBlank(name) && entry.getValue() != null
-                                    && ComponentUtil.getQueryHelper().isApiResponseField(name)) {
-                                if (!first2) {
-                                    buf.append(',');
-                                } else {
-                                    first2 = false;
-                                }
-                                buf.append(escapeJson(name));
-                                buf.append(':');
-                                buf.append(escapeJson(entry.getValue()));
-                            }
-                        }
-                        buf.append('}');
-                        buf.append('\n');
-                        try {
-                            response.getWriter().print(buf.toString());
-                        } catch (final IOException e) {
-                            throw new IORuntimeException(e);
+            final long count = searchService.scrollSearch(params, doc -> {
+                buf.setLength(0);
+                buf.append('{');
+                boolean first2 = true;
+                for (final Map.Entry<String, Object> entry : doc.entrySet()) {
+                    final String name = entry.getKey();
+                    if (StringUtil.isNotBlank(name) && entry.getValue() != null) {
+                        if (!first2) {
+                            buf.append(',');
+                        } else {
+                            first2 = false;
                         }
-                        return true;
-                    }, OptionalThing.empty());
+                        buf.append(escapeJson(name));
+                        buf.append(':');
+                        buf.append(escapeJson(entry.getValue()));
+                    }
+                }
+                buf.append('}');
+                buf.append('\n');
+                try {
+                    response.getWriter().print(buf.toString());
+                } catch (final IOException e) {
+                    throw new IORuntimeException(e);
+                }
+                return true;
+            }, OptionalThing.empty());
             response.flushBuffer();
             if (logger.isDebugEnabled()) {
                 logger.debug("Loaded " + count + " docs");

src/main/java/org/codelibs/fess/app/service/SearchService.java

@@ -195,7 +195,7 @@ public class SearchService {
                     fessConfig.processSearchPreference(searchRequestBuilder, userBean);
                     return SearchConditionBuilder.builder(searchRequestBuilder)
                             .query(StringUtil.isBlank(sortField) ? query : query + " sort:" + sortField).size(pageSize)
-                            .responseFields(queryHelper.getResponseFields()).searchRequestType(params.getType()).build();
+                            .responseFields(queryHelper.getScrollResponseFields()).searchRequestType(params.getType()).build();
                 },
                 (searchResponse, hit) -> {
                     final Map<String, Object> source = hit.getSourceAsMap();

src/main/java/org/codelibs/fess/helper/QueryHelper.java

@@ -107,6 +107,8 @@ public class QueryHelper {
 
     protected String[] responseFields;
 
+    protected String[] scrollResponseFields;
+
     protected String[] cacheResponseFields;
 
     protected String[] highlightedFields;
@@ -162,6 +164,31 @@ public class QueryHelper {
                     fessConfig.getIndexFieldLang(), //
                     fessConfig.getIndexFieldHasCache());
         }
+        if (scrollResponseFields == null) {
+            scrollResponseFields = fessConfig.getQueryAdditionalScrollResponseFields(//
+                    SCORE_FIELD, //
+                    fessConfig.getIndexFieldId(), //
+                    fessConfig.getIndexFieldDocId(), //
+                    fessConfig.getIndexFieldBoost(), //
+                    fessConfig.getIndexFieldContentLength(), //
+                    fessConfig.getIndexFieldHost(), //
+                    fessConfig.getIndexFieldSite(), //
+                    fessConfig.getIndexFieldLastModified(), //
+                    fessConfig.getIndexFieldTimestamp(), //
+                    fessConfig.getIndexFieldMimetype(), //
+                    fessConfig.getIndexFieldFiletype(), //
+                    fessConfig.getIndexFieldFilename(), //
+                    fessConfig.getIndexFieldCreated(), //
+                    fessConfig.getIndexFieldTitle(), //
+                    fessConfig.getIndexFieldDigest(), //
+                    fessConfig.getIndexFieldUrl(), //
+                    fessConfig.getIndexFieldThumbnail(), //
+                    fessConfig.getIndexFieldClickCount(), //
+                    fessConfig.getIndexFieldFavoriteCount(), //
+                    fessConfig.getIndexFieldConfigId(), //
+                    fessConfig.getIndexFieldLang(), //
+                    fessConfig.getIndexFieldHasCache());
+        }
         if (cacheResponseFields == null) {
             cacheResponseFields = fessConfig.getQueryAdditionalCacheResponseFields(//
                     SCORE_FIELD, //
@@ -715,6 +742,14 @@ public class QueryHelper {
         this.responseFields = responseFields;
     }
 
+    public String[] getScrollResponseFields() {
+        return scrollResponseFields;
+    }
+
+    public void setScrollResponseFields(String[] scrollResponseFields) {
+        this.scrollResponseFields = scrollResponseFields;
+    }
+
     public String[] getCacheResponseFields() {
         return cacheResponseFields;
     }

src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java

@@ -590,6 +590,9 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
     /** The key of the configuration. e.g.  */
     String QUERY_ADDITIONAL_API_RESPONSE_FIELDS = "query.additional.api.response.fields";
 
+    /** The key of the configuration. e.g.  */
+    String QUERY_ADDITIONAL_SCROLL_RESPONSE_FIELDS = "query.additional.scroll.response.fields";
+
     /** The key of the configuration. e.g.  */
     String QUERY_ADDITIONAL_CACHE_RESPONSE_FIELDS = "query.additional.cache.response.fields";
 
@@ -3074,6 +3077,21 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
      */
     Integer getQueryAdditionalApiResponseFieldsAsInteger();
 
+    /**
+     * Get the value for the key 'query.additional.scroll.response.fields'. <br>
+     * The value is, e.g.  <br>
+     * @return The value of found property. (NotNull: if not found, exception but basically no way)
+     */
+    String getQueryAdditionalScrollResponseFields();
+
+    /**
+     * Get the value for the key 'query.additional.scroll.response.fields' as {@link Integer}. <br>
+     * The value is, e.g.  <br>
+     * @return The value of found property. (NotNull: if not found, exception but basically no way)
+     * @throws NumberFormatException When the property is not integer.
+     */
+    Integer getQueryAdditionalScrollResponseFieldsAsInteger();
+
     /**
      * Get the value for the key 'query.additional.cache.response.fields'. <br>
      * The value is, e.g.  <br>
@@ -6583,6 +6601,14 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
             return getAsInteger(FessConfig.QUERY_ADDITIONAL_API_RESPONSE_FIELDS);
         }
 
+        public String getQueryAdditionalScrollResponseFields() {
+            return get(FessConfig.QUERY_ADDITIONAL_SCROLL_RESPONSE_FIELDS);
+        }
+
+        public Integer getQueryAdditionalScrollResponseFieldsAsInteger() {
+            return getAsInteger(FessConfig.QUERY_ADDITIONAL_SCROLL_RESPONSE_FIELDS);
+        }
+
         public String getQueryAdditionalCacheResponseFields() {
             return get(FessConfig.QUERY_ADDITIONAL_CACHE_RESPONSE_FIELDS);
         }
@@ -8150,6 +8176,7 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
             defaultMap.put(FessConfig.QUERY_MAX_SEARCH_RESULT_OFFSET, "100000");
             defaultMap.put(FessConfig.QUERY_ADDITIONAL_RESPONSE_FIELDS, "");
             defaultMap.put(FessConfig.QUERY_ADDITIONAL_API_RESPONSE_FIELDS, "");
+            defaultMap.put(FessConfig.QUERY_ADDITIONAL_SCROLL_RESPONSE_FIELDS, "");
             defaultMap.put(FessConfig.QUERY_ADDITIONAL_CACHE_RESPONSE_FIELDS, "");
             defaultMap.put(FessConfig.QUERY_ADDITIONAL_HIGHLIGHTED_FIELDS, "");
             defaultMap.put(FessConfig.QUERY_ADDITIONAL_SEARCH_FIELDS, "");

src/main/java/org/codelibs/fess/mylasta/direction/FessProp.java

@@ -1506,6 +1506,16 @@ public interface FessProp {
         return list.toArray(new String[list.size()]);
     }
 
+    String getQueryAdditionalScrollResponseFields();
+
+    public default String[] getQueryAdditionalScrollResponseFields(final String... fields) {
+        final List<String> list = new ArrayList<>(fields.length + 10);
+        stream(fields).of(stream -> stream.forEach(list::add));
+        split(getQueryAdditionalScrollResponseFields(), ",").of(
+                stream -> stream.filter(StringUtil::isNotBlank).map(s -> s.trim()).forEach(list::add));
+        return list.toArray(new String[list.size()]);
+    }
+
     String getQueryAdditionalCacheResponseFields();
 
     public default String[] getQueryAdditionalCacheResponseFields(final String... fields) {

src/main/resources/fess_config.properties

@@ -293,6 +293,7 @@ query.highlight.content.description.fields=hl_content,digest
 query.max.search.result.offset=100000
 query.additional.response.fields=
 query.additional.api.response.fields=
+query.additional.scroll.response.fields=
 query.additional.cache.response.fields=
 query.additional.highlighted.fields=
 query.additional.search.fields=