fix #841 improve search/get handling

src/main/java/org/codelibs/fess/app/service/SearchService.java

@@ -33,6 +33,7 @@ import org.codelibs.fess.Constants;
 import org.codelibs.fess.entity.QueryContext;
 import org.codelibs.fess.entity.SearchRenderData;
 import org.codelibs.fess.entity.SearchRequestParams;
+import org.codelibs.fess.entity.SearchRequestParams.SearchRequestType;
 import org.codelibs.fess.es.client.FessEsClient;
 import org.codelibs.fess.es.client.FessEsClient.SearchConditionBuilder;
 import org.codelibs.fess.es.client.FessEsClientException;
@@ -52,6 +53,7 @@ import org.elasticsearch.action.bulk.BulkRequestBuilder;
 import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.update.UpdateRequestBuilder;
 import org.elasticsearch.action.update.UpdateResponse;
+import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.lastaflute.taglib.function.LaFunctions;
 
@@ -215,22 +217,41 @@ public class SearchService {
     public OptionalEntity<Map<String, Object>> getDocumentByDocId(final String docId, final String[] fields,
             final OptionalThing<FessUserBean> userBean) {
         return fessEsClient.getDocument(fessConfig.getIndexDocumentSearchIndex(), fessConfig.getIndexDocumentType(), builder -> {
-            builder.setQuery(QueryBuilders.termQuery(fessConfig.getIndexFieldDocId(), docId));
-            builder.setFetchSource(fields, null);
-            fessConfig.processSearchPreference(builder, userBean);
-            return true;
-        });
+            BoolQueryBuilder boolQuery = QueryBuilders.boolQuery().must(QueryBuilders.termQuery(fessConfig.getIndexFieldDocId(), docId));
+            final Set<String> roleSet = ComponentUtil.getRoleQueryHelper().build(SearchRequestType.JSON); // TODO SearchRequestType?
+                if (!roleSet.isEmpty()) {
+                    final BoolQueryBuilder roleQuery = QueryBuilders.boolQuery();
+                    roleSet.stream().forEach(name -> {
+                        roleQuery.should(QueryBuilders.termQuery(fessConfig.getIndexFieldRole(), name));
+                    });
+                    boolQuery.filter(roleQuery);
+                }
+                builder.setQuery(boolQuery);
+                builder.setFetchSource(fields, null);
+                fessConfig.processSearchPreference(builder, userBean);
+                return true;
+            });
+
     }
 
     public List<Map<String, Object>> getDocumentListByDocIds(final String[] docIds, final String[] fields,
             final OptionalThing<FessUserBean> userBean) {
         return fessEsClient.getDocumentList(fessConfig.getIndexDocumentSearchIndex(), fessConfig.getIndexDocumentType(), builder -> {
-            builder.setQuery(QueryBuilders.termsQuery(fessConfig.getIndexFieldDocId(), docIds));
-            builder.setSize(fessConfig.getPagingSearchPageMaxSizeAsInteger().intValue());
-            builder.setFetchSource(fields, null);
-            fessConfig.processSearchPreference(builder, userBean);
-            return true;
-        });
+            BoolQueryBuilder boolQuery = QueryBuilders.boolQuery().must(QueryBuilders.termsQuery(fessConfig.getIndexFieldDocId(), docIds));
+            final Set<String> roleSet = ComponentUtil.getRoleQueryHelper().build(SearchRequestType.JSON); // TODO SearchRequestType?
+                if (!roleSet.isEmpty()) {
+                    final BoolQueryBuilder roleQuery = QueryBuilders.boolQuery();
+                    roleSet.stream().forEach(name -> {
+                        roleQuery.should(QueryBuilders.termQuery(fessConfig.getIndexFieldRole(), name));
+                    });
+                    boolQuery.filter(roleQuery);
+                }
+                builder.setQuery(boolQuery);
+                builder.setSize(fessConfig.getPagingSearchPageMaxSizeAsInteger().intValue());
+                builder.setFetchSource(fields, null);
+                fessConfig.processSearchPreference(builder, userBean);
+                return true;
+            });
     }
 
     public boolean update(final String id, final String field, final Object value) {

src/main/java/org/codelibs/fess/app/web/admin/searchlist/AdminSearchlistAction.java

@@ -429,8 +429,10 @@ public class AdminSearchlistAction extends FessAdminAction {
         case CrudMode.EDIT:
             if (form instanceof EditForm) {
                 final String docId = ((EditForm) form).docId;
-                final QueryBuilder query = QueryBuilders.termQuery(fessConfig.getIndexFieldDocId(), docId);
-                return fessEsClient.getDocumentByQuery(fessConfig.getIndexDocumentUpdateIndex(), fessConfig.getIndexDocumentType(), query);
+                return fessEsClient.getDocument(fessConfig.getIndexDocumentUpdateIndex(), fessConfig.getIndexDocumentType(), builder -> {
+                    builder.setQuery(QueryBuilders.termQuery(fessConfig.getIndexFieldDocId(), docId));
+                    return true;
+                });
             }
             break;
         default:

src/main/java/org/codelibs/fess/app/web/base/FessSearchAction.java

@@ -29,9 +29,9 @@ import org.apache.commons.lang3.StringEscapeUtils;
 import org.codelibs.core.lang.StringUtil;
 import org.codelibs.core.net.URLUtil;
 import org.codelibs.fess.Constants;
+import org.codelibs.fess.app.service.SearchService;
 import org.codelibs.fess.app.web.sso.SsoAction;
 import org.codelibs.fess.entity.SearchRequestParams.SearchRequestType;
-import org.codelibs.fess.es.client.FessEsClient;
 import org.codelibs.fess.helper.LabelTypeHelper;
 import org.codelibs.fess.helper.OpenSearchHelper;
 import org.codelibs.fess.helper.PopularWordHelper;
@@ -53,7 +53,7 @@ public abstract class FessSearchAction extends FessBaseAction {
     protected static final String LABEL_FIELD = "label";
 
     @Resource
-    protected FessEsClient fessEsClient;
+    protected SearchService searchService;
 
     @Resource
     protected ThumbnailManager thumbnailManager;

src/main/java/org/codelibs/fess/app/web/cache/CacheAction.java

@@ -21,8 +21,6 @@ import org.codelibs.fess.Constants;
 import org.codelibs.fess.app.web.base.FessSearchAction;
 import org.codelibs.fess.app.web.error.ErrorAction;
 import org.codelibs.fess.util.DocumentUtil;
-import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.index.query.TermQueryBuilder;
 import org.lastaflute.web.Execute;
 import org.lastaflute.web.response.ActionResponse;
 import org.lastaflute.web.response.StreamResponse;
@@ -56,15 +54,7 @@ public class CacheAction extends FessSearchAction {
 
         Map<String, Object> doc = null;
         try {
-            doc =
-                    fessEsClient.getDocument(fessConfig.getIndexDocumentSearchIndex(), fessConfig.getIndexDocumentType(),
-                            queryRequestBuilder -> {
-                                final TermQueryBuilder termQuery = QueryBuilders.termQuery(fessConfig.getIndexFieldDocId(), form.docId);
-                                queryRequestBuilder.setQuery(termQuery);
-                                queryRequestBuilder.setFetchSource(queryHelper.getCacheResponseFields(), null);
-                                fessConfig.processSearchPreference(queryRequestBuilder, getUserBean());
-                                return true;
-                            }).orElse(null);
+            doc = searchService.getDocumentByDocId(form.docId, queryHelper.getCacheResponseFields(), getUserBean()).orElse(null);
         } catch (final Exception e) {
             logger.warn("Failed to request: " + form.docId, e);
         }

src/main/java/org/codelibs/fess/app/web/go/GoAction.java

@@ -36,8 +36,6 @@ import org.codelibs.fess.helper.ViewHelper;
 import org.codelibs.fess.util.ComponentUtil;
 import org.codelibs.fess.util.DocumentUtil;
 import org.dbflute.util.DfTypeUtil;
-import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.index.query.TermQueryBuilder;
 import org.lastaflute.web.Execute;
 import org.lastaflute.web.response.ActionResponse;
 import org.lastaflute.web.response.HtmlResponse;
@@ -77,17 +75,8 @@ public class GoAction extends FessSearchAction {
         Map<String, Object> doc = null;
         try {
             doc =
-                    fessEsClient.getDocument(
-                            fessConfig.getIndexDocumentSearchIndex(),
-                            fessConfig.getIndexDocumentType(),
-                            queryRequestBuilder -> {
-                                final TermQueryBuilder termQuery = QueryBuilders.termQuery(fessConfig.getIndexFieldDocId(), form.docId);
-                                queryRequestBuilder.setQuery(termQuery);
-                                queryRequestBuilder.setFetchSource(
-                                        new String[] { fessConfig.getIndexFieldUrl(), fessConfig.getIndexFieldConfigId() }, null);
-                                fessConfig.processSearchPreference(queryRequestBuilder, getUserBean());
-                                return true;
-                            }).orElse(null);
+                    searchService.getDocumentByDocId(form.docId,
+                            new String[] { fessConfig.getIndexFieldUrl(), fessConfig.getIndexFieldConfigId() }, getUserBean()).orElse(null);
         } catch (final Exception e) {
             logger.warn("Failed to request: " + form.docId, e);
         }

src/main/java/org/codelibs/fess/app/web/thumbnail/ThumbnailAction.java

@@ -26,8 +26,6 @@ import javax.servlet.http.HttpServletResponse;
 import org.codelibs.core.lang.StringUtil;
 import org.codelibs.fess.app.web.base.FessSearchAction;
 import org.codelibs.fess.util.DocumentUtil;
-import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.index.query.TermQueryBuilder;
 import org.lastaflute.web.Execute;
 import org.lastaflute.web.response.ActionResponse;
 
@@ -54,14 +52,7 @@ public class ThumbnailAction extends FessSearchAction {
         }
 
         final Map<String, Object> doc =
-                fessEsClient.getDocument(fessConfig.getIndexDocumentSearchIndex(), fessConfig.getIndexDocumentType(),
-                        queryRequestBuilder -> {
-                            final TermQueryBuilder termQuery = QueryBuilders.termQuery(fessConfig.getIndexFieldDocId(), form.docId);
-                            queryRequestBuilder.setQuery(termQuery);
-                            queryRequestBuilder.setFetchSource(queryHelper.getResponseFields(), null);
-                            fessConfig.processSearchPreference(queryRequestBuilder, getUserBean());
-                            return true;
-                        }).orElse(null);
+                searchService.getDocumentByDocId(form.docId, queryHelper.getResponseFields(), getUserBean()).orElse(null);
         final String url = DocumentUtil.getValue(doc, fessConfig.getIndexFieldUrl(), String.class);
         if (StringUtil.isBlank(form.queryId) || StringUtil.isBlank(url) || !thumbnailSupport) {
             // 404

src/main/java/org/codelibs/fess/es/client/FessEsClient.java

@@ -135,7 +135,6 @@ import org.elasticsearch.common.settings.Settings.Builder;
 import org.elasticsearch.common.transport.InetSocketTransportAddress;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.index.get.GetField;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.SearchHitField;
@@ -573,7 +572,7 @@ public class FessEsClient implements Client {
         return count;
     }
 
-    public <T> T get(final String index, final String type, final String id, final SearchCondition<GetRequestBuilder> condition,
+    protected <T> T get(final String index, final String type, final String id, final SearchCondition<GetRequestBuilder> condition,
             final SearchResult<T, GetRequestBuilder, GetResponse> searchResult) {
         final long startTime = System.currentTimeMillis();
 
@@ -603,6 +602,9 @@ public class FessEsClient implements Client {
             }
 
             try {
+                if (logger.isDebugEnabled()) {
+                    logger.debug("Query DSL:\n" + searchRequestBuilder.toString());
+                }
                 searchResponse = searchRequestBuilder.execute().actionGet(ComponentUtil.getFessConfig().getIndexSearchTimeout());
             } catch (final SearchPhaseExecutionException e) {
                 throw new InvalidQueryException(messages -> messages.addErrorsInvalidQueryParseError(UserMessages.GLOBAL_PROPERTY_KEY),
@@ -626,6 +628,7 @@ public class FessEsClient implements Client {
                     if (source != null) {
                         final Map<String, Object> docMap = new HashMap<>(source);
                         docMap.put(fessConfig.getIndexFieldId(), hit.getId());
+                        docMap.put(fessConfig.getIndexFieldVersion(), hit.getVersion());
                         return docMap;
                     }
                     final Map<String, SearchHitField> fields = hit.getFields();
@@ -634,15 +637,19 @@ public class FessEsClient implements Client {
                                 fields.entrySet().stream()
                                         .collect(Collectors.toMap(e -> e.getKey(), e -> (Object) e.getValue().getValues()));
                         docMap.put(fessConfig.getIndexFieldId(), hit.getId());
+                        docMap.put(fessConfig.getIndexFieldVersion(), hit.getVersion());
                         return docMap;
                     }
                     return null;
                 });
     }
 
-    public <T> OptionalEntity<T> getDocument(final String index, final String type, final SearchCondition<SearchRequestBuilder> condition,
-            final EntityCreator<T, SearchResponse, SearchHit> creator) {
-        return search(index, type, condition, (queryBuilder, execTime, searchResponse) -> {
+    protected <T> OptionalEntity<T> getDocument(final String index, final String type,
+            final SearchCondition<SearchRequestBuilder> condition, final EntityCreator<T, SearchResponse, SearchHit> creator) {
+        return search(index, type, searchRequestBuilder -> {
+            searchRequestBuilder.setVersion(true);
+            return condition.build(searchRequestBuilder);
+        }, (queryBuilder, execTime, searchResponse) -> {
             return searchResponse.map(response -> {
                 final SearchHit[] hits = response.getHits().hits();
                 if (hits.length > 0) {
@@ -653,65 +660,6 @@ public class FessEsClient implements Client {
         });
     }
 
-    public OptionalEntity<Map<String, Object>> getDocument(final String index, final String type, final String id,
-            final SearchCondition<GetRequestBuilder> condition) {
-        return getDocument(
-                index,
-                type,
-                id,
-                condition,
-                (response, result) -> {
-                    final FessConfig fessConfig = ComponentUtil.getFessConfig();
-                    final Map<String, Object> source = response.getSource();
-                    if (source != null) {
-                        final Map<String, Object> docMap = new HashMap<>(source);
-                        docMap.put(fessConfig.getIndexFieldId(), response.getId());
-                        return docMap;
-                    }
-                    final Map<String, GetField> fields = response.getFields();
-                    if (fields != null) {
-                        final Map<String, Object> docMap =
-                                fields.entrySet().stream()
-                                        .collect(Collectors.toMap(e -> e.getKey(), e -> (Object) e.getValue().getValues()));
-                        docMap.put(fessConfig.getIndexFieldId(), response.getId());
-                        return docMap;
-                    }
-                    return null;
-
-                });
-    }
-
-    public <T> OptionalEntity<T> getDocument(final String index, final String type, final String id,
-            final SearchCondition<GetRequestBuilder> condition, final EntityCreator<T, GetResponse, GetResponse> creator) {
-        return get(index, type, id, condition, (queryBuilder, execTime, getResponse) -> {
-            return getResponse.map(response -> {
-                return creator.build(response, response);
-            });
-        });
-    }
-
-    public OptionalEntity<Map<String, Object>> getDocumentByQuery(final String index, final String type, final QueryBuilder queryBuilder) {
-
-        final FessConfig fessConfig = ComponentUtil.getFessConfig();
-        final SearchResponse response =
-                client.prepareSearch(index).setTypes(type).setSize(1).setQuery(queryBuilder)
-                        .setFetchSource(new String[] { fessConfig.getIndexFieldId() }, null)
-                        .setPreference(Constants.SEARCH_PREFERENCE_PRIMARY).execute().actionGet(fessConfig.getIndexSearchTimeout());
-        final SearchHits hits = response.getHits();
-        if (hits.getTotalHits() != 0) {
-            final SearchHit hit = hits.getAt(0);
-            final String id = hit.getId();
-            final GetResponse getResponse =
-                    client.prepareGet(index, type, id).setPreference(Constants.SEARCH_PREFERENCE_PRIMARY).execute()
-                            .actionGet(fessConfig.getIndexSearchTimeout());
-            final Map<String, Object> source = BeanUtil.copyMapToNewMap(getResponse.getSource());
-            source.put(fessConfig.getIndexFieldId(), id);
-            source.put(fessConfig.getIndexFieldVersion(), getResponse.getVersion());
-            return OptionalEntity.of(source);
-        }
-        return OptionalEntity.empty();
-    }
-
     public List<Map<String, Object>> getDocumentList(final String index, final String type,
             final SearchCondition<SearchRequestBuilder> condition) {
         return getDocumentList(
@@ -738,7 +686,7 @@ public class FessEsClient implements Client {
                 });
     }
 
-    public <T> List<T> getDocumentList(final String index, final String type, final SearchCondition<SearchRequestBuilder> condition,
+    protected <T> List<T> getDocumentList(final String index, final String type, final SearchCondition<SearchRequestBuilder> condition,
             final EntityCreator<T, SearchResponse, SearchHit> creator) {
         return search(index, type, condition, (searchRequestBuilder, execTime, searchResponse) -> {
             final List<T> list = new ArrayList<>();
@@ -823,7 +771,6 @@ public class FessEsClient implements Client {
                     for (int i = 0; i < requests.size(); i++) {
                         final BulkItemResponse resp = items[i];
                         if (resp.isFailed() && resp.getFailure() != null) {
-                            @SuppressWarnings("rawtypes")
                             final ActionRequest req = requests.get(i);
                             final Failure failure = resp.getFailure();
                             logger.debug("Failed Request: " + req + "\n=>" + failure.getMessage());
@@ -967,10 +914,6 @@ public class FessEsClient implements Client {
             }
 
             searchRequestBuilder.setQuery(queryContext.getQueryBuilder());
-            if (logger.isDebugEnabled()) {
-                logger.debug("Query: " + searchRequestBuilder);
-            }
-
             return true;
         }
     }

src/main/java/org/codelibs/fess/helper/IndexingHelper.java

@@ -135,10 +135,11 @@ public class IndexingHelper {
 
     public Map<String, Object> getDocument(final FessEsClient fessEsClient, final String id, final String[] fields) {
         final FessConfig fessConfig = ComponentUtil.getFessConfig();
-        return fessEsClient.getDocument(fessConfig.getIndexDocumentSearchIndex(), fessConfig.getIndexDocumentType(), id,
-                requestBuilder -> {
-                    return true;
-                }).orElse(null);
+        return fessEsClient.getDocument(fessConfig.getIndexDocumentUpdateIndex(), fessConfig.getIndexDocumentType(), builder -> {
+            builder.setQuery(QueryBuilders.termQuery(fessConfig.getIndexFieldDocId(), id));
+            builder.setFetchSource(fields, null);
+            return true;
+        }).orElse(null);
     }
 
     public List<Map<String, Object>> getDocumentListByPrefixId(final FessEsClient fessEsClient, final String id, final String[] fields) {
@@ -164,12 +165,12 @@ public class IndexingHelper {
         final FessConfig fessConfig = ComponentUtil.getFessConfig();
 
         final SearchResponse countResponse =
-                fessEsClient.prepareSearch(fessConfig.getIndexDocumentSearchIndex()).setTypes(fessConfig.getIndexDocumentType())
+                fessEsClient.prepareSearch(fessConfig.getIndexDocumentUpdateIndex()).setTypes(fessConfig.getIndexDocumentType())
                         .setQuery(queryBuilder).setSize(0).execute().actionGet(fessConfig.getIndexSearchTimeout());
         final long numFound = countResponse.getHits().getTotalHits();
         // TODO max threshold
 
-        return fessEsClient.getDocumentList(fessConfig.getIndexDocumentSearchIndex(), fessConfig.getIndexDocumentType(),
+        return fessEsClient.getDocumentList(fessConfig.getIndexDocumentUpdateIndex(), fessConfig.getIndexDocumentType(),
                 requestBuilder -> {
                     requestBuilder.setQuery(queryBuilder).setSize((int) numFound);
                     if (fields != null) {