0ct0pu5/fess
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix #841 improve search/get handling

Browse source
This commit is contained in:
Shinsuke Sugaya 2017-01-21 18:55:57 +09:00
parent 70755c9f13
commit f11de033fd
8 changed files with 62 additions and 125 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

43
src/main/java/org/codelibs/fess/app/service/SearchService.java
View file

@ -33,6 +33,7 @@ import org.codelibs.fess.Constants;
import org.codelibs.fess.entity.QueryContext;
import org.codelibs.fess.entity.SearchRenderData;
import org.codelibs.fess.entity.SearchRequestParams;
import org.codelibs.fess.entity.SearchRequestParams.SearchRequestType;
import org.codelibs.fess.es.client.FessEsClient;
import org.codelibs.fess.es.client.FessEsClient.SearchConditionBuilder;
import org.codelibs.fess.es.client.FessEsClientException;
@ -52,6 +53,7 @@ import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.update.UpdateRequestBuilder;
import org.elasticsearch.action.update.UpdateResponse;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.lastaflute.taglib.function.LaFunctions;
@ -215,22 +217,41 @@ public class SearchService {
public OptionalEntity<Map<String, Object>> getDocumentByDocId(final String docId, final String[] fields,
final OptionalThing<FessUserBean> userBean) {
return fessEsClient.getDocument(fessConfig.getIndexDocumentSearchIndex(), fessConfig.getIndexDocumentType(), builder -> {
builder.setQuery(QueryBuilders.termQuery(fessConfig.getIndexFieldDocId(), docId));
builder.setFetchSource(fields, null);
fessConfig.processSearchPreference(builder, userBean);
return true;
});
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery().must(QueryBuilders.termQuery(fessConfig.getIndexFieldDocId(), docId));
final Set<String> roleSet = ComponentUtil.getRoleQueryHelper().build(SearchRequestType.JSON); // TODO SearchRequestType?
if (!roleSet.isEmpty()) {
final BoolQueryBuilder roleQuery = QueryBuilders.boolQuery();
roleSet.stream().forEach(name -> {
roleQuery.should(QueryBuilders.termQuery(fessConfig.getIndexFieldRole(), name));
});
boolQuery.filter(roleQuery);
}
builder.setQuery(boolQuery);
builder.setFetchSource(fields, null);
fessConfig.processSearchPreference(builder, userBean);
return true;
});
}
public List<Map<String, Object>> getDocumentListByDocIds(final String[] docIds, final String[] fields,
final OptionalThing<FessUserBean> userBean) {
return fessEsClient.getDocumentList(fessConfig.getIndexDocumentSearchIndex(), fessConfig.getIndexDocumentType(), builder -> {
builder.setQuery(QueryBuilders.termsQuery(fessConfig.getIndexFieldDocId(), docIds));
builder.setSize(fessConfig.getPagingSearchPageMaxSizeAsInteger().intValue());
builder.setFetchSource(fields, null);
fessConfig.processSearchPreference(builder, userBean);
return true;
});
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery().must(QueryBuilders.termsQuery(fessConfig.getIndexFieldDocId(), docIds));
final Set<String> roleSet = ComponentUtil.getRoleQueryHelper().build(SearchRequestType.JSON); // TODO SearchRequestType?
if (!roleSet.isEmpty()) {
final BoolQueryBuilder roleQuery = QueryBuilders.boolQuery();
roleSet.stream().forEach(name -> {
roleQuery.should(QueryBuilders.termQuery(fessConfig.getIndexFieldRole(), name));
});
boolQuery.filter(roleQuery);
}
builder.setQuery(boolQuery);
builder.setSize(fessConfig.getPagingSearchPageMaxSizeAsInteger().intValue());
builder.setFetchSource(fields, null);
fessConfig.processSearchPreference(builder, userBean);
return true;
});
}
public boolean update(final String id, final String field, final Object value) {

6
src/main/java/org/codelibs/fess/app/web/admin/searchlist/AdminSearchlistAction.java
View file

@ -429,8 +429,10 @@ public class AdminSearchlistAction extends FessAdminAction {
case CrudMode.EDIT:
if (form instanceof EditForm) {
final String docId = ((EditForm) form).docId;
final QueryBuilder query = QueryBuilders.termQuery(fessConfig.getIndexFieldDocId(), docId);
return fessEsClient.getDocumentByQuery(fessConfig.getIndexDocumentUpdateIndex(), fessConfig.getIndexDocumentType(), query);
return fessEsClient.getDocument(fessConfig.getIndexDocumentUpdateIndex(), fessConfig.getIndexDocumentType(), builder -> {
builder.setQuery(QueryBuilders.termQuery(fessConfig.getIndexFieldDocId(), docId));
return true;
});
}
break;
default:

4
src/main/java/org/codelibs/fess/app/web/base/FessSearchAction.java
View file

@ -29,9 +29,9 @@ import org.apache.commons.lang3.StringEscapeUtils;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.core.net.URLUtil;
import org.codelibs.fess.Constants;
import org.codelibs.fess.app.service.SearchService;
import org.codelibs.fess.app.web.sso.SsoAction;
import org.codelibs.fess.entity.SearchRequestParams.SearchRequestType;
import org.codelibs.fess.es.client.FessEsClient;
import org.codelibs.fess.helper.LabelTypeHelper;
import org.codelibs.fess.helper.OpenSearchHelper;
import org.codelibs.fess.helper.PopularWordHelper;
@ -53,7 +53,7 @@ public abstract class FessSearchAction extends FessBaseAction {
protected static final String LABEL_FIELD = "label";
@Resource
protected FessEsClient fessEsClient;
protected SearchService searchService;
@Resource
protected ThumbnailManager thumbnailManager;

12
src/main/java/org/codelibs/fess/app/web/cache/CacheAction.java vendored
View file

@ -21,8 +21,6 @@ import org.codelibs.fess.Constants;
import org.codelibs.fess.app.web.base.FessSearchAction;
import org.codelibs.fess.app.web.error.ErrorAction;
import org.codelibs.fess.util.DocumentUtil;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.TermQueryBuilder;
import org.lastaflute.web.Execute;
import org.lastaflute.web.response.ActionResponse;
import org.lastaflute.web.response.StreamResponse;
@ -56,15 +54,7 @@ public class CacheAction extends FessSearchAction {
Map<String, Object> doc = null;
try {
doc =
fessEsClient.getDocument(fessConfig.getIndexDocumentSearchIndex(), fessConfig.getIndexDocumentType(),
queryRequestBuilder -> {
final TermQueryBuilder termQuery = QueryBuilders.termQuery(fessConfig.getIndexFieldDocId(), form.docId);
queryRequestBuilder.setQuery(termQuery);
queryRequestBuilder.setFetchSource(queryHelper.getCacheResponseFields(), null);
fessConfig.processSearchPreference(queryRequestBuilder, getUserBean());
return true;
}).orElse(null);
doc = searchService.getDocumentByDocId(form.docId, queryHelper.getCacheResponseFields(), getUserBean()).orElse(null);
} catch (final Exception e) {
logger.warn("Failed to request: " + form.docId, e);
}

15
src/main/java/org/codelibs/fess/app/web/go/GoAction.java
View file

@ -36,8 +36,6 @@ import org.codelibs.fess.helper.ViewHelper;
import org.codelibs.fess.util.ComponentUtil;
import org.codelibs.fess.util.DocumentUtil;
import org.dbflute.util.DfTypeUtil;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.TermQueryBuilder;
import org.lastaflute.web.Execute;
import org.lastaflute.web.response.ActionResponse;
import org.lastaflute.web.response.HtmlResponse;
@ -77,17 +75,8 @@ public class GoAction extends FessSearchAction {
Map<String, Object> doc = null;
try {
doc =
fessEsClient.getDocument(
fessConfig.getIndexDocumentSearchIndex(),
fessConfig.getIndexDocumentType(),
queryRequestBuilder -> {
final TermQueryBuilder termQuery = QueryBuilders.termQuery(fessConfig.getIndexFieldDocId(), form.docId);
queryRequestBuilder.setQuery(termQuery);
queryRequestBuilder.setFetchSource(
new String[] { fessConfig.getIndexFieldUrl(), fessConfig.getIndexFieldConfigId() }, null);
fessConfig.processSearchPreference(queryRequestBuilder, getUserBean());
return true;
}).orElse(null);
searchService.getDocumentByDocId(form.docId,
new String[] { fessConfig.getIndexFieldUrl(), fessConfig.getIndexFieldConfigId() }, getUserBean()).orElse(null);
} catch (final Exception e) {
logger.warn("Failed to request: " + form.docId, e);
}

11
src/main/java/org/codelibs/fess/app/web/thumbnail/ThumbnailAction.java
View file

@ -26,8 +26,6 @@ import javax.servlet.http.HttpServletResponse;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.fess.app.web.base.FessSearchAction;
import org.codelibs.fess.util.DocumentUtil;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.TermQueryBuilder;
import org.lastaflute.web.Execute;
import org.lastaflute.web.response.ActionResponse;
@ -54,14 +52,7 @@ public class ThumbnailAction extends FessSearchAction {
}
final Map<String, Object> doc =
fessEsClient.getDocument(fessConfig.getIndexDocumentSearchIndex(), fessConfig.getIndexDocumentType(),
queryRequestBuilder -> {
final TermQueryBuilder termQuery = QueryBuilders.termQuery(fessConfig.getIndexFieldDocId(), form.docId);
queryRequestBuilder.setQuery(termQuery);
queryRequestBuilder.setFetchSource(queryHelper.getResponseFields(), null);
fessConfig.processSearchPreference(queryRequestBuilder, getUserBean());
return true;
}).orElse(null);
searchService.getDocumentByDocId(form.docId, queryHelper.getResponseFields(), getUserBean()).orElse(null);
final String url = DocumentUtil.getValue(doc, fessConfig.getIndexFieldUrl(), String.class);
if (StringUtil.isBlank(form.queryId) || StringUtil.isBlank(url) || !thumbnailSupport) {
// 404

83
src/main/java/org/codelibs/fess/es/client/FessEsClient.java
View file

@ -135,7 +135,6 @@ import org.elasticsearch.common.settings.Settings.Builder;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.get.GetField;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHitField;
@ -573,7 +572,7 @@ public class FessEsClient implements Client {
return count;
}
public <T> T get(final String index, final String type, final String id, final SearchCondition<GetRequestBuilder> condition,
protected <T> T get(final String index, final String type, final String id, final SearchCondition<GetRequestBuilder> condition,
final SearchResult<T, GetRequestBuilder, GetResponse> searchResult) {
final long startTime = System.currentTimeMillis();
@ -603,6 +602,9 @@ public class FessEsClient implements Client {
}
try {
if (logger.isDebugEnabled()) {
logger.debug("Query DSL:\n" + searchRequestBuilder.toString());
}
searchResponse = searchRequestBuilder.execute().actionGet(ComponentUtil.getFessConfig().getIndexSearchTimeout());
} catch (final SearchPhaseExecutionException e) {
throw new InvalidQueryException(messages -> messages.addErrorsInvalidQueryParseError(UserMessages.GLOBAL_PROPERTY_KEY),
@ -626,6 +628,7 @@ public class FessEsClient implements Client {
if (source != null) {
final Map<String, Object> docMap = new HashMap<>(source);
docMap.put(fessConfig.getIndexFieldId(), hit.getId());
docMap.put(fessConfig.getIndexFieldVersion(), hit.getVersion());
return docMap;
}
final Map<String, SearchHitField> fields = hit.getFields();
@ -634,15 +637,19 @@ public class FessEsClient implements Client {
fields.entrySet().stream()
.collect(Collectors.toMap(e -> e.getKey(), e -> (Object) e.getValue().getValues()));
docMap.put(fessConfig.getIndexFieldId(), hit.getId());
docMap.put(fessConfig.getIndexFieldVersion(), hit.getVersion());
return docMap;
}
return null;
});
}
public <T> OptionalEntity<T> getDocument(final String index, final String type, final SearchCondition<SearchRequestBuilder> condition,
final EntityCreator<T, SearchResponse, SearchHit> creator) {
return search(index, type, condition, (queryBuilder, execTime, searchResponse) -> {
protected <T> OptionalEntity<T> getDocument(final String index, final String type,
final SearchCondition<SearchRequestBuilder> condition, final EntityCreator<T, SearchResponse, SearchHit> creator) {
return search(index, type, searchRequestBuilder -> {
searchRequestBuilder.setVersion(true);
return condition.build(searchRequestBuilder);
}, (queryBuilder, execTime, searchResponse) -> {
return searchResponse.map(response -> {
final SearchHit[] hits = response.getHits().hits();
if (hits.length > 0) {
@ -653,65 +660,6 @@ public class FessEsClient implements Client {
});
}
public OptionalEntity<Map<String, Object>> getDocument(final String index, final String type, final String id,
final SearchCondition<GetRequestBuilder> condition) {
return getDocument(
index,
type,
id,
condition,
(response, result) -> {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final Map<String, Object> source = response.getSource();
if (source != null) {
final Map<String, Object> docMap = new HashMap<>(source);
docMap.put(fessConfig.getIndexFieldId(), response.getId());
return docMap;
}
final Map<String, GetField> fields = response.getFields();
if (fields != null) {
final Map<String, Object> docMap =
fields.entrySet().stream()
.collect(Collectors.toMap(e -> e.getKey(), e -> (Object) e.getValue().getValues()));
docMap.put(fessConfig.getIndexFieldId(), response.getId());
return docMap;
}
return null;
});
}
public <T> OptionalEntity<T> getDocument(final String index, final String type, final String id,
final SearchCondition<GetRequestBuilder> condition, final EntityCreator<T, GetResponse, GetResponse> creator) {
return get(index, type, id, condition, (queryBuilder, execTime, getResponse) -> {
return getResponse.map(response -> {
return creator.build(response, response);
});
});
}
public OptionalEntity<Map<String, Object>> getDocumentByQuery(final String index, final String type, final QueryBuilder queryBuilder) {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final SearchResponse response =
client.prepareSearch(index).setTypes(type).setSize(1).setQuery(queryBuilder)
.setFetchSource(new String[] { fessConfig.getIndexFieldId() }, null)
.setPreference(Constants.SEARCH_PREFERENCE_PRIMARY).execute().actionGet(fessConfig.getIndexSearchTimeout());
final SearchHits hits = response.getHits();
if (hits.getTotalHits() != 0) {
final SearchHit hit = hits.getAt(0);
final String id = hit.getId();
final GetResponse getResponse =
client.prepareGet(index, type, id).setPreference(Constants.SEARCH_PREFERENCE_PRIMARY).execute()
.actionGet(fessConfig.getIndexSearchTimeout());
final Map<String, Object> source = BeanUtil.copyMapToNewMap(getResponse.getSource());
source.put(fessConfig.getIndexFieldId(), id);
source.put(fessConfig.getIndexFieldVersion(), getResponse.getVersion());
return OptionalEntity.of(source);
}
return OptionalEntity.empty();
}
public List<Map<String, Object>> getDocumentList(final String index, final String type,
final SearchCondition<SearchRequestBuilder> condition) {
return getDocumentList(
@ -738,7 +686,7 @@ public class FessEsClient implements Client {
});
}
public <T> List<T> getDocumentList(final String index, final String type, final SearchCondition<SearchRequestBuilder> condition,
protected <T> List<T> getDocumentList(final String index, final String type, final SearchCondition<SearchRequestBuilder> condition,
final EntityCreator<T, SearchResponse, SearchHit> creator) {
return search(index, type, condition, (searchRequestBuilder, execTime, searchResponse) -> {
final List<T> list = new ArrayList<>();
@ -823,7 +771,6 @@ public class FessEsClient implements Client {
for (int i = 0; i < requests.size(); i++) {
final BulkItemResponse resp = items[i];
if (resp.isFailed() && resp.getFailure() != null) {
@SuppressWarnings("rawtypes")
final ActionRequest req = requests.get(i);
final Failure failure = resp.getFailure();
logger.debug("Failed Request: " + req + "\n=>" + failure.getMessage());
@ -967,10 +914,6 @@ public class FessEsClient implements Client {
}
searchRequestBuilder.setQuery(queryContext.getQueryBuilder());
if (logger.isDebugEnabled()) {
logger.debug("Query: " + searchRequestBuilder);
}
return true;
}
}

13
src/main/java/org/codelibs/fess/helper/IndexingHelper.java
View file

@ -135,10 +135,11 @@ public class IndexingHelper {
public Map<String, Object> getDocument(final FessEsClient fessEsClient, final String id, final String[] fields) {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
return fessEsClient.getDocument(fessConfig.getIndexDocumentSearchIndex(), fessConfig.getIndexDocumentType(), id,
requestBuilder -> {
return true;
}).orElse(null);
return fessEsClient.getDocument(fessConfig.getIndexDocumentUpdateIndex(), fessConfig.getIndexDocumentType(), builder -> {
builder.setQuery(QueryBuilders.termQuery(fessConfig.getIndexFieldDocId(), id));
builder.setFetchSource(fields, null);
return true;
}).orElse(null);
}
public List<Map<String, Object>> getDocumentListByPrefixId(final FessEsClient fessEsClient, final String id, final String[] fields) {
@ -164,12 +165,12 @@ public class IndexingHelper {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final SearchResponse countResponse =
fessEsClient.prepareSearch(fessConfig.getIndexDocumentSearchIndex()).setTypes(fessConfig.getIndexDocumentType())
fessEsClient.prepareSearch(fessConfig.getIndexDocumentUpdateIndex()).setTypes(fessConfig.getIndexDocumentType())
.setQuery(queryBuilder).setSize(0).execute().actionGet(fessConfig.getIndexSearchTimeout());
final long numFound = countResponse.getHits().getTotalHits();
// TODO max threshold
return fessEsClient.getDocumentList(fessConfig.getIndexDocumentSearchIndex(), fessConfig.getIndexDocumentType(),
return fessEsClient.getDocumentList(fessConfig.getIndexDocumentUpdateIndex(), fessConfig.getIndexDocumentType(),
requestBuilder -> {
requestBuilder.setQuery(queryBuilder).setSize((int) numFound);
if (fields != null) {