This commit is contained in:
Shinsuke Sugaya 2015-05-28 14:24:59 +09:00
parent 94812173a7
commit debf9296b8
13 changed files with 164 additions and 120 deletions

View file

@ -1,10 +1,10 @@
package org.codelibs.fess.client;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Optional;
import java.util.Set;
import javax.annotation.Resource;
@ -20,7 +20,6 @@ import org.codelibs.fess.entity.GeoInfo;
import org.codelibs.fess.entity.PingResponse;
import org.codelibs.fess.entity.SearchQuery;
import org.codelibs.fess.entity.SearchQuery.SortField;
import org.codelibs.fess.helper.FieldHelper;
import org.codelibs.fess.helper.QueryHelper;
import org.codelibs.fess.helper.RoleQueryHelper;
import org.codelibs.fess.solr.FessSolrQueryException;
@ -37,7 +36,6 @@ import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.update.UpdateResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
@ -47,12 +45,11 @@ import org.elasticsearch.index.query.BoolFilterBuilder;
import org.elasticsearch.index.query.FilterBuilders;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder;
import org.elasticsearch.search.aggregations.bucket.terms.TermsBuilder;
import org.elasticsearch.search.sort.FieldSortBuilder;
import org.elasticsearch.search.sort.SortBuilder;
import org.elasticsearch.search.sort.SortBuilders;
import org.elasticsearch.search.sort.SortOrder;
import org.seasar.framework.container.annotation.tiger.DestroyMethod;
@ -73,10 +70,15 @@ public class SearchClient {
@Resource
protected RoleQueryHelper roleQueryHelper;
protected ElasticsearchClusterRunner runner;
protected List<TransportAddress> transportAddressList = new ArrayList<>();
protected Client client;
protected String index;
protected String type;
@InitMethod
@ -126,35 +128,53 @@ public class SearchClient {
}
}
// TODO
public Map<String, Object> getDocument(final String query) {
return getDocument(query, queryHelper.getResponseFields());
}
public <T> T search(SearchCondition condition, SearchResult<T> searchResult) {
final long startTime = System.currentTimeMillis();
// TODO
public Map<String, Object> getDocument(final String query, final String[] responseFields) {
final List<Map<String, Object>> docList = getDocumentList(query, 0, 1, null, null, responseFields);
if (!docList.isEmpty()) {
return docList.get(0);
}
return null;
}
SearchResponse searchResponse = null;
SearchRequestBuilder queryRequestBuilder = client.prepareSearch(index);
if (condition.build(queryRequestBuilder)) {
// TODO
public List<Map<String, Object>> getDocumentListByDocIds(final String[] docIds, final String[] responseFields,
final String[] docValuesFields, final int pageSize) {
if (docIds == null || docIds.length == 0) {
return Collections.emptyList();
}
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final StringBuilder buf = new StringBuilder(1000);
for (int i = 0; i < docIds.length; i++) {
if (i != 0) {
buf.append(" OR ");
if (queryHelper.getTimeAllowed() >= 0) {
queryRequestBuilder.setTimeout(TimeValue.timeValueMillis(queryHelper.getTimeAllowed()));
}
buf.append(fieldHelper.docIdField + ":").append(docIds[i]);
final Set<Entry<String, String[]>> paramSet = queryHelper.getRequestParameterSet();
if (!paramSet.isEmpty()) {
for (final Map.Entry<String, String[]> entry : paramSet) {
queryRequestBuilder.putHeader(entry.getKey(), entry.getValue());
}
}
searchResponse = queryRequestBuilder.execute().actionGet();
}
return getDocumentList(buf.toString(), 0, pageSize, null, null, responseFields);
final long execTime = System.currentTimeMillis() - startTime;
return searchResult.build(queryRequestBuilder, execTime, Optional.ofNullable(searchResponse));
}
public Optional<Map<String, Object>> getDocument(final SearchCondition condition) {
return search(condition, (queryBuilder, execTime, searchResponse) -> {
return searchResponse.map(response -> {
SearchHit[] hits = response.getHits().hits();
if (hits.length > 0) {
return hits[0].getSource();
}
return null;
});
});
}
public Optional<List<Map<String, Object>>> getDocumentList(final SearchCondition condition) {
return search(condition, (queryRequestBuilder, execTime, searchResponse) -> {
List<Map<String, Object>> list = new ArrayList<>();
searchResponse.ifPresent(response -> {
response.getHits().forEach(hit -> {
list.add(hit.getSource());
});
});
return Optional.of(list);
});
}
// TODO
@ -300,18 +320,6 @@ public class SearchClient {
return queryResponseList;
}
// TODO search
public SearchResponse query(QueryBuilder queryBuilder, AbstractAggregationBuilder aggregationBuilder, SortBuilder sortBuilder) {
SearchRequestBuilder query = client.prepareSearch(index).setQuery(queryBuilder);
if (aggregationBuilder != null) {
query.addAggregation(aggregationBuilder);
}
if (sortBuilder != null) {
query.addSort(sortBuilder);
}
return query.execute().actionGet();
}
public boolean update(String id, String field, Object value) {
try {
return client.prepareUpdate(index, type, id).setDoc(field, value).execute().actionGet().isCreated();
@ -393,4 +401,11 @@ public class SearchClient {
}
}
public interface SearchCondition {
boolean build(SearchRequestBuilder queryRequestBuilder);
}
public interface SearchResult<T> {
T build(SearchRequestBuilder queryRequestBuilder, long execTime, Optional<SearchResponse> searchResponse);
}
}

View file

@ -33,8 +33,6 @@ import org.codelibs.fess.db.exentity.CrawlingSession;
import org.codelibs.fess.db.exentity.CrawlingSessionInfo;
import org.codelibs.fess.service.CrawlingSessionService;
import org.codelibs.fess.util.ComponentUtil;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
@ -154,23 +152,28 @@ public class CrawlingSessionHelper implements Serializable {
}
public List<Map<String, String>> getSessionIdList(final SearchClient searchClient) {
final List<Map<String, String>> sessionIdList = new ArrayList<Map<String, String>>();
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
QueryBuilder queryBuilder = QueryBuilders.matchAllQuery();
TermsBuilder termsBuilder =
AggregationBuilders.terms(fieldHelper.segmentField).field(fieldHelper.segmentField).size(maxSessionIdsInList)
.order(Order.term(false));
final SearchResponse searchResponse = searchClient.query(queryBuilder, termsBuilder, null);
Terms terms = searchResponse.getAggregations().get(fieldHelper.segmentField);
for (Bucket bucket : terms.getBuckets()) {
final Map<String, String> map = new HashMap<String, String>(2);
map.put(fieldHelper.segmentField, bucket.getKey());
map.put(FACET_COUNT_KEY, Long.toString(bucket.getDocCount()));
sessionIdList.add(map);
}
return sessionIdList;
return searchClient.search(
queryRequestBuilder -> {
queryRequestBuilder.setQuery(QueryBuilders.matchAllQuery());
TermsBuilder termsBuilder =
AggregationBuilders.terms(fieldHelper.segmentField).field(fieldHelper.segmentField).size(maxSessionIdsInList)
.order(Order.term(false));
queryRequestBuilder.addAggregation(termsBuilder);
return true;
}, (queryRequestBuilder, execTime, searchResponse) -> {
final List<Map<String, String>> sessionIdList = new ArrayList<Map<String, String>>();
searchResponse.ifPresent(response -> {
Terms terms = response.getAggregations().get(fieldHelper.segmentField);
for (Bucket bucket : terms.getBuckets()) {
final Map<String, String> map = new HashMap<String, String>(2);
map.put(fieldHelper.segmentField, bucket.getKey());
map.put(FACET_COUNT_KEY, Long.toString(bucket.getDocCount()));
sessionIdList.add(map);
}
});
return sessionIdList;
});
}
private String generateId(final String url, final List<String> roleTypeList) {

View file

@ -15,8 +15,6 @@
*/
package org.codelibs.fess.lasta.core.direction;
import org.codelibs.fess.lasta.core.direction.FessEnv;
/**
* @author FreeGen
*/

View file

@ -15,8 +15,8 @@
*/
package org.codelibs.fess.lasta.web.action;
import org.apache.struts.action.ActionMessages;
import org.apache.struts.action.ActionMessage;
import org.apache.struts.action.ActionMessages;
/**
* The keys for message.

View file

@ -91,6 +91,9 @@ import org.codelibs.robot.util.CharUtil;
import org.codelibs.sastruts.core.SSCConstants;
import org.codelibs.sastruts.core.exception.SSCActionMessagesException;
import org.dbflute.optional.OptionalEntity;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.TermQueryBuilder;
import org.seasar.framework.beans.util.Beans;
import org.seasar.framework.container.annotation.tiger.Binding;
import org.seasar.framework.container.annotation.tiger.BindingType;
@ -342,7 +345,12 @@ public class IndexAction {
public String cache() {
Map<String, Object> doc = null;
try {
doc = searchClient.getDocument(fieldHelper.docIdField + ":" + indexForm.docId, queryHelper.getCacheResponseFields());
doc = searchClient.getDocument(queryRequestBuilder -> {
TermQueryBuilder termQuery = QueryBuilders.termQuery(fieldHelper.docIdField, indexForm.docId);
queryRequestBuilder.setQuery(termQuery);
queryRequestBuilder.addFields(queryHelper.getResponseFields());
return true;
}).get();
} catch (final Exception e) {
logger.warn("Failed to request: " + indexForm.docId, e);
}
@ -365,7 +373,12 @@ public class IndexAction {
public String go() throws IOException {
Map<String, Object> doc = null;
try {
doc = searchClient.getDocument(fieldHelper.docIdField + ":" + indexForm.docId, queryHelper.getResponseFields());
doc = searchClient.getDocument(queryRequestBuilder -> {
TermQueryBuilder termQuery = QueryBuilders.termQuery(fieldHelper.docIdField, indexForm.docId);
queryRequestBuilder.setQuery(termQuery);
queryRequestBuilder.addFields(queryHelper.getResponseFields());
return true;
}).get();
} catch (final Exception e) {
logger.warn("Failed to request: " + indexForm.docId, e);
}
@ -497,7 +510,12 @@ public class IndexAction {
OutputStream out = null;
BufferedInputStream in = null;
try {
final Map<String, Object> doc = searchClient.getDocument(fieldHelper.docIdField + ":" + indexForm.docId);
final Map<String, Object> doc = searchClient.getDocument(queryRequestBuilder -> {
TermQueryBuilder termQuery = QueryBuilders.termQuery(fieldHelper.docIdField, indexForm.docId);
queryRequestBuilder.setQuery(termQuery);
queryRequestBuilder.addFields(queryHelper.getResponseFields());
return true;
}).get();
final String url = doc == null ? null : (String) doc.get(fieldHelper.urlField);
if (StringUtil.isBlank(indexForm.queryId) || StringUtil.isBlank(url) || screenShotManager == null) {
// 404
@ -719,9 +737,12 @@ public class IndexAction {
}
try {
final Map<String, Object> doc =
indexForm.docId == null ? null : searchClient.getDocument(fieldHelper.docIdField + ":" + indexForm.docId,
queryHelper.getResponseFields());
final Map<String, Object> doc = indexForm.docId == null ? null : searchClient.getDocument(queryRequestBuilder -> {
TermQueryBuilder termQuery = QueryBuilders.termQuery(fieldHelper.docIdField, indexForm.docId);
queryRequestBuilder.setQuery(termQuery);
queryRequestBuilder.addFields(queryHelper.getResponseFields());
return true;
}).get();
final String userCode = userInfoHelper.getUserCode();
final String favoriteUrl = doc == null ? null : (String) doc.get(fieldHelper.urlField);
@ -790,9 +811,20 @@ public class IndexAction {
}
final String[] docIds = userInfoHelper.getResultDocIds(indexForm.queryId);
final List<Map<String, Object>> docList =
searchClient.getDocumentListByDocIds(docIds, queryHelper.getResponseFields(),
new String[] { fieldHelper.favoriteCountField }, getMaxPageSize());
final List<Map<String, Object>> docList = searchClient.getDocumentList(queryRequstBuilder -> {
if (docIds == null || docIds.length == 0) {
return false;
}
queryRequstBuilder.setFrom(0).setSize(getMaxPageSize());
queryRequstBuilder.addFields(queryHelper.getResponseFields());
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
for (int i = 0; i < docIds.length && i < getMaxPageSize(); i++) {
boolQuery.should(QueryBuilders.termQuery(fieldHelper.docIdField, docIds[i]));
}
queryRequstBuilder.setQuery(boolQuery);
return true;
}).get();
List<String> urlList = new ArrayList<String>(docList.size());
for (final Map<String, Object> doc : docList) {
final Object urlObj = doc.get(fieldHelper.urlField);

View file

@ -35,10 +35,10 @@ import org.codelibs.fess.db.exentity.RoleType;
import org.codelibs.fess.ds.DataStoreFactory;
import org.codelibs.fess.helper.SystemHelper;
import org.codelibs.fess.pager.DataCrawlingConfigPager;
import org.codelibs.fess.service.DataCrawlingConfigService;
import org.codelibs.fess.service.FailureUrlService;
import org.codelibs.fess.service.LabelTypeService;
import org.codelibs.fess.service.RoleTypeService;
import org.codelibs.fess.service.DataCrawlingConfigService;
import org.codelibs.fess.web.base.FessAdminAction;
import org.codelibs.sastruts.core.annotation.Token;
import org.codelibs.sastruts.core.exception.SSCActionMessagesException;

View file

@ -19,7 +19,6 @@ package org.codelibs.fess.web.admin;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
@ -29,9 +28,7 @@ import jp.sf.fess.suggest.SuggestConstants;
import jp.sf.fess.suggest.server.SuggestSolrServer;
import jp.sf.fess.suggest.service.SuggestService;
import org.codelibs.core.util.DynamicProperties;
import org.codelibs.core.util.StringUtil;
import org.codelibs.fess.Constants;
import org.codelibs.fess.client.SearchClient;
import org.codelibs.fess.crud.util.SAStrutsUtil;
import org.codelibs.fess.helper.FieldHelper;
@ -41,11 +38,8 @@ import org.codelibs.fess.helper.WebManagementHelper;
import org.codelibs.fess.util.ComponentUtil;
import org.codelibs.sastruts.core.annotation.Token;
import org.codelibs.sastruts.core.exception.SSCActionMessagesException;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.index.query.MatchAllQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.aggregations.Aggregation;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket;
@ -173,24 +167,32 @@ public class DocumentAction implements Serializable {
}
protected SessionIdList<Map<String, String>> getSessionIdList(final String groupName) {
final SessionIdList<Map<String, String>> sessionIdList = new SessionIdList<Map<String, String>>();
// TODO remove groupName?
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
return searchClient.search(
queryRequestBuilder -> {
queryRequestBuilder.setQuery(QueryBuilders.matchAllQuery());
TermsBuilder termsBuilder =
AggregationBuilders.terms(fieldHelper.segmentField).field(fieldHelper.segmentField).size(100)
.order(Order.count(false));
queryRequestBuilder.addAggregation(termsBuilder);
return true;
}, (queryRequestBuilder, execTime, searchResponse) -> {
final SessionIdList<Map<String, String>> sessionIdList = new SessionIdList<Map<String, String>>();
searchResponse.ifPresent(response -> {
Terms terms = response.getAggregations().get(fieldHelper.segmentField);
for (Bucket bucket : terms.getBuckets()) {
final Map<String, String> map = new HashMap<String, String>(3);
map.put("label", bucket.getKey() + " (" + bucket.getDocCount() + ")");
map.put("value", bucket.getKey());
map.put("count", Long.toString(bucket.getDocCount()));
sessionIdList.add(map);
sessionIdList.addTotalCount(bucket.getDocCount());
}
});
return sessionIdList;
});
QueryBuilder queryBuilder = QueryBuilders.matchAllQuery();
TermsBuilder termsBuilder =
AggregationBuilders.terms(fieldHelper.segmentField).field(fieldHelper.segmentField).size(100).order(Order.count(false));
SearchResponse response = searchClient.query(queryBuilder, termsBuilder, null);
Terms terms = response.getAggregations().get(fieldHelper.segmentField);
for (Bucket bucket : terms.getBuckets()) {
final Map<String, String> map = new HashMap<String, String>(3);
map.put("label", bucket.getKey() + " (" + bucket.getDocCount() + ")");
map.put("value", bucket.getKey());
map.put("count", Long.toString(bucket.getDocCount()));
sessionIdList.add(map);
sessionIdList.addTotalCount(bucket.getDocCount());
}
return sessionIdList;
}
public boolean isSolrProcessRunning() {

View file

@ -16,7 +16,6 @@
package org.codelibs.fess.web.admin;
import java.io.Serializable;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

View file

@ -17,8 +17,8 @@
package org.codelibs.fess.web.admin;
import java.time.LocalDateTime;
import java.util.List;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.annotation.Resource;

View file

@ -35,10 +35,10 @@ import org.codelibs.fess.web.base.FessAdminAction;
import org.codelibs.sastruts.core.annotation.Token;
import org.codelibs.sastruts.core.exception.SSCActionMessagesException;
import org.seasar.framework.beans.util.Beans;
import org.seasar.framework.util.StringUtil;
import org.seasar.struts.annotation.ActionForm;
import org.seasar.struts.annotation.Execute;
import org.seasar.struts.exception.ActionMessagesException;
import org.seasar.framework.util.StringUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

View file

@ -23,27 +23,27 @@ import java.util.Map;
import javax.annotation.Resource;
import org.codelibs.fess.Constants;
import org.codelibs.fess.beans.FessBeans;
import org.codelibs.fess.crud.CommonConstants;
import org.codelibs.fess.crud.CrudMessageException;
import org.codelibs.fess.crud.util.SAStrutsUtil;
import org.codelibs.fess.db.exentity.RoleType;
import org.codelibs.fess.db.exentity.ScheduledJob;
import org.codelibs.fess.helper.JobHelper;
import org.codelibs.fess.helper.SystemHelper;
import org.codelibs.fess.job.JobExecutor;
import org.codelibs.fess.pager.ScheduledJobPager;
import org.codelibs.fess.service.RoleTypeService;
import org.codelibs.fess.service.ScheduledJobService;
import org.codelibs.fess.web.base.FessAdminAction;
import org.codelibs.sastruts.core.annotation.Token;
import org.codelibs.sastruts.core.exception.SSCActionMessagesException;
import org.seasar.framework.beans.util.Beans;
import org.seasar.framework.util.StringUtil;
import org.seasar.struts.annotation.ActionForm;
import org.seasar.struts.annotation.Execute;
import org.seasar.struts.exception.ActionMessagesException;
import org.codelibs.fess.Constants;
import org.codelibs.fess.beans.FessBeans;
import org.codelibs.fess.db.exentity.RoleType;
import org.codelibs.fess.helper.JobHelper;
import org.codelibs.fess.helper.SystemHelper;
import org.codelibs.fess.job.JobExecutor;
import org.codelibs.fess.service.RoleTypeService;
import org.codelibs.fess.web.base.FessAdminAction;
import org.codelibs.sastruts.core.exception.SSCActionMessagesException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

View file

@ -19,6 +19,8 @@ package org.codelibs.fess.web.admin;
import java.io.BufferedWriter;
import java.io.OutputStreamWriter;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
@ -26,34 +28,29 @@ import javax.annotation.Resource;
import javax.servlet.http.HttpServletResponse;
import org.apache.commons.io.IOUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.codelibs.core.util.StringUtil;
import org.codelibs.fess.Constants;
import org.codelibs.fess.crud.CommonConstants;
import org.codelibs.fess.crud.util.SAStrutsUtil;
import org.codelibs.fess.helper.SystemHelper;
import org.codelibs.sastruts.core.exception.SSCActionMessagesException;
import org.codelibs.fess.crud.CrudMessageException;
import org.codelibs.fess.crud.util.SAStrutsUtil;
import org.codelibs.fess.db.exentity.SearchLog;
import org.codelibs.fess.helper.SystemHelper;
import org.codelibs.fess.pager.SearchLogPager;
import org.codelibs.fess.service.SearchLogService;
import org.codelibs.fess.web.base.FessAdminAction;
import org.codelibs.sastruts.core.annotation.Token;
import org.codelibs.sastruts.core.exception.SSCActionMessagesException;
import org.seasar.framework.beans.util.Beans;
import org.seasar.struts.annotation.ActionForm;
import org.seasar.struts.annotation.Execute;
import org.seasar.struts.exception.ActionMessagesException;
import org.seasar.struts.util.RequestUtil;
import org.seasar.struts.util.ResponseUtil;
import org.seasar.struts.annotation.ActionForm;
import org.seasar.struts.exception.ActionMessagesException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.ibm.icu.text.SimpleDateFormat;
import java.util.HashMap;
import java.util.List;
public class SearchLogAction extends FessAdminAction {
private static final Logger logger = LoggerFactory.getLogger(SearchLogAction.class);

View file

@ -35,8 +35,6 @@ import javax.annotation.Resource;
import javax.servlet.http.HttpServletResponse;
import org.apache.commons.io.IOUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.codelibs.core.util.DynamicProperties;
import org.codelibs.fess.Constants;
import org.codelibs.fess.FessSystemException;