0ct0pu5/fess
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

#275

Browse source
This commit is contained in:
Shinsuke Sugaya 2015-04-30 16:30:37 +09:00
parent af506783fd
commit 46f9d3d3dd
42 changed files with 976 additions and 2189 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

61
pom.xml
View file

@ -60,12 +60,12 @@
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<dbflute.version>1.1.0-sp1</dbflute.version>
<s2robot.version>0.8.0</s2robot.version>
<solr.version>4.10.3</solr.version>
<slf4j.version>1.7.7</slf4j.version>
<tika.version>1.6</tika.version>
<poi.version>3.11-beta2</poi.version>
<pdfbox.version>1.8.7</pdfbox.version>
<saflute.version>1.0.0-SNAPSHOT</saflute.version>
<elasticsearch.version>1.5.0</elasticsearch.version>
<!-- Tomcat -->
<tomcat.delegate>true</tomcat.delegate>
<tomcat.useSeparateTomcatClassLoader>true</tomcat.useSeparateTomcatClassLoader>
@ -271,6 +271,17 @@
<groupId>jp.sf.fess</groupId>
<artifactId>fess-suggest</artifactId>
<version>1.3.2</version>
<!-- REMOVE -->
<exclusions>
<exclusion>
<artifactId>solrlib</artifactId>
<groupId>org.codelibs.solr</groupId>
</exclusion>
<exclusion>
<artifactId>solr-solrj</artifactId>
<groupId>org.apache.solr</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>jstl</groupId>
@ -361,39 +372,14 @@
<version>${slf4j.version}</version>
</dependency>
<dependency>
<groupId>org.apache.solr</groupId>
<artifactId>solr-core</artifactId>
<version>${solr.version}</version>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-jdk14</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpmime</artifactId>
</exclusion>
</exclusions>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch</artifactId>
<version>${elasticsearch.version}</version>
</dependency>
<dependency>
<groupId>org.apache.solr</groupId>
<artifactId>solr-solrj</artifactId>
<version>${solr.version}</version>
<exclusions>
<exclusion>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpmime</artifactId>
</exclusion>
</exclusions>
<groupId>org.codelibs</groupId>
<artifactId>elasticsearch-cluster-runner</artifactId>
<version>1.5.0.1</version>
</dependency>
<dependency>
<groupId>xalan</groupId>
@ -470,17 +456,6 @@
<artifactId>corelib</artifactId>
<version>0.2.1</version>
</dependency>
<dependency>
<groupId>org.codelibs.solr</groupId>
<artifactId>solrlib</artifactId>
<version>1.3.2</version>
<exclusions>
<exclusion>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.codelibs.sastruts</groupId>
<artifactId>sastruts-core</artifactId>

35
src/main/java/org/codelibs/fess/Constants.java
View file

@ -22,9 +22,9 @@ import org.codelibs.core.CoreLibConstants;
import org.codelibs.core.util.StringUtil;
public class Constants extends CoreLibConstants {
public static final int MAJOR_VERSION = 9;
public static final int MAJOR_VERSION = 10;
public static final int MINOR_VERSION = 3;
public static final int MINOR_VERSION = 0;
public static final String FESS_VERSION = String.valueOf(MAJOR_VERSION) + "." + String.valueOf(MINOR_VERSION);
@ -70,10 +70,6 @@ public class Constants extends CoreLibConstants {
public static final String MS932 = "MS932";
public static final String OPTIMIZE = "optimize";
public static final String COMMIT = "commit";
public static final String DEFAULT_CRON_EXPRESSION = "0 0 0 * * ?";
public static final String DEFAULT_SEARCH_LOG_CRON_EXPRESSION = "0 * * * * ?";
@ -92,8 +88,6 @@ public class Constants extends CoreLibConstants {
public static final long DEFAULT_CRAWLING_EXECUTION_INTERVAL = 5000L;
public static final long DEFAULT_COMMIT_PER_COUNT = 0L;
/** Solr Status: Active */
public static final String ACTIVE = "ACTIVE";
@ -117,10 +111,6 @@ public class Constants extends CoreLibConstants {
public static final String DAY_FOR_CLEANUP_PROPERTY = "day.for.cleanup";
public static final String COMMIT_PER_COUNT_PROPERTY = "commit.count";
public static final String SERVER_ROTATION_PROPERTY = "server.rotation";
public static final String WEB_API_XML_PROPERTY = "web.api.xml";
public static final String WEB_API_JSON_PROPERTY = "web.api.json";
@ -176,6 +166,7 @@ public class Constants extends CoreLibConstants {
public static final String FIELD_LOGS = "org.codelibs.fess.FieldLogs";
public static final String DEFAULT_DATETIME_FORMAT = "yyyy-MM-dd'T'HH:mm:ss";
public static final String ISO_DATETIME_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'";
public static final int DONE_STATUS = 9999;
@ -212,18 +203,6 @@ public class Constants extends CoreLibConstants {
public static final String DATA_CRAWLER_END_TIME = "DataCrawlEndTime";
public static final String OPTIMIZE_START_TIME = "OptimizeStartTime";
public static final String OPTIMIZE_END_TIME = "OptimizeEndTime";
public static final String OPTIMIZE_EXEC_TIME = "OptimizeExecTime";
public static final String COMMIT_START_TIME = "CommitStartTime";
public static final String COMMIT_END_TIME = "CommitEndTime";
public static final String COMMIT_EXEC_TIME = "CommitExecTime";
public static final String WEB_FS_CRAWLING_EXEC_TIME = "WebFsCrawlExecTime";
public static final String WEB_FS_INDEX_EXEC_TIME = "WebFsIndexExecTime";
@ -281,14 +260,10 @@ public class Constants extends CoreLibConstants {
public static final String FACET_QUERY = "org.codelibs.fess.tag.FacetQuery";
public static final String MLT_QUERY = "org.codelibs.fess.tag.MLTQuery";
public static final String GEO_QUERY = "org.codelibs.fess.tag.GeoQuery";
public static final String FACET_FORM = "org.codelibs.fess.tag.FacetForm";
public static final String MLT_FORM = "org.codelibs.fess.tag.MLTForm";
public static final String GEO_FORM = "org.codelibs.fess.tag.GeoForm";
public static final String LABEL_VALUE_MAP = "org.codelibs.fess.LabelValueMap";
@ -326,4 +301,8 @@ public class Constants extends CoreLibConstants {
public static final String NOW = "NOW";
public static final String FACET_FIELD_PREFIX = "field:";
public static final String FACET_QUERY_PREFIX = "query:";
}

6
src/main/java/org/codelibs/fess/api/BaseApiManager.java
View file

@ -23,8 +23,6 @@ public class BaseApiManager {
protected static final String HOT_SEARCH_WORD_API = "/hotSearchWordApi";
protected static final String ANALYSIS_API = "/analysisApi";
protected static final String SUGGEST_API = "/suggestApi";
protected static final String SPELLCHECK_API = "/spellCheckApi";
@ -32,7 +30,7 @@ public class BaseApiManager {
protected static final String SEARCH_API = "/searchApi";
protected static enum FormatType {
SEARCH, LABEL, SUGGEST, SPELLCHECK, ANALYSIS, HOTSEARCHWORD, FAVORITE, FAVORITES, OTHER, PING;
SEARCH, LABEL, SUGGEST, SPELLCHECK, HOTSEARCHWORD, FAVORITE, FAVORITES, OTHER, PING;
}
public BaseApiManager() {
@ -52,8 +50,6 @@ public class BaseApiManager {
return FormatType.SUGGEST;
} else if (FormatType.SPELLCHECK.name().equals(type)) {
return FormatType.SPELLCHECK;
} else if (FormatType.ANALYSIS.name().equals(type)) {
return FormatType.ANALYSIS;
} else if (FormatType.HOTSEARCHWORD.name().equals(type)) {
return FormatType.HOTSEARCHWORD;
} else if (FormatType.FAVORITE.name().equals(type)) {

117
src/main/java/org/codelibs/fess/api/json/JsonApiManager.java
View file

@ -41,11 +41,9 @@ import org.codelibs.fess.api.BaseApiManager;
import org.codelibs.fess.api.WebApiManager;
import org.codelibs.fess.api.WebApiRequest;
import org.codelibs.fess.api.WebApiResponse;
import org.codelibs.fess.client.SearchClient;
import org.codelibs.fess.db.allcommon.CDef;
import org.codelibs.fess.entity.FieldAnalysisResponse;
import org.codelibs.fess.entity.PingResponse;
import org.codelibs.fess.entity.PingResponse.Target;
import org.codelibs.fess.service.SearchService;
import org.codelibs.fess.util.ComponentUtil;
import org.codelibs.fess.util.FacetResponse;
import org.codelibs.fess.util.FacetResponse.Field;
@ -89,9 +87,6 @@ public class JsonApiManager extends BaseApiManager implements WebApiManager {
case SPELLCHECK:
processSpellCheckRequest(request, response, chain);
break;
case ANALYSIS:
processAnalysisRequest(request, response, chain);
break;
case HOTSEARCHWORD:
processHotSearchWordRequest(request, response, chain);
break;
@ -111,32 +106,12 @@ public class JsonApiManager extends BaseApiManager implements WebApiManager {
}
protected void processPingRequest(final HttpServletRequest request, final HttpServletResponse response, final FilterChain chain) {
final SearchService searchService = ComponentUtil.getSearchService();
final SearchClient searchClient = ComponentUtil.getElasticsearchClient();
int status;
final StringBuilder buf = new StringBuilder(1000);
String errMsg = null;
try {
final PingResponse pingResponse = searchService.ping();
final PingResponse pingResponse = searchClient.ping();
status = pingResponse.getStatus();
buf.append("\"result\":[");
boolean appended = false;
for (final Target target : pingResponse.getTargets()) {
if (appended) {
buf.append(',');
} else {
appended = true;
}
buf.append("{\"status\":");
buf.append(target.getStatus());
buf.append(",\"url\":\"");
buf.append(escapeJson(target.getUrl()));
buf.append("\",\"queryTime\":");
buf.append(target.getQueryTime());
buf.append(",\"searchTime\":");
buf.append(target.getSearchTime());
buf.append("}");
}
buf.append(']');
} catch (final Exception e) {
status = 9;
errMsg = e.getMessage();
@ -148,7 +123,7 @@ public class JsonApiManager extends BaseApiManager implements WebApiManager {
}
}
writeJsonResponse(status, buf.toString(), errMsg);
writeJsonResponse(status, null, errMsg);
}
protected void processSearchRequest(final HttpServletRequest request, final HttpServletResponse response, final FilterChain chain) {
@ -526,83 +501,6 @@ public class JsonApiManager extends BaseApiManager implements WebApiManager {
}
protected void processAnalysisRequest(final HttpServletRequest request, final HttpServletResponse response, final FilterChain chain) {
int status = 0;
String errMsg = StringUtil.EMPTY;
final StringBuilder buf = new StringBuilder(255);
try {
chain.doFilter(new WebApiRequest(request, ANALYSIS_API), new WebApiResponse(response));
WebApiUtil.validate();
final FieldAnalysisResponse fieldAnalysis = WebApiUtil.getObject("fieldAnalysis");
buf.append("\"recordCount\":");
buf.append(fieldAnalysis.size());
if (fieldAnalysis.size() > 0) {
buf.append(',');
buf.append("\"result\":[");
boolean first1 = true;
for (final Map.Entry<String, Map<String, List<Map<String, Object>>>> fEntry : fieldAnalysis.entrySet()) {
if (first1) {
first1 = false;
} else {
buf.append(',');
}
buf.append("{\"field\":").append(escapeJson(fEntry.getKey())).append(",\"analysis\":[");
boolean first2 = true;
for (final Map.Entry<String, List<Map<String, Object>>> aEntry : fEntry.getValue().entrySet()) {
if (first2) {
first2 = false;
} else {
buf.append(',');
}
buf.append("{\"name\":").append(escapeJson(aEntry.getKey())).append(",\"data\":[");
boolean first3 = true;
for (final Map<String, Object> dataMap : aEntry.getValue()) {
if (first3) {
first3 = false;
} else {
buf.append(',');
}
buf.append('{');
boolean first4 = true;
for (final Map.Entry<String, Object> dEntry : dataMap.entrySet()) {
final String key = dEntry.getKey();
final Object value = dEntry.getValue();
if (StringUtil.isNotBlank(key) && value != null) {
if (first4) {
first4 = false;
} else {
buf.append(',');
}
buf.append(escapeJson(key)).append(':').append(escapeJson(value));
}
}
buf.append('}');
}
buf.append("]}");
}
buf.append("]}");
}
buf.append(']');
}
} catch (final Exception e) {
if (e instanceof WebApiException) {
status = ((WebApiException) e).getStatusCode();
} else {
status = 1;
}
errMsg = e.getMessage();
if (logger.isDebugEnabled()) {
logger.debug("Failed to process a suggest request.", e);
}
}
writeJsonResponse(status, buf.toString(), errMsg);
}
protected void processHotSearchWordRequest(final HttpServletRequest request, final HttpServletResponse response, final FilterChain chain) {
int status = 0;
@ -719,10 +617,13 @@ public class JsonApiManager extends BaseApiManager implements WebApiManager {
buf.append(',');
buf.append("\"status\":");
buf.append(status);
buf.append(',');
if (status == 0) {
buf.append(body);
if (StringUtil.isNotBlank(body)) {
buf.append(',');
buf.append(body);
}
} else {
buf.append(',');
buf.append("\"message\":");
buf.append(escapeJson(errMsg));
}

86
src/main/java/org/codelibs/fess/api/xml/XmlApiManager.java
View file

@ -40,11 +40,9 @@ import org.codelibs.fess.api.BaseApiManager;
import org.codelibs.fess.api.WebApiManager;
import org.codelibs.fess.api.WebApiRequest;
import org.codelibs.fess.api.WebApiResponse;
import org.codelibs.fess.client.SearchClient;
import org.codelibs.fess.db.allcommon.CDef;
import org.codelibs.fess.entity.FieldAnalysisResponse;
import org.codelibs.fess.entity.PingResponse;
import org.codelibs.fess.entity.PingResponse.Target;
import org.codelibs.fess.service.SearchService;
import org.codelibs.fess.util.ComponentUtil;
import org.codelibs.fess.util.FacetResponse;
import org.codelibs.fess.util.FacetResponse.Field;
@ -86,9 +84,6 @@ public class XmlApiManager extends BaseApiManager implements WebApiManager {
case SPELLCHECK:
processSpellCheckRequest(request, response, chain);
break;
case ANALYSIS:
processAnalysisRequest(request, response, chain);
break;
case PING:
processPingRequest(request, response, chain);
break;
@ -100,26 +95,12 @@ public class XmlApiManager extends BaseApiManager implements WebApiManager {
}
protected void processPingRequest(final HttpServletRequest request, final HttpServletResponse response, final FilterChain chain) {
final SearchService searchService = ComponentUtil.getSearchService();
final SearchClient searchClient = ComponentUtil.getElasticsearchClient();
int status;
final StringBuilder buf = new StringBuilder(1000);
String errMsg = null;
try {
final PingResponse pingResponse = searchService.ping();
final PingResponse pingResponse = searchClient.ping();
status = pingResponse.getStatus();
buf.append("<result>");
for (final Target target : pingResponse.getTargets()) {
buf.append("<server><status>");
buf.append(target.getStatus());
buf.append("</status><url>");
buf.append(escapeXml(target.getUrl()));
buf.append("</url><query-time>");
buf.append(target.getQueryTime());
buf.append("</query-time><search-time>");
buf.append(target.getSearchTime());
buf.append("</search-time></server>");
}
buf.append("</result>");
} catch (final Exception e) {
status = 9;
errMsg = e.getMessage();
@ -131,7 +112,7 @@ public class XmlApiManager extends BaseApiManager implements WebApiManager {
}
}
writeXmlResponse(status, buf.toString(), errMsg);
writeXmlResponse(status, null, errMsg);
}
protected void processSearchRequest(final HttpServletRequest request, final HttpServletResponse response, final FilterChain chain) {
@ -452,61 +433,6 @@ public class XmlApiManager extends BaseApiManager implements WebApiManager {
writeXmlResponse(status, buf.toString(), errMsg);
}
protected String processAnalysisRequest(final HttpServletRequest request, final HttpServletResponse response, final FilterChain chain) {
int status = 0;
String errMsg = StringUtil.EMPTY;
final StringBuilder buf = new StringBuilder(255);
try {
chain.doFilter(new WebApiRequest(request, ANALYSIS_API), new WebApiResponse(response));
WebApiUtil.validate();
final FieldAnalysisResponse fieldAnalysis = WebApiUtil.getObject("fieldAnalysis");
buf.append("<record-count>");
buf.append(fieldAnalysis.size());
buf.append("</record-count>");
if (fieldAnalysis.size() > 0) {
buf.append("<result>");
for (final Map.Entry<String, Map<String, List<Map<String, Object>>>> fEntry : fieldAnalysis.entrySet()) {
buf.append("<field name=\"").append(escapeXml(fEntry.getKey())).append("\">");
for (final Map.Entry<String, List<Map<String, Object>>> aEntry : fEntry.getValue().entrySet()) {
buf.append("<analysis name=\"").append(escapeXml(aEntry.getKey())).append("\">");
for (final Map<String, Object> dataMap : aEntry.getValue()) {
buf.append("<token>");
for (final Map.Entry<String, Object> dEntry : dataMap.entrySet()) {
final String key = dEntry.getKey();
final Object value = dEntry.getValue();
if (StringUtil.isNotBlank(key) && value != null) {
buf.append("<value name=\"").append(escapeXml(key)).append("\">").append(escapeXml(value))
.append("</value>");
}
}
buf.append("</token>");
}
buf.append("</analysis>");
}
buf.append("</field>");
}
buf.append("</result>");
}
} catch (final Exception e) {
if (e instanceof WebApiException) {
status = ((WebApiException) e).getStatusCode();
} else {
status = 1;
}
errMsg = e.getMessage();
if (logger.isDebugEnabled()) {
logger.debug("Failed to process a suggest request.", e);
}
}
writeXmlResponse(status, buf.toString(), errMsg);
return null;
}
protected void writeXmlResponse(final int status, final String body, final String errMsg) {
final StringBuilder buf = new StringBuilder(1000);
buf.append("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
@ -518,7 +444,9 @@ public class XmlApiManager extends BaseApiManager implements WebApiManager {
buf.append(status);
buf.append("</status>");
if (status == 0) {
buf.append(body);
if (StringUtil.isNotBlank(body)) {
buf.append(body);
}
} else {
buf.append("<message>");
buf.append(escapeXml(errMsg));

368
src/main/java/org/codelibs/fess/client/SearchClient.java Normal file
View file

@ -0,0 +1,368 @@
package org.codelibs.fess.client;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import javax.annotation.Resource;
import org.apache.commons.codec.Charsets;
import org.codelibs.core.util.StringUtil;
import org.codelibs.elasticsearch.runner.ElasticsearchClusterRunner;
import org.codelibs.fess.Constants;
import org.codelibs.fess.FessSystemException;
import org.codelibs.fess.ResultOffsetExceededException;
import org.codelibs.fess.entity.FacetInfo;
import org.codelibs.fess.entity.GeoInfo;
import org.codelibs.fess.entity.PingResponse;
import org.codelibs.fess.entity.SearchQuery;
import org.codelibs.fess.entity.SearchQuery.SortField;
import org.codelibs.fess.helper.FieldHelper;
import org.codelibs.fess.helper.QueryHelper;
import org.codelibs.fess.helper.RoleQueryHelper;
import org.codelibs.fess.solr.FessSolrQueryException;
import org.codelibs.fess.util.ComponentUtil;
import org.codelibs.fess.util.QueryResponseList;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.admin.indices.flush.FlushResponse;
import org.elasticsearch.action.admin.indices.optimize.OptimizeResponse;
import org.elasticsearch.action.admin.indices.refresh.RefreshResponse;
import org.elasticsearch.action.deletebyquery.DeleteByQueryResponse;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.update.UpdateResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.query.BoolFilterBuilder;
import org.elasticsearch.index.query.FilterBuilders;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder;
import org.elasticsearch.search.aggregations.bucket.terms.TermsBuilder;
import org.elasticsearch.search.sort.FieldSortBuilder;
import org.elasticsearch.search.sort.SortBuilder;
import org.elasticsearch.search.sort.SortBuilders;
import org.elasticsearch.search.sort.SortOrder;
import org.seasar.framework.container.annotation.tiger.DestroyMethod;
import org.seasar.framework.container.annotation.tiger.InitMethod;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.io.BaseEncoding;
// TODO rename to SearchClient
public class SearchClient {
private static final Logger logger = LoggerFactory.getLogger(SearchClient.class);
@Resource
protected SearchClient searchClient;
@Resource
protected QueryHelper queryHelper;
@Resource
protected RoleQueryHelper roleQueryHelper;
protected ElasticsearchClusterRunner runner;
protected List<TransportAddress> transportAddressList = new ArrayList<>();
protected Client client;
protected String index;
protected String type;
@InitMethod
public void open() {
if (transportAddressList.isEmpty()) {
if (runner == null) {
throw new FessSystemException("No elasticsearch instance.");
}
client = runner.client();
} else {
TransportClient transportClient = new TransportClient();
for (TransportAddress address : transportAddressList) {
transportClient.addTransportAddress(address);
}
client = transportClient;
}
}
@DestroyMethod
public void close() {
try {
client.close();
} catch (ElasticsearchException e) {
logger.warn("Failed to close Client: " + client, e);
}
}
public void addTransportAddress(String host, int port) {
transportAddressList.add(new InetSocketTransportAddress(host, port));
}
public void deleteByQuery(QueryBuilder queryBuilder) {
DeleteByQueryResponse response = client.prepareDeleteByQuery(index).setQuery(queryBuilder).execute().actionGet();
// TODO
}
public void deleteByQuery(String field, String value) {
// TODO Auto-generated method stub
}
public Map<String, Object> getDocument(final String query) {
return getDocument(query, queryHelper.getResponseFields());
}
public Map<String, Object> getDocument(final String query, final String[] responseFields) {
final List<Map<String, Object>> docList = getDocumentList(query, 0, 1, null, null, responseFields);
if (!docList.isEmpty()) {
return docList.get(0);
}
return null;
}
public List<Map<String, Object>> getDocumentListByDocIds(final String[] docIds, final String[] responseFields,
final String[] docValuesFields, final int pageSize) {
if (docIds == null || docIds.length == 0) {
return Collections.emptyList();
}
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final StringBuilder buf = new StringBuilder(1000);
for (int i = 0; i < docIds.length; i++) {
if (i != 0) {
buf.append(" OR ");
}
buf.append(fieldHelper.docIdField + ":").append(docIds[i]);
}
return getDocumentList(buf.toString(), 0, pageSize, null, null, responseFields);
}
public List<Map<String, Object>> getDocumentList(final String query, final int start, final int rows, final FacetInfo facetInfo,
final GeoInfo geoInfo, final String[] responseFields) {
return getDocumentList(query, start, rows, facetInfo, geoInfo, responseFields, true);
}
public List<Map<String, Object>> getDocumentList(final String query, final int start, final int rows, final FacetInfo facetInfo,
final GeoInfo geoInfo, final String[] responseFields, final boolean forUser) {
if (start > queryHelper.getMaxSearchResultOffset()) {
throw new ResultOffsetExceededException("The number of result size is exceeded.");
}
final long startTime = System.currentTimeMillis();
SearchResponse searchResponse = null;
SearchRequestBuilder queryRequestBuilder = client.prepareSearch(index);
final SearchQuery searchQuery = queryHelper.build(query, forUser);
final String q = searchQuery.getQuery();
if (StringUtil.isNotBlank(q)) {
BoolFilterBuilder boolFilterBuilder = null;
// fields
queryRequestBuilder.addFields(responseFields);
// query
QueryBuilder queryBuilder = QueryBuilders.queryStringQuery(q);
queryRequestBuilder.setFrom(start).setSize(rows);
for (final Map.Entry<String, String[]> entry : queryHelper.getQueryParamMap().entrySet()) {
queryRequestBuilder.putHeader(entry.getKey(), entry.getValue());
}
// filter query
if (searchQuery.hasFilterQueries()) {
if (boolFilterBuilder == null) {
boolFilterBuilder = FilterBuilders.boolFilter();
}
for (String filterQuery : searchQuery.getFilterQueries()) {
boolFilterBuilder.must(FilterBuilders.queryFilter(QueryBuilders.queryStringQuery(filterQuery)));
}
}
// sort
final SortField[] sortFields = searchQuery.getSortFields();
if (sortFields.length != 0) {
for (final SortField sortField : sortFields) {
FieldSortBuilder fieldSort = SortBuilders.fieldSort(sortField.getField());
if (Constants.DESC.equals(sortField.getOrder())) {
fieldSort.order(SortOrder.DESC);
} else {
fieldSort.order(SortOrder.ASC);
}
queryRequestBuilder.addSort(fieldSort);
}
} else if (queryHelper.hasDefaultSortFields()) {
for (final SortField sortField : queryHelper.getDefaultSortFields()) {
FieldSortBuilder fieldSort = SortBuilders.fieldSort(sortField.getField());
if (Constants.DESC.equals(sortField.getOrder())) {
fieldSort.order(SortOrder.DESC);
} else {
fieldSort.order(SortOrder.ASC);
}
queryRequestBuilder.addSort(fieldSort);
}
}
// highlighting
if (queryHelper.getHighlightingFields() != null && queryHelper.getHighlightingFields().length != 0) {
for (final String hf : queryHelper.getHighlightingFields()) {
queryRequestBuilder.addHighlightedField(hf, queryHelper.getHighlightSnippetSize());
}
}
// geo
if (geoInfo != null && geoInfo.isAvailable()) {
if (boolFilterBuilder == null) {
boolFilterBuilder = FilterBuilders.boolFilter();
}
boolFilterBuilder.must(geoInfo.toFilterBuilder());
}
// facets
if (facetInfo != null) {
if (facetInfo.field != null) {
for (final String f : facetInfo.field) {
if (queryHelper.isFacetField(f)) {
String encodedField = BaseEncoding.base64().encode(f.getBytes(Charsets.UTF_8));
TermsBuilder termsBuilder = AggregationBuilders.terms(Constants.FACET_FIELD_PREFIX + encodedField).field(f);
// TODO order
if (facetInfo.limit != null) {
// TODO
termsBuilder.size(Integer.parseInt(facetInfo.limit));
}
queryRequestBuilder.addAggregation(termsBuilder);
} else {
throw new FessSolrQueryException("Invalid facet field: " + f);
}
}
}
if (facetInfo.query != null) {
for (final String fq : facetInfo.query) {
final String facetQuery = queryHelper.buildFacetQuery(fq);
if (StringUtil.isNotBlank(facetQuery)) {
final String encodedFacetQuery = BaseEncoding.base64().encode(facetQuery.getBytes(Charsets.UTF_8));
FilterAggregationBuilder filterBuilder =
AggregationBuilders.filter(Constants.FACET_QUERY_PREFIX + encodedFacetQuery).filter(
FilterBuilders.queryFilter(QueryBuilders.queryStringQuery(facetQuery)));
// TODO order
if (facetInfo.limit != null) {
// TODO
// filterBuilder.size(Integer.parseInt(facetInfo .limit));
}
queryRequestBuilder.addAggregation(filterBuilder);
} else {
throw new FessSolrQueryException("Invalid facet query: " + facetQuery);
}
}
}
}
if (queryHelper.getTimeAllowed() >= 0) {
queryRequestBuilder.setTimeout(TimeValue.timeValueMillis(queryHelper.getTimeAllowed()));
}
final Set<Entry<String, String[]>> paramSet = queryHelper.getRequestParameterSet();
if (!paramSet.isEmpty()) {
for (final Map.Entry<String, String[]> entry : paramSet) {
queryRequestBuilder.putHeader(entry.getKey(), entry.getValue());
}
}
if (boolFilterBuilder != null) {
queryBuilder = QueryBuilders.filteredQuery(queryBuilder, boolFilterBuilder);
}
searchResponse = queryRequestBuilder.setQuery(queryBuilder).execute().actionGet();
}
final long execTime = System.currentTimeMillis() - startTime;
final QueryResponseList queryResponseList = ComponentUtil.getQueryResponseList();
queryResponseList.init(searchResponse, start, rows);
queryResponseList.setSearchQuery(q);
if (queryRequestBuilder != null) {
queryResponseList.setSolrQuery(queryRequestBuilder.toString());
}
queryResponseList.setExecTime(execTime);
return queryResponseList;
}
// TODO search
public SearchResponse query(QueryBuilder queryBuilder, AbstractAggregationBuilder aggregationBuilder, SortBuilder sortBuilder) {
SearchRequestBuilder query = client.prepareSearch(index).setQuery(queryBuilder);
if (aggregationBuilder != null) {
query.addAggregation(aggregationBuilder);
}
if (sortBuilder != null) {
query.addSort(sortBuilder);
}
return query.execute().actionGet();
}
public void update(String id, String field, Object value) {
UpdateResponse response = client.prepareUpdate(index, type, id).setDoc(field, value).execute().actionGet();
// TODO
}
public void refresh() {
client.admin().indices().prepareRefresh(index).execute(new ActionListener<RefreshResponse>() {
@Override
public void onResponse(RefreshResponse response) {
// TODO Auto-generated method stub
}
@Override
public void onFailure(Throwable e) {
// TODO Auto-generated method stub
}
});
}
public void flush() {
client.admin().indices().prepareFlush(index).execute(new ActionListener<FlushResponse>() {
@Override
public void onResponse(FlushResponse response) {
// TODO Auto-generated method stub
}
@Override
public void onFailure(Throwable e) {
// TODO Auto-generated method stub
}
});
}
public void optimize() {
client.admin().indices().prepareOptimize(index).execute(new ActionListener<OptimizeResponse>() {
@Override
public void onResponse(OptimizeResponse response) {
// TODO Auto-generated method stub
}
@Override
public void onFailure(Throwable e) {
// TODO Auto-generated method stub
}
});
}
public PingResponse ping() {
ClusterHealthResponse response = client.admin().cluster().prepareHealth().execute().actionGet();
return new PingResponse(response);
}
public void addAll(List<Map<String, Object>> docList) {
// TODO Auto-generated method stub
}
}

10
src/main/java/org/codelibs/fess/ds/IndexUpdateCallback.java
View file

@ -18,22 +18,18 @@ package org.codelibs.fess.ds;
import java.util.Map;
import org.codelibs.solr.lib.SolrGroup;
import org.codelibs.fess.client.SearchClient;
public interface IndexUpdateCallback {
boolean store(Map<String, Object> dataMap);
void setSolrGroup(SolrGroup solrGroup);
void setElasticsearchClient(SearchClient solrGroup);
SolrGroup getSolrGroup();
void setCommitPerCount(long commitPerCount);
SearchClient getElasticsearchClient();
long getDocumentSize();
long getExecuteTime();
void commit();
}

31
src/main/java/org/codelibs/fess/ds/impl/FileListDataStoreImpl.java
View file

@ -25,6 +25,7 @@ import java.util.Map;
import jp.sf.orangesignal.csv.CsvConfig;
import org.codelibs.fess.Constants;
import org.codelibs.fess.client.SearchClient;
import org.codelibs.fess.db.exentity.DataCrawlingConfig;
import org.codelibs.fess.ds.DataStoreCrawlingException;
import org.codelibs.fess.ds.DataStoreException;
@ -45,7 +46,6 @@ import org.codelibs.robot.rule.Rule;
import org.codelibs.robot.rule.RuleManager;
import org.codelibs.robot.transformer.Transformer;
import org.codelibs.robot.util.LruHashMap;
import org.codelibs.solr.lib.SolrGroup;
import org.seasar.framework.container.SingletonS2Container;
import org.seasar.framework.util.SerializeUtil;
import org.slf4j.Logger;
@ -240,7 +240,7 @@ public class FileListDataStoreImpl extends CsvDataStoreImpl {
if (deleteIdList.size() >= maxDeleteDocumentCacheSize) {
final IndexingHelper indexingHelper = ComponentUtil.getIndexingHelper();
for (final String id : deleteIdList) {
indexingHelper.deleteDocument(indexUpdateCallback.getSolrGroup(), id);
indexingHelper.deleteDocument(indexUpdateCallback.getElasticsearchClient(), id);
}
if (logger.isDebugEnabled()) {
logger.debug("Deleted " + deleteIdList);
@ -253,13 +253,8 @@ public class FileListDataStoreImpl extends CsvDataStoreImpl {
}
@Override
public void setSolrGroup(final SolrGroup solrGroup) {
indexUpdateCallback.setSolrGroup(solrGroup);
}
@Override
public void setCommitPerCount(final long commitPerCount) {
indexUpdateCallback.setCommitPerCount(commitPerCount);
public void setElasticsearchClient(final SearchClient searchClient) {
indexUpdateCallback.setElasticsearchClient(searchClient);
}
@Override
@ -273,22 +268,8 @@ public class FileListDataStoreImpl extends CsvDataStoreImpl {
}
@Override
public void commit() {
if (!deleteIdList.isEmpty()) {
final IndexingHelper indexingHelper = ComponentUtil.getIndexingHelper();
for (final String id : deleteIdList) {
indexingHelper.deleteDocument(indexUpdateCallback.getSolrGroup(), id);
}
if (logger.isDebugEnabled()) {
logger.debug("Deleted " + deleteIdList);
}
}
indexUpdateCallback.commit();
}
@Override
public SolrGroup getSolrGroup() {
return indexUpdateCallback.getSolrGroup();
public SearchClient getElasticsearchClient() {
return indexUpdateCallback.getElasticsearchClient();
}
}
}

89
src/main/java/org/codelibs/fess/ds/impl/IndexUpdateCallbackImpl.java
View file

@ -21,8 +21,8 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.solr.common.SolrInputDocument;
import org.codelibs.fess.FessSystemException;
import org.codelibs.fess.client.SearchClient;
import org.codelibs.fess.ds.IndexUpdateCallback;
import org.codelibs.fess.helper.CrawlingSessionHelper;
import org.codelibs.fess.helper.FieldHelper;
@ -30,7 +30,6 @@ import org.codelibs.fess.helper.IndexingHelper;
import org.codelibs.fess.helper.SearchLogHelper;
import org.codelibs.fess.helper.SystemHelper;
import org.codelibs.fess.util.ComponentUtil;
import org.codelibs.solr.lib.SolrGroup;
import org.seasar.framework.container.annotation.tiger.InitMethod;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -38,7 +37,7 @@ import org.slf4j.LoggerFactory;
public class IndexUpdateCallbackImpl implements IndexUpdateCallback {
private static final Logger logger = LoggerFactory.getLogger(IndexUpdateCallbackImpl.class);
protected SolrGroup solrGroup;
protected SearchClient searchClient;
public int maxDocumentCacheSize = 5;
@ -48,11 +47,9 @@ public class IndexUpdateCallbackImpl implements IndexUpdateCallback {
protected volatile AtomicLong documentSize = new AtomicLong(0);
protected volatile long commitPerCount = 0;
protected volatile long executeTime = 0;
final List<SolrInputDocument> docList = new ArrayList<SolrInputDocument>();
final List<Map<String, Object>> docList = new ArrayList<>();
private FieldHelper fieldHelper;
@ -82,23 +79,20 @@ public class IndexUpdateCallbackImpl implements IndexUpdateCallback {
final CrawlingSessionHelper crawlingSessionHelper = ComponentUtil.getCrawlingSessionHelper();
dataMap.put(fieldHelper.idField, crawlingSessionHelper.generateId(dataMap));
final SolrInputDocument doc = createSolrDocument(dataMap);
updateDocument(dataMap);
docList.add(doc);
docList.add(dataMap);
if (logger.isDebugEnabled()) {
logger.debug("Added the document. " + "The number of a document cache is " + docList.size() + ".");
}
if (docList.size() >= maxDocumentCacheSize) {
indexingHelper.sendDocuments(solrGroup, docList);
indexingHelper.sendDocuments(searchClient, docList);
}
documentSize.getAndIncrement();
// commit
if (commitPerCount > 0 && documentSize.get() % commitPerCount == 0) {
if (!docList.isEmpty()) {
indexingHelper.sendDocuments(solrGroup, docList);
}
commitDocuments();
if (!docList.isEmpty()) {
indexingHelper.sendDocuments(searchClient, docList);
}
if (logger.isDebugEnabled()) {
logger.debug("The number of an added document is " + documentSize.get() + ".");
@ -108,73 +102,37 @@ public class IndexUpdateCallbackImpl implements IndexUpdateCallback {
return true;
}
protected SolrInputDocument createSolrDocument(final Map<String, Object> dataMap) {
protected void updateDocument(final Map<String, Object> dataMap) {
final String url = dataMap.get(fieldHelper.urlField).toString();
final SolrInputDocument doc = new SolrInputDocument();
for (final Map.Entry<String, Object> entry : dataMap.entrySet()) {
if (fieldHelper.boostField.equals(entry.getKey())) {
// boost
final float documentBoost = Float.valueOf(entry.getValue().toString());
doc.setDocumentBoost(documentBoost);
if (logger.isDebugEnabled()) {
logger.debug("Set a document boost (" + documentBoost + ").");
}
}
doc.addField(entry.getKey(), entry.getValue());
}
if (clickCountEnabled) {
addClickCountField(doc, url, fieldHelper.clickCountField);
addClickCountField(dataMap, url, fieldHelper.clickCountField);
}
if (favoriteCountEnabled) {
addFavoriteCountField(doc, url, fieldHelper.favoriteCountField);
addFavoriteCountField(dataMap, url, fieldHelper.favoriteCountField);
}
if (!dataMap.containsKey(fieldHelper.docIdField)) {
final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
doc.addField(fieldHelper.docIdField, systemHelper.generateDocId(dataMap));
dataMap.put(fieldHelper.docIdField, systemHelper.generateDocId(dataMap));
}
return doc;
}
@Override
public void commit() {
if (!docList.isEmpty()) {
final IndexingHelper indexingHelper = ComponentUtil.getIndexingHelper();
indexingHelper.sendDocuments(solrGroup, docList);
}
commitDocuments();
}
protected void commitDocuments() {
final long execTime = System.currentTimeMillis();
if (logger.isInfoEnabled()) {
logger.info("Committing documents. ");
}
synchronized (solrGroup) {
solrGroup.commit(true, true, false, true);
}
if (logger.isInfoEnabled()) {
logger.info("Committed documents. The execution time is " + (System.currentTimeMillis() - execTime) + "ms.");
}
}
protected void addClickCountField(final SolrInputDocument doc, final String url, final String clickCountField) {
protected void addClickCountField(final Map<String, Object> doc, final String url, final String clickCountField) {
final SearchLogHelper searchLogHelper = ComponentUtil.getSearchLogHelper();
final int count = searchLogHelper.getClickCount(url);
doc.addField(clickCountField, count);
doc.put(clickCountField, count);
if (logger.isDebugEnabled()) {
logger.debug("Click Count: " + count + ", url: " + url);
}
}
protected void addFavoriteCountField(final SolrInputDocument doc, final String url, final String favoriteCountField) {
protected void addFavoriteCountField(final Map<String, Object> doc, final String url, final String favoriteCountField) {
final SearchLogHelper searchLogHelper = ComponentUtil.getSearchLogHelper();
final long count = searchLogHelper.getFavoriteCount(url);
doc.addField(favoriteCountField, count);
doc.put(favoriteCountField, count);
if (logger.isDebugEnabled()) {
logger.debug("Favorite Count: " + count + ", url: " + url);
}
@ -191,18 +149,13 @@ public class IndexUpdateCallbackImpl implements IndexUpdateCallback {
}
@Override
public SolrGroup getSolrGroup() {
return solrGroup;
public SearchClient getElasticsearchClient() {
return searchClient;
}
@Override
public void setSolrGroup(final SolrGroup solrGroup) {
this.solrGroup = solrGroup;
}
@Override
public void setCommitPerCount(final long commitPerCount) {
this.commitPerCount = commitPerCount;
public void setElasticsearchClient(final SearchClient searchClient) {
this.searchClient = searchClient;
}
}

92
src/main/java/org/codelibs/fess/entity/FieldAnalysisResponse.java
View file

@ -1,92 +0,0 @@
/*
* Copyright 2009-2015 the CodeLibs Project and the Others.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
* either express or implied. See the License for the specific language
* governing permissions and limitations under the License.
*/
package org.codelibs.fess.entity;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
public class FieldAnalysisResponse extends LinkedHashMap<String, Map<String, List<Map<String, Object>>>> {
private static final long serialVersionUID = 1L;
protected long execTime;
public FieldAnalysisResponse(final NamedList<Object> response) {
final NamedList<?> fnList = getNamedList(getNamedList(response, "analysis"), "field_names");
if (fnList != null) {
for (int i = 0; i < fnList.size(); i++) {
final String fieldName = fnList.getName(i);
final NamedList<?> analysisList = getNamedList(fnList.getVal(i), "index");
final Map<String, List<Map<String, Object>>> analysisMap = new LinkedHashMap<String, List<Map<String, Object>>>();
put(fieldName, analysisMap);
if (analysisList != null) {
for (int j = 0; j < analysisList.size(); j++) {
final String analysisName = analysisList.getName(j);
final Object analysisDataList = analysisList.getVal(j);
if (analysisDataList instanceof List<?>) {
final List<Map<String, Object>> dataMapList = getDataMap((List<Object>) analysisDataList);
analysisMap.put(analysisName, dataMapList);
}
}
}
}
}
}
private List<Map<String, Object>> getDataMap(final List<Object> analysisDataList) {
final List<Map<String, Object>> dataMapList = new ArrayList<Map<String, Object>>();
for (final Object dataList : analysisDataList) {
if (dataList instanceof SimpleOrderedMap<?>) {
final Map<String, Object> dataMap = new LinkedHashMap<String, Object>();
for (int k = 0; k < ((SimpleOrderedMap<?>) dataList).size(); k++) {
final String dataName = ((SimpleOrderedMap<?>) dataList).getName(k);
final Object dataObj = ((SimpleOrderedMap<?>) dataList).getVal(k);
dataMap.put(dataName, dataObj);
}
dataMapList.add(dataMap);
}
}
return dataMapList;
}
public long getExecTime() {
return execTime;
}
public void setExecTime(final long execTime) {
this.execTime = execTime;
}
private NamedList<?> getNamedList(final Object obj, final String key) {
if (obj instanceof NamedList<?>) {
return (NamedList<?>) ((NamedList<?>) obj).get(key);
}
return null;
}
@Override
public String toString() {
return "FieldAnalysisResponse [execTime=" + execTime + "]";
}
}

40
src/main/java/org/codelibs/fess/entity/GeoInfo.java
View file

@ -17,6 +17,9 @@
package org.codelibs.fess.entity;
import org.codelibs.core.util.StringUtil;
import org.elasticsearch.common.unit.DistanceUnit;
import org.elasticsearch.index.query.FilterBuilder;
import org.elasticsearch.index.query.FilterBuilders;
import org.seasar.struts.annotation.Mask;
import org.seasar.struts.annotation.Maxbytelength;
@ -35,14 +38,14 @@ public class GeoInfo {
private boolean isInit = false;
private FilterBuilder builder;
private void init() {
if (!isInit) {
isInit = true;
if (StringUtil.isBlank(latitude) || StringUtil.isBlank(longitude) || StringUtil.isBlank(distance)) {
latitude = null;
longitude = null;
distance = null;
clear();
return;
}
@ -51,10 +54,9 @@ public class GeoInfo {
double lat = Double.parseDouble(latitude);
double lon = Double.parseDouble(longitude);
if (dist > 0) {
distance = Double.toString(dist);
} else {
distance = null;
if (dist <= 0) {
clear();
return;
}
if (lat > 90) {
@ -75,24 +77,28 @@ public class GeoInfo {
}
}
latitude = Double.toString(lat);
longitude = Double.toString(lon);
builder = FilterBuilders.geoDistanceFilter("geo_info").distance(dist, DistanceUnit.KILOMETERS).lat(lat).lon(lon);
} catch (final NumberFormatException e) {
latitude = null;
longitude = null;
distance = null;
clear();
}
}
}
public boolean isAvailable() {
init();
return StringUtil.isNotBlank(latitude) && StringUtil.isNotBlank(longitude) && StringUtil.isNotBlank(distance);
private void clear() {
latitude = null;
longitude = null;
distance = null;
builder = null;
}
public String toGeoQueryString() {
public boolean isAvailable() {
init();
return "{!geofilt pt=" + latitude + "," + longitude + " sfield=location d=" + distance + "}";
return builder != null;
}
public FilterBuilder toFilterBuilder() {
init();
return builder;
}
@Override

35
src/main/java/org/codelibs/fess/entity/MoreLikeThisInfo.java
View file

@ -1,35 +0,0 @@
/*
* Copyright 2009-2015 the CodeLibs Project and the Others.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
* either express or implied. See the License for the specific language
* governing permissions and limitations under the License.
*/
package org.codelibs.fess.entity;
import java.util.Arrays;
import org.seasar.struts.annotation.IntegerType;
import org.seasar.struts.annotation.Maxbytelength;
public class MoreLikeThisInfo {
@Maxbytelength(maxbytelength = 255)
public String[] field;
@IntegerType
public String count;
@Override
public String toString() {
return "MoreLikeThisInfo [field=" + Arrays.toString(field) + ", count=" + count + "]";
}
}

62
src/main/java/org/codelibs/fess/entity/PingResponse.java
View file

@ -16,70 +16,18 @@
package org.codelibs.fess.entity;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import org.apache.solr.client.solrj.response.SolrPingResponse;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus;
public class PingResponse {
private final int status = 0;
private final Target[] targets;
public PingResponse(final Collection<SolrPingResponse> responses) {
final List<Target> targetList = new ArrayList<>();
for (final SolrPingResponse response : responses) {
int status = response.getStatus();
if (status != 0) {
status = 1;
}
targetList.add(new Target(status, response.getRequestUrl(), response.getElapsedTime(), response.getQTime()));
}
targets = targetList.toArray(new Target[targetList.size()]);
}
public static class Target {
private final int status;
private final String url;
private final long searchTime;
private final int queryTime;
public Target(final int status, final String url, final long elapsedTime, final int qTime) {
this.status = status;
this.url = url;
searchTime = elapsedTime;
queryTime = qTime;
}
public int getStatus() {
return status;
}
public String getUrl() {
return url;
}
public long getSearchTime() {
return searchTime;
}
public int getQueryTime() {
return queryTime;
}
private int status = 0;
public PingResponse(final ClusterHealthResponse response) {
status = response.getStatus() == ClusterHealthStatus.RED ? 1 : 0;
}
public int getStatus() {
return status;
}
public Target[] getTargets() {
return targets;
}
}

100
src/main/java/org/codelibs/fess/exec/Crawler.java
View file

@ -37,7 +37,7 @@ import org.codelibs.core.CoreLibConstants;
import org.codelibs.core.util.DynamicProperties;
import org.codelibs.core.util.StringUtil;
import org.codelibs.fess.Constants;
import org.codelibs.fess.FessSystemException;
import org.codelibs.fess.client.SearchClient;
import org.codelibs.fess.db.allcommon.CDef;
import org.codelibs.fess.helper.CrawlingSessionHelper;
import org.codelibs.fess.helper.DataIndexHelper;
@ -50,13 +50,10 @@ import org.codelibs.fess.helper.WebFsIndexHelper;
import org.codelibs.fess.screenshot.ScreenShotManager;
import org.codelibs.fess.service.CrawlingSessionService;
import org.codelibs.fess.service.PathMappingService;
import org.codelibs.fess.taglib.FessFunctions;
import org.codelibs.fess.util.ComponentUtil;
import org.codelibs.fess.util.ResourceUtil;
import org.codelibs.solr.lib.SolrGroup;
import org.codelibs.solr.lib.SolrGroupManager;
import org.codelibs.solr.lib.policy.QueryType;
import org.codelibs.solr.lib.policy.StatusPolicy;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.kohsuke.args4j.CmdLineException;
import org.kohsuke.args4j.CmdLineParser;
import org.kohsuke.args4j.Option;
@ -91,7 +88,7 @@ public class Crawler implements Serializable {
private static final String MAIL_TEMPLATE_NAME = "crawler";
@Resource
protected SolrGroupManager solrGroupManager;
protected SearchClient searchClient;
@Binding(bindingType = BindingType.MAY)
@Resource
@ -137,9 +134,6 @@ public class Crawler implements Serializable {
@Option(name = "-p", aliases = "--properties", metaVar = "properties", usage = "Properties File")
protected String propertiesPath;
@Option(name = "-o", aliases = "--operation", metaVar = "operation", usage = "Opration when crawlwer is finised")
protected String operation;
@Option(name = "-e", aliases = "--expires", metaVar = "expires", usage = "Expires for documents")
protected String expires;
@ -182,13 +176,6 @@ public class Crawler implements Serializable {
return idList;
}
public boolean isOptimize() {
return Constants.OPTIMIZE.equalsIgnoreCase(operation);
}
public boolean isCommit() {
return Constants.COMMIT.equalsIgnoreCase(operation);
}
}
public static void main(final String[] args) {
@ -374,11 +361,6 @@ public class Crawler implements Serializable {
try {
writeTimeToSessionInfo(crawlingSessionHelper, Constants.CRAWLER_START_TIME);
final SolrGroup updateSolrGroup = solrGroupManager.getSolrGroup(QueryType.ADD);
if (!updateSolrGroup.isActive(QueryType.ADD)) {
throw new FessSystemException("SolrGroup " + updateSolrGroup.getGroupName() + " is not available.");
}
// setup path mapping
final List<CDef.ProcessType> ptList = new ArrayList<CDef.ProcessType>();
ptList.add(CDef.ProcessType.Crawling);
@ -411,7 +393,7 @@ public class Crawler implements Serializable {
public void run() {
// crawl web
writeTimeToSessionInfo(crawlingSessionHelper, Constants.WEB_FS_CRAWLER_START_TIME);
webFsIndexHelper.crawl(options.sessionId, webConfigIdList, fileConfigIdList, updateSolrGroup);
webFsIndexHelper.crawl(options.sessionId, webConfigIdList, fileConfigIdList);
writeTimeToSessionInfo(crawlingSessionHelper, Constants.WEB_FS_CRAWLER_END_TIME);
}
}, WEB_FS_CRAWLING_PROCESS);
@ -424,7 +406,7 @@ public class Crawler implements Serializable {
public void run() {
// crawl data system
writeTimeToSessionInfo(crawlingSessionHelper, Constants.DATA_CRAWLER_START_TIME);
dataIndexHelper.crawl(options.sessionId, dataConfigIdList, updateSolrGroup);
dataIndexHelper.crawl(options.sessionId, dataConfigIdList);
writeTimeToSessionInfo(crawlingSessionHelper, Constants.DATA_CRAWLER_END_TIME);
}
}, DATA_CRAWLING_PROCESS);
@ -435,32 +417,18 @@ public class Crawler implements Serializable {
joinCrawlerThread(dataCrawlerThread);
// clean up
QueryBuilder queryBuilder =
QueryBuilders.boolQuery().must(QueryBuilders.rangeQuery(fieldHelper.expiresField).to(new Date()))
.mustNot(QueryBuilders.termQuery(fieldHelper.segmentField, options.sessionId));
try {
updateSolrGroup.deleteByQuery(fieldHelper.expiresField + ":[* TO " + FessFunctions.formatDate(new Date())
+ "] NOT segment:" + options.sessionId);
searchClient.deleteByQuery(queryBuilder);
} catch (final Exception e) {
if (logger.isWarnEnabled()) {
logger.warn("Could not delete expired sessions in " + updateSolrGroup.getGroupName(), e);
logger.warn("Could not delete expired sessions: " + queryBuilder.toString(), e);
}
exitCode = Constants.EXIT_FAIL;
}
if (options.isCommit()) {
commit(crawlingSessionHelper, updateSolrGroup);
} else if (options.isOptimize()) {
optimize(crawlingSessionHelper, updateSolrGroup);
} else {
if (logger.isInfoEnabled()) {
logger.info("No index commit.");
}
}
final String serverRotationStr = crawlerProperties.getProperty(Constants.SERVER_ROTATION_PROPERTY, Constants.FALSE);
if (Constants.TRUE.equalsIgnoreCase(serverRotationStr)) {
// apply
solrGroupManager.applyNewSolrGroup();
}
if (logger.isInfoEnabled()) {
logger.info("Finished Crawler");
}
@ -479,52 +447,6 @@ public class Crawler implements Serializable {
}
}
protected void optimize(final CrawlingSessionHelper crawlingSessionHelper, final SolrGroup solrGroup) {
writeTimeToSessionInfo(crawlingSessionHelper, Constants.OPTIMIZE_START_TIME);
long startTime = System.currentTimeMillis();
solrGroup.optimize();
startTime = System.currentTimeMillis() - startTime;
writeTimeToSessionInfo(crawlingSessionHelper, Constants.OPTIMIZE_END_TIME);
if (crawlingSessionHelper != null) {
crawlingSessionHelper.putToInfoMap(Constants.OPTIMIZE_EXEC_TIME, Long.toString(startTime));
}
// update status
final StatusPolicy statusPolicy = solrGroup.getStatusPolicy();
for (final String serverName : solrGroup.getServerNames()) {
if (statusPolicy.isActive(QueryType.OPTIMIZE, serverName)) {
statusPolicy.activate(QueryType.OPTIMIZE, serverName);
}
}
if (logger.isInfoEnabled()) {
logger.info("[EXEC TIME] index optimize time: " + startTime + "ms");
}
}
protected void commit(final CrawlingSessionHelper crawlingSessionHelper, final SolrGroup solrGroup) {
writeTimeToSessionInfo(crawlingSessionHelper, Constants.COMMIT_START_TIME);
long startTime = System.currentTimeMillis();
solrGroup.commit(true, true, false, true);
startTime = System.currentTimeMillis() - startTime;
writeTimeToSessionInfo(crawlingSessionHelper, Constants.COMMIT_END_TIME);
if (crawlingSessionHelper != null) {
crawlingSessionHelper.putToInfoMap(Constants.COMMIT_EXEC_TIME, Long.toString(startTime));
}
// update status
final StatusPolicy statusPolicy = solrGroup.getStatusPolicy();
for (final String serverName : solrGroup.getServerNames()) {
if (statusPolicy.isActive(QueryType.COMMIT, serverName)) {
statusPolicy.activate(QueryType.COMMIT, serverName);
}
}
if (logger.isInfoEnabled()) {
logger.info("[EXEC TIME] index commit time: " + startTime + "ms");
}
}
private void writeTimeToSessionInfo(final CrawlingSessionHelper crawlingSessionHelper, final String key) {
if (crawlingSessionHelper != null) {
final SimpleDateFormat dateFormat = new SimpleDateFormat(CoreLibConstants.DATE_FORMAT_ISO_8601_EXTEND);

46
src/main/java/org/codelibs/fess/helper/CrawlingSessionHelper.java
View file

@ -25,18 +25,22 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrQuery.ORDER;
import org.apache.solr.client.solrj.response.FacetField;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.codelibs.core.util.StringUtil;
import org.codelibs.fess.Constants;
import org.codelibs.fess.FessSystemException;
import org.codelibs.fess.client.SearchClient;
import org.codelibs.fess.db.exentity.CrawlingSession;
import org.codelibs.fess.db.exentity.CrawlingSessionInfo;
import org.codelibs.fess.service.CrawlingSessionService;
import org.codelibs.fess.util.ComponentUtil;
import org.codelibs.solr.lib.SolrGroup;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket;
import org.elasticsearch.search.aggregations.bucket.terms.Terms.Order;
import org.elasticsearch.search.aggregations.bucket.terms.TermsBuilder;
import org.seasar.framework.container.SingletonS2Container;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -52,6 +56,8 @@ public class CrawlingSessionHelper implements Serializable {
protected LocalDateTime documentExpires;
private int maxSessionIdsInList = 100;
protected CrawlingSessionService getCrawlingSessionService() {
return SingletonS2Container.getComponent(CrawlingSessionService.class);
}
@ -147,28 +153,22 @@ public class CrawlingSessionHelper implements Serializable {
return generateId(url, roleTypeList);
}
public List<Map<String, String>> getSessionIdList(final SolrGroup serverGroup) {
public List<Map<String, String>> getSessionIdList(final SearchClient searchClient) {
final List<Map<String, String>> sessionIdList = new ArrayList<Map<String, String>>();
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final SolrQuery query = new SolrQuery();
query.setQuery("*:*");
query.setFacet(true);
query.addFacetField(fieldHelper.segmentField);
query.addSort(fieldHelper.segmentField, ORDER.desc);
QueryBuilder queryBuilder = QueryBuilders.matchAllQuery();
TermsBuilder termsBuilder =
AggregationBuilders.terms(fieldHelper.segmentField).field(fieldHelper.segmentField).size(maxSessionIdsInList)
.order(Order.term(false));
final QueryResponse queryResponse = serverGroup.query(query);
final List<FacetField> facets = queryResponse.getFacetFields();
for (final FacetField facet : facets) {
final List<FacetField.Count> facetEntries = facet.getValues();
if (facetEntries != null) {
for (final FacetField.Count fcount : facetEntries) {
final Map<String, String> map = new HashMap<String, String>(2);
map.put(fieldHelper.segmentField, fcount.getName());
map.put(FACET_COUNT_KEY, Long.toString(fcount.getCount()));
sessionIdList.add(map);
}
}
final SearchResponse searchResponse = searchClient.query(queryBuilder, termsBuilder, null);
Terms terms = searchResponse.getAggregations().get(fieldHelper.segmentField);
for (Bucket bucket : terms.getBuckets()) {
final Map<String, String> map = new HashMap<String, String>(2);
map.put(fieldHelper.segmentField, bucket.getKey());
map.put(FACET_COUNT_KEY, Long.toString(bucket.getDocCount()));
sessionIdList.add(map);
}
return sessionIdList;
}

36
src/main/java/org/codelibs/fess/helper/DataIndexHelper.java
View file

@ -25,7 +25,6 @@ import java.util.Map;
import javax.annotation.Resource;
import org.apache.solr.client.solrj.util.ClientUtils;
import org.codelibs.core.util.DynamicProperties;
import org.codelibs.core.util.StringUtil;
import org.codelibs.fess.Constants;
@ -35,7 +34,8 @@ import org.codelibs.fess.ds.DataStoreFactory;
import org.codelibs.fess.ds.IndexUpdateCallback;
import org.codelibs.fess.service.DataCrawlingConfigService;
import org.codelibs.fess.util.ComponentUtil;
import org.codelibs.solr.lib.SolrGroup;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.seasar.framework.container.SingletonS2Container;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -63,7 +63,7 @@ public class DataIndexHelper implements Serializable {
private final List<DataCrawlingThread> dataCrawlingThreadList = Collections.synchronizedList(new ArrayList<DataCrawlingThread>());
public void crawl(final String sessionId, final SolrGroup solrGroup) {
public void crawl(final String sessionId) {
final List<DataCrawlingConfig> configList = dataCrawlingConfigService.getAllDataCrawlingConfigList();
if (configList.isEmpty()) {
@ -74,10 +74,10 @@ public class DataIndexHelper implements Serializable {
return;
}
crawl(sessionId, solrGroup, configList);
doCrawl(sessionId, configList);
}
public void crawl(final String sessionId, final List<Long> configIdList, final SolrGroup solrGroup) {
public void crawl(final String sessionId, final List<Long> configIdList) {
final List<DataCrawlingConfig> configList = dataCrawlingConfigService.getDataCrawlingConfigListByIds(configIdList);
if (configList.isEmpty()) {
@ -88,10 +88,10 @@ public class DataIndexHelper implements Serializable {
return;
}
crawl(sessionId, solrGroup, configList);
doCrawl(sessionId, configList);
}
protected void crawl(final String sessionId, final SolrGroup solrGroup, final List<DataCrawlingConfig> configList) {
protected void doCrawl(final String sessionId, final List<DataCrawlingConfig> configList) {
int multiprocessCrawlingCount = 5;
String value = crawlerProperties.getProperty(Constants.CRAWLING_THREAD_COUNT_PROPERTY, "5");
try {
@ -100,19 +100,9 @@ public class DataIndexHelper implements Serializable {
// NOP
}
long commitPerCount = Constants.DEFAULT_COMMIT_PER_COUNT;
value = crawlerProperties.getProperty(Constants.COMMIT_PER_COUNT_PROPERTY, Long.toString(Constants.DEFAULT_COMMIT_PER_COUNT));
try {
commitPerCount = Long.parseLong(value);
} catch (final NumberFormatException e) {
// NOP
}
final long startTime = System.currentTimeMillis();
final IndexUpdateCallback indexUpdateCallback = SingletonS2Container.getComponent(IndexUpdateCallback.class);
indexUpdateCallback.setSolrGroup(solrGroup);
indexUpdateCallback.setCommitPerCount(commitPerCount);
final List<String> sessionIdList = new ArrayList<String>();
final Map<String, String> initParamMap = new HashMap<String, String>();
@ -193,8 +183,6 @@ public class DataIndexHelper implements Serializable {
dataCrawlingThreadList.clear();
dataCrawlingThreadStatusList.clear();
indexUpdateCallback.commit();
// put cralwing info
final CrawlingSessionHelper crawlingSessionHelper = ComponentUtil.getCrawlingSessionHelper();
@ -265,13 +253,11 @@ public class DataIndexHelper implements Serializable {
return;
}
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final StringBuilder buf = new StringBuilder(100);
buf.append(fieldHelper.configIdField).append(':').append(dataCrawlingConfig.getConfigId());
buf.append(" NOT ");
buf.append(fieldHelper.segmentField).append(':').append(ClientUtils.escapeQueryChars(sessionId));
QueryBuilder queryBuilder =
QueryBuilders.boolQuery().must(QueryBuilders.termQuery(fieldHelper.configIdField, dataCrawlingConfig.getConfigId()))
.mustNot(QueryBuilders.termQuery(fieldHelper.segmentField, sessionId));
try {
indexUpdateCallback.getSolrGroup().deleteByQuery(buf.toString());
ComponentUtil.getElasticsearchClient().deleteByQuery(queryBuilder);
} catch (final Exception e) {
logger.error("Could not delete old docs at " + dataCrawlingConfig, e);
}

29
src/main/java/org/codelibs/fess/helper/DocumentHelper.java
View file

@ -18,40 +18,19 @@ package org.codelibs.fess.helper;
import javax.annotation.Resource;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.common.SolrInputDocument;
import org.codelibs.fess.FessSystemException;
import org.codelibs.solr.lib.SolrGroup;
import org.codelibs.solr.lib.SolrGroupManager;
import org.codelibs.solr.lib.policy.QueryType;
import org.codelibs.fess.client.SearchClient;
//TODO needed?
public class DocumentHelper {
@Resource
protected FieldHelper fieldHelper;
@Resource
protected SolrGroupManager solrGroupManager;
protected SearchClient searchClient;
public void update(final String docId, final String fieldName, final long num) {
final SolrGroup solrGroup = solrGroupManager.getSolrGroup(QueryType.ADD);
if (!solrGroup.isActive(QueryType.ADD)) {
throw new FessSystemException("SolrGroup " + solrGroup.getGroupName() + " is not available.");
}
final SolrInputDocument doc = new SolrInputDocument();
doc.setField(fieldHelper.idField, "none");
doc.setField(fieldHelper.urlField, "none");
doc.setField(fieldHelper.docIdField, docId);
doc.setField(fieldName, num);
final UpdateRequest req = new UpdateRequest();
req.add(doc);
req.setParam("excmd", "update");
req.setParam("term", fieldHelper.docIdField);
solrGroup.request(req);
solrGroup.commit(false, false, true);
searchClient.update(docId, fieldName, num);
}
}

461
src/main/java/org/codelibs/fess/helper/IndexingHelper.java
View file

@ -16,18 +16,10 @@
package org.codelibs.fess.helper;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.client.solrj.response.UpdateResponse;
import org.apache.solr.client.solrj.util.ClientUtils;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrInputDocument;
import org.codelibs.fess.util.ComponentUtil;
import org.codelibs.solr.lib.SolrGroup;
import org.codelibs.fess.client.SearchClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -40,14 +32,14 @@ public class IndexingHelper {
public long requestInterval = 500;
public void sendDocuments(final SolrGroup solrGroup, final List<SolrInputDocument> docList) {
public void sendDocuments(final SearchClient searchClient, final List<Map<String, Object>> docList) {
final long execTime = System.currentTimeMillis();
if (logger.isDebugEnabled()) {
logger.debug("Sending " + docList.size() + " documents to a server.");
}
synchronized (solrGroup) {
deleteOldDocuments(solrGroup, docList);
solrGroup.add(docList);
synchronized (searchClient) {
deleteOldDocuments(searchClient, docList);
searchClient.addAll(docList);
}
if (logger.isInfoEnabled()) {
logger.info("Sent " + docList.size() + " docs (Solr: " + (System.currentTimeMillis() - execTime) + "ms)");
@ -55,223 +47,238 @@ public class IndexingHelper {
docList.clear();
}
private void deleteOldDocuments(final SolrGroup solrGroup, final List<SolrInputDocument> docList) {
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final List<String> docIdList = new ArrayList<String>();
final StringBuilder q = new StringBuilder(1000);
final StringBuilder fq = new StringBuilder(100);
for (final SolrInputDocument inputDoc : docList) {
final Object idValue = inputDoc.getFieldValue(fieldHelper.idField);
if (idValue == null) {
continue;
}
final Object configIdValue = inputDoc.getFieldValue(fieldHelper.configIdField);
if (configIdValue == null) {
continue;
}
q.setLength(0);
q.append(fieldHelper.urlField).append(":\"");
q.append(ClientUtils.escapeQueryChars((String) inputDoc.getFieldValue(fieldHelper.urlField)));
q.append('"');
fq.setLength(0);
fq.append(fieldHelper.configIdField).append(':');
fq.append(configIdValue.toString());
final SolrDocumentList docs =
getSolrDocumentList(solrGroup, fq.toString(), q.toString(),
new String[] { fieldHelper.idField, fieldHelper.docIdField });
for (final SolrDocument doc : docs) {
final Object oldIdValue = doc.getFieldValue(fieldHelper.idField);
if (!idValue.equals(oldIdValue) && oldIdValue != null) {
final Object oldDocIdValue = doc.getFieldValue(fieldHelper.docIdField);
if (oldDocIdValue != null) {
docIdList.add(oldDocIdValue.toString());
}
}
}
if (logger.isDebugEnabled()) {
logger.debug(q + " in " + fq + " => " + docs);
}
}
if (!docIdList.isEmpty()) {
deleteDocumentsByDocId(solrGroup, docIdList);
}
private void deleteOldDocuments(final SearchClient searchClient, final List<Map<String, Object>> docList) {
// TODO move searchClient
// final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
//
// final List<String> docIdList = new ArrayList<String>();
// final StringBuilder q = new StringBuilder(1000);
// final StringBuilder fq = new StringBuilder(100);
// for (final Map<String, Object> inputDoc : docList) {
// final Object idValue = inputDoc.get(fieldHelper.idField);
// if (idValue == null) {
// continue;
// }
//
// final Object configIdValue = inputDoc.get(fieldHelper.configIdField);
// if (configIdValue == null) {
// continue;
// }
//
// q.setLength(0);
// q.append(fieldHelper.urlField).append(":\"");
// q.append(QueryUtil.escapeValue((String) inputDoc.get(fieldHelper.urlField)));
// q.append('"');
//
// fq.setLength(0);
// fq.append(fieldHelper.configIdField).append(':');
// fq.append(configIdValue.toString());
//
// final List<Map<String, Object>> docs =
// getSolrDocumentList(searchClient, fq.toString(), q.toString(), new String[] { fieldHelper.idField,
// fieldHelper.docIdField });
// for (final SolrDocument doc : docs) {
// final Object oldIdValue = doc.getFieldValue(fieldHelper.idField);
// if (!idValue.equals(oldIdValue) && oldIdValue != null) {
// final Object oldDocIdValue = doc.getFieldValue(fieldHelper.docIdField);
// if (oldDocIdValue != null) {
// docIdList.add(oldDocIdValue.toString());
// }
// }
// }
// if (logger.isDebugEnabled()) {
// logger.debug(q + " in " + fq + " => " + docs);
// }
// }
// if (!docIdList.isEmpty()) {
// deleteDocumentsByDocId(searchClient, docIdList);
// }
}
public SolrDocumentList getSolrDocumentList(final SolrGroup solrGroup, final String fq, final String q, final String[] fields) {
return getSolrDocumentList(solrGroup, fq, q, fields, defaultRowSize);
public List<Map<String, Object>> getSolrDocumentList(final SearchClient searchClient, final String fq, final String q,
final String[] fields) {
return getSolrDocumentList(searchClient, fq, q, fields, defaultRowSize);
}
protected SolrDocumentList getSolrDocumentList(final SolrGroup solrGroup, final String fq, final String q, final String[] fields,
protected List<Map<String, Object>> getSolrDocumentList(final SearchClient searchClient, final String fq, final String q,
final String[] fields, final int row) {
// TODO move searchClient
// final SolrQuery sq = new SolrQuery();
// if (fq != null) {
// sq.setFilterQueries(fq);
// }
// sq.setQuery(q);
// if (fields != null) {
// sq.setFields(fields);
// }
// sq.setRows(row);
// e.
// final List<Map<String,Object>> docList = searchClient.query(sq).getResults();
// if (docList.getNumFound() <= row) {
// return docList;
// }
// return getSolrDocumentList(searchClient, fq, q, fields, (int) docList.getNumFound());
return null;
}
public void deleteDocument(final SearchClient searchClient, final String id) {
// TODO move searchClient
// final long start = System.currentTimeMillis();
// final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
// final String query = "{!raw f=" + fieldHelper.idField + "}" + id;
// for (int i = 0; i < maxRetryCount; i++) {
// boolean done = true;
// try {
// for (final UpdateResponse response : searchClient.deleteByQuery(query)) {
// if (response.getStatus() != 0) {
// logger.warn("Failed to delete: " + response);
// done = false;
// }
// }
// } catch (final Exception e) {
// logger.warn("Could not delete a document from Solr." + " It might be busy. " + "Retrying.. id:" + id + ", cause: "
// + e.getMessage());
// done = false;
// }
// if (done) {
// if (logger.isDebugEnabled()) {
// logger.info("Deleted 1 doc (Solr: " + (System.currentTimeMillis() - start) + "ms) => id:" + id);
// } else {
// logger.info("Deleted 1 doc (Solr: " + (System.currentTimeMillis() - start) + "ms)");
// }
// return;
// }
// try {
// Thread.sleep(requestInterval);
// } catch (final InterruptedException e) {}
// }
}
public void deleteDocumentsByDocId(final SearchClient searchClient, final List<String> docIdList) {
// TODO move searchClient
// final long start = System.currentTimeMillis();
// final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
// final StringBuilder buf = new StringBuilder(500);
// for (final String docId : docIdList) {
// if (buf.length() != 0) {
// buf.append(" OR ");
// }
// buf.append(fieldHelper.docIdField).append(':').append(docId);
// }
// final String query = buf.toString();
// for (int i = 0; i < maxRetryCount; i++) {
// boolean done = true;
// try {
// for (final UpdateResponse response : searchClient.deleteByQuery(query)) {
// if (response.getStatus() != 0) {
// logger.warn("Failed to delete: " + response);
// done = false;
// }
// }
// } catch (final Exception e) {
// logger.warn("Could not delete a document from Solr." + " It might be busy. " + "Retrying.. id:" + docIdList + ", cause: "
// + e.getMessage());
// done = false;
// }
// if (done) {
// if (logger.isDebugEnabled()) {
// logger.info("Deleted " + docIdList.size() + " docs (Solr: " + (System.currentTimeMillis() - start) + "ms) => docId:"
// + docIdList);
// } else {
// logger.info("Deleted " + docIdList.size() + " docs (Solr: " + (System.currentTimeMillis() - start) + "ms)");
// }
// return;
// }
// try {
// Thread.sleep(requestInterval);
// } catch (final InterruptedException e) {}
// }
}
public Map<String, Object> getSolrDocument(final SearchClient searchClient, final String id, final String[] fields) {
// TODO move searchClient
// final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
//
// final SolrQuery solrQuery = new SolrQuery();
// final StringBuilder queryBuf = new StringBuilder(500);
// queryBuf.append("{!raw f=").append(fieldHelper.idField).append("}");
// queryBuf.append(id);
// solrQuery.setQuery(queryBuf.toString());
// if (fields != null) {
// solrQuery.setFields(fields);
// }
// final QueryResponse response = searchClient.query(solrQuery);
// final List<Map<String, Object>> docList = response.getResults();
// if (docList.isEmpty()) {
// return null;
// }
// if (docList.size() > 1) {
// logger.error("Invalid multiple docs for " + id);
// for (final SolrDocument doc : docList) {
// final Object idValue = doc.getFieldValue(fieldHelper.idField);
// if (idValue != null) {
// deleteDocument(searchClient, idValue.toString());
// }
// }
// return null;
// }
// return docList.get(0);
return null;
}
public List<Map<String, Object>> getSolrDocumentListByPrefixId(final SearchClient searchClient, final String id, final String[] fields) {
// TODO move searchClient
// final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
// final SolrQuery solrQuery = new SolrQuery();
// final StringBuilder queryBuf = new StringBuilder(500);
// queryBuf.append("{!prefix f=").append(fieldHelper.idField).append("}");
// queryBuf.append(id);
// solrQuery.setQuery(queryBuf.toString());
// if (fields != null) {
// solrQuery.setFields(fields);
// }
// final QueryResponse response = searchClient.query(solrQuery);
// final List<Map<String, Object>> docList = response.getResults();
// if (docList.isEmpty()) {
// return null;
// }
// if (logger.isDebugEnabled()) {
// logger.debug("Found solr documents: " + docList);
// }
// return docList;
return null;
}
public void deleteChildSolrDocument(final SearchClient searchClient, final String id) {
// TODO move searchClient
// final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
// final String query = "{!raw f=" + fieldHelper.parentIdField + " v=\"" + id + "\"}";
// for (final UpdateResponse response : searchClient.deleteByQuery(query)) {
// if (response.getStatus() != 0) {
// logger.warn("Failed to delete: " + response);
// }
// }
}
public List<Map<String, Object>> getChildSolrDocumentList(final SearchClient searchClient, final String id, final String[] fields) {
return getChildSolrDocumentList(searchClient, id, fields, defaultRowSize);
}
protected List<Map<String, Object>> getChildSolrDocumentList(final SearchClient searchClient, final String id, final String[] fields,
final int row) {
final SolrQuery sq = new SolrQuery();
if (fq != null) {
sq.setFilterQueries(fq);
}
sq.setQuery(q);
if (fields != null) {
sq.setFields(fields);
}
sq.setRows(row);
final SolrDocumentList docList = solrGroup.query(sq).getResults();
if (docList.getNumFound() <= row) {
return docList;
}
return getSolrDocumentList(solrGroup, fq, q, fields, (int) docList.getNumFound());
}
public void deleteDocument(final SolrGroup solrGroup, final String id) {
final long start = System.currentTimeMillis();
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final String query = "{!raw f=" + fieldHelper.idField + "}" + id;
for (int i = 0; i < maxRetryCount; i++) {
boolean done = true;
try {
for (final UpdateResponse response : solrGroup.deleteByQuery(query)) {
if (response.getStatus() != 0) {
logger.warn("Failed to delete: " + response);
done = false;
}
}
} catch (final Exception e) {
logger.warn("Could not delete a document from Solr." + " It might be busy. " + "Retrying.. id:" + id + ", cause: "
+ e.getMessage());
done = false;
}
if (done) {
if (logger.isDebugEnabled()) {
logger.info("Deleted 1 doc (Solr: " + (System.currentTimeMillis() - start) + "ms) => id:" + id);
} else {
logger.info("Deleted 1 doc (Solr: " + (System.currentTimeMillis() - start) + "ms)");
}
return;
}
try {
Thread.sleep(requestInterval);
} catch (final InterruptedException e) {}
}
}
public void deleteDocumentsByDocId(final SolrGroup solrGroup, final List<String> docIdList) {
final long start = System.currentTimeMillis();
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final StringBuilder buf = new StringBuilder(500);
for (final String docId : docIdList) {
if (buf.length() != 0) {
buf.append(" OR ");
}
buf.append(fieldHelper.docIdField).append(':').append(docId);
}
final String query = buf.toString();
for (int i = 0; i < maxRetryCount; i++) {
boolean done = true;
try {
for (final UpdateResponse response : solrGroup.deleteByQuery(query)) {
if (response.getStatus() != 0) {
logger.warn("Failed to delete: " + response);
done = false;
}
}
} catch (final Exception e) {
logger.warn("Could not delete a document from Solr." + " It might be busy. " + "Retrying.. id:" + docIdList + ", cause: "
+ e.getMessage());
done = false;
}
if (done) {
if (logger.isDebugEnabled()) {
logger.info("Deleted " + docIdList.size() + " docs (Solr: " + (System.currentTimeMillis() - start) + "ms) => docId:"
+ docIdList);
} else {
logger.info("Deleted " + docIdList.size() + " docs (Solr: " + (System.currentTimeMillis() - start) + "ms)");
}
return;
}
try {
Thread.sleep(requestInterval);
} catch (final InterruptedException e) {}
}
}
public SolrDocument getSolrDocument(final SolrGroup solrGroup, final String id, final String[] fields) {
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final SolrQuery solrQuery = new SolrQuery();
final StringBuilder queryBuf = new StringBuilder(500);
queryBuf.append("{!raw f=").append(fieldHelper.idField).append("}");
queryBuf.append(id);
solrQuery.setQuery(queryBuf.toString());
if (fields != null) {
solrQuery.setFields(fields);
}
final QueryResponse response = solrGroup.query(solrQuery);
final SolrDocumentList docList = response.getResults();
if (docList.isEmpty()) {
return null;
}
if (docList.size() > 1) {
logger.error("Invalid multiple docs for " + id);
for (final SolrDocument doc : docList) {
final Object idValue = doc.getFieldValue(fieldHelper.idField);
if (idValue != null) {
deleteDocument(solrGroup, idValue.toString());
}
}
return null;
}
return docList.get(0);
}
public SolrDocumentList getSolrDocumentListByPrefixId(final SolrGroup solrGroup, final String id, final String[] fields) {
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final SolrQuery solrQuery = new SolrQuery();
final StringBuilder queryBuf = new StringBuilder(500);
queryBuf.append("{!prefix f=").append(fieldHelper.idField).append("}");
queryBuf.append(id);
solrQuery.setQuery(queryBuf.toString());
if (fields != null) {
solrQuery.setFields(fields);
}
final QueryResponse response = solrGroup.query(solrQuery);
final SolrDocumentList docList = response.getResults();
if (docList.isEmpty()) {
return null;
}
if (logger.isDebugEnabled()) {
logger.debug("Found solr documents: " + docList);
}
return docList;
}
public void deleteChildSolrDocument(final SolrGroup solrGroup, final String id) {
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final String query = "{!raw f=" + fieldHelper.parentIdField + " v=\"" + id + "\"}";
for (final UpdateResponse response : solrGroup.deleteByQuery(query)) {
if (response.getStatus() != 0) {
logger.warn("Failed to delete: " + response);
}
}
}
public SolrDocumentList getChildSolrDocumentList(final SolrGroup solrGroup, final String id, final String[] fields) {
return getChildSolrDocumentList(solrGroup, id, fields, defaultRowSize);
}
protected SolrDocumentList getChildSolrDocumentList(final SolrGroup solrGroup, final String id, final String[] fields, final int row) {
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final SolrQuery solrQuery = new SolrQuery();
solrQuery.setQuery("{!raw f=" + fieldHelper.parentIdField + " v=\"" + id + "\"}");
if (fields != null) {
solrQuery.setFields(fields);
}
solrQuery.setRows(row);
final SolrDocumentList docList = solrGroup.query(solrQuery).getResults();
if (docList.getNumFound() <= row) {
return docList;
}
return getChildSolrDocumentList(solrGroup, id, fields, (int) docList.getNumFound());
// TODO move searchClient
// final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
// final SolrQuery solrQuery = new SolrQuery();
// solrQuery.setQuery("{!raw f=" + fieldHelper.parentIdField + " v=\"" + id + "\"}");
// if (fields != null) {
// solrQuery.setFields(fields);
// }
// solrQuery.setRows(row);
// final List<Map<String,Object>> docList = searchClient.query(solrQuery).getResults();
// if (docList.getNumFound() <= row) {
// return docList;
// }
// return getChildSolrDocumentList(searchClient, id, fields, (int) docList.getNumFound());
return null;
}
}

8
src/main/java/org/codelibs/fess/helper/KeyMatchHelper.java
View file

@ -23,9 +23,9 @@ import java.util.List;
import java.util.Map;
import org.codelibs.core.util.StringUtil;
import org.codelibs.fess.client.SearchClient;
import org.codelibs.fess.db.exentity.KeyMatch;
import org.codelibs.fess.service.KeyMatchService;
import org.codelibs.fess.service.SearchService;
import org.codelibs.fess.util.ComponentUtil;
import org.seasar.framework.container.SingletonS2Container;
import org.seasar.framework.container.annotation.tiger.InitMethod;
@ -81,11 +81,11 @@ public class KeyMatchHelper {
}
protected List<Map<String, Object>> getDocumentList(final KeyMatch keyMatch) {
final SearchService searchService = ComponentUtil.getSearchService();
final SearchClient searchClient = ComponentUtil.getElasticsearchClient();
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final List<Map<String, Object>> documentList =
searchService.getDocumentList(keyMatch.getQuery(), 0, keyMatch.getMaxSize(), null, null, null,
new String[] { fieldHelper.docIdField }, null, false);
searchClient.getDocumentList(keyMatch.getQuery(), 0, keyMatch.getMaxSize(), null, null,
new String[] { fieldHelper.docIdField }, false);
return documentList;
}

43
src/main/java/org/codelibs/fess/helper/QueryHelper.java
View file

@ -51,7 +51,6 @@ import org.codelibs.fess.Constants;
import org.codelibs.fess.InvalidQueryException;
import org.codelibs.fess.entity.FacetInfo;
import org.codelibs.fess.entity.GeoInfo;
import org.codelibs.fess.entity.MoreLikeThisInfo;
import org.codelibs.fess.entity.SearchQuery;
import org.codelibs.fess.entity.SearchQuery.SortField;
import org.codelibs.fess.util.QueryUtil;
@ -125,20 +124,16 @@ public class QueryHelper implements Serializable {
protected int highlightSnippetSize = 5;
protected String shards;
protected boolean useBigram = true;
protected String additionalQuery;
protected int maxFilterQueriesForRole = Integer.MAX_VALUE;
protected int timeAllowed = -1;
protected long timeAllowed = -1;
protected Map<String, String[]> requestParameterMap = new HashMap<String, String[]>();
protected String additionalGeoQuery;
protected Map<String, String> fieldLanguageMap = new HashMap<String, String>();
protected int maxSearchResultOffset = 100000;
@ -153,8 +148,6 @@ public class QueryHelper implements Serializable {
protected FacetInfo defaultFacetInfo;
protected MoreLikeThisInfo defaultMoreLikeThisInfo;
protected GeoInfo defaultGeoInfo;
protected String defaultQueryLanguage;
@ -1219,20 +1212,6 @@ public class QueryHelper implements Serializable {
this.highlightSnippetSize = highlightSnippetSize;
}
/**
* @return the shards
*/
public String getShards() {
return shards;
}
/**
* @param shards the shards to set
*/
public void setShards(final String shards) {
this.shards = shards;
}
/**
* @return the useBigram
*/
@ -1272,14 +1251,14 @@ public class QueryHelper implements Serializable {
/**
* @return the timeAllowed
*/
public int getTimeAllowed() {
public long getTimeAllowed() {
return timeAllowed;
}
/**
* @param timeAllowed the timeAllowed to set
*/
public void setTimeAllowed(final int timeAllowed) {
public void setTimeAllowed(final long timeAllowed) {
this.timeAllowed = timeAllowed;
}
@ -1297,14 +1276,6 @@ public class QueryHelper implements Serializable {
return requestParameterMap.entrySet();
}
public String getAdditionalGeoQuery() {
return additionalGeoQuery;
}
public void setAdditionalGeoQuery(final String additionalGeoQuery) {
this.additionalGeoQuery = additionalGeoQuery;
}
public void addFieldLanguage(final String lang, final String fieldLang) {
fieldLanguageMap.put(lang, fieldLang);
}
@ -1404,14 +1375,6 @@ public class QueryHelper implements Serializable {
this.defaultFacetInfo = defaultFacetInfo;
}
public MoreLikeThisInfo getDefaultMoreLikeThisInfo() {
return defaultMoreLikeThisInfo;
}
public void setDefaultMoreLikeThisInfo(final MoreLikeThisInfo defaultMoreLikeThisInfo) {
this.defaultMoreLikeThisInfo = defaultMoreLikeThisInfo;
}
public GeoInfo getDefaultGeoInfo() {
return defaultGeoInfo;
}

5
src/main/java/org/codelibs/fess/helper/SuggestHelper.java
View file

@ -35,17 +35,12 @@ import org.codelibs.fess.db.exbhv.SuggestBadWordBhv;
import org.codelibs.fess.db.exbhv.SuggestElevateWordBhv;
import org.codelibs.fess.db.exentity.SuggestBadWord;
import org.codelibs.fess.db.exentity.SuggestElevateWord;
import org.codelibs.fess.entity.PingResponse;
import org.codelibs.fess.service.SearchService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class SuggestHelper {
private static final Logger logger = LoggerFactory.getLogger(SuggestHelper.class);
@Resource
protected SearchService searchService;
@Resource
protected SuggestService suggestService;

11
src/main/java/org/codelibs/fess/helper/SystemHelper.java
View file

@ -40,9 +40,6 @@ import org.codelibs.fess.Constants;
import org.codelibs.fess.db.exentity.RoleType;
import org.codelibs.fess.service.RoleTypeService;
import org.codelibs.robot.util.CharUtil;
import org.codelibs.solr.lib.SolrGroup;
import org.codelibs.solr.lib.policy.QueryType;
import org.codelibs.solr.lib.policy.StatusPolicy;
import org.seasar.framework.container.SingletonS2Container;
import org.seasar.framework.container.annotation.tiger.InitMethod;
import org.seasar.struts.util.MessageResourcesUtil;
@ -326,14 +323,6 @@ public class SystemHelper implements Serializable {
this.maxTextLength = maxTextLength;
}
public void updateStatus(final SolrGroup solrGroup, final QueryType queryType) {
final StatusPolicy statusPolicy = solrGroup.getStatusPolicy();
for (final String serverName : solrGroup.getServerNames()) {
statusPolicy.activate(queryType, serverName);
}
}
public boolean isForceStop() {
return forceStop.get();
}

10
src/main/java/org/codelibs/fess/helper/ViewHelper.java
View file

@ -103,8 +103,6 @@ public class ViewHelper implements Serializable {
private final Map<String, String> initFacetParamMap = new HashMap<String, String>();
private final Map<String, String> initMltParamMap = new HashMap<String, String>();
private final Map<String, String> initGeoParamMap = new HashMap<String, String>();
private final List<FacetQueryView> facetQueryViewList = new ArrayList<FacetQueryView>();
@ -475,14 +473,6 @@ public class ViewHelper implements Serializable {
return initFacetParamMap;
}
public void addInitMltParam(final String key, final String value) {
initMltParamMap.put(value, key);
}
public Map<String, String> getInitMltParamMap() {
return initMltParamMap;
}
public void addInitGeoParam(final String key, final String value) {
initGeoParamMap.put(value, key);
}

23
src/main/java/org/codelibs/fess/helper/WebFsIndexHelper.java
View file

@ -42,7 +42,6 @@ import org.codelibs.robot.S2RobotContext;
import org.codelibs.robot.service.DataService;
import org.codelibs.robot.service.UrlFilterService;
import org.codelibs.robot.service.UrlQueueService;
import org.codelibs.solr.lib.SolrGroup;
import org.seasar.framework.container.SingletonS2Container;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -84,7 +83,9 @@ public class WebFsIndexHelper implements Serializable {
private final List<S2Robot> s2RobotList = Collections.synchronizedList(new ArrayList<S2Robot>());
public void crawl(final String sessionId, final SolrGroup solrGroup) {
// needed?
@Deprecated
public void crawl(final String sessionId) {
final List<WebCrawlingConfig> webConfigList = webCrawlingConfigService.getAllWebCrawlingConfigList();
final List<FileCrawlingConfig> fileConfigList = fileCrawlingConfigService.getAllFileCrawlingConfigList();
@ -96,10 +97,10 @@ public class WebFsIndexHelper implements Serializable {
return;
}
crawl(sessionId, solrGroup, webConfigList, fileConfigList);
doCrawl(sessionId, webConfigList, fileConfigList);
}
public void crawl(final String sessionId, final List<Long> webConfigIdList, final List<Long> fileConfigIdList, final SolrGroup solrGroup) {
public void crawl(final String sessionId, final List<Long> webConfigIdList, final List<Long> fileConfigIdList) {
final boolean runAll = webConfigIdList == null && fileConfigIdList == null;
final List<WebCrawlingConfig> webConfigList;
if (runAll || webConfigIdList != null) {
@ -122,10 +123,10 @@ public class WebFsIndexHelper implements Serializable {
return;
}
crawl(sessionId, solrGroup, webConfigList, fileConfigList);
doCrawl(sessionId, webConfigList, fileConfigList);
}
protected void crawl(final String sessionId, final SolrGroup solrGroup, final List<WebCrawlingConfig> webConfigList,
protected void doCrawl(final String sessionId, final List<WebCrawlingConfig> webConfigList,
final List<FileCrawlingConfig> fileConfigList) {
int multiprocessCrawlingCount = 5;
String value = crawlerProperties.getProperty(Constants.CRAWLING_THREAD_COUNT_PROPERTY, "5");
@ -135,14 +136,6 @@ public class WebFsIndexHelper implements Serializable {
// NOP
}
long commitPerCount = Constants.DEFAULT_COMMIT_PER_COUNT;
value = crawlerProperties.getProperty(Constants.COMMIT_PER_COUNT_PROPERTY, Long.toString(Constants.DEFAULT_COMMIT_PER_COUNT));
try {
commitPerCount = Long.parseLong(value);
} catch (final NumberFormatException e) {
// NOP
}
final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
final long startTime = System.currentTimeMillis();
@ -401,9 +394,7 @@ public class WebFsIndexHelper implements Serializable {
indexUpdater.setName("IndexUpdater");
indexUpdater.setPriority(indexUpdaterPriority);
indexUpdater.setSessionIdList(sessionIdList);
indexUpdater.setSolrGroup(solrGroup);
indexUpdater.setDaemon(true);
indexUpdater.setCommitPerCount(commitPerCount);
indexUpdater.setS2RobotList(s2RobotList);
for (final BoostDocumentRule rule : boostDocumentRuleService.getAvailableBoostDocumentRuleList()) {
indexUpdater.addBoostDocumentRule(new org.codelibs.fess.solr.BoostDocumentRule(rule));

1
src/main/java/org/codelibs/fess/job/CrawlJob.java
View file

@ -118,7 +118,6 @@ public class CrawlJob {
public String execute(final JobExecutor jobExecutor) {
jobExecutor(jobExecutor);
operation(Constants.COMMIT);
return execute();
}

29
src/main/java/org/codelibs/fess/robot/FessS2RobotThread.java
View file

@ -29,10 +29,9 @@ import jcifs.smb.ACE;
import jcifs.smb.SID;
import org.apache.commons.io.IOUtils;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.codelibs.core.util.DynamicProperties;
import org.codelibs.fess.Constants;
import org.codelibs.fess.client.SearchClient;
import org.codelibs.fess.db.exentity.CrawlingConfig;
import org.codelibs.fess.helper.CrawlingConfigHelper;
import org.codelibs.fess.helper.CrawlingSessionHelper;
@ -49,9 +48,6 @@ import org.codelibs.robot.entity.RequestData;
import org.codelibs.robot.entity.ResponseData;
import org.codelibs.robot.entity.UrlQueue;
import org.codelibs.robot.log.LogType;
import org.codelibs.solr.lib.SolrGroup;
import org.codelibs.solr.lib.SolrGroupManager;
import org.codelibs.solr.lib.policy.QueryType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -71,11 +67,9 @@ public class FessS2RobotThread extends S2RobotThread {
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final SambaHelper sambaHelper = ComponentUtil.getSambaHelper();
final IndexingHelper indexingHelper = ComponentUtil.getIndexingHelper();
final SolrGroupManager solrGroupManager = ComponentUtil.getSolrGroupManager();
final SearchClient searchClient = ComponentUtil.getElasticsearchClient();
final boolean useAclAsRole = crawlerProperties.getProperty(Constants.USE_ACL_AS_ROLE, Constants.FALSE).equals(Constants.TRUE);
final SolrGroup solrGroup = solrGroupManager.getSolrGroup(QueryType.ADD);
final String url = urlQueue.getUrl();
ResponseData responseData = null;
try {
@ -107,20 +101,20 @@ public class FessS2RobotThread extends S2RobotThread {
dataMap.put(fieldHelper.roleField, roleTypeList);
final String id = crawlingSessionHelper.generateId(dataMap);
final SolrDocument solrDocument =
indexingHelper.getSolrDocument(solrGroup, id, new String[] { fieldHelper.idField, fieldHelper.lastModifiedField,
final Map<String, Object> solrDocument =
indexingHelper.getSolrDocument(searchClient, id, new String[] { fieldHelper.idField, fieldHelper.lastModifiedField,
fieldHelper.anchorField, fieldHelper.segmentField, fieldHelper.expiresField, fieldHelper.clickCountField,
fieldHelper.favoriteCountField });
if (solrDocument == null) {
storeChildUrlsToQueue(urlQueue, getChildUrlSet(solrGroup, id));
storeChildUrlsToQueue(urlQueue, getChildUrlSet(searchClient, id));
return true;
}
final Date expires = (Date) solrDocument.get(fieldHelper.expiresField);
if (expires != null && expires.getTime() < System.currentTimeMillis()) {
final Object idValue = solrDocument.getFieldValue(fieldHelper.idField);
final Object idValue = solrDocument.get(fieldHelper.idField);
if (idValue != null) {
indexingHelper.deleteDocument(solrGroup, idValue.toString());
indexingHelper.deleteDocument(searchClient, idValue.toString());
}
return true;
}
@ -159,7 +153,7 @@ public class FessS2RobotThread extends S2RobotThread {
final int httpStatusCode = responseData.getHttpStatusCode();
if (httpStatusCode == 404) {
storeChildUrlsToQueue(urlQueue, getAnchorSet(solrDocument.get(fieldHelper.anchorField)));
indexingHelper.deleteDocument(solrGroup, id);
indexingHelper.deleteDocument(searchClient, id);
return false;
} else if (responseData.getLastModified() == null) {
return true;
@ -218,10 +212,11 @@ public class FessS2RobotThread extends S2RobotThread {
return childUrlSet;
}
protected Set<RequestData> getChildUrlSet(final SolrGroup solrGroup, final String id) {
protected Set<RequestData> getChildUrlSet(final SearchClient searchClient, final String id) {
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final IndexingHelper indexingHelper = ComponentUtil.getIndexingHelper();
final SolrDocumentList docList = indexingHelper.getChildSolrDocumentList(solrGroup, id, new String[] { fieldHelper.urlField });
final List<Map<String, Object>> docList =
indexingHelper.getChildSolrDocumentList(searchClient, id, new String[] { fieldHelper.urlField });
if (docList.isEmpty()) {
return null;
}
@ -229,7 +224,7 @@ public class FessS2RobotThread extends S2RobotThread {
logger.debug("Found solr documents: " + docList);
}
final Set<RequestData> urlSet = new HashSet<>(docList.size());
for (final SolrDocument doc : docList) {
for (final Map<String, Object> doc : docList) {
final Object obj = doc.get(fieldHelper.urlField);
if (obj != null) {
urlSet.add(RequestDataBuilder.newRequestData().get().url(obj.toString()).build());

268
src/main/java/org/codelibs/fess/service/SearchService.java
View file

@ -1,268 +0,0 @@
/*
* Copyright 2009-2015 the CodeLibs Project and the Others.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
* either express or implied. See the License for the specific language
* governing permissions and limitations under the License.
*/
package org.codelibs.fess.service;
import java.io.Serializable;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import javax.annotation.Resource;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.request.FieldAnalysisRequest;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.util.NamedList;
import org.codelibs.core.util.StringUtil;
import org.codelibs.fess.Constants;
import org.codelibs.fess.ResultOffsetExceededException;
import org.codelibs.fess.entity.FacetInfo;
import org.codelibs.fess.entity.FieldAnalysisResponse;
import org.codelibs.fess.entity.GeoInfo;
import org.codelibs.fess.entity.MoreLikeThisInfo;
import org.codelibs.fess.entity.PingResponse;
import org.codelibs.fess.entity.SearchQuery;
import org.codelibs.fess.entity.SearchQuery.SortField;
import org.codelibs.fess.helper.FieldHelper;
import org.codelibs.fess.helper.QueryHelper;
import org.codelibs.fess.helper.RoleQueryHelper;
import org.codelibs.fess.solr.FessSolrQueryException;
import org.codelibs.fess.util.ComponentUtil;
import org.codelibs.fess.util.QueryResponseList;
import org.codelibs.solr.lib.SolrGroup;
import org.codelibs.solr.lib.SolrGroupManager;
import org.codelibs.solr.lib.policy.QueryType;
public class SearchService implements Serializable {
private static final long serialVersionUID = 1L;
@Resource
protected SolrGroupManager solrGroupManager;
@Resource
protected QueryHelper queryHelper;
@Resource
protected RoleQueryHelper roleQueryHelper;
public PingResponse ping() {
final SolrGroup solrGroup = solrGroupManager.getSolrGroup(QueryType.QUERY);
return new PingResponse(solrGroup.ping());
}
public Map<String, Object> getDocument(final String query) {
return getDocument(query, queryHelper.getResponseFields(), null);
}
public Map<String, Object> getDocument(final String query, final String[] responseFields, final String[] docValuesFields) {
final List<Map<String, Object>> docList = getDocumentList(query, 0, 1, null, null, null, responseFields, docValuesFields);
if (!docList.isEmpty()) {
return docList.get(0);
}
return null;
}
public List<Map<String, Object>> getDocumentListByDocIds(final String[] docIds, final String[] responseFields,
final String[] docValuesFields, final int pageSize) {
if (docIds == null || docIds.length == 0) {
return Collections.emptyList();
}
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final StringBuilder buf = new StringBuilder(1000);
for (int i = 0; i < docIds.length; i++) {
if (i != 0) {
buf.append(" OR ");
}
buf.append(fieldHelper.docIdField + ":").append(docIds[i]);
}
return getDocumentList(buf.toString(), 0, pageSize, null, null, null, responseFields, docValuesFields);
}
public List<Map<String, Object>> getDocumentList(final String query, final int start, final int rows, final FacetInfo facetInfo,
final GeoInfo geoInfo, final MoreLikeThisInfo mltInfo, final String[] responseFields, final String[] docValuesFields) {
return getDocumentList(query, start, rows, facetInfo, geoInfo, mltInfo, responseFields, docValuesFields, true);
}
public List<Map<String, Object>> getDocumentList(final String query, final int start, final int rows, final FacetInfo facetInfo,
final GeoInfo geoInfo, final MoreLikeThisInfo mltInfo, final String[] responseFields, final String[] docValuesFields,
final boolean forUser) {
if (start > queryHelper.getMaxSearchResultOffset()) {
throw new ResultOffsetExceededException("The number of result size is exceeded.");
}
final long startTime = System.currentTimeMillis();
final SolrGroup solrGroup = solrGroupManager.getSolrGroup(QueryType.QUERY);
QueryResponse queryResponse = null;
final SolrQuery solrQuery = new SolrQuery();
final SearchQuery searchQuery = queryHelper.build(query, forUser);
final String q = searchQuery.getQuery();
if (StringUtil.isNotBlank(q)) {
// fields
solrQuery.setFields(responseFields);
// query
solrQuery.setQuery(q);
solrQuery.setStart(start);
solrQuery.setRows(rows);
solrQuery.set("mm", searchQuery.getMinimumShouldMatch());
solrQuery.set("defType", searchQuery.getDefType());
for (final Map.Entry<String, String[]> entry : queryHelper.getQueryParamMap().entrySet()) {
solrQuery.set(entry.getKey(), entry.getValue());
}
// filter query
if (searchQuery.hasFilterQueries()) {
solrQuery.addFilterQuery(searchQuery.getFilterQueries());
}
// sort
final SortField[] sortFields = searchQuery.getSortFields();
if (sortFields.length != 0) {
for (final SortField sortField : sortFields) {
solrQuery.addSort(sortField.getField(), Constants.DESC.equals(sortField.getOrder()) ? SolrQuery.ORDER.desc
: SolrQuery.ORDER.asc);
}
} else if (queryHelper.hasDefaultSortFields()) {
for (final SortField sortField : queryHelper.getDefaultSortFields()) {
solrQuery.addSort(sortField.getField(), Constants.DESC.equals(sortField.getOrder()) ? SolrQuery.ORDER.desc
: SolrQuery.ORDER.asc);
}
}
// highlighting
if (queryHelper.getHighlightingFields() != null && queryHelper.getHighlightingFields().length != 0) {
for (final String hf : queryHelper.getHighlightingFields()) {
solrQuery.addHighlightField(hf);
}
solrQuery.setHighlightSnippets(queryHelper.getHighlightSnippetSize());
}
// shards
if (queryHelper.getShards() != null) {
solrQuery.setParam("shards", queryHelper.getShards());
}
// geo
if (geoInfo != null && geoInfo.isAvailable()) {
solrQuery.addFilterQuery(geoInfo.toGeoQueryString());
final String additionalGeoQuery = queryHelper.getAdditionalGeoQuery();
if (StringUtil.isNotBlank(additionalGeoQuery)) {
solrQuery.addFilterQuery(additionalGeoQuery);
}
}
// facets
if (facetInfo != null) {
solrQuery.setFacet(true);
if (facetInfo.field != null) {
for (final String f : facetInfo.field) {
if (queryHelper.isFacetField(f)) {
solrQuery.addFacetField(f);
} else {
throw new FessSolrQueryException("EFESS0002", new Object[] { f });
}
}
}
if (facetInfo.query != null) {
for (final String fq : facetInfo.query) {
final String facetQuery = queryHelper.buildFacetQuery(fq);
if (StringUtil.isNotBlank(facetQuery)) {
solrQuery.addFacetQuery(facetQuery);
} else {
throw new FessSolrQueryException("EFESS0003", new Object[] { fq, facetQuery });
}
}
}
if (facetInfo.limit != null) {
solrQuery.setFacetLimit(Integer.parseInt(facetInfo.limit));
}
if (facetInfo.minCount != null) {
solrQuery.setFacetMinCount(Integer.parseInt(facetInfo.minCount));
}
if (facetInfo.missing != null) {
solrQuery.setFacetMissing(Boolean.parseBoolean(facetInfo.missing));
}
if (facetInfo.prefix != null) {
solrQuery.setFacetPrefix(facetInfo.prefix);
}
if (facetInfo.sort != null && queryHelper.isFacetSortValue(facetInfo.sort)) {
solrQuery.setFacetSort(facetInfo.sort);
}
}
// mlt
if (mltInfo != null) {
final String mltField = queryHelper.getMoreLikeThisField(mltInfo.field);
if (mltField != null) {
solrQuery.set("mlt", true);
if (mltInfo.count != null) {
solrQuery.set("mlt.count", Integer.parseInt(mltInfo.count));
}
solrQuery.set("mlt.fl", mltField);
}
}
if (queryHelper.getTimeAllowed() >= 0) {
solrQuery.setTimeAllowed(queryHelper.getTimeAllowed());
}
final Set<Entry<String, String[]>> paramSet = queryHelper.getRequestParameterSet();
if (!paramSet.isEmpty()) {
for (final Map.Entry<String, String[]> entry : paramSet) {
solrQuery.set(entry.getKey(), entry.getValue());
}
}
if (docValuesFields != null) {
for (final String docValuesField : docValuesFields) {
solrQuery.add(Constants.DCF, docValuesField);
}
}
queryResponse = solrGroup.query(solrQuery, SolrRequest.METHOD.POST);
}
final long execTime = System.currentTimeMillis() - startTime;
final QueryResponseList queryResponseList = ComponentUtil.getQueryResponseList();
queryResponseList.init(queryResponse, rows);
queryResponseList.setSearchQuery(q);
queryResponseList.setSolrQuery(solrQuery.toString());
queryResponseList.setExecTime(execTime);
return queryResponseList;
}
public FieldAnalysisResponse getFieldAnalysisResponse(final String[] fieldNames, final String fieldValue) {
final FieldAnalysisRequest request = new FieldAnalysisRequest();
for (final String fieldName : fieldNames) {
if (!queryHelper.isAnalysisFieldName(fieldName)) {
throw new FessSolrQueryException("EFESS0001", new Object[] { fieldName });
}
request.addFieldName(fieldName);
}
request.setFieldValue(fieldValue);
final long startTime = System.currentTimeMillis();
final SolrGroup solrGroup = solrGroupManager.getSolrGroup(QueryType.REQUEST);
final NamedList<Object> response = solrGroup.request(request);
final long execTime = System.currentTimeMillis() - startTime;
final FieldAnalysisResponse fieldAnalysisResponse = new FieldAnalysisResponse(response);
fieldAnalysisResponse.setExecTime(execTime);
return fieldAnalysisResponse;
}
}

21
src/main/java/org/codelibs/fess/solr/FessSolrQueryException.java
View file

@ -16,15 +16,22 @@
package org.codelibs.fess.solr;
import org.codelibs.solr.lib.exception.SolrLibQueryException;
import org.codelibs.fess.FessSystemException;
public class FessSolrQueryException extends FessSystemException {
public class FessSolrQueryException extends SolrLibQueryException {
/**
*
*/
private static final long serialVersionUID = 1L;
public FessSolrQueryException(final String messageCode, final Object[] args) {
super(messageCode, args);
public FessSolrQueryException(String message, Throwable cause) {
super(message, cause);
}
public FessSolrQueryException(String message) {
super(message);
}
public FessSolrQueryException(Throwable cause) {
super(cause);
}
}

102
src/main/java/org/codelibs/fess/solr/IndexUpdater.java
View file

@ -23,10 +23,10 @@ import java.util.Map;
import javax.annotation.Resource;
import org.apache.solr.common.SolrInputDocument;
import org.codelibs.core.util.StringUtil;
import org.codelibs.fess.Constants;
import org.codelibs.fess.FessSystemException;
import org.codelibs.fess.client.SearchClient;
import org.codelibs.fess.db.exbhv.ClickLogBhv;
import org.codelibs.fess.db.exbhv.FavoriteLogBhv;
import org.codelibs.fess.db.exbhv.pmbean.FavoriteUrlCountPmb;
@ -47,8 +47,6 @@ import org.codelibs.robot.service.DataService;
import org.codelibs.robot.service.UrlFilterService;
import org.codelibs.robot.service.UrlQueueService;
import org.codelibs.robot.transformer.Transformer;
import org.codelibs.solr.lib.SolrGroup;
import org.codelibs.solr.lib.exception.SolrLibException;
import org.seasar.framework.container.SingletonS2Container;
import org.seasar.framework.container.annotation.tiger.Binding;
import org.seasar.framework.container.annotation.tiger.BindingType;
@ -60,7 +58,7 @@ public class IndexUpdater extends Thread {
protected List<String> sessionIdList;
protected SolrGroup solrGroup;
protected SearchClient searchClient;
@Resource
protected DataService dataService;
@ -104,8 +102,6 @@ public class IndexUpdater extends Thread {
protected long documentSize;
protected long commitPerCount = 0;
protected int maxSolrErrorCount = 0;
protected int maxErrorCount = 2;
@ -184,13 +180,12 @@ public class IndexUpdater extends Thread {
cb.fetchFirst(maxDocumentCacheSize);
cb.fetchPage(1);
final List<SolrInputDocument> docList = new ArrayList<SolrInputDocument>();
final List<Map<String, Object>> docList = new ArrayList<>();
final List<org.codelibs.robot.entity.AccessResult> accessResultList = new ArrayList<org.codelibs.robot.entity.AccessResult>();
final List<org.codelibs.robot.db.exentity.AccessResultData> accessResultDataList =
new ArrayList<org.codelibs.robot.db.exentity.AccessResultData>();
long updateTime = System.currentTimeMillis();
int solrErrorCount = 0;
int errorCount = 0;
int emptyListCount = 0;
while (!finishCrawling || !accessResultList.isEmpty()) {
@ -240,7 +235,7 @@ public class IndexUpdater extends Thread {
}
if (!docList.isEmpty()) {
indexingHelper.sendDocuments(solrGroup, docList);
indexingHelper.sendDocuments(searchClient, docList);
}
synchronized (finishedSessionIdList) {
@ -255,14 +250,7 @@ public class IndexUpdater extends Thread {
}
// reset count
solrErrorCount = 0;
errorCount = 0;
} catch (final SolrLibException e) {
if (solrErrorCount > maxSolrErrorCount) {
throw e;
}
solrErrorCount++;
logger.warn("Failed to access a solr group. Retry to access.. " + solrErrorCount, e);
} catch (final Exception e) {
if (errorCount > maxErrorCount) {
throw e;
@ -318,7 +306,7 @@ public class IndexUpdater extends Thread {
}
}
private void processAccessResults(final List<SolrInputDocument> docList,
private void processAccessResults(final List<Map<String, Object>> docList,
final List<org.codelibs.robot.entity.AccessResult> accessResultList,
final List<org.codelibs.robot.db.exentity.AccessResultData> accessResultDataList, final PagingResultBean<AccessResult> arList) {
for (final AccessResult accessResult : arList) {
@ -364,29 +352,20 @@ public class IndexUpdater extends Thread {
map.remove(Constants.INDEXING_TARGET);
}
final SolrInputDocument doc = createSolrDocument(map);
updateDocument(map);
docList.add(doc);
docList.add(map);
if (logger.isDebugEnabled()) {
logger.debug("Added the document. " + "The number of a document cache is " + docList.size() + ".");
}
if (docList.size() >= maxDocumentCacheSize) {
indexingHelper.sendDocuments(solrGroup, docList);
indexingHelper.sendDocuments(searchClient, docList);
}
documentSize++;
// commit
if (commitPerCount > 0 && documentSize % commitPerCount == 0) {
if (!docList.isEmpty()) {
indexingHelper.sendDocuments(solrGroup, docList);
}
commitDocuments();
}
if (logger.isDebugEnabled()) {
logger.debug("The number of an added document is " + documentSize + ".");
}
} catch (final SolrLibException e) {
throw e;
} catch (final Exception e) {
logger.warn("Could not add a doc: " + accessResult.getUrl(), e);
}
@ -399,25 +378,13 @@ public class IndexUpdater extends Thread {
}
}
protected SolrInputDocument createSolrDocument(final Map<String, Object> map) {
final SolrInputDocument doc = new SolrInputDocument();
float documentBoost = 0.0f;
// add data
for (final Map.Entry<String, Object> entry : map.entrySet()) {
if (fieldHelper.boostField.equals(entry.getKey())) {
// boost
documentBoost = Float.valueOf(entry.getValue().toString());
} else {
doc.addField(entry.getKey(), entry.getValue());
}
}
protected void updateDocument(final Map<String, Object> map) {
if (clickCountEnabled) {
addClickCountField(map, doc);
addClickCountField(map);
}
if (favoriteCountEnabled) {
addFavoriteCountField(map, doc);
addFavoriteCountField(map);
}
// default values
@ -429,6 +396,7 @@ public class IndexUpdater extends Thread {
}
}
float documentBoost = 0.0f;
for (final BoostDocumentRule rule : boostRuleList) {
if (rule.match(map)) {
documentBoost = rule.getValue(map);
@ -437,39 +405,35 @@ public class IndexUpdater extends Thread {
}
if (documentBoost > 0) {
addBoostValue(map, documentBoost, doc);
addBoostValue(map, documentBoost);
}
if (!map.containsKey(fieldHelper.docIdField)) {
doc.addField(fieldHelper.docIdField, systemHelper.generateDocId(map));
map.put(fieldHelper.docIdField, systemHelper.generateDocId(map));
}
return doc;
}
protected void addBoostValue(final Map<String, Object> map, final float documentBoost, final SolrInputDocument doc) {
doc.addField(fieldHelper.boostField, documentBoost);
doc.setDocumentBoost(documentBoost);
protected void addBoostValue(final Map<String, Object> map, final float documentBoost) {
map.put(fieldHelper.boostField, documentBoost);
if (logger.isDebugEnabled()) {
logger.debug("Set a document boost (" + documentBoost + ").");
}
}
protected void addClickCountField(final Map<String, Object> map, final SolrInputDocument doc) {
final String url = (String) map.get(fieldHelper.urlField);
protected void addClickCountField(final Map<String, Object> doc) {
final String url = (String) doc.get(fieldHelper.urlField);
if (StringUtil.isNotBlank(url)) {
final int count = clickLogBhv.selectCount(cb -> {
cb.query().setUrl_Equal(url);
});
doc.addField(fieldHelper.clickCountField, count);
map.put(fieldHelper.clickCountField, count);
doc.put(fieldHelper.clickCountField, count);
if (logger.isDebugEnabled()) {
logger.debug("Click Count: " + count + ", url: " + url);
}
}
}
protected void addFavoriteCountField(final Map<String, Object> map, final SolrInputDocument doc) {
protected void addFavoriteCountField(final Map<String, Object> map) {
final String url = (String) map.get(fieldHelper.urlField);
if (StringUtil.isNotBlank(url)) {
final FavoriteUrlCountPmb pmb = new FavoriteUrlCountPmb();
@ -481,7 +445,6 @@ public class IndexUpdater extends Thread {
count = list.get(0).getCnt().longValue();
}
doc.addField(fieldHelper.favoriteCountField, count);
map.put(fieldHelper.favoriteCountField, count);
if (logger.isDebugEnabled()) {
logger.debug("Favorite Count: " + count + ", url: " + url);
@ -561,19 +524,6 @@ public class IndexUpdater extends Thread {
}
}
private void commitDocuments() {
final long execTime = System.currentTimeMillis();
if (logger.isInfoEnabled()) {
logger.info("Committing documents. ");
}
synchronized (solrGroup) {
solrGroup.commit(true, true, false, true);
}
if (logger.isInfoEnabled()) {
logger.info("Committed documents. The execution time is " + (System.currentTimeMillis() - execTime) + "ms.");
}
}
private void forceStop() {
systemHelper.setForceStop(true);
for (final S2Robot s2Robot : s2RobotList) {
@ -593,14 +543,6 @@ public class IndexUpdater extends Thread {
this.sessionIdList = sessionIdList;
}
public SolrGroup getSolrGroup() {
return solrGroup;
}
public void setSolrGroup(final SolrGroup solrGroup) {
this.solrGroup = solrGroup;
}
public void setFinishCrawling(final boolean finishCrawling) {
this.finishCrawling = finishCrawling;
}
@ -609,10 +551,6 @@ public class IndexUpdater extends Thread {
return documentSize;
}
public void setCommitPerCount(final long commitPerCount) {
this.commitPerCount = commitPerCount;
}
@Binding(bindingType = BindingType.MAY)
@Override
public void setUncaughtExceptionHandler(final UncaughtExceptionHandler eh) {

181
src/main/java/org/codelibs/fess/solr/policy/FessStatusPolicy.java
View file

@ -1,181 +0,0 @@
/*
* Copyright 2009-2015 the CodeLibs Project and the Others.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
* either express or implied. See the License for the specific language
* governing permissions and limitations under the License.
*/
package org.codelibs.fess.solr.policy;
import java.io.File;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.HashMap;
import java.util.Map;
import org.codelibs.core.util.DynamicProperties;
import org.codelibs.core.util.StringUtil;
import org.codelibs.fess.Constants;
import org.codelibs.fess.helper.MailHelper;
import org.codelibs.fess.util.ComponentUtil;
import org.codelibs.fess.util.ResourceUtil;
import org.codelibs.solr.lib.policy.QueryType;
import org.codelibs.solr.lib.policy.impl.StatusPolicyImpl;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.github.jknack.handlebars.Context;
import com.github.jknack.handlebars.Handlebars;
import com.github.jknack.handlebars.Template;
import com.github.jknack.handlebars.io.FileTemplateLoader;
public class FessStatusPolicy extends StatusPolicyImpl {
private static final String MAIL_TEMPLATE_NAME = "solr_status";
private static final Logger logger = LoggerFactory.getLogger(FessStatusPolicy.class);
public String activateSubject = "[FESS] Solr status changed";
public String deactivateSubject = "[FESS] Solr status changed";
@Override
public void activate(final QueryType queryType, final String serverName) {
final String statusValue = solrGroupProperties.getProperty(getStatusKey(serverName));
final String indexValue = solrGroupProperties.getProperty(getIndexKey(serverName));
final Map<String, Object> dataMap = new HashMap<String, Object>();
if (StringUtil.isNotBlank(statusValue) && INACTIVE.equals(statusValue)) {
// status: INACTIVE -> ACTIVE
dataMap.put("statusBefore", Constants.INACTIVE);
dataMap.put("statusAfter", Constants.ACTIVE);
}
switch (queryType) {
case COMMIT:
case OPTIMIZE:
case ROLLBACK:
if (StringUtil.isNotBlank(indexValue) && UNFINISHED.equals(indexValue)) {
// index: UNFINISHED -> COMPLETED
dataMap.put("indexBefore", UNFINISHED.toUpperCase());
dataMap.put("indexAfter", COMPLETED.toUpperCase());
}
break;
case ADD:
case DELETE:
if (StringUtil.isNotBlank(indexValue) && UNFINISHED.equals(indexValue)) {
// index: UNFINISHED -> READY
dataMap.put("indexBefore", UNFINISHED.toUpperCase());
dataMap.put("indexAfter", READY.toUpperCase());
}
break;
case PING:
case QUERY:
case REQUEST:
default:
break;
}
if (!dataMap.isEmpty()) {
if (logger.isDebugEnabled()) {
logger.debug("Server: " + serverName + "\n" + dataMap);
}
dataMap.put("server", serverName);
send(activateSubject, dataMap);
}
super.activate(queryType, serverName);
}
@Override
public void deactivate(final QueryType queryType, final String serverName) {
final String statusValue = solrGroupProperties.getProperty(getStatusKey(serverName));
final String indexValue = solrGroupProperties.getProperty(getIndexKey(serverName));
final Map<String, Object> dataMap = new HashMap<String, Object>();
if (StringUtil.isNotBlank(statusValue) && ACTIVE.equals(statusValue)) {
// status: ACTIVE -> INACTIVE
dataMap.put("statusBefore", Constants.ACTIVE);
dataMap.put("statusAfter", Constants.INACTIVE);
}
switch (queryType) {
case COMMIT:
case OPTIMIZE:
case ROLLBACK:
case ADD:
case DELETE:
if (StringUtil.isNotBlank(indexValue) && !UNFINISHED.equals(indexValue)) {
// index: READY/COMPLETED -> UNFINISHED
dataMap.put("indexBefore", indexValue == null ? "UNKNOWN" : indexValue.toUpperCase());
dataMap.put("indexAfter", UNFINISHED.toUpperCase());
}
break;
case PING:
case QUERY:
case REQUEST:
default:
break;
}
if (!dataMap.isEmpty()) {
if (logger.isDebugEnabled()) {
logger.debug("Server: " + serverName + "\n" + dataMap);
}
dataMap.put("server", serverName);
send(deactivateSubject, dataMap);
}
super.deactivate(queryType, serverName);
}
protected void send(final String subject, final Map<String, Object> dataMap) {
final DynamicProperties crawlerProperties = ComponentUtil.getCrawlerProperties();
if (crawlerProperties == null) {
logger.info("crawlerProperties is not found.");
return;
}
final MailHelper mailHelper = ComponentUtil.getMailHelper();
if (mailHelper == null) {
logger.info("mailHelper is not found.");
return;
}
final String toStrs = (String) crawlerProperties.get(Constants.NOTIFICATION_TO_PROPERTY);
if (StringUtil.isNotBlank(toStrs)) {
final String[] toAddresses = toStrs.split(",");
try {
dataMap.put("hostname", InetAddress.getLocalHost().getHostAddress());
} catch (final UnknownHostException e) {
// ignore
}
final FileTemplateLoader loader = new FileTemplateLoader(new File(ResourceUtil.getMailTemplatePath(StringUtil.EMPTY)));
final Handlebars handlebars = new Handlebars(loader);
try {
final Template template = handlebars.compile(MAIL_TEMPLATE_NAME);
final Context hbsContext = Context.newContext(dataMap);
final String body = template.apply(hbsContext);
mailHelper.send(toAddresses, subject, body);
} catch (final Exception e) {
logger.warn("Failed to send the notification.", e);
}
}
}
}

17
src/main/java/org/codelibs/fess/taglib/FessFunctions.java
View file

@ -17,6 +17,7 @@
package org.codelibs.fess.taglib;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Collections;
@ -29,7 +30,6 @@ import java.util.Map;
import javax.servlet.http.HttpServletRequest;
import org.apache.commons.lang.StringEscapeUtils;
import org.apache.solr.common.util.DateUtil;
import org.codelibs.core.util.DynamicProperties;
import org.codelibs.fess.Constants;
import org.codelibs.fess.entity.FacetQueryView;
@ -41,6 +41,7 @@ import org.seasar.struts.util.RequestUtil;
import org.seasar.struts.util.URLEncoderUtil;
public class FessFunctions {
private static final String GEO_PREFIX = "geo.";
private static final String FACET_PREFIX = "facet.";
@ -142,28 +143,24 @@ public class FessFunctions {
return null;
}
try {
return DateUtil.parseDate(value);
return new SimpleDateFormat(Constants.ISO_DATETIME_FORMAT).parse(value);
} catch (final ParseException e) {
return null;
}
}
public static String formatDate(final Date date) {
return DateUtil.getThreadLocalDateFormat().format(date);
return new SimpleDateFormat(Constants.ISO_DATETIME_FORMAT).format(date);
}
public static String formatDate(final LocalDateTime date) {
return date.format(DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.ROOT));
return date.format(DateTimeFormatter.ofPattern(Constants.ISO_DATETIME_FORMAT, Locale.ROOT));
}
public static String facetQuery() {
return createQuery(Constants.FACET_QUERY, FACET_PREFIX);
}
public static String mltQuery() {
return createQuery(Constants.MLT_QUERY, MLT_PREFIX);
}
public static String geoQuery() {
return createQuery(Constants.GEO_QUERY, GEO_PREFIX);
}
@ -172,10 +169,6 @@ public class FessFunctions {
return createForm(Constants.FACET_FORM, FACET_PREFIX);
}
public static String mltForm() {
return createForm(Constants.MLT_FORM, MLT_PREFIX);
}
public static String geoForm() {
return createForm(Constants.GEO_FORM, GEO_PREFIX);
}

18
src/main/java/org/codelibs/fess/util/ComponentUtil.java
View file

@ -19,6 +19,7 @@ package org.codelibs.fess.util;
import org.codelibs.core.crypto.CachedCipher;
import org.codelibs.core.util.DynamicProperties;
import org.codelibs.fess.api.WebApiManagerFactory;
import org.codelibs.fess.client.SearchClient;
import org.codelibs.fess.ds.DataStoreFactory;
import org.codelibs.fess.helper.AdRoleHelper;
import org.codelibs.fess.helper.CrawlingConfigHelper;
@ -43,10 +44,8 @@ import org.codelibs.fess.helper.SystemHelper;
import org.codelibs.fess.helper.UserAgentHelper;
import org.codelibs.fess.helper.ViewHelper;
import org.codelibs.fess.job.JobExecutor;
import org.codelibs.fess.service.SearchService;
import org.codelibs.fess.solr.IndexUpdater;
import org.codelibs.robot.extractor.ExtractorFactory;
import org.codelibs.solr.lib.SolrGroupManager;
import org.seasar.framework.container.SingletonS2Container;
public final class ComponentUtil {
@ -116,6 +115,8 @@ public final class ComponentUtil {
private static final String FIELD_HELPER = "fieldHelper";
private static final String ELASTICSEARCH_CLIENT = "searchClient";
private ComponentUtil() {
}
@ -135,10 +136,6 @@ public final class ComponentUtil {
return SingletonS2Container.getComponent(CRAWLER_PROPERTIES);
}
public static SolrGroupManager getSolrGroupManager() {
return SingletonS2Container.getComponent(SOLR_GROUP_MANAGER);
}
public static SystemHelper getSystemHelper() {
return SingletonS2Container.getComponent(SYSTEM_HELPER);
}
@ -239,10 +236,6 @@ public final class ComponentUtil {
return SingletonS2Container.getComponent(USER_AGENT_NAME);
}
public static SearchService getSearchService() {
return SingletonS2Container.getComponent(SEARCH_SERVICE);
}
public static KeyMatchHelper getKeyMatchHelper() {
return SingletonS2Container.getComponent(KEY_MATCH_HELPER);
}
@ -255,7 +248,12 @@ public final class ComponentUtil {
return SingletonS2Container.getComponent(FIELD_HELPER);
}
public static SearchClient getElasticsearchClient() {
return SingletonS2Container.getComponent(ELASTICSEARCH_CLIENT);
}
public static <T> T getComponent(Class<T> clazz) {
return SingletonS2Container.getComponent(clazz);
}
}

48
src/main/java/org/codelibs/fess/util/FacetResponse.java
View file

@ -21,29 +21,31 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import org.apache.solr.client.solrj.response.FacetField;
import org.apache.solr.client.solrj.response.FacetField.Count;
import org.codelibs.fess.Constants;
import org.elasticsearch.common.base.Charsets;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.aggregations.bucket.filter.Filter;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import com.google.common.io.BaseEncoding;
public class FacetResponse {
protected Map<String, Long> queryCountMap;
protected Map<String, Long> queryCountMap = new LinkedHashMap<String, Long>();
protected List<Field> fieldList;
protected List<Field> fieldList = new ArrayList<FacetResponse.Field>();
public FacetResponse(final List<FacetField> facetFields, final Map<String, Integer> facetQueryMap) {
if (facetFields != null && !facetFields.isEmpty()) {
fieldList = new ArrayList<FacetResponse.Field>();
for (final FacetField facetField : facetFields) {
if (facetField.getValues() != null) {
fieldList.add(new Field(facetField));
}
public FacetResponse(final Aggregations aggregations) {
aggregations.forEach(aggregation -> {
if (aggregation.getName().startsWith(Constants.FACET_FIELD_PREFIX)) {
final Terms termFacet = (Terms) aggregation;
fieldList.add(new Field(termFacet));
} else if (aggregation.getName().startsWith(Constants.FACET_QUERY_PREFIX)) {
final Filter queryFacet = (Filter) aggregation;
String encodedQuery = queryFacet.getName().substring(Constants.FACET_QUERY_PREFIX.length());
queryCountMap.put(new String(BaseEncoding.base64().decode(encodedQuery), Charsets.UTF_8), queryFacet.getDocCount());
}
}
if (facetQueryMap != null && !facetQueryMap.isEmpty()) {
queryCountMap = new LinkedHashMap<String, Long>();
for (final Map.Entry<String, Integer> entry : facetQueryMap.entrySet()) {
queryCountMap.put(entry.getKey(), entry.getValue().longValue());
}
}
});
}
public boolean hasFacetResponse() {
@ -55,11 +57,11 @@ public class FacetResponse {
protected String name;
public Field(final FacetField facetField) {
name = facetField.getName();
valueCountMap = new LinkedHashMap<String, Long>();
for (final Count count : facetField.getValues()) {
valueCountMap.put(count.getName(), count.getCount());
public Field(final Terms termFacet) {
String encodedField = termFacet.getName().substring(Constants.FACET_FIELD_PREFIX.length());
name = new String(BaseEncoding.base64().decode(encodedField), Charsets.UTF_8);
for (final Terms.Bucket tfEntry : termFacet.getBuckets()) {
valueCountMap.put(tfEntry.getKeyAsText().string(), tfEntry.getDocCount());
}
}

120
src/main/java/org/codelibs/fess/util/QueryResponseList.java
View file

@ -25,24 +25,19 @@ import java.util.ListIterator;
import java.util.Map;
import org.apache.commons.lang.StringUtils;
import org.apache.solr.client.solrj.response.FacetField;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.codelibs.core.util.StringUtil;
import org.codelibs.fess.helper.FieldHelper;
import org.codelibs.fess.helper.QueryHelper;
import org.codelibs.fess.helper.ViewHelper;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.common.text.Text;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.highlight.HighlightField;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class QueryResponseList implements List<Map<String, Object>> {
private static final String PARTIAL_RESULTS = "partialResults";
private static final String MORE_LIKE_THIS = "moreLikeThis";
private static final String DOC_VALUES = "docValues";
private static final Logger logger = LoggerFactory.getLogger(QueryResponseList.class);
@ -80,9 +75,7 @@ public class QueryResponseList implements List<Map<String, Object>> {
protected boolean partialResults = false;
protected int queryTime;
protected long searchTime;
protected long queryTime;
public QueryResponseList() {
parent = new ArrayList<Map<String, Object>>();
@ -93,18 +86,14 @@ public class QueryResponseList implements List<Map<String, Object>> {
this.parent = parent;
}
public void init(final QueryResponse queryResponse, final int pageSize) {
long start = 0;
public void init(final SearchResponse searchResponse, final long start, final int pageSize) {
long numFound = 0;
if (queryResponse != null) {
final SolrDocumentList sdList = queryResponse.getResults();
start = sdList.getStart();
numFound = sdList.getNumFound();
queryTime = queryResponse.getQTime();
searchTime = queryResponse.getElapsedTime();
if (searchResponse != null) {
SearchHits searchHits = searchResponse.getHits();
numFound = searchHits.getTotalHits();
queryTime = searchResponse.getTookInMillis();
final Object partialResultsValue = queryResponse.getResponseHeader().get(PARTIAL_RESULTS);
if (partialResultsValue != null && ((Boolean) partialResultsValue).booleanValue()) {
if (searchResponse.getTotalShards() != searchResponse.getSuccessfulShards()) {
partialResults = true;
}
@ -112,19 +101,23 @@ public class QueryResponseList implements List<Map<String, Object>> {
final QueryHelper queryHelper = ComponentUtil.getQueryHelper();
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final String hlPrefix = queryHelper.getHighlightingPrefix();
for (final SolrDocument solrDocMap : sdList) {
for (final SearchHit searchHit : searchHits.getHits()) {
final Map<String, Object> docMap = new HashMap<String, Object>();
docMap.putAll(solrDocMap);
docMap.putAll(searchHit.getSource());
Map<String, HighlightField> highlightFields = searchHit.getHighlightFields();
try {
final Object idValue = docMap.get(fieldHelper.idField);
if (queryResponse.getHighlighting().get(idValue) != null) {
for (final String hf : queryHelper.getHighlightingFields()) {
final List<String> highlightSnippets = queryResponse.getHighlighting().get(idValue).get(hf);
String value = null;
if (highlightSnippets != null && !highlightSnippets.isEmpty()) {
value = StringUtils.join(highlightSnippets, "...");
docMap.put(hlPrefix + hf, value);
if (highlightFields != null) {
for (Map.Entry<String, HighlightField> entry : highlightFields.entrySet()) {
HighlightField highlightField = entry.getValue();
Text[] fragments = highlightField.fragments();
if (fragments != null && fragments.length != 0) {
String[] texts = new String[fragments.length];
for (int i = 0; i < fragments.length; i++) {
texts[i] = fragments[i].string();
}
String value = StringUtils.join(texts, "...");
docMap.put(hlPrefix + highlightField.getName(), value);
}
}
}
@ -147,46 +140,11 @@ public class QueryResponseList implements List<Map<String, Object>> {
}
// facet
final List<FacetField> facetFields = queryResponse.getFacetFields();
final Map<String, Integer> facetQueryMap = queryResponse.getFacetQuery();
if (facetFields != null || facetQueryMap != null) {
facetResponse = new FacetResponse(facetFields, facetQueryMap);
Aggregations aggregations = searchResponse.getAggregations();
if (aggregations != null) {
facetResponse = new FacetResponse(aggregations);
}
// mlt
final Object moreLikeThisMap = queryResponse.getResponse().get(MORE_LIKE_THIS);
if (moreLikeThisMap instanceof SimpleOrderedMap) {
moreLikeThisResponse = new MoreLikeThisResponse();
final int size = ((SimpleOrderedMap<?>) moreLikeThisMap).size();
for (int i = 0; i < size; i++) {
final String id = ((SimpleOrderedMap<?>) moreLikeThisMap).getName(i);
final Object docList = ((SimpleOrderedMap<?>) moreLikeThisMap).getVal(i);
if (StringUtil.isNotBlank(id) && docList instanceof SolrDocumentList) {
final List<Map<String, Object>> docMapList =
new ArrayList<Map<String, Object>>(((SolrDocumentList) docList).size());
for (final SolrDocument solrDoc : (SolrDocumentList) docList) {
final Map<String, Object> docMap = new HashMap<String, Object>();
docMap.putAll(solrDoc);
docMapList.add(docMap);
}
moreLikeThisResponse.put(id, docMapList);
}
}
}
// docValues
final Object docValuesObj = queryResponse.getResponse().get(DOC_VALUES);
if (docValuesObj instanceof SimpleOrderedMap) {
@SuppressWarnings("unchecked")
final SimpleOrderedMap<List<Long>> docValuesMap = (SimpleOrderedMap<List<Long>>) docValuesObj;
for (int i = 0; i < docValuesMap.size(); i++) {
final String name = docValuesMap.getName(i);
final List<Long> valueList = docValuesMap.getVal(i);
for (int j = 0; j < valueList.size() && j < parent.size(); j++) {
parent.get(j).put(name, valueList.get(j));
}
}
}
}
calculatePageInfo(start, pageSize, numFound);
}
@ -415,22 +373,4 @@ public class QueryResponseList implements List<Map<String, Object>> {
return partialResults;
}
public int getQueryTime() {
return queryTime;
}
public long getSearchTime() {
return searchTime;
}
@Override
public String toString() {
return "parent=" + parent + ", pageSize=" + pageSize + ", currentPageNumber=" + currentPageNumber + ", allRecordCount="
+ allRecordCount + ", allPageCount=" + allPageCount + ", existNextPage=" + existNextPage + ", existPrevPage="
+ existPrevPage + ", currentStartRecordNumber=" + currentStartRecordNumber + ", currentEndRecordNumber="
+ currentEndRecordNumber + ", pageNumberList=" + pageNumberList + ", searchQuery=" + searchQuery + ", solrQuery="
+ solrQuery + ", execTime=" + execTime + ", facetResponse=" + facetResponse + ", moreLikeThisResponse="
+ moreLikeThisResponse + ", partialResults=" + partialResults + ", queryTime=" + queryTime + ", searchTime=" + searchTime;
}
}

61
src/main/java/org/codelibs/fess/web/IndexAction.java
View file

@ -61,11 +61,11 @@ import org.codelibs.fess.Constants;
import org.codelibs.fess.InvalidQueryException;
import org.codelibs.fess.ResultOffsetExceededException;
import org.codelibs.fess.UnsupportedSearchException;
import org.codelibs.fess.client.SearchClient;
import org.codelibs.fess.db.allcommon.CDef;
import org.codelibs.fess.db.exentity.ClickLog;
import org.codelibs.fess.db.exentity.SearchLog;
import org.codelibs.fess.db.exentity.UserInfo;
import org.codelibs.fess.entity.FieldAnalysisResponse;
import org.codelibs.fess.entity.LoginInfo;
import org.codelibs.fess.helper.CrawlingConfigHelper;
import org.codelibs.fess.helper.DocumentHelper;
@ -82,7 +82,6 @@ import org.codelibs.fess.helper.UserInfoHelper;
import org.codelibs.fess.helper.ViewHelper;
import org.codelibs.fess.screenshot.ScreenShotManager;
import org.codelibs.fess.service.FavoriteLogService;
import org.codelibs.fess.service.SearchService;
import org.codelibs.fess.util.ComponentUtil;
import org.codelibs.fess.util.FacetResponse;
import org.codelibs.fess.util.MoreLikeThisResponse;
@ -91,7 +90,6 @@ import org.codelibs.fess.util.WebApiUtil;
import org.codelibs.robot.util.CharUtil;
import org.codelibs.sastruts.core.SSCConstants;
import org.codelibs.sastruts.core.exception.SSCActionMessagesException;
import org.codelibs.solr.lib.exception.SolrLibQueryException;
import org.dbflute.optional.OptionalEntity;
import org.seasar.framework.beans.util.Beans;
import org.seasar.framework.container.annotation.tiger.Binding;
@ -135,7 +133,7 @@ public class IndexAction {
protected IndexForm indexForm;
@Resource
protected SearchService searchService;
protected SearchClient searchClient;
@Resource
protected FavoriteLogService favoriteLogService;
@ -344,7 +342,7 @@ public class IndexAction {
public String cache() {
Map<String, Object> doc = null;
try {
doc = searchService.getDocument(fieldHelper.docIdField + ":" + indexForm.docId, queryHelper.getCacheResponseFields(), null);
doc = searchClient.getDocument(fieldHelper.docIdField + ":" + indexForm.docId, queryHelper.getCacheResponseFields());
} catch (final Exception e) {
logger.warn("Failed to request: " + indexForm.docId, e);
}
@ -367,9 +365,7 @@ public class IndexAction {
public String go() throws IOException {
Map<String, Object> doc = null;
try {
doc =
searchService.getDocument(fieldHelper.docIdField + ":" + indexForm.docId, queryHelper.getResponseFields(),
new String[] { fieldHelper.clickCountField });
doc = searchClient.getDocument(fieldHelper.docIdField + ":" + indexForm.docId, queryHelper.getResponseFields());
} catch (final Exception e) {
logger.warn("Failed to request: " + indexForm.docId, e);
}
@ -501,7 +497,7 @@ public class IndexAction {
OutputStream out = null;
BufferedInputStream in = null;
try {
final Map<String, Object> doc = searchService.getDocument(fieldHelper.docIdField + ":" + indexForm.docId);
final Map<String, Object> doc = searchClient.getDocument(fieldHelper.docIdField + ":" + indexForm.docId);
final String url = doc == null ? null : (String) doc.get(fieldHelper.urlField);
if (StringUtil.isBlank(indexForm.queryId) || StringUtil.isBlank(url) || screenShotManager == null) {
// 404
@ -682,33 +678,6 @@ public class IndexAction {
return null;
}
@Execute(validator = false)
public String analysisApi() {
if (Constants.FALSE.equals(crawlerProperties.getProperty(Constants.WEB_API_ANALYSIS_PROPERTY, Constants.TRUE))) {
WebApiUtil.setError(9, "Unsupported operation.");
return null;
}
if (indexForm.fn == null || indexForm.fn.length == 0) {
WebApiUtil.setError(2, "The field name is empty.");
return null;
}
if (StringUtil.isBlank(indexForm.query)) {
WebApiUtil.setError(3, "Your query is empty.");
return null;
}
try {
final String[] fieldNames = indexForm.fn;
final FieldAnalysisResponse fieldAnalysis = searchService.getFieldAnalysisResponse(fieldNames, indexForm.query);
WebApiUtil.setObject("fieldAnalysis", fieldAnalysis);
} catch (final Exception e) {
WebApiUtil.setError(1, e);
}
return null;
}
@Execute(validator = false)
public String hotSearchWordApi() {
if (Constants.FALSE.equals(crawlerProperties.getProperty(Constants.WEB_API_HOT_SEARCH_WORD_PROPERTY, Constants.TRUE))) {
@ -751,8 +720,8 @@ public class IndexAction {
try {
final Map<String, Object> doc =
indexForm.docId == null ? null : searchService.getDocument(fieldHelper.docIdField + ":" + indexForm.docId,
queryHelper.getResponseFields(), new String[] { fieldHelper.favoriteCountField });
indexForm.docId == null ? null : searchClient.getDocument(fieldHelper.docIdField + ":" + indexForm.docId,
queryHelper.getResponseFields());
final String userCode = userInfoHelper.getUserCode();
final String favoriteUrl = doc == null ? null : (String) doc.get(fieldHelper.urlField);
@ -822,7 +791,7 @@ public class IndexAction {
final String[] docIds = userInfoHelper.getResultDocIds(indexForm.queryId);
final List<Map<String, Object>> docList =
searchService.getDocumentListByDocIds(docIds, queryHelper.getResponseFields(),
searchClient.getDocumentListByDocIds(docIds, queryHelper.getResponseFields(),
new String[] { fieldHelper.favoriteCountField }, getMaxPageSize());
List<String> urlList = new ArrayList<String>(docList.size());
for (final Map<String, Object> doc : docList) {
@ -969,13 +938,8 @@ public class IndexAction {
final int pageNum = Integer.parseInt(indexForm.num);
try {
documentItems =
searchService.getDocumentList(query, pageStart, pageNum, indexForm.facet, indexForm.geo, indexForm.mlt,
queryHelper.getResponseFields(), queryHelper.getResponseDocValuesFields());
} catch (final SolrLibQueryException e) {
if (logger.isDebugEnabled()) {
logger.debug(e.getMessage(), e);
}
throw new SSCActionMessagesException(e, "errors.invalid_query_unknown");
searchClient
.getDocumentList(query, pageStart, pageNum, indexForm.facet, indexForm.geo, queryHelper.getResponseFields());
} catch (final InvalidQueryException e) {
if (logger.isDebugEnabled()) {
logger.debug(e.getMessage(), e);
@ -1121,10 +1085,6 @@ public class IndexAction {
indexForm.facet = queryHelper.getDefaultFacetInfo();
}
if (indexForm.mlt == null) {
indexForm.mlt = queryHelper.getDefaultMoreLikeThisInfo();
}
if (indexForm.geo == null) {
indexForm.geo = queryHelper.getDefaultGeoInfo();
}
@ -1259,7 +1219,6 @@ public class IndexAction {
protected void buildInitParams() {
buildInitParamMap(viewHelper.getInitFacetParamMap(), Constants.FACET_QUERY, Constants.FACET_FORM);
buildInitParamMap(viewHelper.getInitMltParamMap(), Constants.MLT_QUERY, Constants.MLT_FORM);
buildInitParamMap(viewHelper.getInitGeoParamMap(), Constants.GEO_QUERY, Constants.GEO_FORM);
}

5
src/main/java/org/codelibs/fess/web/IndexForm.java
View file

@ -20,7 +20,6 @@ import java.io.Serializable;
import org.codelibs.fess.entity.FacetInfo;
import org.codelibs.fess.entity.GeoInfo;
import org.codelibs.fess.entity.MoreLikeThisInfo;
import org.codelibs.fess.util.SearchParamMap;
import org.seasar.struts.annotation.IntegerType;
import org.seasar.struts.annotation.Maxbytelength;
@ -95,10 +94,6 @@ public class IndexForm implements Serializable {
public FacetInfo facet;
// mlt
public MoreLikeThisInfo mlt;
// advance
public SearchParamMap options = new SearchParamMap();

4
src/main/java/org/codelibs/fess/web/admin/CrawlAction.java
View file

@ -60,7 +60,6 @@ public class CrawlAction implements Serializable {
protected String showIndex(final boolean redirect) {
crawlForm.diffCrawling = crawlerProperties.getProperty(Constants.DIFF_CRAWLING_PROPERTY, Constants.TRUE);
crawlForm.useAclAsRole = crawlerProperties.getProperty(Constants.USE_ACL_AS_ROLE, Constants.FALSE);
crawlForm.serverRotation = crawlerProperties.getProperty(Constants.SERVER_ROTATION_PROPERTY, Constants.FALSE);
crawlForm.dayForCleanup = crawlerProperties.getProperty(Constants.DAY_FOR_CLEANUP_PROPERTY, "1");
crawlForm.crawlingThreadCount = crawlerProperties.getProperty(Constants.CRAWLING_THREAD_COUNT_PROPERTY, "5");
crawlForm.searchLog = crawlerProperties.getProperty(Constants.SEARCH_LOG_PROPERTY, Constants.TRUE);
@ -105,9 +104,6 @@ public class CrawlAction implements Serializable {
crawlForm.diffCrawling != null && Constants.ON.equalsIgnoreCase(crawlForm.diffCrawling) ? Constants.TRUE : Constants.FALSE);
crawlerProperties.setProperty(Constants.USE_ACL_AS_ROLE,
crawlForm.useAclAsRole != null && Constants.ON.equalsIgnoreCase(crawlForm.useAclAsRole) ? Constants.TRUE : Constants.FALSE);
crawlerProperties.setProperty(Constants.SERVER_ROTATION_PROPERTY,
crawlForm.serverRotation != null && Constants.ON.equalsIgnoreCase(crawlForm.serverRotation) ? Constants.TRUE
: Constants.FALSE);
crawlerProperties.setProperty(Constants.DAY_FOR_CLEANUP_PROPERTY, crawlForm.dayForCleanup);
crawlerProperties.setProperty(Constants.CRAWLING_THREAD_COUNT_PROPERTY, crawlForm.crawlingThreadCount);
crawlerProperties.setProperty(Constants.SEARCH_LOG_PROPERTY,

252
src/main/java/org/codelibs/fess/web/admin/DocumentAction.java
View file

@ -29,13 +29,10 @@ import jp.sf.fess.suggest.SuggestConstants;
import jp.sf.fess.suggest.server.SuggestSolrServer;
import jp.sf.fess.suggest.service.SuggestService;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrQuery.ORDER;
import org.apache.solr.client.solrj.response.FacetField;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.codelibs.core.util.DynamicProperties;
import org.codelibs.core.util.StringUtil;
import org.codelibs.fess.Constants;
import org.codelibs.fess.client.SearchClient;
import org.codelibs.fess.crud.util.SAStrutsUtil;
import org.codelibs.fess.helper.FieldHelper;
import org.codelibs.fess.helper.JobHelper;
@ -44,11 +41,16 @@ import org.codelibs.fess.helper.WebManagementHelper;
import org.codelibs.fess.util.ComponentUtil;
import org.codelibs.sastruts.core.annotation.Token;
import org.codelibs.sastruts.core.exception.SSCActionMessagesException;
import org.codelibs.solr.lib.SolrGroup;
import org.codelibs.solr.lib.SolrGroupManager;
import org.codelibs.solr.lib.SolrLibConstants;
import org.codelibs.solr.lib.policy.QueryType;
import org.codelibs.solr.lib.policy.impl.StatusPolicyImpl;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.index.query.MatchAllQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.aggregations.Aggregation;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket;
import org.elasticsearch.search.aggregations.bucket.terms.Terms.Order;
import org.elasticsearch.search.aggregations.bucket.terms.TermsBuilder;
import org.seasar.struts.annotation.ActionForm;
import org.seasar.struts.annotation.Execute;
import org.seasar.struts.taglib.S2Functions;
@ -71,10 +73,7 @@ public class DocumentAction implements Serializable {
protected DocumentForm documentForm;
@Resource
protected DynamicProperties solrProperties;
@Resource
protected SolrGroupManager solrGroupManager;
protected SearchClient searchClient;
@Resource
protected WebManagementHelper webManagementHelper;
@ -98,60 +97,7 @@ public class DocumentAction implements Serializable {
}
protected String showIndex(final boolean redirect) {
final Map<String, DynamicProperties> groupPropMap = new HashMap<String, DynamicProperties>();
for (final String groupName : solrGroupManager.getSolrGroupNames()) {
final DynamicProperties props = ComponentUtil.getSolrGroupProperties(groupName);
if (props != null) {
groupPropMap.put(groupName, props);
}
}
final String[] serverNames = solrGroupManager.getSolrServerNames();
for (final String name : serverNames) {
final String[] names = name.split(":");
if (names.length == 2) {
final Map<String, String> map = new HashMap<String, String>(4);
map.put("groupName", names[0]);
map.put("serverName", names[1]);
final DynamicProperties props = groupPropMap.get(names[0]);
if (props != null) {
String status = props.getProperty(StatusPolicyImpl.STATUS_PREFIX + names[1]);
if (StringUtil.isBlank(status)) {
status = StatusPolicyImpl.ACTIVE;
}
map.put("status", status);
String index = props.getProperty(StatusPolicyImpl.INDEX_PREFIX + names[1]);
if (StringUtil.isBlank(index)) {
index = StatusPolicyImpl.READY;
}
map.put("index", index);
} else {
map.put("status", StatusPolicyImpl.ACTIVE);
map.put("index", StatusPolicyImpl.READY);
}
documentForm.serverStatusList.add(map);
}
}
suggestDocumentNums = getSuggestDocumentNum();
// select group status
documentForm.currentServerForSelect = solrProperties.getProperty(SolrLibConstants.SELECT_GROUP);
final SolrGroup selectSolrGroup = solrGroupManager.getSolrGroup(documentForm.currentServerForSelect);
if (selectSolrGroup != null && selectSolrGroup.isActive(QueryType.QUERY)) {
documentForm.currentServerStatusForSelect = Constants.ACTIVE;
} else {
documentForm.currentServerStatusForSelect = Constants.INACTIVE;
}
// update group status
documentForm.currentServerForUpdate = solrProperties.getProperty(SolrLibConstants.SELECT_GROUP);
final SolrGroup updateSolrGroup = solrGroupManager.getSolrGroup(documentForm.currentServerForUpdate);
if (updateSolrGroup != null && updateSolrGroup.isActive(QueryType.QUERY)) {
documentForm.currentServerStatusForUpdate = Constants.ACTIVE;
} else {
documentForm.currentServerStatusForUpdate = Constants.INACTIVE;
}
// TODO
if (redirect) {
return "index?redirect=true";
@ -169,87 +115,36 @@ public class DocumentAction implements Serializable {
@Token(save = false, validate = true)
@Execute(validator = true, input = "index")
public String commit() {
// TODO change to flush
if (jobHelper.isCrawlProcessRunning()) {
throw new SSCActionMessagesException("errors.failed_to_start_solr_process_because_of_running");
}
final SolrGroup solrGroup = solrGroupManager.getSolrGroup(documentForm.groupName);
if (solrGroup == null) {
throw new SSCActionMessagesException("errors.failed_to_commit_solr_index");
} else {
final Thread thread = new Thread(new Runnable() {
@Override
public void run() {
if (!jobHelper.isCrawlProcessRunning()) {
final long execTime = System.currentTimeMillis();
try {
systemHelper.updateStatus(solrGroup, QueryType.ADD);
solrGroup.commit(true, true, false, true);
systemHelper.updateStatus(solrGroup, QueryType.COMMIT);
if (logger.isInfoEnabled()) {
logger.info("[EXEC TIME] index commit time: " + (System.currentTimeMillis() - execTime) + "ms");
}
} catch (final Exception e) {
logger.error("Failed to commit index.", e);
}
} else {
if (logger.isInfoEnabled()) {
logger.info("could not start index cleanup process" + " because of running solr process.");
}
}
}
});
thread.start();
SAStrutsUtil.addSessionMessage("success.commit_solr_index");
}
searchClient.flush();
SAStrutsUtil.addSessionMessage("success.commit_solr_index");
return showIndex(true);
}
@Token(save = false, validate = true)
@Execute(validator = true, input = "index")
public String optimize() {
// TODO change to optimize
if (jobHelper.isCrawlProcessRunning()) {
throw new SSCActionMessagesException("errors.failed_to_start_solr_process_because_of_running");
}
final SolrGroup solrGroup = solrGroupManager.getSolrGroup(documentForm.groupName);
if (solrGroup == null) {
throw new SSCActionMessagesException("errors.failed_to_optimize_solr_index");
} else {
final Thread thread = new Thread(new Runnable() {
@Override
public void run() {
if (!jobHelper.isCrawlProcessRunning()) {
final long execTime = System.currentTimeMillis();
try {
systemHelper.updateStatus(solrGroup, QueryType.ADD);
solrGroup.optimize();
systemHelper.updateStatus(solrGroup, QueryType.OPTIMIZE);
if (logger.isInfoEnabled()) {
logger.info("[EXEC TIME] index optimize time: " + (System.currentTimeMillis() - execTime) + "ms");
}
} catch (final Exception e) {
logger.error("Failed to optimize index.", e);
}
} else {
if (logger.isInfoEnabled()) {
logger.info("could not start index cleanup process" + " because of running solr process.");
}
}
}
});
thread.start();
SAStrutsUtil.addSessionMessage("success.optimize_solr_index");
}
searchClient.optimize();
SAStrutsUtil.addSessionMessage("success.optimize_solr_index");
return showIndex(true);
}
@Token(save = false, validate = true)
@Execute(validator = true, input = "index")
public String delete() {
String deleteQuery;
QueryBuilder deleteQuery;
if ("*".equals(documentForm.sessionId)) {
deleteQuery = "*:*";
deleteQuery = QueryBuilders.matchAllQuery();
} else {
deleteQuery = fieldHelper.segmentField + ":" + documentForm.sessionId;
deleteQuery = QueryBuilders.termQuery(fieldHelper.segmentField, documentForm.sessionId);
}
return deleteByQuery(deleteQuery);
}
@ -264,99 +159,37 @@ public class DocumentAction implements Serializable {
@Token(save = false, validate = true)
@Execute(validator = true, input = "index")
public String deleteByUrl() {
final String deleteUrl = documentForm.deleteUrl;
final String deleteQuery = fieldHelper.urlField + ":\"" + deleteUrl + "\"";
return deleteByQuery(deleteQuery);
return deleteByQuery(QueryBuilders.termQuery(fieldHelper.urlField, documentForm.deleteUrl));
}
private String deleteByQuery(final String deleteQuery) {
private String deleteByQuery(final QueryBuilder queryBuilder) {
if (jobHelper.isCrawlProcessRunning()) {
throw new SSCActionMessagesException("errors.failed_to_start_solr_process_because_of_running");
}
final SolrGroup solrGroup = solrGroupManager.getSolrGroup(documentForm.groupName);
if (solrGroup == null) {
throw new SSCActionMessagesException("errors.failed_to_delete_solr_index");
} else {
final Thread thread = new Thread(new Runnable() {
@Override
public void run() {
if (!jobHelper.isCrawlProcessRunning()) {
final long execTime = System.currentTimeMillis();
try {
systemHelper.updateStatus(solrGroup, QueryType.DELETE);
solrGroup.deleteByQuery(deleteQuery);
solrGroup.commit(true, true, false, true);
systemHelper.updateStatus(solrGroup, QueryType.OPTIMIZE);
if (logger.isInfoEnabled()) {
logger.info("[EXEC TIME] index cleanup time: " + (System.currentTimeMillis() - execTime) + "ms");
}
} catch (final Exception e) {
logger.error("Failed to delete index (query=" + deleteQuery + ").", e);
}
} else {
if (logger.isInfoEnabled()) {
logger.info("could not start index cleanup process" + " because of running solr process.");
}
}
}
});
thread.start();
SAStrutsUtil.addSessionMessage("success.delete_solr_index");
}
return showIndex(true);
}
public List<Map<String, Object>> getGroupActionItems() {
final List<Map<String, Object>> groupActionItems = new ArrayList<Map<String, Object>>();
for (final String groupName : solrGroupManager.getSolrGroupNames()) {
try {
final Map<String, Object> map = new HashMap<String, Object>();
map.put("groupName", groupName);
final SessionIdList<Map<String, String>> sessionIdList = getSessionIdList(groupName);
map.put("sessionIdItems", sessionIdList);
map.put("totalCount", sessionIdList.getTotalCount());
groupActionItems.add(map);
} catch (final Exception e) {
logger.info("could not get server groups.", e);
}
}
return groupActionItems;
searchClient.deleteByQuery(queryBuilder);
SAStrutsUtil.addSessionMessage("success.delete_solr_index");
return showIndex(true);
}
protected SessionIdList<Map<String, String>> getSessionIdList(final String groupName) {
final SessionIdList<Map<String, String>> sessionIdList = new SessionIdList<Map<String, String>>();
SolrGroup serverGroup;
try {
serverGroup = solrGroupManager.getSolrGroup(groupName);
} catch (final Exception e) {
if (logger.isInfoEnabled()) {
logger.info(e.getMessage());
}
return sessionIdList;
QueryBuilder queryBuilder = QueryBuilders.matchAllQuery();
TermsBuilder termsBuilder =
AggregationBuilders.terms(fieldHelper.segmentField).field(fieldHelper.segmentField).size(100).order(Order.count(false));
SearchResponse response = searchClient.query(queryBuilder, termsBuilder, null);
Terms terms = response.getAggregations().get(fieldHelper.segmentField);
for (Bucket bucket : terms.getBuckets()) {
final Map<String, String> map = new HashMap<String, String>(3);
map.put("label", bucket.getKey() + " (" + bucket.getDocCount() + ")");
map.put("value", bucket.getKey());
map.put("count", Long.toString(bucket.getDocCount()));
sessionIdList.add(map);
sessionIdList.addTotalCount(bucket.getDocCount());
}
final SolrQuery query = new SolrQuery();
query.setQuery("*:*");
query.setFacet(true);
query.addFacetField(fieldHelper.segmentField);
query.addSort(fieldHelper.segmentField, ORDER.desc);
final QueryResponse queryResponse = serverGroup.query(query);
final List<FacetField> facets = queryResponse.getFacetFields();
for (final FacetField facet : facets) {
final List<FacetField.Count> facetEntries = facet.getValues();
if (facetEntries != null) {
for (final FacetField.Count fcount : facetEntries) {
final Map<String, String> map = new HashMap<String, String>(3);
map.put("label", fcount.getName() + " (" + fcount.getCount() + ")");
map.put("value", fcount.getName());
map.put("count", Long.toString(fcount.getCount()));
sessionIdList.add(map);
sessionIdList.addTotalCount(fcount.getCount());
}
}
}
return sessionIdList;
}
@ -392,6 +225,8 @@ public class DocumentAction implements Serializable {
}
if (StringUtil.isNotBlank(query)) {
// TODO
/*
final Thread thread = new Thread(new Runnable() {
@Override
public void run() {
@ -415,6 +250,7 @@ public class DocumentAction implements Serializable {
});
thread.start();
SAStrutsUtil.addSessionMessage("success.delete_solr_index");
*/
}
return showIndex(true);
}

56
src/main/java/org/codelibs/fess/web/admin/SearchListAction.java
View file

@ -28,18 +28,15 @@ import org.codelibs.core.util.DynamicProperties;
import org.codelibs.core.util.StringUtil;
import org.codelibs.fess.InvalidQueryException;
import org.codelibs.fess.ResultOffsetExceededException;
import org.codelibs.fess.client.SearchClient;
import org.codelibs.fess.crud.util.SAStrutsUtil;
import org.codelibs.fess.helper.FieldHelper;
import org.codelibs.fess.helper.JobHelper;
import org.codelibs.fess.helper.QueryHelper;
import org.codelibs.fess.helper.SystemHelper;
import org.codelibs.fess.service.SearchService;
import org.codelibs.fess.util.QueryResponseList;
import org.codelibs.sastruts.core.annotation.Token;
import org.codelibs.sastruts.core.exception.SSCActionMessagesException;
import org.codelibs.solr.lib.SolrGroup;
import org.codelibs.solr.lib.SolrGroupManager;
import org.codelibs.solr.lib.policy.QueryType;
import org.seasar.framework.beans.util.Beans;
import org.seasar.struts.annotation.ActionForm;
import org.seasar.struts.annotation.Execute;
@ -62,10 +59,7 @@ public class SearchListAction implements Serializable {
protected SearchListForm searchListForm;
@Resource
protected SolrGroupManager solrGroupManager;
@Resource
protected SearchService searchService;
protected SearchClient searchClient;
@Resource
protected DynamicProperties crawlerProperties;
@ -159,9 +153,7 @@ public class SearchListAction implements Serializable {
final int offset = Integer.parseInt(searchListForm.start);
final int size = Integer.parseInt(searchListForm.num);
try {
documentItems =
searchService.getDocumentList(query, offset, size, null, null, null, queryHelper.getResponseFields(), new String[] {
fieldHelper.clickCountField, fieldHelper.favoriteCountField }, false);
documentItems = searchClient.getDocumentList(query, offset, size, null, null, queryHelper.getResponseFields(), false);
} catch (final InvalidQueryException e) {
if (logger.isDebugEnabled()) {
logger.debug(e.getMessage(), e);
@ -254,34 +246,28 @@ public class SearchListAction implements Serializable {
if (jobHelper.isCrawlProcessRunning()) {
throw new SSCActionMessagesException("errors.failed_to_start_solr_process_because_of_running");
}
final SolrGroup solrGroup = solrGroupManager.getSolrGroup(QueryType.DELETE);
if (solrGroup == null) {
throw new SSCActionMessagesException("errors.failed_to_delete_solr_index");
} else {
final Thread thread = new Thread(new Runnable() {
@Override
public void run() {
if (!jobHelper.isCrawlProcessRunning()) {
final long time = System.currentTimeMillis();
try {
solrGroup.deleteByQuery(fieldHelper.docIdField + ":" + docId);
solrGroup.commit(true, true, false, true);
if (logger.isInfoEnabled()) {
logger.info("[EXEC TIME] index cleanup time: " + (System.currentTimeMillis() - time) + "ms");
}
} catch (final Exception e) {
logger.error("Failed to delete index (query=" + fieldHelper.docIdField + ":" + docId + ").", e);
}
} else {
final Thread thread = new Thread(new Runnable() {
@Override
public void run() {
if (!jobHelper.isCrawlProcessRunning()) {
final long time = System.currentTimeMillis();
try {
searchClient.deleteByQuery(fieldHelper.docIdField, docId);
if (logger.isInfoEnabled()) {
logger.info("could not start index cleanup process" + " because of running solr process.");
logger.info("[EXEC TIME] index cleanup time: " + (System.currentTimeMillis() - time) + "ms");
}
} catch (final Exception e) {
logger.error("Failed to delete index (query=" + fieldHelper.docIdField + ":" + docId + ").", e);
}
} else {
if (logger.isInfoEnabled()) {
logger.info("could not start index cleanup process" + " because of running solr process.");
}
}
});
thread.start();
SAStrutsUtil.addSessionMessage("success.delete_solr_index");
}
}
});
thread.start();
SAStrutsUtil.addSessionMessage("success.delete_solr_index");
return "search?query=" + S2Functions.u(searchListForm.query) + "&redirect=true";
}

135
src/main/java/org/codelibs/fess/web/admin/SystemAction.java
View file

@ -18,7 +18,6 @@ package org.codelibs.fess.web.admin;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -27,23 +26,16 @@ import java.util.Set;
import javax.annotation.Resource;
import javax.servlet.http.HttpSession;
import org.codelibs.core.util.DynamicProperties;
import org.codelibs.core.util.StringUtil;
import org.codelibs.fess.Constants;
import org.codelibs.fess.client.SearchClient;
import org.codelibs.fess.crud.util.SAStrutsUtil;
import org.codelibs.fess.db.exentity.ScheduledJob;
import org.codelibs.fess.helper.JobHelper;
import org.codelibs.fess.helper.SystemHelper;
import org.codelibs.fess.helper.WebManagementHelper;
import org.codelibs.fess.service.ScheduledJobService;
import org.codelibs.fess.util.ComponentUtil;
import org.codelibs.sastruts.core.annotation.Token;
import org.codelibs.sastruts.core.exception.SSCActionMessagesException;
import org.codelibs.solr.lib.SolrGroup;
import org.codelibs.solr.lib.SolrGroupManager;
import org.codelibs.solr.lib.SolrLibConstants;
import org.codelibs.solr.lib.policy.QueryType;
import org.codelibs.solr.lib.policy.impl.StatusPolicyImpl;
import org.seasar.struts.annotation.ActionForm;
import org.seasar.struts.annotation.Execute;
import org.seasar.struts.util.RequestUtil;
@ -62,10 +54,7 @@ public class SystemAction implements Serializable {
protected SystemForm systemForm;
@Resource
protected DynamicProperties solrProperties;
@Resource
protected SolrGroupManager solrGroupManager;
protected SearchClient searchClient;
@Resource
protected WebManagementHelper webManagementHelper;
@ -84,59 +73,6 @@ public class SystemAction implements Serializable {
}
protected String showIndex(final boolean redirect) {
final Map<String, DynamicProperties> groupPropMap = new HashMap<String, DynamicProperties>();
for (final String groupName : solrGroupManager.getSolrGroupNames()) {
final DynamicProperties props = ComponentUtil.getSolrGroupProperties(groupName);
if (props != null) {
groupPropMap.put(groupName, props);
}
}
final String[] serverNames = solrGroupManager.getSolrServerNames();
final List<String> serverNameList = new ArrayList<String>();
serverNameList.addAll(Arrays.asList(serverNames));
for (final String name : serverNameList) {
final String[] names = name.split(":");
if (names.length == 2) {
final Map<String, String> map = new HashMap<String, String>(4);
map.put("groupName", names[0]);
map.put("serverName", names[1]);
final DynamicProperties props = groupPropMap.get(names[0]);
if (props != null) {
String status = props.getProperty(StatusPolicyImpl.STATUS_PREFIX + names[1]);
if (StringUtil.isBlank(status)) {
status = StatusPolicyImpl.ACTIVE;
}
map.put("status", status);
String index = props.getProperty(StatusPolicyImpl.INDEX_PREFIX + names[1]);
if (StringUtil.isBlank(index)) {
index = StatusPolicyImpl.READY;
}
map.put("index", index);
} else {
map.put("status", StatusPolicyImpl.ACTIVE);
map.put("index", StatusPolicyImpl.READY);
}
systemForm.serverStatusList.add(map);
}
}
// select group status
systemForm.currentServerForSelect = solrProperties.getProperty(SolrLibConstants.SELECT_GROUP);
final SolrGroup selectSolrGroup = solrGroupManager.getSolrGroup(systemForm.currentServerForSelect);
if (selectSolrGroup != null && selectSolrGroup.isActive(QueryType.QUERY)) {
systemForm.currentServerStatusForSelect = Constants.ACTIVE;
} else {
systemForm.currentServerStatusForSelect = Constants.INACTIVE;
}
// update group status
systemForm.currentServerForUpdate = solrProperties.getProperty(SolrLibConstants.SELECT_GROUP);
final SolrGroup updateSolrGroup = solrGroupManager.getSolrGroup(systemForm.currentServerForUpdate);
if (updateSolrGroup != null && updateSolrGroup.isActive(QueryType.QUERY)) {
systemForm.currentServerStatusForUpdate = Constants.ACTIVE;
} else {
systemForm.currentServerStatusForUpdate = Constants.INACTIVE;
}
if (redirect) {
return "index?redirect=true";
@ -151,71 +87,16 @@ public class SystemAction implements Serializable {
return showIndex(false);
}
@Token(save = false, validate = true)
@Execute(validator = true, input = "index.jsp")
public String update() {
// load solr group properties
final Map<String, DynamicProperties> groupPropMap = new HashMap<String, DynamicProperties>();
for (final String groupName : solrGroupManager.getSolrGroupNames()) {
final DynamicProperties props = ComponentUtil.getSolrGroupProperties(groupName);
if (props != null) {
groupPropMap.put(groupName, props);
}
}
try {
// server status
for (final Map<String, String> statusMap : systemForm.serverStatusList) {
for (final Map.Entry<String, String> entry : statusMap.entrySet()) {
final String[] names = entry.getKey().split("/");
if (names.length == 3) {
final DynamicProperties props = groupPropMap.get(names[0]);
if (props != null) {
final String value = entry.getValue();
String key;
if ("status".equals(names[2])) {
key = StatusPolicyImpl.STATUS_PREFIX + names[1];
} else if ("index".equals(names[2])) {
key = StatusPolicyImpl.INDEX_PREFIX + names[1];
} else {
logger.error("Invalid parameter: " + entry.getKey());
throw new SSCActionMessagesException("errors.failed_to_update_solr_params");
}
props.setProperty(key, value);
props.store();
} else {
logger.warn("Solr group properties is not found: " + names[0]);
}
}
}
}
SAStrutsUtil.addSessionMessage("success.update_solr_params");
} catch (final Exception e) {
logger.error("Failed to update solr parameters.", e);
throw new SSCActionMessagesException(e, "errors.failed_to_update_solr_params", e);
}
return showIndex(true);
}
@Token(save = false, validate = true)
@Execute(validator = true, input = "index")
public String start() {
final String groupName = solrProperties.getProperty(SolrLibConstants.UPDATE_GROUP);
final SolrGroup solrGroup = solrGroupManager.getSolrGroup(groupName);
if (solrGroup != null) {
if (!jobHelper.isCrawlProcessRunning()) {
final List<ScheduledJob> scheduledJobList = scheduledJobService.getCrawloerJobList();
for (final ScheduledJob scheduledJob : scheduledJobList) {
scheduledJob.start();
}
SAStrutsUtil.addSessionMessage("success.start_crawl_process");
RequestUtil.getRequest().getSession().setAttribute(STARTING_CRAWL_PROCESS, Boolean.TRUE);
} else {
SAStrutsUtil.addSessionMessage("success.failed_to_start_crawl_process");
if (!jobHelper.isCrawlProcessRunning()) {
final List<ScheduledJob> scheduledJobList = scheduledJobService.getCrawloerJobList();
for (final ScheduledJob scheduledJob : scheduledJobList) {
scheduledJob.start();
}
SAStrutsUtil.addSessionMessage("success.start_crawl_process");
RequestUtil.getRequest().getSession().setAttribute(STARTING_CRAWL_PROCESS, Boolean.TRUE);
} else {
SAStrutsUtil.addSessionMessage("success.failed_to_start_crawl_process");
}

23
src/test/java/org/codelibs/fess/entity/Document.java
View file

@ -21,65 +21,50 @@ import java.util.Date;
import java.util.List;
import org.apache.commons.lang.builder.ReflectionToStringBuilder;
import org.apache.solr.client.solrj.beans.Field;
import org.codelibs.fess.taglib.FessFunctions;
public class Document implements Serializable {
private static final long serialVersionUID = 1L;
@Field
public String id;
// @Field
//
// public String segment;
@Field
public String digest;
// @Field
//
// public String boost;
@Field
public String host;
@Field
public String site;
@Field
public String url;
@Field
public String content;
@Field
public String title;
@Field
public String cache;
@Field
public String tstamp;
@Field
public List<String> anchor;
@Field
public List<String> type;
@Field
public Long contentLength;
@Field
public String lastModified;
// @Field
//
// public String date;
// @Field
//
// public String lang;
@Field
public String mimetype;
// public String getContentTitle() {

20
src/test/java/org/codelibs/fess/entity/GeoInfoTest.java
View file

@ -26,8 +26,8 @@ public class GeoInfoTest extends S2TestCase {
final GeoInfo geoInfo = create(latitude, lonitude, distance);
assertTrue(geoInfo.isAvailable());
assertEquals("{!geofilt pt=" + Double.toString(Double.parseDouble(latitude)) + "," + Double.toString(Double.parseDouble(lonitude))
+ " sfield=location d=" + distance + ".0}", geoInfo.toGeoQueryString());
// assertEquals("{!geofilt pt=" + Double.toString(Double.parseDouble(latitude)) + "," + Double.toString(Double.parseDouble(lonitude))
// + " sfield=location d=" + distance + ".0}", geoInfo.toGeoQueryString());
}
public void test_90_180_10() {
@ -37,8 +37,8 @@ public class GeoInfoTest extends S2TestCase {
final GeoInfo geoInfo = create(latitude, lonitude, distance);
assertTrue(geoInfo.isAvailable());
assertEquals("{!geofilt pt=" + Double.toString(Double.parseDouble(latitude)) + "," + Double.toString(Double.parseDouble(lonitude))
+ " sfield=location d=" + distance + ".0}", geoInfo.toGeoQueryString());
// assertEquals("{!geofilt pt=" + Double.toString(Double.parseDouble(latitude)) + "," + Double.toString(Double.parseDouble(lonitude))
// + " sfield=location d=" + distance + ".0}", geoInfo.toGeoQueryString());
}
public void test_91_181_10() {
@ -48,7 +48,7 @@ public class GeoInfoTest extends S2TestCase {
final GeoInfo geoInfo = create(latitude, lonitude, distance);
assertTrue(geoInfo.isAvailable());
assertEquals("{!geofilt pt=90.0,-179.0 sfield=location d=10.0}", geoInfo.toGeoQueryString());
// assertEquals("{!geofilt pt=90.0,-179.0 sfield=location d=10.0}", geoInfo.toGeoQueryString());
}
public void test_91_361_10() {
@ -58,7 +58,7 @@ public class GeoInfoTest extends S2TestCase {
final GeoInfo geoInfo = create(latitude, lonitude, distance);
assertTrue(geoInfo.isAvailable());
assertEquals("{!geofilt pt=90.0,1.0 sfield=location d=100.0}", geoInfo.toGeoQueryString());
// assertEquals("{!geofilt pt=90.0,1.0 sfield=location d=100.0}", geoInfo.toGeoQueryString());
}
public void test__90__180_10() {
@ -68,8 +68,8 @@ public class GeoInfoTest extends S2TestCase {
final GeoInfo geoInfo = create(latitude, lonitude, distance);
assertTrue(geoInfo.isAvailable());
assertEquals("{!geofilt pt=" + Double.toString(Double.parseDouble(latitude)) + "," + Double.toString(Double.parseDouble(lonitude))
+ " sfield=location d=" + distance + ".0}", geoInfo.toGeoQueryString());
// assertEquals("{!geofilt pt=" + Double.toString(Double.parseDouble(latitude)) + "," + Double.toString(Double.parseDouble(lonitude))
// + " sfield=location d=" + distance + ".0}", geoInfo.toGeoQueryString());
}
public void test__91__181_10() {
@ -79,7 +79,7 @@ public class GeoInfoTest extends S2TestCase {
final GeoInfo geoInfo = create(latitude, lonitude, distance);
assertTrue(geoInfo.isAvailable());
assertEquals("{!geofilt pt=-90.0,179.0 sfield=location d=10.0}", geoInfo.toGeoQueryString());
// assertEquals("{!geofilt pt=-90.0,179.0 sfield=location d=10.0}", geoInfo.toGeoQueryString());
}
public void test__91__361_10() {
@ -89,7 +89,7 @@ public class GeoInfoTest extends S2TestCase {
final GeoInfo geoInfo = create(latitude, lonitude, distance);
assertTrue(geoInfo.isAvailable());
assertEquals("{!geofilt pt=-90.0,-1.0 sfield=location d=100.0}", geoInfo.toGeoQueryString());
// assertEquals("{!geofilt pt=-90.0,-1.0 sfield=location d=100.0}", geoInfo.toGeoQueryString());
}
public void test_0_0_0() {