0ct0pu5/fess
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

#275

Browse source
This commit is contained in:
Shinsuke Sugaya 2015-06-18 16:45:13 +09:00
parent 55115343b0
commit 84db9d8a48
61 changed files with 838 additions and 2458 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
pom.xml
View file

@ -144,9 +144,9 @@
</configuration>
</plugin>
<plugin>
<groupId>com.marvinformatics.formatter</groupId>
<groupId>net.revelc.code</groupId>
<artifactId>formatter-maven-plugin</artifactId>
<version>1.4.0</version>
<version>0.5.2</version>
<executions>
<execution>
<goals>
@ -154,6 +154,9 @@
</goals>
</execution>
</executions>
<configuration>
<configFile>${basedir}/src/config/eclipse/formatter/java.xml</configFile>
</configuration>
</plugin>
<plugin>
<artifactId>maven-javadoc-plugin</artifactId>

6
src/main/java/org/codelibs/fess/api/json/JsonApiManager.java
View file

@ -37,7 +37,7 @@ import org.codelibs.fess.api.BaseApiManager;
import org.codelibs.fess.api.WebApiManager;
import org.codelibs.fess.api.WebApiRequest;
import org.codelibs.fess.api.WebApiResponse;
import org.codelibs.fess.client.SearchClient;
import org.codelibs.fess.client.FessEsClient;
import org.codelibs.fess.db.allcommon.CDef;
import org.codelibs.fess.entity.PingResponse;
import org.codelibs.fess.util.ComponentUtil;
@ -99,11 +99,11 @@ public class JsonApiManager extends BaseApiManager implements WebApiManager {
}
protected void processPingRequest(final HttpServletRequest request, final HttpServletResponse response, final FilterChain chain) {
final SearchClient searchClient = ComponentUtil.getElasticsearchClient();
final FessEsClient fessEsClient = ComponentUtil.getElasticsearchClient();
int status;
String errMsg = null;
try {
final PingResponse pingResponse = searchClient.ping();
final PingResponse pingResponse = fessEsClient.ping();
status = pingResponse.getStatus();
} catch (final Exception e) {
status = 9;

6
src/main/java/org/codelibs/fess/api/xml/XmlApiManager.java
View file

@ -35,7 +35,7 @@ import org.codelibs.fess.api.BaseApiManager;
import org.codelibs.fess.api.WebApiManager;
import org.codelibs.fess.api.WebApiRequest;
import org.codelibs.fess.api.WebApiResponse;
import org.codelibs.fess.client.SearchClient;
import org.codelibs.fess.client.FessEsClient;
import org.codelibs.fess.db.allcommon.CDef;
import org.codelibs.fess.entity.PingResponse;
import org.codelibs.fess.util.ComponentUtil;
@ -87,11 +87,11 @@ public class XmlApiManager extends BaseApiManager implements WebApiManager {
}
protected void processPingRequest(final HttpServletRequest request, final HttpServletResponse response, final FilterChain chain) {
final SearchClient searchClient = ComponentUtil.getElasticsearchClient();
final FessEsClient fessEsClient = ComponentUtil.getElasticsearchClient();
int status;
String errMsg = null;
try {
final PingResponse pingResponse = searchClient.ping();
final PingResponse pingResponse = fessEsClient.ping();
status = pingResponse.getStatus();
} catch (final Exception e) {
status = 9;

22
src/main/java/org/codelibs/fess/beans/FessCopy.java
View file

@ -16,22 +16,13 @@
package org.codelibs.fess.beans;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import org.codelibs.fess.Constants;
import org.codelibs.fess.beans.converter.LocalDateTimeConverter;
import org.seasar.framework.beans.Converter;
import org.seasar.framework.beans.util.Copy;
import org.seasar.framework.util.TimestampConversionUtil;
public class FessCopy extends Copy {
protected static final Converter DEFAULT_LOCALDATETIME_CONVERTER = new LocalDateTimeConverter(TimestampConversionUtil.getPattern(Locale
.getDefault()));
public FessCopy(final Object src, final Object dest) {
super(src, dest);
}
@ -48,25 +39,16 @@ public class FessCopy extends Copy {
list.add("createdTime");
list.add("updatedBy");
list.add("updatedTime");
list.add("deletedBy");
list.add("deletedTime");
if (propertyNames.length > 0) {
for (final CharSequence propertyName : propertyNames) {
list.add(propertyName);
}
}
return super.excludes(list.toArray(new CharSequence[list.size()])).dateConverter(Constants.DEFAULT_DATETIME_FORMAT, "createdTime",
"updatedTime", "deletedTime");
return super.excludes(list.toArray(new CharSequence[list.size()]));
}
public Copy commonColumnDateConverter() {
return converter(new LocalDateTimeConverter(Constants.DEFAULT_DATETIME_FORMAT), "createdTime", "updatedTime", "deletedTime");
return this;
}
protected Converter findDefaultConverter(Class<?> clazz) {
if (LocalDateTime.class.isAssignableFrom(clazz)) {
return DEFAULT_LOCALDATETIME_CONVERTER;
}
return super.findDefaultConverter(clazz);
}
}

311
src/main/java/org/codelibs/fess/client/SearchClient.java → src/main/java/org/codelibs/fess/client/FessEsClient.java
View file

@ -10,6 +10,7 @@ import java.util.Optional;
import java.util.Set;
import org.apache.commons.codec.Charsets;
import org.codelibs.core.beans.util.BeanUtil;
import org.codelibs.core.io.FileUtil;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.elasticsearch.runner.ElasticsearchClusterRunner;
@ -28,19 +29,25 @@ import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.ShardOperationFailedException;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
import org.elasticsearch.action.admin.indices.flush.FlushResponse;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
import org.elasticsearch.action.admin.indices.optimize.OptimizeResponse;
import org.elasticsearch.action.admin.indices.refresh.RefreshResponse;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.delete.DeleteResponse;
import org.elasticsearch.action.index.IndexRequest.OpType;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.cluster.metadata.MappingMetaData;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.settings.ImmutableSettings.Builder;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.common.unit.TimeValue;
@ -48,6 +55,8 @@ import org.elasticsearch.index.query.BoolFilterBuilder;
import org.elasticsearch.index.query.FilterBuilders;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.indices.IndexAlreadyExistsException;
import org.elasticsearch.indices.IndexMissingException;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder;
@ -62,8 +71,8 @@ import org.slf4j.LoggerFactory;
import com.google.common.io.BaseEncoding;
public class SearchClient {
private static final Logger logger = LoggerFactory.getLogger(SearchClient.class);
public class FessEsClient {
private static final Logger logger = LoggerFactory.getLogger(FessEsClient.class);
protected ElasticsearchClusterRunner runner;
@ -71,34 +80,22 @@ public class SearchClient {
protected Client client;
protected String index = "fess";
protected String type = "doc";
protected String clusterName = "elasticsearch";
protected Map<String, String> settings;
protected String indexConfigPath = "fess_indices";
protected List<String> indexConfigList = new ArrayList<>();
public void addIndexConfig(String path) {
indexConfigList.add(path);
}
public void setSettings(Map<String, String> settings) {
this.settings = settings;
}
public String getIndex() {
return index;
}
public void setIndex(String index) {
this.index = index;
}
public String getType() {
return type;
}
public void setType(String type) {
this.type = type;
}
public String getClusterName() {
return clusterName;
}
@ -117,12 +114,12 @@ public class SearchClient {
@InitMethod
public void open() {
String transportAddressesValue = System.getProperty(Constants.FESS_ES_TRANSPORT_ADDRESSES);
final String transportAddressesValue = System.getProperty(Constants.FESS_ES_TRANSPORT_ADDRESSES);
if (StringUtil.isNotBlank(transportAddressesValue)) {
for (String transportAddressValue : transportAddressesValue.split(",")) {
String[] addressPair = transportAddressValue.trim().split(":");
for (final String transportAddressValue : transportAddressesValue.split(",")) {
final String[] addressPair = transportAddressValue.trim().split(":");
if (addressPair.length < 3) {
String host = addressPair[0];
final String host = addressPair[0];
int port = 9300;
if (addressPair.length == 2) {
port = Integer.parseInt(addressPair[1]);
@ -137,17 +134,14 @@ public class SearchClient {
if (transportAddressList.isEmpty()) {
if (runner == null) {
runner = new ElasticsearchClusterRunner();
Configs config = newConfigs().clusterName(clusterName).numOfNode(1);
String esDir = System.getProperty("fess.es.dir");
final Configs config = newConfigs().clusterName(clusterName).numOfNode(1);
final String esDir = System.getProperty("fess.es.dir");
if (esDir != null) {
config.basePath(esDir);
}
runner.onBuild(new ElasticsearchClusterRunner.Builder() {
@Override
public void build(final int number, final Builder settingsBuilder) {
if (settings != null) {
settingsBuilder.put(settings);
}
runner.onBuild((number, settingsBuilder) -> {
if (settings != null) {
settingsBuilder.put(settings);
}
});
runner.build(config);
@ -155,24 +149,24 @@ public class SearchClient {
client = runner.client();
addTransportAddress("localhost", runner.node().settings().getAsInt("transport.tcp.port", 9300));
} else {
Builder settingsBuilder = ImmutableSettings.settingsBuilder();
final Builder settingsBuilder = ImmutableSettings.settingsBuilder();
settingsBuilder.put("cluster.name", clusterName);
Settings settings = settingsBuilder.build();
TransportClient transportClient = new TransportClient(settings);
for (TransportAddress address : transportAddressList) {
final Settings settings = settingsBuilder.build();
final TransportClient transportClient = new TransportClient(settings);
for (final TransportAddress address : transportAddressList) {
transportClient.addTransportAddress(address);
}
client = transportClient;
}
if (StringUtil.isBlank(transportAddressesValue)) {
StringBuilder buf = new StringBuilder();
for (TransportAddress transportAddress : transportAddressList) {
final StringBuilder buf = new StringBuilder();
for (final TransportAddress transportAddress : transportAddressList) {
if (transportAddress instanceof InetSocketTransportAddress) {
if (buf.length() > 0) {
buf.append(',');
}
InetSocketTransportAddress inetTransportAddress = (InetSocketTransportAddress) transportAddress;
final InetSocketTransportAddress inetTransportAddress = (InetSocketTransportAddress) transportAddress;
buf.append(inetTransportAddress.address().getHostName());
buf.append(':');
buf.append(inetTransportAddress.address().getPort());
@ -185,27 +179,68 @@ public class SearchClient {
waitForYellowStatus();
if (!isIndexExists()) {
createIndex();
waitForYellowStatus();
}
}
indexConfigList.forEach(configName -> {
final String[] values = configName.split("/");
if (values.length == 2) {
final String configIndex = values[0];
final String configType = values[1];
boolean exists = false;
try {
client.prepareExists(configIndex).execute().actionGet();
exists = true;
} catch (final IndexMissingException e) {
// ignore
}
if (!exists) {
try {
String source = null;
final String indexConfigFile = indexConfigPath + "/" + configIndex + ".json";
try {
source = FileUtil.readUTF8(indexConfigFile);
} catch (final Exception e) {
logger.warn(indexConfigFile + " is not found.", e);
}
final CreateIndexResponse indexResponse =
client.admin().indices().prepareCreate(configIndex).setSource(source).execute().actionGet();
if (indexResponse.isAcknowledged()) {
logger.info("Created " + configIndex + " index.");
} else if (logger.isDebugEnabled()) {
logger.debug("Failed to create " + configIndex + " index.");
}
} catch (final IndexAlreadyExistsException e) {
// ignore
}
}
private void createIndex() {
String source = FileUtil.readText("json/index.json");
CreateIndexResponse response = client.admin().indices().prepareCreate(index).setSource(source).execute().actionGet();
if (!response.isAcknowledged()) {
logger.warn("Failed to create {0}.", index);
final GetMappingsResponse getMappingsResponse =
client.admin().indices().prepareGetMappings(configIndex).setTypes(configType).execute().actionGet();
final ImmutableOpenMap<String, MappingMetaData> indexMappings = getMappingsResponse.mappings().get(configIndex);
if (indexMappings == null || !indexMappings.containsKey(configType)) {
String source = null;
final String mappingFile = indexConfigPath + "/" + configIndex + "/" + configType + ".json";
try {
source = FileUtil.readUTF8(mappingFile);
} catch (final Exception e) {
logger.warn(mappingFile + " is not found.", e);
}
final PutMappingResponse putMappingResponse =
client.admin().indices().preparePutMapping(configIndex).setType(configType).setSource(source).execute().actionGet();
if (putMappingResponse.isAcknowledged()) {
logger.info("Created " + configIndex + "/" + configType + " mapping.");
} else {
logger.warn("Failed to create " + configIndex + "/" + configType + " mapping.");
}
} else if (logger.isDebugEnabled()) {
logger.debug(configIndex + "/" + configType + " mapping exists.");
}
} else {
logger.warn("Invalid index config name: " + configName);
}
}
private boolean isIndexExists() {
IndicesExistsResponse response = client.admin().indices().prepareExists(index).execute().actionGet();
return response.isExists();
}) ;
}
private void waitForYellowStatus() {
ClusterHealthResponse response = client.admin().cluster().prepareHealth().setWaitForYellowStatus().execute().actionGet();
final ClusterHealthResponse response = client.admin().cluster().prepareHealth().setWaitForYellowStatus().execute().actionGet();
if (logger.isDebugEnabled()) {
logger.debug("Elasticsearch Cluster Status: {0}", response.getStatus());
}
@ -215,35 +250,35 @@ public class SearchClient {
public void close() {
try {
client.close();
} catch (ElasticsearchException e) {
} catch (final ElasticsearchException e) {
logger.warn("Failed to close Client: " + client, e);
}
}
public void deleteByQuery(QueryBuilder queryBuilder) {
public void deleteByQuery(String index, String type, QueryBuilder queryBuilder) {
try {
// TODO replace with deleting bulk ids with scroll/scan
client.prepareDeleteByQuery(index).setQuery(queryBuilder).execute().actionGet().forEach(res -> {
ShardOperationFailedException[] failures = res.getFailures();
client.prepareDeleteByQuery(index).setQuery(queryBuilder).setTypes(type).execute().actionGet().forEach(res -> {
final ShardOperationFailedException[] failures = res.getFailures();
if (failures.length > 0) {
StringBuilder buf = new StringBuilder(200);
final StringBuilder buf = new StringBuilder(200);
buf.append("Failed to delete documents in some shards.");
for (ShardOperationFailedException failure : failures) {
for (final ShardOperationFailedException failure : failures) {
buf.append('\n').append(failure.toString());
}
throw new SearchException(buf.toString());
throw new FessEsClientException(buf.toString());
}
});
} catch (ElasticsearchException e) {
throw new SearchException("Failed to delete documents.", e);
} catch (final ElasticsearchException e) {
throw new FessEsClientException("Failed to delete documents.", e);
}
}
public <T> T search(SearchCondition condition, SearchResult<T> searchResult) {
public <T> T search(String index, String type, SearchCondition condition, SearchResult<T> searchResult) {
final long startTime = System.currentTimeMillis();
SearchResponse searchResponse = null;
SearchRequestBuilder searchRequestBuilder = client.prepareSearch(index);
final SearchRequestBuilder searchRequestBuilder = client.prepareSearch(index).setTypes(type);
if (condition.build(searchRequestBuilder)) {
if (ComponentUtil.getQueryHelper().getTimeAllowed() >= 0) {
@ -268,113 +303,125 @@ public class SearchClient {
return searchResult.build(searchRequestBuilder, execTime, Optional.ofNullable(searchResponse));
}
public Optional<Map<String, Object>> getDocument(final SearchCondition condition) {
return search(condition, (queryBuilder, execTime, searchResponse) -> {
public Optional<Map<String, Object>> getDocument(String index, String type, final SearchCondition condition) {
return getDocument(index, type, condition, (response, hit) -> {
return hit.getSource();
});
}
public <T> Optional<T> getDocument(String index, String type, final SearchCondition condition, EntityCreator<T> creator) {
return search(index, type, condition, (queryBuilder, execTime, searchResponse) -> {
return searchResponse.map(response -> {
SearchHit[] hits = response.getHits().hits();
final SearchHit[] hits = response.getHits().hits();
if (hits.length > 0) {
return hits[0].getSource();
return creator.build(response, hits[0]);
}
return null;
});
});
}
public Optional<List<Map<String, Object>>> getDocumentList(final SearchCondition condition) {
return search(condition, (searchRequestBuilder, execTime, searchResponse) -> {
List<Map<String, Object>> list = new ArrayList<>();
searchResponse.ifPresent(response -> {
response.getHits().forEach(hit -> {
list.add(hit.getSource());
});
});
return Optional.of(list);
public List<Map<String, Object>> getDocumentList(String index, String type, final SearchCondition condition) {
return getDocumentList(index, type, condition, (response, hit) -> {
return hit.getSource();
});
}
public boolean update(String id, String field, Object value) {
public <T> List<T> getDocumentList(String index, String type, final SearchCondition condition, EntityCreator<T> creator) {
return search(index, type, condition, (searchRequestBuilder, execTime, searchResponse) -> {
final List<T> list = new ArrayList<>();
searchResponse.ifPresent(response -> {
response.getHits().forEach(hit -> {
list.add(creator.build(response, hit));
});
});
return list;
});
}
public boolean update(String index, String type, String id, String field, Object value) {
try {
return client.prepareUpdate(index, type, id).setDoc(field, value).execute().actionGet().isCreated();
} catch (ElasticsearchException e) {
throw new SearchException("Failed to set " + value + " to " + field + " for doc " + id, e);
} catch (final ElasticsearchException e) {
throw new FessEsClientException("Failed to set " + value + " to " + field + " for doc " + id, e);
}
}
public void refresh() {
public void refresh(String index) {
client.admin().indices().prepareRefresh(index).execute(new ActionListener<RefreshResponse>() {
@Override
public void onResponse(RefreshResponse response) {
if (logger.isDebugEnabled()) {
logger.debug("Refreshed " + index + "/" + type + ".");
logger.debug("Refreshed " + index + ".");
}
}
@Override
public void onFailure(Throwable e) {
logger.error("Failed to refresh " + index + "/" + type + ".", e);
logger.error("Failed to refresh " + index + ".", e);
}
});
}
public void flush() {
public void flush(String index) {
client.admin().indices().prepareFlush(index).execute(new ActionListener<FlushResponse>() {
@Override
public void onResponse(FlushResponse response) {
if (logger.isDebugEnabled()) {
logger.debug("Flushed " + index + "/" + type + ".");
logger.debug("Flushed " + index + ".");
}
}
@Override
public void onFailure(Throwable e) {
logger.error("Failed to flush " + index + "/" + type + ".", e);
logger.error("Failed to flush " + index + ".", e);
}
});
}
public void optimize() {
public void optimize(String index) {
client.admin().indices().prepareOptimize(index).execute(new ActionListener<OptimizeResponse>() {
@Override
public void onResponse(OptimizeResponse response) {
if (logger.isDebugEnabled()) {
logger.debug("Optimzed " + index + "/" + type + ".");
logger.debug("Optimzed " + index + ".");
}
}
@Override
public void onFailure(Throwable e) {
logger.error("Failed to optimze " + index + "/" + type + ".", e);
logger.error("Failed to optimze " + index + ".", e);
}
});
}
public PingResponse ping() {
try {
ClusterHealthResponse response = client.admin().cluster().prepareHealth().execute().actionGet();
final ClusterHealthResponse response = client.admin().cluster().prepareHealth().execute().actionGet();
return new PingResponse(response);
} catch (ElasticsearchException e) {
throw new SearchException("Failed to process a ping request.", e);
} catch (final ElasticsearchException e) {
throw new FessEsClientException("Failed to process a ping request.", e);
}
}
public void addAll(List<Map<String, Object>> docList) {
BulkRequestBuilder bulkRequestBuilder = client.prepareBulk();
for (Map<String, Object> doc : docList) {
public void addAll(String index, String type, List<Map<String, Object>> docList) {
final BulkRequestBuilder bulkRequestBuilder = client.prepareBulk();
for (final Map<String, Object> doc : docList) {
bulkRequestBuilder.add(client.prepareIndex(index, type).setSource(doc));
}
BulkResponse response = bulkRequestBuilder.execute().actionGet();
String failureMessage = response.buildFailureMessage();
final BulkResponse response = bulkRequestBuilder.execute().actionGet();
final String failureMessage = response.buildFailureMessage();
if (StringUtil.isNotBlank(failureMessage)) {
throw new SearchException(failureMessage);
throw new FessEsClientException(failureMessage);
}
}
public static class SearchConditionBuilder {
private SearchRequestBuilder searchRequestBuilder;
private final SearchRequestBuilder searchRequestBuilder;
private String query;
private boolean administrativeAccess = false;
private String[] responseFields;
@ -447,7 +494,7 @@ public class SearchClient {
final SortField[] sortFields = searchQuery.getSortFields();
if (sortFields.length != 0) {
for (final SortField sortField : sortFields) {
FieldSortBuilder fieldSort = SortBuilders.fieldSort(sortField.getField());
final FieldSortBuilder fieldSort = SortBuilders.fieldSort(sortField.getField());
if (Constants.DESC.equals(sortField.getOrder())) {
fieldSort.order(SortOrder.DESC);
} else {
@ -457,7 +504,7 @@ public class SearchClient {
}
} else if (ComponentUtil.getQueryHelper().hasDefaultSortFields()) {
for (final SortField sortField : ComponentUtil.getQueryHelper().getDefaultSortFields()) {
FieldSortBuilder fieldSort = SortBuilders.fieldSort(sortField.getField());
final FieldSortBuilder fieldSort = SortBuilders.fieldSort(sortField.getField());
if (Constants.DESC.equals(sortField.getOrder())) {
fieldSort.order(SortOrder.DESC);
} else {
@ -479,8 +526,9 @@ public class SearchClient {
if (facetInfo.field != null) {
for (final String f : facetInfo.field) {
if (ComponentUtil.getQueryHelper().isFacetField(f)) {
String encodedField = BaseEncoding.base64().encode(f.getBytes(Charsets.UTF_8));
TermsBuilder termsBuilder = AggregationBuilders.terms(Constants.FACET_FIELD_PREFIX + encodedField).field(f);
final String encodedField = BaseEncoding.base64().encode(f.getBytes(Charsets.UTF_8));
final TermsBuilder termsBuilder =
AggregationBuilders.terms(Constants.FACET_FIELD_PREFIX + encodedField).field(f);
// TODO order
if (facetInfo.limit != null) {
// TODO
@ -497,7 +545,7 @@ public class SearchClient {
final String facetQuery = ComponentUtil.getQueryHelper().buildFacetQuery(fq);
if (StringUtil.isNotBlank(facetQuery)) {
final String encodedFacetQuery = BaseEncoding.base64().encode(facetQuery.getBytes(Charsets.UTF_8));
FilterAggregationBuilder filterBuilder =
final FilterAggregationBuilder filterBuilder =
AggregationBuilders.filter(Constants.FACET_QUERY_PREFIX + encodedFacetQuery).filter(
FilterBuilders.queryFilter(QueryBuilders.queryStringQuery(facetQuery)));
// TODO order
@ -522,7 +570,7 @@ public class SearchClient {
if (boolFilterBuilder == null) {
boolFilterBuilder = FilterBuilders.boolFilter();
}
for (String filterQuery : searchQuery.getFilterQueries()) {
for (final String filterQuery : searchQuery.getFilterQueries()) {
boolFilterBuilder.must(FilterBuilders.queryFilter(QueryBuilders.queryStringQuery(filterQuery)));
}
}
@ -544,6 +592,42 @@ public class SearchClient {
}
}
public boolean store(String index, String type, Object obj) {
final Map<String, Object> source = BeanUtil.copyBeanToNewMap(obj);
final String id = (String) source.remove("id");
final Long version = (Long) source.remove("version");
IndexResponse response;
try {
if (id == null) {
// create
response =
client.prepareIndex(index, type).setSource(source).setRefresh(true).setOpType(OpType.CREATE).execute().actionGet();
} else {
// update
response =
client.prepareIndex(index, type, id).setSource(source).setRefresh(true).setOpType(OpType.INDEX).setVersion(version)
.execute().actionGet();
}
return response.isCreated();
} catch (final ElasticsearchException e) {
throw new FessEsClientException("Failed to store: " + obj, e);
}
}
public boolean delete(String index, String type, String id, long version) {
try {
final DeleteResponse response =
client.prepareDelete(index, type, id).setVersion(version).setRefresh(true).execute().actionGet();
return response.isFound();
} catch (final ElasticsearchException e) {
throw new FessEsClientException("Failed to delete: " + index + "/" + type + "/" + id + "/" + version, e);
}
}
public void setIndexConfigPath(String indexConfigPath) {
this.indexConfigPath = indexConfigPath;
}
public interface SearchCondition {
boolean build(SearchRequestBuilder searchRequestBuilder);
}
@ -551,4 +635,9 @@ public class SearchClient {
public interface SearchResult<T> {
T build(SearchRequestBuilder searchRequestBuilder, long execTime, Optional<SearchResponse> searchResponse);
}
public interface EntityCreator<T> {
T build(SearchResponse response, SearchHit hit);
}
}

6
src/main/java/org/codelibs/fess/client/SearchException.java → src/main/java/org/codelibs/fess/client/FessEsClientException.java
View file

@ -2,15 +2,15 @@ package org.codelibs.fess.client;
import org.codelibs.fess.FessSystemException;
public class SearchException extends FessSystemException {
public class FessEsClientException extends FessSystemException {
private static final long serialVersionUID = 1L;
public SearchException(String message, Throwable cause) {
public FessEsClientException(String message, Throwable cause) {
super(message, cause);
}
public SearchException(String message) {
public FessEsClientException(String message) {
super(message);
}

7
src/main/java/org/codelibs/fess/db/exentity/ScheduledJob.java
View file

@ -52,11 +52,6 @@ public class ScheduledJob extends BsScheduledJob {
public void start() {
final ScheduledJob scheduledJob = this;
new Thread(new Runnable() {
@Override
public void run() {
new TriggeredJob().execute(scheduledJob);
}
}).start();
new Thread(() -> new TriggeredJob().execute(scheduledJob)).start();
}
}

9
src/main/java/org/codelibs/fess/dict/synonym/SynonymLocator.java
View file

@ -19,7 +19,6 @@ package org.codelibs.fess.dict.synonym;
import java.io.File;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import org.codelibs.fess.dict.DictionaryFile;
@ -50,13 +49,7 @@ public class SynonymLocator extends DictionaryLocator {
fileList.add(new SynonymFile(file));
}
}
Collections.sort(fileList, new Comparator<DictionaryFile<? extends DictionaryItem>>() {
@Override
public int compare(final DictionaryFile<? extends DictionaryItem> o1, final DictionaryFile<? extends DictionaryItem> o2) {
return o1.getName().compareTo(o2.getName());
}
});
Collections.sort(fileList, (o1, o2) -> o1.getName().compareTo(o2.getName()));
return fileList;
}
}

9
src/main/java/org/codelibs/fess/dict/userdict/UserDictLocator.java
View file

@ -19,7 +19,6 @@ package org.codelibs.fess.dict.userdict;
import java.io.File;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import org.codelibs.fess.dict.DictionaryFile;
@ -50,13 +49,7 @@ public class UserDictLocator extends DictionaryLocator {
fileList.add(new UserDictFile(file));
}
}
Collections.sort(fileList, new Comparator<DictionaryFile<? extends DictionaryItem>>() {
@Override
public int compare(final DictionaryFile<? extends DictionaryItem> o1, final DictionaryFile<? extends DictionaryItem> o2) {
return o1.getName().compareTo(o2.getName());
}
});
Collections.sort(fileList, (o1, o2) -> o1.getName().compareTo(o2.getName()));
return fileList;
}
}

6
src/main/java/org/codelibs/fess/ds/IndexUpdateCallback.java
View file

@ -18,15 +18,15 @@ package org.codelibs.fess.ds;
import java.util.Map;
import org.codelibs.fess.client.SearchClient;
import org.codelibs.fess.client.FessEsClient;
public interface IndexUpdateCallback {
boolean store(Map<String, Object> dataMap);
void setElasticsearchClient(SearchClient solrGroup);
void setEsClient(FessEsClient fessEsClient);
SearchClient getElasticsearchClient();
FessEsClient getsClient();
long getDocumentSize();

7
src/main/java/org/codelibs/fess/ds/impl/CsvDataStoreImpl.java
View file

@ -95,12 +95,7 @@ public class CsvDataStoreImpl extends AbstractDataStoreImpl {
for (final String path : values) {
final File dir = new File(path);
if (dir.isDirectory()) {
final File[] files = dir.listFiles(new FilenameFilter() {
@Override
public boolean accept(final File file, final String name) {
return isCsvFile(file, name);
}
});
final File[] files = dir.listFiles((FilenameFilter) (file, name) -> isCsvFile(file, name));
for (final File file : files) {
fileList.add(file);
}

22
src/main/java/org/codelibs/fess/ds/impl/FileListDataStoreImpl.java
View file

@ -23,7 +23,7 @@ import java.util.List;
import java.util.Map;
import org.codelibs.fess.Constants;
import org.codelibs.fess.client.SearchClient;
import org.codelibs.fess.client.FessEsClient;
import org.codelibs.fess.db.exentity.DataCrawlingConfig;
import org.codelibs.fess.ds.DataStoreCrawlingException;
import org.codelibs.fess.ds.DataStoreException;
@ -103,7 +103,9 @@ public class FileListDataStoreImpl extends CsvDataStoreImpl {
protected void storeData(final DataCrawlingConfig dataConfig, final IndexUpdateCallback callback, final Map<String, String> paramMap,
final Map<String, String> scriptMap, final Map<String, Object> defaultDataMap) {
super.storeData(dataConfig, new FileListIndexUpdateCallback(callback), paramMap, scriptMap, defaultDataMap);
final FileListIndexUpdateCallback fileListIndexUpdateCallback = new FileListIndexUpdateCallback(callback);
super.storeData(dataConfig, fileListIndexUpdateCallback, paramMap, scriptMap, defaultDataMap);
fileListIndexUpdateCallback.commit();
}
@Override
@ -160,7 +162,7 @@ public class FileListDataStoreImpl extends CsvDataStoreImpl {
protected boolean addDocument(final Map<String, Object> dataMap) {
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
synchronized (indexUpdateCallback) {
// required check
// required check
if (!dataMap.containsKey(fieldHelper.urlField) || dataMap.get(fieldHelper.urlField) == null) {
logger.warn("Could not add a doc. Invalid data: " + dataMap);
return false;
@ -228,7 +230,7 @@ public class FileListDataStoreImpl extends CsvDataStoreImpl {
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
// required check
// required check
if (!dataMap.containsKey(fieldHelper.urlField) || dataMap.get(fieldHelper.urlField) == null) {
logger.warn("Could not delete a doc. Invalid data: " + dataMap);
return false;
@ -240,7 +242,7 @@ public class FileListDataStoreImpl extends CsvDataStoreImpl {
if (deleteIdList.size() >= maxDeleteDocumentCacheSize) {
final IndexingHelper indexingHelper = ComponentUtil.getIndexingHelper();
for (final String id : deleteIdList) {
indexingHelper.deleteDocument(indexUpdateCallback.getElasticsearchClient(), id);
indexingHelper.deleteDocument(indexUpdateCallback.getsClient(), id);
}
if (logger.isDebugEnabled()) {
logger.debug("Deleted " + deleteIdList);
@ -257,7 +259,7 @@ public class FileListDataStoreImpl extends CsvDataStoreImpl {
if (!deleteIdList.isEmpty()) {
final IndexingHelper indexingHelper = ComponentUtil.getIndexingHelper();
for (final String id : deleteIdList) {
indexingHelper.deleteDocument(indexUpdateCallback.getElasticsearchClient(), id);
indexingHelper.deleteDocument(indexUpdateCallback.getsClient(), id);
}
if (logger.isDebugEnabled()) {
logger.debug("Deleted " + deleteIdList);
@ -267,8 +269,8 @@ public class FileListDataStoreImpl extends CsvDataStoreImpl {
}
@Override
public void setElasticsearchClient(final SearchClient searchClient) {
indexUpdateCallback.setElasticsearchClient(searchClient);
public void setEsClient(final FessEsClient fessEsClient) {
indexUpdateCallback.setEsClient(fessEsClient);
}
@Override
@ -282,8 +284,8 @@ public class FileListDataStoreImpl extends CsvDataStoreImpl {
}
@Override
public SearchClient getElasticsearchClient() {
return indexUpdateCallback.getElasticsearchClient();
public FessEsClient getsClient() {
return indexUpdateCallback.getsClient();
}
}
}

18
src/main/java/org/codelibs/fess/ds/impl/IndexUpdateCallbackImpl.java
View file

@ -22,7 +22,7 @@ import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
import org.codelibs.fess.FessSystemException;
import org.codelibs.fess.client.SearchClient;
import org.codelibs.fess.client.FessEsClient;
import org.codelibs.fess.ds.IndexUpdateCallback;
import org.codelibs.fess.helper.CrawlingSessionHelper;
import org.codelibs.fess.helper.FieldHelper;
@ -37,7 +37,7 @@ import org.slf4j.LoggerFactory;
public class IndexUpdateCallbackImpl implements IndexUpdateCallback {
private static final Logger logger = LoggerFactory.getLogger(IndexUpdateCallbackImpl.class);
protected SearchClient searchClient;
protected FessEsClient fessEsClient;
public int maxDocumentCacheSize = 5;
@ -87,12 +87,12 @@ public class IndexUpdateCallbackImpl implements IndexUpdateCallback {
}
if (docList.size() >= maxDocumentCacheSize) {
indexingHelper.sendDocuments(searchClient, docList);
indexingHelper.sendDocuments(fessEsClient, docList);
}
documentSize.getAndIncrement();
if (!docList.isEmpty()) {
indexingHelper.sendDocuments(searchClient, docList);
indexingHelper.sendDocuments(fessEsClient, docList);
}
if (logger.isDebugEnabled()) {
logger.debug("The number of an added document is " + documentSize.get() + ".");
@ -124,7 +124,7 @@ public class IndexUpdateCallbackImpl implements IndexUpdateCallback {
public void commit() {
if (!docList.isEmpty()) {
final IndexingHelper indexingHelper = ComponentUtil.getIndexingHelper();
indexingHelper.sendDocuments(searchClient, docList);
indexingHelper.sendDocuments(fessEsClient, docList);
}
}
@ -157,13 +157,13 @@ public class IndexUpdateCallbackImpl implements IndexUpdateCallback {
}
@Override
public SearchClient getElasticsearchClient() {
return searchClient;
public FessEsClient getsClient() {
return fessEsClient;
}
@Override
public void setElasticsearchClient(final SearchClient searchClient) {
this.searchClient = searchClient;
public void setEsClient(final FessEsClient fessEsClient) {
this.fessEsClient = fessEsClient;
}
}

26
src/main/java/org/codelibs/fess/exec/Crawler.java
View file

@ -37,7 +37,7 @@ import org.codelibs.core.CoreLibConstants;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.core.misc.DynamicProperties;
import org.codelibs.fess.Constants;
import org.codelibs.fess.client.SearchClient;
import org.codelibs.fess.client.FessEsClient;
import org.codelibs.fess.db.allcommon.CDef;
import org.codelibs.fess.helper.CrawlingSessionHelper;
import org.codelibs.fess.helper.DataIndexHelper;
@ -88,7 +88,7 @@ public class Crawler implements Serializable {
private static final String MAIL_TEMPLATE_NAME = "crawler";
@Resource
protected SearchClient searchClient;
protected FessEsClient fessEsClient;
@Binding(bindingType = BindingType.MAY)
@Resource
@ -388,28 +388,22 @@ public class Crawler implements Serializable {
Thread dataCrawlerThread = null;
if (runAll || webConfigIdList != null || fileConfigIdList != null) {
webFsCrawlerThread = new Thread(new Runnable() {
@Override
public void run() {
// crawl web
webFsCrawlerThread = new Thread((Runnable) () -> {
// crawl web
writeTimeToSessionInfo(crawlingSessionHelper, Constants.WEB_FS_CRAWLER_START_TIME);
webFsIndexHelper.crawl(options.sessionId, webConfigIdList, fileConfigIdList);
writeTimeToSessionInfo(crawlingSessionHelper, Constants.WEB_FS_CRAWLER_END_TIME);
}
}, WEB_FS_CRAWLING_PROCESS);
}, WEB_FS_CRAWLING_PROCESS);
webFsCrawlerThread.start();
}
if (runAll || dataConfigIdList != null) {
dataCrawlerThread = new Thread(new Runnable() {
@Override
public void run() {
// crawl data system
dataCrawlerThread = new Thread((Runnable) () -> {
// crawl data system
writeTimeToSessionInfo(crawlingSessionHelper, Constants.DATA_CRAWLER_START_TIME);
dataIndexHelper.crawl(options.sessionId, dataConfigIdList);
writeTimeToSessionInfo(crawlingSessionHelper, Constants.DATA_CRAWLER_END_TIME);
}
}, DATA_CRAWLING_PROCESS);
}, DATA_CRAWLING_PROCESS);
dataCrawlerThread.start();
}
@ -417,11 +411,11 @@ public class Crawler implements Serializable {
joinCrawlerThread(dataCrawlerThread);
// clean up
QueryBuilder queryBuilder =
final QueryBuilder queryBuilder =
QueryBuilders.boolQuery().must(QueryBuilders.rangeQuery(fieldHelper.expiresField).to(new Date()))
.mustNot(QueryBuilders.termQuery(fieldHelper.segmentField, options.sessionId));
try {
searchClient.deleteByQuery(queryBuilder);
fessEsClient.deleteByQuery(fieldHelper.docIndex, fieldHelper.docType, queryBuilder);
} catch (final Exception e) {
if (logger.isWarnEnabled()) {
logger.warn("Could not delete expired sessions: " + queryBuilder.toString(), e);

14
src/main/java/org/codelibs/fess/helper/CrawlingSessionHelper.java
View file

@ -28,7 +28,7 @@ import java.util.Map;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.fess.Constants;
import org.codelibs.fess.FessSystemException;
import org.codelibs.fess.client.SearchClient;
import org.codelibs.fess.client.FessEsClient;
import org.codelibs.fess.db.exentity.CrawlingSession;
import org.codelibs.fess.db.exentity.CrawlingSessionInfo;
import org.codelibs.fess.service.CrawlingSessionService;
@ -54,7 +54,7 @@ public class CrawlingSessionHelper implements Serializable {
protected LocalDateTime documentExpires;
private int maxSessionIdsInList = 100;
private final int maxSessionIdsInList = 100;
protected CrawlingSessionService getCrawlingSessionService() {
return SingletonS2Container.getComponent(CrawlingSessionService.class);
@ -151,12 +151,12 @@ public class CrawlingSessionHelper implements Serializable {
return generateId(url, roleTypeList);
}
public List<Map<String, String>> getSessionIdList(final SearchClient searchClient) {
public List<Map<String, String>> getSessionIdList(final FessEsClient fessEsClient) {
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
return searchClient.search(
return fessEsClient.search(fieldHelper.docIndex, fieldHelper.docType,
queryRequestBuilder -> {
queryRequestBuilder.setQuery(QueryBuilders.matchAllQuery());
TermsBuilder termsBuilder =
final TermsBuilder termsBuilder =
AggregationBuilders.terms(fieldHelper.segmentField).field(fieldHelper.segmentField).size(maxSessionIdsInList)
.order(Order.term(false));
queryRequestBuilder.addAggregation(termsBuilder);
@ -164,8 +164,8 @@ public class CrawlingSessionHelper implements Serializable {
}, (queryRequestBuilder, execTime, searchResponse) -> {
final List<Map<String, String>> sessionIdList = new ArrayList<Map<String, String>>();
searchResponse.ifPresent(response -> {
Terms terms = response.getAggregations().get(fieldHelper.segmentField);
for (Bucket bucket : terms.getBuckets()) {
final Terms terms = response.getAggregations().get(fieldHelper.segmentField);
for (final Bucket bucket : terms.getBuckets()) {
final Map<String, String> map = new HashMap<String, String>(2);
map.put(fieldHelper.segmentField, bucket.getKey());
map.put(FACET_COUNT_KEY, Long.toString(bucket.getDocCount()));

6
src/main/java/org/codelibs/fess/helper/DataIndexHelper.java
View file

@ -93,7 +93,7 @@ public class DataIndexHelper implements Serializable {
protected void doCrawl(final String sessionId, final List<DataCrawlingConfig> configList) {
int multiprocessCrawlingCount = 5;
String value = crawlerProperties.getProperty(Constants.CRAWLING_THREAD_COUNT_PROPERTY, "5");
final String value = crawlerProperties.getProperty(Constants.CRAWLING_THREAD_COUNT_PROPERTY, "5");
try {
multiprocessCrawlingCount = Integer.parseInt(value);
} catch (final NumberFormatException e) {
@ -254,11 +254,11 @@ public class DataIndexHelper implements Serializable {
return;
}
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
QueryBuilder queryBuilder =
final QueryBuilder queryBuilder =
QueryBuilders.boolQuery().must(QueryBuilders.termQuery(fieldHelper.configIdField, dataCrawlingConfig.getConfigId()))
.mustNot(QueryBuilders.termQuery(fieldHelper.segmentField, sessionId));
try {
ComponentUtil.getElasticsearchClient().deleteByQuery(queryBuilder);
ComponentUtil.getElasticsearchClient().deleteByQuery(fieldHelper.docIndex, fieldHelper.docType, queryBuilder);
} catch (final Exception e) {
logger.error("Could not delete old docs at " + dataCrawlingConfig, e);
}

36
src/main/java/org/codelibs/fess/helper/DocumentHelper.java
View file

@ -1,36 +0,0 @@
/*
* Copyright 2009-2015 the CodeLibs Project and the Others.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
* either express or implied. See the License for the specific language
* governing permissions and limitations under the License.
*/
package org.codelibs.fess.helper;
import javax.annotation.Resource;
import org.codelibs.fess.client.SearchClient;
//TODO needed?
public class DocumentHelper {
@Resource
protected FieldHelper fieldHelper;
@Resource
protected SearchClient searchClient;
public void update(final String docId, final String fieldName, final long num) {
searchClient.update(docId, fieldName, num);
}
}

9
src/main/java/org/codelibs/fess/helper/FieldHelper.java
View file

@ -16,6 +16,7 @@
package org.codelibs.fess.helper;
// TODO move to properties file
public class FieldHelper {
public String favoriteCountField = "favorite_count";
@ -69,4 +70,12 @@ public class FieldHelper {
public String filetypeField = "filetype";
public String docIndex = "fess";
public String docType = "doc";
public String configIndex = ".fess_config";
public String boostDocumentRuleType = "boost_document_rule";
}

431
src/main/java/org/codelibs/fess/helper/IndexingHelper.java
View file

@ -19,7 +19,8 @@ package org.codelibs.fess.helper;
import java.util.List;
import java.util.Map;
import org.codelibs.fess.client.SearchClient;
import org.codelibs.fess.client.FessEsClient;
import org.codelibs.fess.util.ComponentUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -32,14 +33,15 @@ public class IndexingHelper {
public long requestInterval = 500;
public void sendDocuments(final SearchClient searchClient, final List<Map<String, Object>> docList) {
public void sendDocuments(final FessEsClient fessEsClient, final List<Map<String, Object>> docList) {
final long execTime = System.currentTimeMillis();
if (logger.isDebugEnabled()) {
logger.debug("Sending " + docList.size() + " documents to a server.");
}
synchronized (searchClient) {
deleteOldDocuments(searchClient, docList);
searchClient.addAll(docList);
synchronized (fessEsClient) {
deleteOldDocuments(fessEsClient, docList);
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
fessEsClient.addAll(fieldHelper.docIndex, fieldHelper.docType, docList);
}
if (logger.isInfoEnabled()) {
logger.info("Sent " + docList.size() + " docs (Solr: " + (System.currentTimeMillis() - execTime) + "ms)");
@ -47,238 +49,253 @@ public class IndexingHelper {
docList.clear();
}
private void deleteOldDocuments(final SearchClient searchClient, final List<Map<String, Object>> docList) {
// TODO move searchClient
// final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
private void deleteOldDocuments(final FessEsClient fessEsClient, final List<Map<String, Object>> docList) {
// TODO move fessEsClient
// final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
//
// final List<String> docIdList = new ArrayList<String>();
// final StringBuilder q = new StringBuilder(1000);
// final StringBuilder fq = new StringBuilder(100);
// for (final Map<String, Object> inputDoc : docList) {
// final Object idValue = inputDoc.get(fieldHelper.idField);
// if (idValue == null) {
// continue;
// }
// final List<String> docIdList = new ArrayList<String>();
// final StringBuilder q = new StringBuilder(1000);
// final StringBuilder fq = new StringBuilder(100);
// for (final Map<String, Object> inputDoc : docList) {
// final Object idValue = inputDoc.get(fieldHelper.idField);
// if (idValue == null) {
// continue;
// }
//
// final Object configIdValue = inputDoc.get(fieldHelper.configIdField);
// if (configIdValue == null) {
// continue;
// }
// final Object configIdValue = inputDoc.get(fieldHelper.configIdField);
// if (configIdValue == null) {
// continue;
// }
//
// q.setLength(0);
// q.append(fieldHelper.urlField).append(":\"");
// q.append(QueryUtil.escapeValue((String) inputDoc.get(fieldHelper.urlField)));
// q.append('"');
// q.setLength(0);
// q.append(fieldHelper.urlField).append(":\"");
// q.append(QueryUtil.escapeValue((String)
// inputDoc.get(fieldHelper.urlField)));
// q.append('"');
//
// fq.setLength(0);
// fq.append(fieldHelper.configIdField).append(':');
// fq.append(configIdValue.toString());
// fq.setLength(0);
// fq.append(fieldHelper.configIdField).append(':');
// fq.append(configIdValue.toString());
//
// final List<Map<String, Object>> docs =
// getSolrDocumentList(searchClient, fq.toString(), q.toString(), new String[] { fieldHelper.idField,
// fieldHelper.docIdField });
// for (final SolrDocument doc : docs) {
// final Object oldIdValue = doc.getFieldValue(fieldHelper.idField);
// if (!idValue.equals(oldIdValue) && oldIdValue != null) {
// final Object oldDocIdValue = doc.getFieldValue(fieldHelper.docIdField);
// if (oldDocIdValue != null) {
// docIdList.add(oldDocIdValue.toString());
// }
// }
// }
// if (logger.isDebugEnabled()) {
// logger.debug(q + " in " + fq + " => " + docs);
// }
// }
// if (!docIdList.isEmpty()) {
// deleteDocumentsByDocId(searchClient, docIdList);
// }
// final List<Map<String, Object>> docs =
// getSolrDocumentList(fessEsClient, fq.toString(), q.toString(), new
// String[] { fieldHelper.idField,
// fieldHelper.docIdField });
// for (final SolrDocument doc : docs) {
// final Object oldIdValue = doc.getFieldValue(fieldHelper.idField);
// if (!idValue.equals(oldIdValue) && oldIdValue != null) {
// final Object oldDocIdValue =
// doc.getFieldValue(fieldHelper.docIdField);
// if (oldDocIdValue != null) {
// docIdList.add(oldDocIdValue.toString());
// }
// }
// }
// if (logger.isDebugEnabled()) {
// logger.debug(q + " in " + fq + " => " + docs);
// }
// }
// if (!docIdList.isEmpty()) {
// deleteDocumentsByDocId(fessEsClient, docIdList);
// }
}
public List<Map<String, Object>> getSolrDocumentList(final SearchClient searchClient, final String fq, final String q,
public List<Map<String, Object>> getSolrDocumentList(final FessEsClient fessEsClient, final String fq, final String q,
final String[] fields) {
return getSolrDocumentList(searchClient, fq, q, fields, defaultRowSize);
return getSolrDocumentList(fessEsClient, fq, q, fields, defaultRowSize);
}
protected List<Map<String, Object>> getSolrDocumentList(final SearchClient searchClient, final String fq, final String q,
protected List<Map<String, Object>> getSolrDocumentList(final FessEsClient fessEsClient, final String fq, final String q,
final String[] fields, final int row) {
// TODO move searchClient
// final SolrQuery sq = new SolrQuery();
// if (fq != null) {
// sq.setFilterQueries(fq);
// }
// sq.setQuery(q);
// if (fields != null) {
// sq.setFields(fields);
// }
// sq.setRows(row);
// e.
// final List<Map<String,Object>> docList = searchClient.query(sq).getResults();
// if (docList.getNumFound() <= row) {
// return docList;
// }
// return getSolrDocumentList(searchClient, fq, q, fields, (int) docList.getNumFound());
// TODO move fessEsClient
// final SolrQuery sq = new SolrQuery();
// if (fq != null) {
// sq.setFilterQueries(fq);
// }
// sq.setQuery(q);
// if (fields != null) {
// sq.setFields(fields);
// }
// sq.setRows(row);
// e.
// final List<Map<String,Object>> docList =
// fessEsClient.query(sq).getResults();
// if (docList.getNumFound() <= row) {
// return docList;
// }
// return getSolrDocumentList(fessEsClient, fq, q, fields, (int)
// docList.getNumFound());
return null;
}
public void deleteDocument(final SearchClient searchClient, final String id) {
// TODO move searchClient
// final long start = System.currentTimeMillis();
// final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
// final String query = "{!raw f=" + fieldHelper.idField + "}" + id;
// for (int i = 0; i < maxRetryCount; i++) {
// boolean done = true;
// try {
// for (final UpdateResponse response : searchClient.deleteByQuery(query)) {
// if (response.getStatus() != 0) {
// logger.warn("Failed to delete: " + response);
// done = false;
// }
// }
// } catch (final Exception e) {
// logger.warn("Could not delete a document from Solr." + " It might be busy. " + "Retrying.. id:" + id + ", cause: "
// + e.getMessage());
// done = false;
// }
// if (done) {
// if (logger.isDebugEnabled()) {
// logger.info("Deleted 1 doc (Solr: " + (System.currentTimeMillis() - start) + "ms) => id:" + id);
// } else {
// logger.info("Deleted 1 doc (Solr: " + (System.currentTimeMillis() - start) + "ms)");
// }
// return;
// }
// try {
// Thread.sleep(requestInterval);
// } catch (final InterruptedException e) {}
// }
public void deleteDocument(final FessEsClient fessEsClient, final String id) {
// TODO move fessEsClient
// final long start = System.currentTimeMillis();
// final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
// final String query = "{!raw f=" + fieldHelper.idField + "}" + id;
// for (int i = 0; i < maxRetryCount; i++) {
// boolean done = true;
// try {
// for (final UpdateResponse response : fessEsClient.deleteByQuery(query)) {
// if (response.getStatus() != 0) {
// logger.warn("Failed to delete: " + response);
// done = false;
// }
// }
// } catch (final Exception e) {
// logger.warn("Could not delete a document from Solr." +
// " It might be busy. " + "Retrying.. id:" + id + ", cause: "
// + e.getMessage());
// done = false;
// }
// if (done) {
// if (logger.isDebugEnabled()) {
// logger.info("Deleted 1 doc (Solr: " + (System.currentTimeMillis() -
// start) + "ms) => id:" + id);
// } else {
// logger.info("Deleted 1 doc (Solr: " + (System.currentTimeMillis() -
// start) + "ms)");
// }
// return;
// }
// try {
// Thread.sleep(requestInterval);
// } catch (final InterruptedException e) {}
// }
}
public void deleteDocumentsByDocId(final SearchClient searchClient, final List<String> docIdList) {
// TODO move searchClient
// final long start = System.currentTimeMillis();
// final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
// final StringBuilder buf = new StringBuilder(500);
// for (final String docId : docIdList) {
// if (buf.length() != 0) {
// buf.append(" OR ");
// }
// buf.append(fieldHelper.docIdField).append(':').append(docId);
// }
// final String query = buf.toString();
// for (int i = 0; i < maxRetryCount; i++) {
// boolean done = true;
// try {
// for (final UpdateResponse response : searchClient.deleteByQuery(query)) {
// if (response.getStatus() != 0) {
// logger.warn("Failed to delete: " + response);
// done = false;
// }
// }
// } catch (final Exception e) {
// logger.warn("Could not delete a document from Solr." + " It might be busy. " + "Retrying.. id:" + docIdList + ", cause: "
// + e.getMessage());
// done = false;
// }
// if (done) {
// if (logger.isDebugEnabled()) {
// logger.info("Deleted " + docIdList.size() + " docs (Solr: " + (System.currentTimeMillis() - start) + "ms) => docId:"
// + docIdList);
// } else {
// logger.info("Deleted " + docIdList.size() + " docs (Solr: " + (System.currentTimeMillis() - start) + "ms)");
// }
// return;
// }
// try {
// Thread.sleep(requestInterval);
// } catch (final InterruptedException e) {}
// }
public void deleteDocumentsByDocId(final FessEsClient fessEsClient, final List<String> docIdList) {
// TODO move fessEsClient
// final long start = System.currentTimeMillis();
// final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
// final StringBuilder buf = new StringBuilder(500);
// for (final String docId : docIdList) {
// if (buf.length() != 0) {
// buf.append(" OR ");
// }
// buf.append(fieldHelper.docIdField).append(':').append(docId);
// }
// final String query = buf.toString();
// for (int i = 0; i < maxRetryCount; i++) {
// boolean done = true;
// try {
// for (final UpdateResponse response : fessEsClient.deleteByQuery(query)) {
// if (response.getStatus() != 0) {
// logger.warn("Failed to delete: " + response);
// done = false;
// }
// }
// } catch (final Exception e) {
// logger.warn("Could not delete a document from Solr." +
// " It might be busy. " + "Retrying.. id:" + docIdList + ", cause: "
// + e.getMessage());
// done = false;
// }
// if (done) {
// if (logger.isDebugEnabled()) {
// logger.info("Deleted " + docIdList.size() + " docs (Solr: " +
// (System.currentTimeMillis() - start) + "ms) => docId:"
// + docIdList);
// } else {
// logger.info("Deleted " + docIdList.size() + " docs (Solr: " +
// (System.currentTimeMillis() - start) + "ms)");
// }
// return;
// }
// try {
// Thread.sleep(requestInterval);
// } catch (final InterruptedException e) {}
// }
}
public Map<String, Object> getSolrDocument(final SearchClient searchClient, final String id, final String[] fields) {
// TODO move searchClient
// final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
public Map<String, Object> getSolrDocument(final FessEsClient fessEsClient, final String id, final String[] fields) {
// TODO move fessEsClient
// final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
//
// final SolrQuery solrQuery = new SolrQuery();
// final StringBuilder queryBuf = new StringBuilder(500);
// queryBuf.append("{!raw f=").append(fieldHelper.idField).append("}");
// queryBuf.append(id);
// solrQuery.setQuery(queryBuf.toString());
// if (fields != null) {
// solrQuery.setFields(fields);
// }
// final QueryResponse response = searchClient.query(solrQuery);
// final List<Map<String, Object>> docList = response.getResults();
// if (docList.isEmpty()) {
// return null;
// }
// if (docList.size() > 1) {
// logger.error("Invalid multiple docs for " + id);
// for (final SolrDocument doc : docList) {
// final Object idValue = doc.getFieldValue(fieldHelper.idField);
// if (idValue != null) {
// deleteDocument(searchClient, idValue.toString());
// }
// }
// return null;
// }
// return docList.get(0);
// final SolrQuery solrQuery = new SolrQuery();
// final StringBuilder queryBuf = new StringBuilder(500);
// queryBuf.append("{!raw f=").append(fieldHelper.idField).append("}");
// queryBuf.append(id);
// solrQuery.setQuery(queryBuf.toString());
// if (fields != null) {
// solrQuery.setFields(fields);
// }
// final QueryResponse response = fessEsClient.query(solrQuery);
// final List<Map<String, Object>> docList = response.getResults();
// if (docList.isEmpty()) {
// return null;
// }
// if (docList.size() > 1) {
// logger.error("Invalid multiple docs for " + id);
// for (final SolrDocument doc : docList) {
// final Object idValue = doc.getFieldValue(fieldHelper.idField);
// if (idValue != null) {
// deleteDocument(fessEsClient, idValue.toString());
// }
// }
// return null;
// }
// return docList.get(0);
return null;
}
public List<Map<String, Object>> getSolrDocumentListByPrefixId(final SearchClient searchClient, final String id, final String[] fields) {
// TODO move searchClient
// final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
// final SolrQuery solrQuery = new SolrQuery();
// final StringBuilder queryBuf = new StringBuilder(500);
// queryBuf.append("{!prefix f=").append(fieldHelper.idField).append("}");
// queryBuf.append(id);
// solrQuery.setQuery(queryBuf.toString());
// if (fields != null) {
// solrQuery.setFields(fields);
// }
// final QueryResponse response = searchClient.query(solrQuery);
// final List<Map<String, Object>> docList = response.getResults();
// if (docList.isEmpty()) {
// return null;
// }
// if (logger.isDebugEnabled()) {
// logger.debug("Found solr documents: " + docList);
// }
// return docList;
public List<Map<String, Object>> getSolrDocumentListByPrefixId(final FessEsClient fessEsClient, final String id, final String[] fields) {
// TODO move fessEsClient
// final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
// final SolrQuery solrQuery = new SolrQuery();
// final StringBuilder queryBuf = new StringBuilder(500);
// queryBuf.append("{!prefix f=").append(fieldHelper.idField).append("}");
// queryBuf.append(id);
// solrQuery.setQuery(queryBuf.toString());
// if (fields != null) {
// solrQuery.setFields(fields);
// }
// final QueryResponse response = fessEsClient.query(solrQuery);
// final List<Map<String, Object>> docList = response.getResults();
// if (docList.isEmpty()) {
// return null;
// }
// if (logger.isDebugEnabled()) {
// logger.debug("Found solr documents: " + docList);
// }
// return docList;
return null;
}
public void deleteChildSolrDocument(final SearchClient searchClient, final String id) {
// TODO move searchClient
// final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
// final String query = "{!raw f=" + fieldHelper.parentIdField + " v=\"" + id + "\"}";
// for (final UpdateResponse response : searchClient.deleteByQuery(query)) {
// if (response.getStatus() != 0) {
// logger.warn("Failed to delete: " + response);
// }
// }
public void deleteChildSolrDocument(final FessEsClient fessEsClient, final String id) {
// TODO move fessEsClient
// final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
// final String query = "{!raw f=" + fieldHelper.parentIdField + " v=\""
// + id + "\"}";
// for (final UpdateResponse response : fessEsClient.deleteByQuery(query)) {
// if (response.getStatus() != 0) {
// logger.warn("Failed to delete: " + response);
// }
// }
}
public List<Map<String, Object>> getChildSolrDocumentList(final SearchClient searchClient, final String id, final String[] fields) {
return getChildSolrDocumentList(searchClient, id, fields, defaultRowSize);
public List<Map<String, Object>> getChildSolrDocumentList(final FessEsClient fessEsClient, final String id, final String[] fields) {
return getChildSolrDocumentList(fessEsClient, id, fields, defaultRowSize);
}
protected List<Map<String, Object>> getChildSolrDocumentList(final SearchClient searchClient, final String id, final String[] fields,
protected List<Map<String, Object>> getChildSolrDocumentList(final FessEsClient fessEsClient, final String id, final String[] fields,
final int row) {
// TODO move searchClient
// final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
// final SolrQuery solrQuery = new SolrQuery();
// solrQuery.setQuery("{!raw f=" + fieldHelper.parentIdField + " v=\"" + id + "\"}");
// if (fields != null) {
// solrQuery.setFields(fields);
// }
// solrQuery.setRows(row);
// final List<Map<String,Object>> docList = searchClient.query(solrQuery).getResults();
// if (docList.getNumFound() <= row) {
// return docList;
// }
// return getChildSolrDocumentList(searchClient, id, fields, (int) docList.getNumFound());
// TODO move fessEsClient
// final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
// final SolrQuery solrQuery = new SolrQuery();
// solrQuery.setQuery("{!raw f=" + fieldHelper.parentIdField + " v=\"" +
// id + "\"}");
// if (fields != null) {
// solrQuery.setFields(fields);
// }
// solrQuery.setRows(row);
// final List<Map<String,Object>> docList =
// fessEsClient.query(solrQuery).getResults();
// if (docList.getNumFound() <= row) {
// return docList;
// }
// return getChildSolrDocumentList(fessEsClient, id, fields, (int)
// docList.getNumFound());
return null;
}
}

35
src/main/java/org/codelibs/fess/helper/JobHelper.java
View file

@ -77,26 +77,23 @@ public class JobHelper {
final CountDownLatch latch = new CountDownLatch(1);
final Process process = jobProcess.getProcess();
new Thread(new Runnable() {
@Override
public void run() {
try {
IOUtils.closeQuietly(process.getInputStream());
} catch (final Exception e) {
logger.warn("Could not close a process input stream.", e);
}
try {
IOUtils.closeQuietly(process.getErrorStream());
} catch (final Exception e) {
logger.warn("Could not close a process error stream.", e);
}
try {
IOUtils.closeQuietly(process.getOutputStream());
} catch (final Exception e) {
logger.warn("Could not close a process output stream.", e);
}
latch.countDown();
new Thread((Runnable) () -> {
try {
IOUtils.closeQuietly(process.getInputStream());
} catch (final Exception e1) {
logger.warn("Could not close a process input stream.", e1);
}
try {
IOUtils.closeQuietly(process.getErrorStream());
} catch (final Exception e2) {
logger.warn("Could not close a process error stream.", e2);
}
try {
IOUtils.closeQuietly(process.getOutputStream());
} catch (final Exception e3) {
logger.warn("Could not close a process output stream.", e3);
}
latch.countDown();
}, "ProcessCloser").start();
try {

22
src/main/java/org/codelibs/fess/helper/KeyMatchHelper.java
View file

@ -23,8 +23,8 @@ import java.util.List;
import java.util.Map;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.fess.client.SearchClient;
import org.codelibs.fess.client.SearchClient.SearchConditionBuilder;
import org.codelibs.fess.client.FessEsClient;
import org.codelibs.fess.client.FessEsClient.SearchConditionBuilder;
import org.codelibs.fess.db.exentity.KeyMatch;
import org.codelibs.fess.service.KeyMatchService;
import org.codelibs.fess.util.ComponentUtil;
@ -44,12 +44,7 @@ public class KeyMatchHelper {
}
public void update() {
new Thread(new Runnable() {
@Override
public void run() {
reload(reloadInterval);
}
}).start();
new Thread(() -> reload(reloadInterval)).start();
}
protected void reload(final long interval) {
@ -82,14 +77,13 @@ public class KeyMatchHelper {
}
protected List<Map<String, Object>> getDocumentList(final KeyMatch keyMatch) {
final SearchClient searchClient = ComponentUtil.getElasticsearchClient();
final FessEsClient fessEsClient = ComponentUtil.getElasticsearchClient();
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final List<Map<String, Object>> documentList =
searchClient.getDocumentList(
searchRequestBuilder -> {
return SearchConditionBuilder.builder(searchRequestBuilder).administrativeAccess().size(keyMatch.getMaxSize())
.responseFields(new String[] { fieldHelper.docIdField }).build();
}).get();
fessEsClient.getDocumentList(fieldHelper.docIndex, fieldHelper.docType, searchRequestBuilder -> {
return SearchConditionBuilder.builder(searchRequestBuilder).administrativeAccess().size(keyMatch.getMaxSize())
.responseFields(new String[] { fieldHelper.docIdField }).build();
});
return documentList;
}

11
src/main/java/org/codelibs/fess/helper/SuggestHelper.java
View file

@ -43,10 +43,8 @@ public class SuggestHelper {
public String badwordFileDir = "./solr/core1/conf/";
public void init() {
final Thread th = new Thread(new Runnable() {
@Override
public void run() {
// TODO replace with Elasticsearch
final Thread th = new Thread(() -> {
// TODO replace with Elasticsearch
/*
while (true) {
final PingResponse response = searchService.ping();
@ -65,8 +63,7 @@ public class SuggestHelper {
}
}
*/
}
});
});
th.start();
}
@ -146,7 +143,7 @@ public class SuggestHelper {
}
public void updateSolrBadwordFile() {
final List<SuggestBadWord> list = suggestBadWordBhv.selectList(cb -> {
suggestBadWordBhv.selectList(cb -> {
cb.query().setDeletedBy_IsNull();
});

6
src/main/java/org/codelibs/fess/helper/SystemHelper.java
View file

@ -44,8 +44,6 @@ import org.seasar.framework.container.SingletonS2Container;
import org.seasar.framework.container.annotation.tiger.InitMethod;
import org.seasar.struts.util.MessageResourcesUtil;
import org.seasar.struts.util.RequestUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
@ -133,6 +131,10 @@ public class SystemHelper implements Serializable {
return LocalDateTime.now();
}
public long getCurrentTimeAsLong() {
return System.currentTimeMillis();
}
public String getLogFilePath() {
return logFilePath;
}

4
src/main/java/org/codelibs/fess/helper/WebFsIndexHelper.java
View file

@ -26,9 +26,9 @@ import javax.annotation.Resource;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.core.misc.DynamicProperties;
import org.codelibs.fess.Constants;
import org.codelibs.fess.db.exentity.BoostDocumentRule;
import org.codelibs.fess.db.exentity.FileCrawlingConfig;
import org.codelibs.fess.db.exentity.WebCrawlingConfig;
import org.codelibs.fess.entity.BoostDocumentRule;
import org.codelibs.fess.interval.FessIntervalController;
import org.codelibs.fess.service.BoostDocumentRuleService;
import org.codelibs.fess.service.FailureUrlService;
@ -129,7 +129,7 @@ public class WebFsIndexHelper implements Serializable {
protected void doCrawl(final String sessionId, final List<WebCrawlingConfig> webConfigList,
final List<FileCrawlingConfig> fileConfigList) {
int multiprocessCrawlingCount = 5;
String value = crawlerProperties.getProperty(Constants.CRAWLING_THREAD_COUNT_PROPERTY, "5");
final String value = crawlerProperties.getProperty(Constants.CRAWLING_THREAD_COUNT_PROPERTY, "5");
try {
multiprocessCrawlingCount = Integer.parseInt(value);
} catch (final NumberFormatException e) {

14
src/main/java/org/codelibs/fess/helper/impl/SearchLogHelperImpl.java
View file

@ -26,13 +26,13 @@ import java.util.Queue;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.fess.Constants;
import org.codelibs.fess.beans.FessBeans;
import org.codelibs.fess.client.FessEsClient;
import org.codelibs.fess.db.exbhv.ClickLogBhv;
import org.codelibs.fess.db.exbhv.SearchLogBhv;
import org.codelibs.fess.db.exbhv.UserInfoBhv;
import org.codelibs.fess.db.exentity.ClickLog;
import org.codelibs.fess.db.exentity.SearchLog;
import org.codelibs.fess.db.exentity.UserInfo;
import org.codelibs.fess.helper.DocumentHelper;
import org.codelibs.fess.helper.FieldHelper;
import org.codelibs.fess.helper.SearchLogHelper;
import org.codelibs.fess.service.SearchLogService;
@ -76,15 +76,12 @@ public class SearchLogHelperImpl extends SearchLogHelper {
botNames = value.split(",");
}
final boolean suggestAvailable =
Constants.TRUE.equals(crawlerProperties.getProperty(Constants.SUGGEST_SEARCH_LOG_PROPERTY, Constants.TRUE));
Constants.TRUE.equals(crawlerProperties.getProperty(Constants.SUGGEST_SEARCH_LOG_PROPERTY, Constants.TRUE));
final String dayForCleanupStr = crawlerProperties.getProperty(Constants.PURGE_SUGGEST_SEARCH_LOG_DAY_PROPERTY, "30");
int dayForCleanup = -1;
try {
dayForCleanup = Integer.parseInt(dayForCleanupStr);
Integer.parseInt(dayForCleanupStr);
} catch (final NumberFormatException e) {}
boolean addedSuggest = false;
final Map<String, UserInfo> userInfoMap = new HashMap<String, UserInfo>();
for (final SearchLog searchLog : queue) {
boolean add = true;
@ -193,12 +190,13 @@ public class SearchLogHelperImpl extends SearchLogHelper {
}
}
final DocumentHelper documentHelper = ComponentUtil.getDocumentHelper();
final FessEsClient fessEsClient = ComponentUtil.getElasticsearchClient();
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
for (final Map.Entry<String, Long> entry : clickCountMap.entrySet()) {
try {
// TODO buik update
documentHelper.update(entry.getKey(), fieldHelper.clickCountField, entry.getValue() + 1);
fessEsClient.update(fieldHelper.docIndex, fieldHelper.docType, entry.getKey(), fieldHelper.clickCountField,
entry.getValue() + 1);
} catch (final Exception e) {
logger.warn("Failed to update a clickCount(" + entry.getValue() + ") for " + entry.getKey(), e);
}

17
src/main/java/org/codelibs/fess/job/CrawlJob.java
View file

@ -33,7 +33,6 @@ import org.codelibs.fess.FessSystemException;
import org.codelibs.fess.exec.Crawler;
import org.codelibs.fess.helper.JobHelper;
import org.codelibs.fess.helper.SystemHelper;
import org.codelibs.fess.job.JobExecutor.ShutdownListener;
import org.codelibs.fess.util.ComponentUtil;
import org.codelibs.fess.util.InputStreamThread;
import org.codelibs.fess.util.JobProcess;
@ -193,12 +192,7 @@ public class CrawlJob {
}
if (jobExecutor != null) {
jobExecutor.addShutdownListener(new ShutdownListener() {
@Override
public void onShutdown() {
ComponentUtil.getJobHelper().destroyCrawlerProcess(sessionId);
}
});
jobExecutor.addShutdownListener(() -> ComponentUtil.getJobHelper().destroyCrawlerProcess(sessionId));
}
try {
@ -256,7 +250,7 @@ public class CrawlJob {
}
crawlerCmdList.add(buf.toString());
String transportAddresses = System.getProperty(Constants.FESS_ES_TRANSPORT_ADDRESSES);
final String transportAddresses = System.getProperty(Constants.FESS_ES_TRANSPORT_ADDRESSES);
if (StringUtil.isNotBlank(transportAddresses)) {
crawlerCmdList.add("-D" + Constants.FESS_ES_TRANSPORT_ADDRESSES + "=" + transportAddresses);
}
@ -374,12 +368,7 @@ public class CrawlJob {
}
protected void appendJarFile(final String cpSeparator, final StringBuilder buf, final File libDir, final String basePath) {
final File[] jarFiles = libDir.listFiles(new FilenameFilter() {
@Override
public boolean accept(final File dir, final String name) {
return name.toLowerCase().endsWith(".jar");
}
});
final File[] jarFiles = libDir.listFiles((FilenameFilter) (dir, name) -> name.toLowerCase().endsWith(".jar"));
if (jarFiles != null) {
for (final File file : jarFiles) {
buf.append(cpSeparator);

8
src/main/java/org/codelibs/fess/lasta/core/direction/FessConfig.java
View file

@ -37,6 +37,7 @@ public interface FessConfig extends FessEnv {
* @param propertyKey The key of the property. (NotNull)
* @return The value of found property. (NullAllowed: if null, not found)
*/
@Override
String get(String propertyKey);
/**
@ -44,6 +45,7 @@ public interface FessConfig extends FessEnv {
* @param propertyKey The key of the property which is boolean type. (NotNull)
* @return The determination, true or false. (if the property can be true, returns true)
*/
@Override
boolean is(String propertyKey);
/**
@ -106,31 +108,37 @@ public interface FessConfig extends FessEnv {
private static final long serialVersionUID = 1L;
/** {@inheritDoc} */
@Override
public String getDomainTitle() {
return get(FessConfig.DOMAIN_TITLE);
}
/** {@inheritDoc} */
@Override
public String getCookieDefaultPath() {
return get(FessConfig.COOKIE_DEFAULT_PATH);
}
/** {@inheritDoc} */
@Override
public String getCookieDefaultExpire() {
return get(FessConfig.COOKIE_DEFAULT_EXPIRE);
}
/** {@inheritDoc} */
@Override
public Integer getCookieDefaultExpireAsInteger() {
return getAsInteger(FessConfig.COOKIE_DEFAULT_EXPIRE);
}
/** {@inheritDoc} */
@Override
public String getCookieEternalExpire() {
return get(FessConfig.COOKIE_ETERNAL_EXPIRE);
}
/** {@inheritDoc} */
@Override
public Integer getCookieEternalExpireAsInteger() {
return getAsInteger(FessConfig.COOKIE_ETERNAL_EXPIRE);
}

19
src/main/java/org/codelibs/fess/lasta/core/direction/FessFwAssistantDirector.java
View file

@ -23,13 +23,10 @@ import org.codelibs.fess.lasta.core.direction.sponsor.FessActionAdjustmentProvid
import org.codelibs.fess.lasta.core.direction.sponsor.FessTimeResourceProvider;
import org.codelibs.fess.lasta.core.direction.sponsor.FessUserLocaleProcessProvider;
import org.codelibs.fess.lasta.core.direction.sponsor.FessUserTimeZoneProcessProvider;
import org.dbflute.saflute.core.direction.BootProcessCallback;
import org.dbflute.saflute.core.direction.CachedFwAssistantDirector;
import org.dbflute.saflute.core.direction.FwAssistantDirector;
import org.dbflute.saflute.core.direction.OptionalAssistDirection;
import org.dbflute.saflute.core.direction.OptionalCoreDirection;
import org.dbflute.saflute.core.security.InvertibleCipher;
import org.dbflute.saflute.core.security.SecurityResourceProvider;
import org.dbflute.saflute.db.dbflute.OptionalDBFluteDirection;
import org.dbflute.saflute.web.action.OptionalActionDirection;
import org.dbflute.saflute.web.servlet.OptionalServletDirection;
@ -108,11 +105,7 @@ public class FessFwAssistantDirector extends CachedFwAssistantDirector {
direction.directFrameworkDebug(fessConfig.isFrameworkDebug()); // basically false
// you can add your own process when your application is booting
direction.directBootProcessCallback(new BootProcessCallback() {
public void callback(FwAssistantDirector assistantDirector) {
processDBFluteSystem();
}
});
direction.directBootProcessCallback(assistantDirector -> processDBFluteSystem());
}
protected void processDBFluteSystem() {
@ -120,6 +113,7 @@ public class FessFwAssistantDirector extends CachedFwAssistantDirector {
DBFluteSystem.setFinalTimeZoneProvider(new DfFinalTimeZoneProvider() {
protected final TimeZone provided = FessUserTimeZoneProcessProvider.centralTimeZone;
@Override
public TimeZone provide() {
return provided;
}
@ -138,11 +132,7 @@ public class FessFwAssistantDirector extends CachedFwAssistantDirector {
protected void prepareSecurity(OptionalCoreDirection direction) {
final String key = getPrimarySecurityWord();
final InvertibleCipher primaryInvertibleCipher = InvertibleCipher.createAesCipher(key); // AES for now
direction.directSecurity(new SecurityResourceProvider() {
public InvertibleCipher providePrimaryInvertibleCipher() {
return primaryInvertibleCipher;
}
});
direction.directSecurity(() -> primaryInvertibleCipher);
}
protected String getPrimarySecurityWord() {
@ -233,14 +223,17 @@ public class FessFwAssistantDirector extends CachedFwAssistantDirector {
final Integer cookieDefaultExpire = fessConfig.getCookieDefaultExpireAsInteger();
final InvertibleCipher cookieCipher = InvertibleCipher.createAesCipher(key); // AES for now
direction.directCookie(new CookieResourceProvider() {
@Override
public String provideDefaultPath() {
return cookieDefaultPath;
}
@Override
public Integer provideDefaultExpire() {
return cookieDefaultExpire;
}
@Override
public InvertibleCipher provideCipher() {
return cookieCipher;
}

7
src/main/java/org/codelibs/fess/lasta/core/direction/sponsor/FessActionAdjustmentProvider.java
View file

@ -31,30 +31,37 @@ public class FessActionAdjustmentProvider implements ActionAdjustmentProvider {
private static final int INDEXED_PROPERTY_SIZE_LIMIT = 200; // hard coding for now
@Override
public int provideIndexedPropertySizeLimit() {
return INDEXED_PROPERTY_SIZE_LIMIT;
}
@Override
public String decodeUrlParameterPropertyValue(Object bean, String name, String value) {
return null;
}
@Override
public String filterJspPath(String path, ActionMappingWrapper actionMappingWrapper) {
return null;
}
@Override
public List<String> prepareJspRetryWordList(String requestPath, List<String> wordList) {
return null;
}
@Override
public boolean isForcedRoutingTarget(HttpServletRequest request, String requestPath) {
return false;
}
@Override
public boolean isForcedSuppressRedirectWithSlash(HttpServletRequest request, String requestPath, S2ExecuteConfig executeConfig) {
return false;
}
@Override
public String customizeActionMappingRequestPath(String requestPath) {
return null;
}

8
src/main/java/org/codelibs/fess/lasta/core/direction/sponsor/FessTimeResourceProvider.java
View file

@ -46,6 +46,7 @@ public class FessTimeResourceProvider implements TimeResourceProvider {
// ===================================================================================
// Basic Handling
// ==============
@Override
public BusinessTimeHandler provideBusinessTimeHandler(TimeManager timeManager) {
return new TypicalBusinessTimeHandler(() -> {
return timeManager.getCurrentMillis();
@ -54,6 +55,7 @@ public class FessTimeResourceProvider implements TimeResourceProvider {
});
}
@Override
public boolean isCurrentIgnoreTransaction() {
// this project uses transaction time for current date
return false; // fixedly
@ -62,17 +64,19 @@ public class FessTimeResourceProvider implements TimeResourceProvider {
// ===================================================================================
// Time Adjustment
// ===============
@Override
public boolean isAdjustAbsoluteMode() { // *1
final String exp = maihamaConfig.getTimeAdjustTimeMillis();
return exp.startsWith("$"); // means absolute e.g. $(2014/07/10)
}
@Override
public long provideAdjustTimeMillis() { // *1
final String exp = maihamaConfig.getTimeAdjustTimeMillis();
try {
return doProvideAdjustTimeMillis(exp);
} catch (RuntimeException e) {
String msg = "Illegal property for time.adjust.time.millis: " + exp;
} catch (final RuntimeException e) {
final String msg = "Illegal property for time.adjust.time.millis: " + exp;
throw new IllegalStateException(msg);
}
}

4
src/main/java/org/codelibs/fess/lasta/core/direction/sponsor/FessUserLocaleProcessProvider.java
View file

@ -31,6 +31,7 @@ public class FessUserLocaleProcessProvider implements UserLocaleProcessProvider
/**
* {@inheritDoc}
*/
@Override
public boolean isAcceptCookieLocale() {
return false;
}
@ -38,6 +39,7 @@ public class FessUserLocaleProcessProvider implements UserLocaleProcessProvider
/**
* {@inheritDoc}
*/
@Override
public Locale findBusinessLocale(ActionExecuteMeta executeMeta, RequestManager requestManager) {
return null;
}
@ -45,6 +47,7 @@ public class FessUserLocaleProcessProvider implements UserLocaleProcessProvider
/**
* {@inheritDoc}
*/
@Override
public Locale getRequestedLocale(RequestManager requestManager) {
return null; // null means browser default
}
@ -52,6 +55,7 @@ public class FessUserLocaleProcessProvider implements UserLocaleProcessProvider
/**
* {@inheritDoc}
*/
@Override
public Locale getFallbackLocale() {
return centralLocale;
}

5
src/main/java/org/codelibs/fess/lasta/core/direction/sponsor/FessUserTimeZoneProcessProvider.java
View file

@ -31,6 +31,7 @@ public class FessUserTimeZoneProcessProvider implements UserTimeZoneProcessProvi
/**
* {@inheritDoc}
*/
@Override
public boolean isUseTimeZoneHandling() {
return false;
}
@ -38,6 +39,7 @@ public class FessUserTimeZoneProcessProvider implements UserTimeZoneProcessProvi
/**
* {@inheritDoc}
*/
@Override
public boolean isAcceptCookieTimeZone() {
return false;
}
@ -45,6 +47,7 @@ public class FessUserTimeZoneProcessProvider implements UserTimeZoneProcessProvi
/**
* {@inheritDoc}
*/
@Override
public TimeZone findBusinessTimeZone(ActionExecuteMeta executeMeta, RequestManager requestManager) {
return null;
}
@ -52,6 +55,7 @@ public class FessUserTimeZoneProcessProvider implements UserTimeZoneProcessProvi
/**
* {@inheritDoc}
*/
@Override
public TimeZone getRequestedTimeZone(RequestManager requestManager) {
return centralTimeZone; // same as fall-back
}
@ -59,6 +63,7 @@ public class FessUserTimeZoneProcessProvider implements UserTimeZoneProcessProvi
/**
* {@inheritDoc}
*/
@Override
public TimeZone getFallbackTimeZone() {
return centralTimeZone;
}

2
src/main/java/org/codelibs/fess/lasta/web/action/FessMessages.java
View file

@ -671,7 +671,7 @@ public class FessMessages extends ActionMessages {
*/
protected void assertPropertyNotNull(String property) {
if (property == null) {
String msg = "The argument 'property' for message should not be null.";
final String msg = "The argument 'property' for message should not be null.";
throw new IllegalArgumentException(msg);
}
}

23
src/main/java/org/codelibs/fess/robot/FessS2RobotThread.java
View file

@ -28,7 +28,7 @@ import java.util.Set;
import org.apache.commons.io.IOUtils;
import org.codelibs.core.misc.DynamicProperties;
import org.codelibs.fess.Constants;
import org.codelibs.fess.client.SearchClient;
import org.codelibs.fess.client.FessEsClient;
import org.codelibs.fess.db.exentity.CrawlingConfig;
import org.codelibs.fess.helper.CrawlingConfigHelper;
import org.codelibs.fess.helper.CrawlingSessionHelper;
@ -67,7 +67,7 @@ public class FessS2RobotThread extends S2RobotThread {
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final SambaHelper sambaHelper = ComponentUtil.getSambaHelper();
final IndexingHelper indexingHelper = ComponentUtil.getIndexingHelper();
final SearchClient searchClient = ComponentUtil.getElasticsearchClient();
final FessEsClient fessEsClient = ComponentUtil.getElasticsearchClient();
final boolean useAclAsRole = crawlerProperties.getProperty(Constants.USE_ACL_AS_ROLE, Constants.FALSE).equals(Constants.TRUE);
final String url = urlQueue.getUrl();
@ -81,7 +81,7 @@ public class FessS2RobotThread extends S2RobotThread {
roleTypeList.add(roleType);
}
if (useAclAsRole && url.startsWith("smb://")) {
// head method
// head method
responseData = client.execute(RequestDataBuilder.newRequestData().head().url(url).build());
if (responseData == null) {
return true;
@ -102,11 +102,11 @@ public class FessS2RobotThread extends S2RobotThread {
final String id = crawlingSessionHelper.generateId(dataMap);
final Map<String, Object> solrDocument =
indexingHelper.getSolrDocument(searchClient, id, new String[] { fieldHelper.idField, fieldHelper.lastModifiedField,
indexingHelper.getSolrDocument(fessEsClient, id, new String[] { fieldHelper.idField, fieldHelper.lastModifiedField,
fieldHelper.anchorField, fieldHelper.segmentField, fieldHelper.expiresField, fieldHelper.clickCountField,
fieldHelper.favoriteCountField });
if (solrDocument == null) {
storeChildUrlsToQueue(urlQueue, getChildUrlSet(searchClient, id));
storeChildUrlsToQueue(urlQueue, getChildUrlSet(fessEsClient, id));
return true;
}
@ -114,7 +114,7 @@ public class FessS2RobotThread extends S2RobotThread {
if (expires != null && expires.getTime() < System.currentTimeMillis()) {
final Object idValue = solrDocument.get(fieldHelper.idField);
if (idValue != null) {
indexingHelper.deleteDocument(searchClient, idValue.toString());
indexingHelper.deleteDocument(fessEsClient, idValue.toString());
}
return true;
}
@ -143,7 +143,7 @@ public class FessS2RobotThread extends S2RobotThread {
}
if (responseData == null) {
// head method
// head method
responseData = client.execute(RequestDataBuilder.newRequestData().head().url(url).build());
if (responseData == null) {
return true;
@ -153,7 +153,7 @@ public class FessS2RobotThread extends S2RobotThread {
final int httpStatusCode = responseData.getHttpStatusCode();
if (httpStatusCode == 404) {
storeChildUrlsToQueue(urlQueue, getAnchorSet(solrDocument.get(fieldHelper.anchorField)));
indexingHelper.deleteDocument(searchClient, id);
indexingHelper.deleteDocument(fessEsClient, id);
return false;
} else if (responseData.getLastModified() == null) {
return true;
@ -183,7 +183,7 @@ public class FessS2RobotThread extends S2RobotThread {
protected void storeChildUrlsToQueue(final UrlQueue urlQueue, final Set<RequestData> childUrlSet) {
if (childUrlSet != null) {
synchronized (robotContext.getAccessCountLock()) {
// add an url
// add an url
storeChildUrls(childUrlSet, urlQueue.getUrl(), urlQueue.getDepth() != null ? urlQueue.getDepth() + 1 : 1);
}
}
@ -212,11 +212,11 @@ public class FessS2RobotThread extends S2RobotThread {
return childUrlSet;
}
protected Set<RequestData> getChildUrlSet(final SearchClient searchClient, final String id) {
protected Set<RequestData> getChildUrlSet(final FessEsClient fessEsClient, final String id) {
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final IndexingHelper indexingHelper = ComponentUtil.getIndexingHelper();
final List<Map<String, Object>> docList =
indexingHelper.getChildSolrDocumentList(searchClient, id, new String[] { fieldHelper.urlField });
indexingHelper.getChildSolrDocumentList(fessEsClient, id, new String[] { fieldHelper.urlField });
if (docList.isEmpty()) {
return null;
}
@ -232,5 +232,4 @@ public class FessS2RobotThread extends S2RobotThread {
}
return urlSet;
}
}

19
src/main/java/org/codelibs/fess/screenshot/ScreenShotManager.java
View file

@ -88,17 +88,14 @@ public class ScreenShotManager {
}
generating = true;
screenshotGeneratorThread = new Thread(new Runnable() {
@Override
public void run() {
while (generating) {
try {
screenShotTaskQueue.take().generate();
} catch (final InterruptedException e) {
logger.debug("Interupted task.", e);
} catch (final Exception e) {
logger.warn("Failed to generage a screenshot.", e);
}
screenshotGeneratorThread = new Thread((Runnable) () -> {
while (generating) {
try {
screenShotTaskQueue.take().generate();
} catch (final InterruptedException e1) {
logger.debug("Interupted task.", e1);
} catch (final Exception e2) {
logger.warn("Failed to generage a screenshot.", e2);
}
}
}, "ScreenShotGenerator");

1290
src/main/java/org/codelibs/fess/service/DatabaseService.java
View file

File diff suppressed because it is too large Load diff

13
src/main/java/org/codelibs/fess/service/StatsService.java
View file

@ -25,9 +25,6 @@ import java.util.Map;
import javax.annotation.Resource;
import org.codelibs.fess.crud.CommonConstants;
import org.codelibs.fess.db.bsbhv.BsClickLogBhv;
import org.codelibs.fess.db.bsbhv.BsFavoriteLogBhv;
import org.codelibs.fess.db.bsbhv.BsSearchLogBhv;
import org.codelibs.fess.db.exbhv.ClickLogBhv;
import org.codelibs.fess.db.exbhv.FavoriteLogBhv;
import org.codelibs.fess.db.exbhv.SearchLogBhv;
@ -112,7 +109,6 @@ public class StatsService implements Serializable {
pmb.setFromRequestedTime(statsPager.getFromRequestedTime());
pmb.setToRequestedTime(statsPager.getToRequestedTime());
final String path = BsSearchLogBhv.PATH_selectSearchWordRanking;
final PagingResultBean<SearchWordRanking> statsList = searchLogBhv.outsideSql().selectPage(pmb);
// update pager
@ -140,7 +136,6 @@ public class StatsService implements Serializable {
pmb.setFromRequestedTime(statsPager.getFromRequestedTime());
pmb.setToRequestedTime(statsPager.getToRequestedTime());
final String path = BsSearchLogBhv.PATH_selectSearchQueryRanking;
final PagingResultBean<SearchQueryRanking> statsList = searchLogBhv.outsideSql().selectPage(pmb);
// update pager
@ -168,7 +163,6 @@ public class StatsService implements Serializable {
pmb.setFromRequestedTime(statsPager.getFromRequestedTime());
pmb.setToRequestedTime(statsPager.getToRequestedTime());
final String path = BsSearchLogBhv.PATH_selectSolrQueryRanking;
final PagingResultBean<SolrQueryRanking> statsList = searchLogBhv.outsideSql().selectPage(pmb);
// update pager
@ -196,7 +190,6 @@ public class StatsService implements Serializable {
pmb.setFromRequestedTime(statsPager.getFromRequestedTime());
pmb.setToRequestedTime(statsPager.getToRequestedTime());
final String path = BsSearchLogBhv.PATH_selectUserAgentRanking;
final PagingResultBean<UserAgentRanking> statsList = searchLogBhv.outsideSql().selectPage(pmb);
// update pager
@ -224,7 +217,6 @@ public class StatsService implements Serializable {
pmb.setFromRequestedTime(statsPager.getFromRequestedTime());
pmb.setToRequestedTime(statsPager.getToRequestedTime());
final String path = BsSearchLogBhv.PATH_selectRefererRanking;
final PagingResultBean<RefererRanking> statsList = searchLogBhv.outsideSql().selectPage(pmb);
// update pager
@ -252,7 +244,6 @@ public class StatsService implements Serializable {
pmb.setFromRequestedTime(statsPager.getFromRequestedTime());
pmb.setToRequestedTime(statsPager.getToRequestedTime());
final String path = BsSearchLogBhv.PATH_selectClientIpRanking;
final PagingResultBean<ClientIpRanking> statsList = searchLogBhv.outsideSql().selectPage(pmb);
// update pager
@ -280,7 +271,6 @@ public class StatsService implements Serializable {
pmb.setFromRequestedTime(statsPager.getFromRequestedTime());
pmb.setToRequestedTime(statsPager.getToRequestedTime());
final String path = BsClickLogBhv.PATH_selectClickUrlRanking;
final PagingResultBean<ClickUrlRanking> statsList = clicklogBhv.outsideSql().selectPage(pmb);
// update pager
@ -308,7 +298,6 @@ public class StatsService implements Serializable {
pmb.setFromCreatedTime(statsPager.getFromRequestedTime());
pmb.setToCreatedTime(statsPager.getToRequestedTime());
final String path = BsFavoriteLogBhv.PATH_selectFavoriteUrlRanking;
final PagingResultBean<FavoriteUrlRanking> statsList = favoriteLogBhv.outsideSql().selectPage(pmb);
// update pager
@ -337,7 +326,6 @@ public class StatsService implements Serializable {
pmb.setFromRequestedTime(statsPager.getFromRequestedTime());
pmb.setToRequestedTime(statsPager.getToRequestedTime());
final String path = BsSearchLogBhv.PATH_selectSearchFieldRanking;
final PagingResultBean<SearchFieldRanking> statsList = searchLogBhv.outsideSql().selectPage(pmb);
// update pager
@ -355,5 +343,4 @@ public class StatsService implements Serializable {
return mapList;
}
}

11
src/main/java/org/codelibs/fess/servlet/H2ConfigServlet.java
View file

@ -25,7 +25,6 @@ import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
import org.h2.tools.Server;
import org.seasar.framework.util.Disposable;
import org.seasar.framework.util.DisposableUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -91,12 +90,9 @@ public class H2ConfigServlet extends HttpServlet {
@Override
public void destroy() {
if (System.getProperty("java.specification.version").equals("1.7")) {
DisposableUtil.add(new Disposable() {
@Override
public void dispose() {
if (server != null) {
server.stop();
}
DisposableUtil.add(() -> {
if (server != null) {
server.stop();
}
});
} else {
@ -105,5 +101,4 @@ public class H2ConfigServlet extends HttpServlet {
}
}
}
}

227
src/main/java/org/codelibs/fess/servlet/Tomcat6ConfigServlet.java
View file

@ -1,227 +0,0 @@
/*
* Copyright 2009-2015 the CodeLibs Project and the Others.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
* either express or implied. See the License for the specific language
* governing permissions and limitations under the License.
*/
package org.codelibs.fess.servlet;
import java.lang.ref.Reference;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.List;
import javax.servlet.http.HttpServlet;
import org.seasar.framework.util.DriverManagerUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class Tomcat6ConfigServlet extends HttpServlet {
private static final Logger logger = LoggerFactory.getLogger(Tomcat6ConfigServlet.class);
private static final long serialVersionUID = 1L;
@Override
public void destroy() {
if (logger.isInfoEnabled()) {
logger.info("Removing HTTP connection manager...");
}
shutdownCommonsHttpClient();
if (logger.isInfoEnabled()) {
logger.info("Removing all drivers...");
}
DriverManagerUtil.deregisterAllDrivers();
cleanupAllThreads();
}
private void shutdownCommonsHttpClient() {
try {
final Class<?> clazz = Class.forName("org.apache.commons.httpclient.MultiThreadedHttpConnectionManager");
final Method method = clazz.getMethod("shutdownAll", null);
method.invoke(null, null);
} catch (final ClassNotFoundException e) {
// ignore
} catch (final Exception e) {
logger.warn("Could not shutdown Commons HttpClient.", e);
}
}
@SuppressWarnings("deprecation")
private void cleanupAllThreads() {
final Thread[] threads = getThreads();
final ClassLoader cl = this.getClass().getClassLoader();
try {
cl.getResource(null);
} catch (final Exception e) {}
final List<String> jvmThreadGroupList = new ArrayList<String>();
jvmThreadGroupList.add("system");
jvmThreadGroupList.add("RMI Runtime");
// Iterate over the set of threads
for (final Thread thread : threads) {
if (thread != null) {
final ClassLoader ccl = thread.getContextClassLoader();
if (ccl != null && ccl.equals(cl)) {
// Don't warn about this thread
if (thread == Thread.currentThread()) {
continue;
}
// Don't warn about JVM controlled threads
final ThreadGroup tg = thread.getThreadGroup();
if (tg != null && jvmThreadGroupList.contains(tg.getName())) {
continue;
}
waitThread(thread);
// Skip threads that have already died
if (!thread.isAlive()) {
continue;
}
if (logger.isInfoEnabled()) {
logger.info("Interrupting a thread [" + thread.getName() + "]...");
}
thread.interrupt();
waitThread(thread);
// Skip threads that have already died
if (!thread.isAlive()) {
continue;
}
if (logger.isInfoEnabled()) {
logger.info("Stopping a thread [" + thread.getName() + "]...");
}
thread.stop();
}
}
}
Field threadLocalsField = null;
Field inheritableThreadLocalsField = null;
Field tableField = null;
try {
threadLocalsField = Thread.class.getDeclaredField("threadLocals");
threadLocalsField.setAccessible(true);
inheritableThreadLocalsField = Thread.class.getDeclaredField("inheritableThreadLocals");
inheritableThreadLocalsField.setAccessible(true);
// Make the underlying array of ThreadLoad.ThreadLocalMap.Entry objects
// accessible
final Class<?> tlmClass = Class.forName("java.lang.ThreadLocal$ThreadLocalMap");
tableField = tlmClass.getDeclaredField("table");
tableField.setAccessible(true);
} catch (final Exception e) {
// ignore
}
for (final Thread thread : threads) {
if (thread != null) {
Object threadLocalMap;
try {
// Clear the first map
threadLocalMap = threadLocalsField.get(thread);
clearThreadLocalMap(cl, threadLocalMap, tableField);
} catch (final Exception e) {
// ignore
}
try { // Clear the second map
threadLocalMap = inheritableThreadLocalsField.get(thread);
clearThreadLocalMap(cl, threadLocalMap, tableField);
} catch (final Exception e) {
// ignore
}
}
}
}
private void waitThread(final Thread thread) {
int count = 0;
while (thread.isAlive() && count < 5) {
try {
Thread.sleep(100);
} catch (final InterruptedException e) {}
count++;
}
}
/*
* Get the set of current threads as an array.
*/
private Thread[] getThreads() {
// Get the current thread group
ThreadGroup tg = Thread.currentThread().getThreadGroup();
// Find the root thread group
while (tg.getParent() != null) {
tg = tg.getParent();
}
int threadCountGuess = tg.activeCount() + 50;
Thread[] threads = new Thread[threadCountGuess];
int threadCountActual = tg.enumerate(threads);
// Make sure we don't miss any threads
while (threadCountActual == threadCountGuess) {
threadCountGuess *= 2;
threads = new Thread[threadCountGuess];
// Note tg.enumerate(Thread[]) silently ignores any threads that
// can't fit into the array
threadCountActual = tg.enumerate(threads);
}
return threads;
}
private void clearThreadLocalMap(final ClassLoader cl, final Object map, final Field internalTableField) throws NoSuchMethodException,
IllegalAccessException, NoSuchFieldException, InvocationTargetException {
if (map != null) {
final Method mapRemove = map.getClass().getDeclaredMethod("remove", ThreadLocal.class);
mapRemove.setAccessible(true);
final Object[] table = (Object[]) internalTableField.get(map);
if (table != null) {
for (final Object element : table) {
if (element != null) {
boolean remove = false;
// Check the key
final Field keyField = Reference.class.getDeclaredField("referent");
keyField.setAccessible(true);
final Object key = keyField.get(element);
if (cl.equals(key) || key != null && cl == key.getClass().getClassLoader()) {
remove = true;
}
// Check the value
final Field valueField = element.getClass().getDeclaredField("value");
valueField.setAccessible(true);
final Object value = valueField.get(element);
if (cl.equals(value) || value != null && cl == value.getClass().getClassLoader()) {
remove = true;
}
if (remove) {
final Object entry = ((Reference<?>) element).get();
if (logger.isInfoEnabled()) {
logger.info("Removing " + key.toString() + " from a thread local...");
}
mapRemove.invoke(map, entry);
}
}
}
}
}
}
}

8
src/main/java/org/codelibs/fess/solr/IndexUpdater.java
View file

@ -26,7 +26,7 @@ import javax.annotation.Resource;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.fess.Constants;
import org.codelibs.fess.FessSystemException;
import org.codelibs.fess.client.SearchClient;
import org.codelibs.fess.client.FessEsClient;
import org.codelibs.fess.db.exbhv.ClickLogBhv;
import org.codelibs.fess.db.exbhv.FavoriteLogBhv;
import org.codelibs.fess.db.exbhv.pmbean.FavoriteUrlCountPmb;
@ -58,7 +58,7 @@ public class IndexUpdater extends Thread {
protected List<String> sessionIdList;
protected SearchClient searchClient;
protected FessEsClient fessEsClient;
@Resource
protected DataService dataService;
@ -235,7 +235,7 @@ public class IndexUpdater extends Thread {
}
if (!docList.isEmpty()) {
indexingHelper.sendDocuments(searchClient, docList);
indexingHelper.sendDocuments(fessEsClient, docList);
}
synchronized (finishedSessionIdList) {
@ -360,7 +360,7 @@ public class IndexUpdater extends Thread {
}
if (docList.size() >= maxDocumentCacheSize) {
indexingHelper.sendDocuments(searchClient, docList);
indexingHelper.sendDocuments(fessEsClient, docList);
}
documentSize++;
if (logger.isDebugEnabled()) {

17
src/main/java/org/codelibs/fess/util/ComponentUtil.java
View file

@ -19,13 +19,12 @@ package org.codelibs.fess.util;
import org.codelibs.core.crypto.CachedCipher;
import org.codelibs.core.misc.DynamicProperties;
import org.codelibs.fess.api.WebApiManagerFactory;
import org.codelibs.fess.client.SearchClient;
import org.codelibs.fess.client.FessEsClient;
import org.codelibs.fess.ds.DataStoreFactory;
import org.codelibs.fess.helper.AdRoleHelper;
import org.codelibs.fess.helper.CrawlingConfigHelper;
import org.codelibs.fess.helper.CrawlingSessionHelper;
import org.codelibs.fess.helper.DatabaseHelper;
import org.codelibs.fess.helper.DocumentHelper;
import org.codelibs.fess.helper.FieldHelper;
import org.codelibs.fess.helper.FileTypeHelper;
import org.codelibs.fess.helper.HotSearchWordHelper;
@ -69,8 +68,6 @@ public final class ComponentUtil {
private static final String WEB_API_MANAGER_FACTORY = "webApiManagerFactory";
private static final String DOCUMENT_HELPER = "documentHelper";
private static final String JOB_HELPER = "jobHelper";
private static final String OVERLAPPING_HOST_HELPER = "overlappingHostHelper";
@ -97,8 +94,6 @@ public final class ComponentUtil {
private static final String AD_ROLE_HELPER = "adRoleHelper";
private static final String SOLR_GROUP_MANAGER = "solrGroupManager";
private static final String CRAWLER_PROPERTIES = "crawlerProperties";
private static final String PROPERTIES_SUFFIX = "Properties";
@ -107,15 +102,13 @@ public final class ComponentUtil {
private static final String JOB_EXECUTOR_SUFFIX = "JobExecutor";
private static final String SEARCH_SERVICE = "searchService";
private static final String KEY_MATCH_HELPER = "keyMatchHelper";
private static final String INDEXING_HELPER = "indexingHelper";
private static final String FIELD_HELPER = "fieldHelper";
private static final String ELASTICSEARCH_CLIENT = "searchClient";
private static final String ELASTICSEARCH_CLIENT = "fessEsClient";
private ComponentUtil() {
}
@ -184,10 +177,6 @@ public final class ComponentUtil {
return SingletonS2Container.getComponent(JOB_HELPER);
}
public static DocumentHelper getDocumentHelper() {
return SingletonS2Container.getComponent(DOCUMENT_HELPER);
}
public static WebApiManagerFactory getWebApiManagerFactory() {
return SingletonS2Container.getComponent(WEB_API_MANAGER_FACTORY);
}
@ -248,7 +237,7 @@ public final class ComponentUtil {
return SingletonS2Container.getComponent(FIELD_HELPER);
}
public static SearchClient getElasticsearchClient() {
public static FessEsClient getElasticsearchClient() {
return SingletonS2Container.getComponent(ELASTICSEARCH_CLIENT);
}

4
src/main/java/org/codelibs/fess/util/FacetResponse.java
View file

@ -41,7 +41,7 @@ public class FacetResponse {
fieldList.add(new Field(termFacet));
} else if (aggregation.getName().startsWith(Constants.FACET_QUERY_PREFIX)) {
final Filter queryFacet = (Filter) aggregation;
String encodedQuery = queryFacet.getName().substring(Constants.FACET_QUERY_PREFIX.length());
final String encodedQuery = queryFacet.getName().substring(Constants.FACET_QUERY_PREFIX.length());
queryCountMap.put(new String(BaseEncoding.base64().decode(encodedQuery), Charsets.UTF_8), queryFacet.getDocCount());
}
@ -58,7 +58,7 @@ public class FacetResponse {
protected String name;
public Field(final Terms termFacet) {
String encodedField = termFacet.getName().substring(Constants.FACET_FIELD_PREFIX.length());
final String encodedField = termFacet.getName().substring(Constants.FACET_FIELD_PREFIX.length());
name = new String(BaseEncoding.base64().decode(encodedField), Charsets.UTF_8);
for (final Terms.Bucket tfEntry : termFacet.getBuckets()) {
valueCountMap.put(tfEntry.getKeyAsText().string(), tfEntry.getDocCount());

19
src/main/java/org/codelibs/fess/util/QueryResponseList.java
View file

@ -25,7 +25,6 @@ import java.util.ListIterator;
import java.util.Map;
import org.apache.commons.lang.StringUtils;
import org.codelibs.fess.helper.FieldHelper;
import org.codelibs.fess.helper.QueryHelper;
import org.codelibs.fess.helper.ViewHelper;
import org.elasticsearch.action.search.SearchResponse;
@ -89,7 +88,7 @@ public class QueryResponseList implements List<Map<String, Object>> {
public void init(final SearchResponse searchResponse, final long start, final int pageSize) {
long numFound = 0;
if (searchResponse != null) {
SearchHits searchHits = searchResponse.getHits();
final SearchHits searchHits = searchResponse.getHits();
numFound = searchHits.getTotalHits();
queryTime = searchResponse.getTookInMillis();
@ -99,24 +98,24 @@ public class QueryResponseList implements List<Map<String, Object>> {
// build highlighting fields
final QueryHelper queryHelper = ComponentUtil.getQueryHelper();
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
ComponentUtil.getFieldHelper();
final String hlPrefix = queryHelper.getHighlightingPrefix();
for (final SearchHit searchHit : searchHits.getHits()) {
final Map<String, Object> docMap = new HashMap<String, Object>();
docMap.putAll(searchHit.getSource());
Map<String, HighlightField> highlightFields = searchHit.getHighlightFields();
final Map<String, HighlightField> highlightFields = searchHit.getHighlightFields();
try {
if (highlightFields != null) {
for (Map.Entry<String, HighlightField> entry : highlightFields.entrySet()) {
HighlightField highlightField = entry.getValue();
Text[] fragments = highlightField.fragments();
for (final Map.Entry<String, HighlightField> entry : highlightFields.entrySet()) {
final HighlightField highlightField = entry.getValue();
final Text[] fragments = highlightField.fragments();
if (fragments != null && fragments.length != 0) {
String[] texts = new String[fragments.length];
final String[] texts = new String[fragments.length];
for (int i = 0; i < fragments.length; i++) {
texts[i] = fragments[i].string();
}
String value = StringUtils.join(texts, "...");
final String value = StringUtils.join(texts, "...");
docMap.put(hlPrefix + highlightField.getName(), value);
}
}
@ -140,7 +139,7 @@ public class QueryResponseList implements List<Map<String, Object>> {
}
// facet
Aggregations aggregations = searchResponse.getAggregations();
final Aggregations aggregations = searchResponse.getAggregations();
if (aggregations != null) {
facetResponse = new FacetResponse(aggregations);
}

7
src/main/java/org/codelibs/fess/util/ResourceUtil.java
View file

@ -83,12 +83,7 @@ public class ResourceUtil {
return new File[0];
}
final File libDir = new File(libPath);
return libDir.listFiles(new FilenameFilter() {
@Override
public boolean accept(final File file, final String name) {
return name.startsWith(namePrefix);
}
});
return libDir.listFiles((FilenameFilter) (file, name) -> name.startsWith(namePrefix));
}
public static String resolve(final String value) {

12
src/main/java/org/codelibs/fess/util/UserDictCSVUtil.java
View file

@ -38,16 +38,18 @@ public final class UserDictCSVUtil {
/**
* Parse CSV line
* @param line line containing csv-encoded data
*
* @param line
* line containing csv-encoded data
* @return Array of values
*/
public static String[] parse(String line) {
boolean insideQuote = false;
ArrayList<String> result = new ArrayList<>();
final ArrayList<String> result = new ArrayList<>();
int quoteCount = 0;
StringBuilder sb = new StringBuilder();
final StringBuilder sb = new StringBuilder();
for (int i = 0; i < line.length(); i++) {
char c = line.charAt(i);
final char c = line.charAt(i);
if (c == QUOTE) {
insideQuote = !insideQuote;
@ -80,7 +82,7 @@ public final class UserDictCSVUtil {
// Unquote
if (result.indexOf('\"') >= 0) {
Matcher m = QUOTE_REPLACE_PATTERN.matcher(original);
final Matcher m = QUOTE_REPLACE_PATTERN.matcher(original);
if (m.matches()) {
result = m.group(1);
}

85
src/main/java/org/codelibs/fess/web/IndexAction.java
View file

@ -56,15 +56,14 @@ import org.codelibs.fess.Constants;
import org.codelibs.fess.InvalidQueryException;
import org.codelibs.fess.ResultOffsetExceededException;
import org.codelibs.fess.UnsupportedSearchException;
import org.codelibs.fess.client.SearchClient;
import org.codelibs.fess.client.SearchClient.SearchConditionBuilder;
import org.codelibs.fess.client.FessEsClient;
import org.codelibs.fess.client.FessEsClient.SearchConditionBuilder;
import org.codelibs.fess.db.allcommon.CDef;
import org.codelibs.fess.db.exentity.ClickLog;
import org.codelibs.fess.db.exentity.SearchLog;
import org.codelibs.fess.db.exentity.UserInfo;
import org.codelibs.fess.entity.LoginInfo;
import org.codelibs.fess.helper.CrawlingConfigHelper;
import org.codelibs.fess.helper.DocumentHelper;
import org.codelibs.fess.helper.FieldHelper;
import org.codelibs.fess.helper.HotSearchWordHelper;
import org.codelibs.fess.helper.HotSearchWordHelper.Range;
@ -132,7 +131,7 @@ public class IndexAction {
protected IndexForm indexForm;
@Resource
protected SearchClient searchClient;
protected FessEsClient fessEsClient;
@Resource
protected FavoriteLogService favoriteLogService;
@ -335,8 +334,8 @@ public class IndexAction {
public String cache() {
Map<String, Object> doc = null;
try {
doc = searchClient.getDocument(queryRequestBuilder -> {
TermQueryBuilder termQuery = QueryBuilders.termQuery(fieldHelper.docIdField, indexForm.docId);
doc = fessEsClient.getDocument(fieldHelper.docIndex, fieldHelper.docType, queryRequestBuilder -> {
final TermQueryBuilder termQuery = QueryBuilders.termQuery(fieldHelper.docIdField, indexForm.docId);
queryRequestBuilder.setQuery(termQuery);
queryRequestBuilder.addFields(queryHelper.getResponseFields());
return true;
@ -363,8 +362,8 @@ public class IndexAction {
public String go() throws IOException {
Map<String, Object> doc = null;
try {
doc = searchClient.getDocument(queryRequestBuilder -> {
TermQueryBuilder termQuery = QueryBuilders.termQuery(fieldHelper.docIdField, indexForm.docId);
doc = fessEsClient.getDocument(fieldHelper.docIndex, fieldHelper.docType, queryRequestBuilder -> {
final TermQueryBuilder termQuery = QueryBuilders.termQuery(fieldHelper.docIdField, indexForm.docId);
queryRequestBuilder.setQuery(termQuery);
queryRequestBuilder.addFields(queryHelper.getResponseFields());
return true;
@ -390,7 +389,7 @@ public class IndexAction {
final SearchLogHelper searchLogHelper = ComponentUtil.getSearchLogHelper();
final ClickLog clickLog = new ClickLog();
clickLog.setUrl(url);
LocalDateTime now = systemHelper.getCurrentTime();
final LocalDateTime now = systemHelper.getCurrentTime();
clickLog.setRequestedTime(now);
clickLog.setQueryRequestedTime(LocalDateTime.ofInstant(Instant.ofEpochMilli(Long.parseLong(indexForm.rt)),
ZoneId.systemDefault()));
@ -500,8 +499,8 @@ public class IndexAction {
OutputStream out = null;
BufferedInputStream in = null;
try {
final Map<String, Object> doc = searchClient.getDocument(queryRequestBuilder -> {
TermQueryBuilder termQuery = QueryBuilders.termQuery(fieldHelper.docIdField, indexForm.docId);
final Map<String, Object> doc = fessEsClient.getDocument(fieldHelper.docIndex, fieldHelper.docType, queryRequestBuilder -> {
final TermQueryBuilder termQuery = QueryBuilders.termQuery(fieldHelper.docIdField, indexForm.docId);
queryRequestBuilder.setQuery(termQuery);
queryRequestBuilder.addFields(queryHelper.getResponseFields());
return true;
@ -592,19 +591,17 @@ public class IndexAction {
final List<String> suggestFieldName = Arrays.asList(fieldNames);
WebApiUtil.setObject("suggestFieldName", suggestFieldName);
final List<String> labelList;
if (labels == null) {
labelList = new ArrayList<String>();
new ArrayList<String>();
} else {
labelList = Arrays.asList(labels);
Arrays.asList(labels);
}
try {
final Set<String> roleSet;
if (roleQueryHelper != null) {
roleSet = roleQueryHelper.build();
roleQueryHelper.build();
} else {
roleSet = new HashSet<>();
new HashSet<>();
}
// TODO
@ -663,12 +660,14 @@ public class IndexAction {
}
try {
final Map<String, Object> doc = indexForm.docId == null ? null : searchClient.getDocument(queryRequestBuilder -> {
TermQueryBuilder termQuery = QueryBuilders.termQuery(fieldHelper.docIdField, indexForm.docId);
queryRequestBuilder.setQuery(termQuery);
queryRequestBuilder.addFields(queryHelper.getResponseFields());
return true;
}).get();
final Map<String, Object> doc =
indexForm.docId == null ? null : fessEsClient.getDocument(fieldHelper.docIndex, fieldHelper.docType,
queryRequestBuilder -> {
final TermQueryBuilder termQuery = QueryBuilders.termQuery(fieldHelper.docIdField, indexForm.docId);
queryRequestBuilder.setQuery(termQuery);
queryRequestBuilder.addFields(queryHelper.getResponseFields());
return true;
}).get();
final String userCode = userInfoHelper.getUserCode();
final String favoriteUrl = doc == null ? null : (String) doc.get(fieldHelper.urlField);
@ -703,10 +702,11 @@ public class IndexAction {
return null;
}
final DocumentHelper documentHelper = ComponentUtil.getDocumentHelper();
final FessEsClient fessEsClient = ComponentUtil.getElasticsearchClient();
final Object count = doc.get(fieldHelper.favoriteCountField);
if (count instanceof Long) {
documentHelper.update(indexForm.docId, fieldHelper.favoriteCountField, ((Long) count).longValue() + 1);
fessEsClient.update(fieldHelper.docIndex, fieldHelper.docType, indexForm.docId, fieldHelper.favoriteCountField,
((Long) count).longValue() + 1);
} else {
WebApiUtil.setError(7, "Failed to update count: " + favoriteUrl);
return null;
@ -737,20 +737,21 @@ public class IndexAction {
}
final String[] docIds = userInfoHelper.getResultDocIds(indexForm.queryId);
final List<Map<String, Object>> docList = searchClient.getDocumentList(queryRequstBuilder -> {
if (docIds == null || docIds.length == 0) {
return false;
}
final List<Map<String, Object>> docList =
fessEsClient.getDocumentList(fieldHelper.docIndex, fieldHelper.docType, queryRequstBuilder -> {
if (docIds == null || docIds.length == 0) {
return false;
}
queryRequstBuilder.setFrom(0).setSize(getMaxPageSize());
queryRequstBuilder.addFields(queryHelper.getResponseFields());
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
for (int i = 0; i < docIds.length && i < getMaxPageSize(); i++) {
boolQuery.should(QueryBuilders.termQuery(fieldHelper.docIdField, docIds[i]));
}
queryRequstBuilder.setQuery(boolQuery);
return true;
}).get();
queryRequstBuilder.setFrom(0).setSize(getMaxPageSize());
queryRequstBuilder.addFields(queryHelper.getResponseFields());
final BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
for (int i = 0; i < docIds.length && i < getMaxPageSize(); i++) {
boolQuery.should(QueryBuilders.termQuery(fieldHelper.docIdField, docIds[i]));
}
queryRequstBuilder.setQuery(boolQuery);
return true;
});
List<String> urlList = new ArrayList<String>(docList.size());
for (final Map<String, Object> doc : docList) {
final Object urlObj = doc.get(fieldHelper.urlField);
@ -896,12 +897,12 @@ public class IndexAction {
final int pageNum = Integer.parseInt(indexForm.num);
try {
documentItems =
searchClient.getDocumentList(
fessEsClient.getDocumentList(fieldHelper.docIndex, fieldHelper.docType,
searchRequestBuilder -> {
return SearchConditionBuilder.builder(searchRequestBuilder).query(query).offset(pageStart).size(pageNum)
.facetInfo(indexForm.facet).geoInfo(indexForm.geo).responseFields(queryHelper.getResponseFields())
.build();
}).get();
});
} catch (final InvalidQueryException e) {
if (logger.isDebugEnabled()) {
logger.debug(e.getMessage(), e);
@ -1223,13 +1224,13 @@ public class IndexAction {
}
protected int getMaxPageSize() {
Object maxPageSize = crawlerProperties.get(Constants.SEARCH_RESULT_MAX_PAGE_SIZE);
final Object maxPageSize = crawlerProperties.get(Constants.SEARCH_RESULT_MAX_PAGE_SIZE);
if (maxPageSize == null) {
return MAX_PAGE_SIZE;
}
try {
return Integer.parseInt(maxPageSize.toString());
} catch (NumberFormatException e) {
} catch (final NumberFormatException e) {
return MAX_PAGE_SIZE;
}
}

93
src/main/java/org/codelibs/fess/web/admin/DataAction.java
View file

@ -32,7 +32,6 @@ import java.text.SimpleDateFormat;
import java.util.Date;
import javax.annotation.Resource;
import javax.servlet.ServletOutputStream;
import javax.servlet.http.HttpServletResponse;
import org.apache.commons.io.IOUtils;
@ -44,7 +43,6 @@ import org.codelibs.fess.crud.util.SAStrutsUtil;
import org.codelibs.fess.helper.SystemHelper;
import org.codelibs.fess.service.ClickLogService;
import org.codelibs.fess.service.CrawlingSessionService;
import org.codelibs.fess.service.DatabaseService;
import org.codelibs.fess.service.SearchLogService;
import org.codelibs.robot.util.StreamUtil;
import org.codelibs.sastruts.core.exception.SSCActionMessagesException;
@ -65,9 +63,6 @@ public class DataAction implements Serializable {
@ActionForm
protected DataForm dataForm;
@Resource
protected DatabaseService databaseService;
@Resource
protected CrawlingSessionService crawlingSessionService;
@ -94,33 +89,6 @@ public class DataAction implements Serializable {
return "index.jsp";
}
@Execute(validator = false)
public String download() {
final DateFormat df = new SimpleDateFormat(CoreLibConstants.DATE_FORMAT_DIGIT_ONLY);
final StringBuilder buf = new StringBuilder();
buf.append("backup-");
buf.append(df.format(new Date()));
buf.append(".xml");
final HttpServletResponse response = ResponseUtil.getResponse();
response.setContentType("application/octet-stream");
response.setHeader("Content-Disposition", "attachment; filename=\"" + buf.toString() + "\"");
try {
final ServletOutputStream sos = response.getOutputStream();
try {
databaseService.exportData(sos);
sos.flush();
} finally {
sos.close();
}
return null;
} catch (final Exception e) {
logger.error("Failed to export data.", e);
throw new SSCActionMessagesException(e, "errors.failed_to_export_data");
}
}
@Execute(validator = false)
public String downloadCrawlingSession() {
final DateFormat df = new SimpleDateFormat(CoreLibConstants.DATE_FORMAT_DIGIT_ONLY);
@ -208,17 +176,7 @@ public class DataAction implements Serializable {
@Execute(validator = true, input = "index")
public String upload() {
final String fileName = dataForm.uploadedFile.getFileName();
if (fileName.endsWith(".xml")) {
try {
databaseService.importData(dataForm.uploadedFile.getInputStream(),
dataForm.overwrite != null && "on".equalsIgnoreCase(dataForm.overwrite));
SAStrutsUtil.addSessionMessage("success.importing_data");
return "index?redirect=true";
} catch (final Exception e) {
logger.error("Failed to import data.", e);
throw new SSCActionMessagesException(e, "errors.failed_to_import_data");
}
} else if (fileName.endsWith(".csv")) {
if (fileName.endsWith(".csv")) {
BufferedInputStream is = null;
File tempFile = null;
FileOutputStream fos = null;
@ -252,33 +210,30 @@ public class DataAction implements Serializable {
throw new SSCActionMessagesException("errors.unknown_import_file");
}
final String enc = crawlerProperties.getProperty(Constants.CSV_FILE_ENCODING_PROPERTY, Constants.UTF_8);
new Thread(new Runnable() {
@Override
public void run() {
Reader reader = null;
try {
reader = new BufferedReader(new InputStreamReader(new FileInputStream(oFile), enc));
if (head.startsWith("SessionId,")) {
// Crawling Session
crawlingSessionService.importCsv(reader);
} else if (head.startsWith("SearchWord,")) {
// Search Log
searchLogService.importCsv(reader);
} else if (head.startsWith("SearchId,")) {
// Click Log
clickLogService.importCsv(reader);
}
} catch (final Exception e) {
logger.error("Failed to import data.", e);
throw new FessSystemException("Failed to import data.", e);
} finally {
if (!oFile.delete()) {
logger.warn("Could not delete " + oFile.getAbsolutePath());
}
IOUtils.closeQuietly(reader);
}
new Thread(() -> {
Reader reader = null;
try {
reader = new BufferedReader(new InputStreamReader(new FileInputStream(oFile), enc));
if (head.startsWith("SessionId,")) {
// Crawling Session
crawlingSessionService.importCsv(reader);
} else if (head.startsWith("SearchWord,")) {
// Search Log
searchLogService.importCsv(reader);
} else if (head.startsWith("SearchId,")) {
// Click Log
clickLogService.importCsv(reader);
}
}).start();
} catch (final Exception e) {
logger.error("Failed to import data.", e);
throw new FessSystemException("Failed to import data.", e);
} finally {
if (!oFile.delete()) {
logger.warn("Could not delete " + oFile.getAbsolutePath());
}
IOUtils.closeQuietly(reader);
}
} ).start();
} catch (final ActionMessagesException e) {
if (!oFile.delete()) {
logger.warn("Could not delete " + oFile.getAbsolutePath());

102
src/main/java/org/codelibs/fess/web/admin/DocumentAction.java
View file

@ -24,7 +24,7 @@ import java.util.Set;
import javax.annotation.Resource;
import org.codelibs.fess.client.SearchClient;
import org.codelibs.fess.client.FessEsClient;
import org.codelibs.fess.crud.util.SAStrutsUtil;
import org.codelibs.fess.helper.FieldHelper;
import org.codelibs.fess.helper.JobHelper;
@ -47,12 +47,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class DocumentAction implements Serializable {
private static final String SUGGEST_TYPE_ALL = "all";
private static final String SUGGEST_TYPE_SEARCH_LOG = "searchLog";
private static final String SUGGEST_TYPE_CONTENT = "content";
private static final Logger logger = LoggerFactory.getLogger(DocumentAction.class);
private static final long serialVersionUID = 1L;
@ -62,7 +56,7 @@ public class DocumentAction implements Serializable {
protected DocumentForm documentForm;
@Resource
protected SearchClient searchClient;
protected FessEsClient fessEsClient;
@Resource
protected WebManagementHelper webManagementHelper;
@ -106,7 +100,7 @@ public class DocumentAction implements Serializable {
throw new SSCActionMessagesException("errors.failed_to_start_solr_process_because_of_running");
}
searchClient.flush();
fessEsClient.flush(ComponentUtil.getFieldHelper().docIndex);
SAStrutsUtil.addSessionMessage("success.commit_solr_index");
return showIndex(true);
}
@ -118,7 +112,7 @@ public class DocumentAction implements Serializable {
if (jobHelper.isCrawlProcessRunning()) {
throw new SSCActionMessagesException("errors.failed_to_start_solr_process_because_of_running");
}
searchClient.optimize();
fessEsClient.optimize(ComponentUtil.getFieldHelper().docIndex);
SAStrutsUtil.addSessionMessage("success.optimize_solr_index");
return showIndex(true);
}
@ -153,7 +147,7 @@ public class DocumentAction implements Serializable {
throw new SSCActionMessagesException("errors.failed_to_start_solr_process_because_of_running");
}
searchClient.deleteByQuery(queryBuilder);
fessEsClient.deleteByQuery(fieldHelper.docIndex, fieldHelper.docType, queryBuilder);
SAStrutsUtil.addSessionMessage("success.delete_solr_index");
return showIndex(true);
}
@ -161,10 +155,12 @@ public class DocumentAction implements Serializable {
protected SessionIdList<Map<String, String>> getSessionIdList(final String groupName) {
// TODO remove groupName?
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
return searchClient.search(
return fessEsClient.search(
fieldHelper.docIndex,
fieldHelper.docType,
queryRequestBuilder -> {
queryRequestBuilder.setQuery(QueryBuilders.matchAllQuery());
TermsBuilder termsBuilder =
final TermsBuilder termsBuilder =
AggregationBuilders.terms(fieldHelper.segmentField).field(fieldHelper.segmentField).size(100)
.order(Order.count(false));
queryRequestBuilder.addAggregation(termsBuilder);
@ -172,8 +168,8 @@ public class DocumentAction implements Serializable {
}, (queryRequestBuilder, execTime, searchResponse) -> {
final SessionIdList<Map<String, String>> sessionIdList = new SessionIdList<Map<String, String>>();
searchResponse.ifPresent(response -> {
Terms terms = response.getAggregations().get(fieldHelper.segmentField);
for (Bucket bucket : terms.getBuckets()) {
final Terms terms = response.getAggregations().get(fieldHelper.segmentField);
for (final Bucket bucket : terms.getBuckets()) {
final Map<String, String> map = new HashMap<String, String>(3);
map.put("label", bucket.getKey() + " (" + bucket.getDocCount() + ")");
map.put("value", bucket.getKey());
@ -197,9 +193,11 @@ public class DocumentAction implements Serializable {
protected Map<String, Long> getSuggestDocumentNum() {
final Map<String, Long> map = new HashMap<String, Long>();
// map.put(SUGGEST_TYPE_CONTENT, suggestService.getContentDocumentNum());
// map.put(SUGGEST_TYPE_SEARCH_LOG, suggestService.getSearchLogDocumentNum());
// map.put(SUGGEST_TYPE_ALL, suggestService.getDocumentNum());
// map.put(SUGGEST_TYPE_CONTENT,
// suggestService.getContentDocumentNum());
// map.put(SUGGEST_TYPE_SEARCH_LOG,
// suggestService.getSearchLogDocumentNum());
// map.put(SUGGEST_TYPE_ALL, suggestService.getDocumentNum());
return map;
}
@ -208,44 +206,32 @@ public class DocumentAction implements Serializable {
public String deleteSuggest() {
// TODO
/*
final SuggestSolrServer suggestSolrServer = suggestService.getSuggestSolrServer();
final String query;
if (SUGGEST_TYPE_CONTENT.equals(documentForm.deleteSuggestType)) {
query =
"*:* NOT " + SuggestConstants.SuggestFieldNames.SEGMENT + ":" + SuggestConstants.SEGMENT_ELEVATE + " NOT "
+ SuggestConstants.SuggestFieldNames.SEGMENT + ":" + SuggestConstants.SEGMENT_QUERY;
} else if (SUGGEST_TYPE_SEARCH_LOG.equals(documentForm.deleteSuggestType)) {
query = SuggestConstants.SuggestFieldNames.SEGMENT + ":" + SuggestConstants.SEGMENT_QUERY;
} else {
query = "";
}
if (StringUtil.isNotBlank(query)) {
final Thread thread = new Thread(new Runnable() {
@Override
public void run() {
if (!jobHelper.isCrawlProcessRunning()) {
final long execTime = System.currentTimeMillis();
try {
suggestSolrServer.deleteByQuery(query);
suggestSolrServer.commit();
if (logger.isInfoEnabled()) {
logger.info("[EXEC TIME] suggest index cleanup time: " + (System.currentTimeMillis() - execTime) + "ms");
}
} catch (final Exception e) {
logger.error("Failed to delete suggest index (query=" + query + ").", e);
}
} else {
if (logger.isInfoEnabled()) {
logger.info("could not start index cleanup process" + " because of running solr process.");
}
}
}
});
thread.start();
SAStrutsUtil.addSessionMessage("success.delete_solr_index");
}
*/
* final SuggestSolrServer suggestSolrServer =
* suggestService.getSuggestSolrServer(); final String query; if
* (SUGGEST_TYPE_CONTENT.equals(documentForm.deleteSuggestType)) { query
* = "*:* NOT " + SuggestConstants.SuggestFieldNames.SEGMENT + ":" +
* SuggestConstants.SEGMENT_ELEVATE + " NOT " +
* SuggestConstants.SuggestFieldNames.SEGMENT + ":" +
* SuggestConstants.SEGMENT_QUERY; } else if
* (SUGGEST_TYPE_SEARCH_LOG.equals(documentForm.deleteSuggestType)) {
* query = SuggestConstants.SuggestFieldNames.SEGMENT + ":" +
* SuggestConstants.SEGMENT_QUERY; } else { query = ""; }
*
* if (StringUtil.isNotBlank(query)) { final Thread thread = new
* Thread(new Runnable() {
*
* @Override public void run() { if (!jobHelper.isCrawlProcessRunning())
* { final long execTime = System.currentTimeMillis(); try {
* suggestSolrServer.deleteByQuery(query); suggestSolrServer.commit();
* if (logger.isInfoEnabled()) {
* logger.info("[EXEC TIME] suggest index cleanup time: " +
* (System.currentTimeMillis() - execTime) + "ms"); } } catch (final
* Exception e) { logger.error("Failed to delete suggest index (query="
* + query + ").", e); } } else { if (logger.isInfoEnabled()) {
* logger.info("could not start index cleanup process" +
* " because of running solr process."); } } } }); thread.start();
* SAStrutsUtil.addSessionMessage("success.delete_solr_index"); }
*/
return showIndex(true);
}
@ -259,9 +245,5 @@ public class DocumentAction implements Serializable {
totalCount += count;
}
public long getTotalCount() {
return totalCount;
}
}
}

25
src/main/java/org/codelibs/fess/web/admin/LogAction.java
View file

@ -25,7 +25,6 @@ import java.io.UnsupportedEncodingException;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
@ -104,26 +103,20 @@ public class LogAction implements Serializable {
logger.warn("Log directory does not exist: " + parentDir.getAbsolutePath());
return logFileItems;
}
final File[] files = parentDir.listFiles(new FilenameFilter() {
@Override
public boolean accept(final File dir, final String name) {
if (name.indexOf(".out") > 0) {
return true;
}
return false;
final File[] files = parentDir.listFiles((FilenameFilter) (dir, name) -> {
if (name.indexOf(".out") > 0) {
return true;
}
return false;
});
if (files == null) {
return logFileItems;
}
Arrays.sort(files, new Comparator<File>() {
@Override
public int compare(final File o1, final File o2) {
if (o1.lastModified() < o2.lastModified()) {
return 1;
} else {
return -1;
}
Arrays.sort(files, (o1, o2) -> {
if (o1.lastModified() < o2.lastModified()) {
return 1;
} else {
return -1;
}
});
for (final File logFile : files) {

47
src/main/java/org/codelibs/fess/web/admin/SearchListAction.java
View file

@ -29,19 +29,16 @@ import org.codelibs.core.misc.DynamicProperties;
import org.codelibs.fess.Constants;
import org.codelibs.fess.InvalidQueryException;
import org.codelibs.fess.ResultOffsetExceededException;
import org.codelibs.fess.client.SearchClient;
import org.codelibs.fess.client.SearchClient.SearchConditionBuilder;
import org.codelibs.fess.client.FessEsClient;
import org.codelibs.fess.client.FessEsClient.SearchConditionBuilder;
import org.codelibs.fess.crud.util.SAStrutsUtil;
import org.codelibs.fess.entity.SearchQuery;
import org.codelibs.fess.helper.FieldHelper;
import org.codelibs.fess.helper.JobHelper;
import org.codelibs.fess.helper.QueryHelper;
import org.codelibs.fess.helper.SystemHelper;
import org.codelibs.fess.util.ComponentUtil;
import org.codelibs.fess.util.QueryResponseList;
import org.codelibs.sastruts.core.annotation.Token;
import org.codelibs.sastruts.core.exception.SSCActionMessagesException;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.seasar.framework.beans.util.Beans;
@ -62,7 +59,7 @@ public class SearchListAction implements Serializable {
protected SearchListForm searchListForm;
@Resource
protected SearchClient searchClient;
protected FessEsClient fessEsClient;
@Resource
protected DynamicProperties crawlerProperties;
@ -157,11 +154,10 @@ public class SearchListAction implements Serializable {
final int size = Integer.parseInt(searchListForm.num);
try {
documentItems =
searchClient.getDocumentList(
searchRequestBuilder -> {
return SearchConditionBuilder.builder(searchRequestBuilder).administrativeAccess().offset(offset)
.size(size).responseFields(queryHelper.getResponseFields()).build();
}).get();
fessEsClient.getDocumentList(fieldHelper.docIndex, fieldHelper.docType, searchRequestBuilder -> {
return SearchConditionBuilder.builder(searchRequestBuilder).administrativeAccess().offset(offset).size(size)
.responseFields(queryHelper.getResponseFields()).build();
});
} catch (final InvalidQueryException e) {
if (logger.isDebugEnabled()) {
logger.debug(e.getMessage(), e);
@ -254,24 +250,21 @@ public class SearchListAction implements Serializable {
if (jobHelper.isCrawlProcessRunning()) {
throw new SSCActionMessagesException("errors.failed_to_start_solr_process_because_of_running");
}
final Thread thread = new Thread(new Runnable() {
@Override
public void run() {
if (!jobHelper.isCrawlProcessRunning()) {
final long time = System.currentTimeMillis();
try {
QueryBuilder query = QueryBuilders.termQuery(fieldHelper.docIdField, docId);
searchClient.deleteByQuery(query);
if (logger.isInfoEnabled()) {
logger.info("[EXEC TIME] index cleanup time: " + (System.currentTimeMillis() - time) + "ms");
}
} catch (final Exception e) {
logger.error("Failed to delete index (query=" + fieldHelper.docIdField + ":" + docId + ").", e);
}
} else {
final Thread thread = new Thread(() -> {
if (!jobHelper.isCrawlProcessRunning()) {
final long time = System.currentTimeMillis();
try {
final QueryBuilder query = QueryBuilders.termQuery(fieldHelper.docIdField, docId);
fessEsClient.deleteByQuery(fieldHelper.docIndex, fieldHelper.docType, query);
if (logger.isInfoEnabled()) {
logger.info("could not start index cleanup process" + " because of running solr process.");
logger.info("[EXEC TIME] index cleanup time: " + (System.currentTimeMillis() - time) + "ms");
}
} catch (final Exception e) {
logger.error("Failed to delete index (query=" + fieldHelper.docIdField + ":" + docId + ").", e);
}
} else {
if (logger.isInfoEnabled()) {
logger.info("could not start index cleanup process" + " because of running solr process.");
}
}
});

30
src/main/java/org/codelibs/fess/web/admin/SuggestBadWordAction.java
View file

@ -428,23 +428,21 @@ public class SuggestBadWordAction extends FessAdminAction {
throw new SSCActionMessagesException("errors.unknown_import_file");
}
final String enc = crawlerProperties.getProperty(Constants.CSV_FILE_ENCODING_PROPERTY, Constants.UTF_8);
new Thread(new Runnable() {
public void run() {
Reader reader = null;
try {
reader = new BufferedReader(new InputStreamReader(new FileInputStream(oFile), enc));
suggestBadWordService.importCsv(reader);
} catch (final Exception e) {
logger.error("Failed to import data.", e);
throw new FessSystemException("Failed to import data.", e);
} finally {
if (!oFile.delete()) {
logger.warn("Could not delete " + oFile.getAbsolutePath());
}
IOUtils.closeQuietly(reader);
suggestHelper.deleteAllBadWord();
suggestHelper.updateSolrBadwordFile();
new Thread(() -> {
Reader reader = null;
try {
reader = new BufferedReader(new InputStreamReader(new FileInputStream(oFile), enc));
suggestBadWordService.importCsv(reader);
} catch (final Exception e) {
logger.error("Failed to import data.", e);
throw new FessSystemException("Failed to import data.", e);
} finally {
if (!oFile.delete()) {
logger.warn("Could not delete " + oFile.getAbsolutePath());
}
IOUtils.closeQuietly(reader);
suggestHelper.deleteAllBadWord();
suggestHelper.updateSolrBadwordFile();
}
}).start();
} catch (final ActionMessagesException e) {

28
src/main/java/org/codelibs/fess/web/admin/SuggestElevateWordAction.java
View file

@ -425,22 +425,20 @@ public class SuggestElevateWordAction extends FessAdminAction {
throw new SSCActionMessagesException("errors.unknown_import_file");
}
final String enc = crawlerProperties.getProperty(Constants.CSV_FILE_ENCODING_PROPERTY, Constants.UTF_8);
new Thread(new Runnable() {
public void run() {
Reader reader = null;
try {
reader = new BufferedReader(new InputStreamReader(new FileInputStream(oFile), enc));
suggestElevateWordService.importCsv(reader);
} catch (final Exception e) {
logger.error("Failed to import data.", e);
throw new FessSystemException("Failed to import data.", e);
} finally {
if (!oFile.delete()) {
logger.warn("Could not delete " + oFile.getAbsolutePath());
}
IOUtils.closeQuietly(reader);
suggestHelper.storeAllElevateWords();
new Thread(() -> {
Reader reader = null;
try {
reader = new BufferedReader(new InputStreamReader(new FileInputStream(oFile), enc));
suggestElevateWordService.importCsv(reader);
} catch (final Exception e) {
logger.error("Failed to import data.", e);
throw new FessSystemException("Failed to import data.", e);
} finally {
if (!oFile.delete()) {
logger.warn("Could not delete " + oFile.getAbsolutePath());
}
IOUtils.closeQuietly(reader);
suggestHelper.storeAllElevateWords();
}
}).start();
} catch (final ActionMessagesException e) {

5
src/main/java/org/codelibs/fess/web/admin/SystemAction.java
View file

@ -24,12 +24,11 @@ import javax.annotation.Resource;
import javax.servlet.http.HttpSession;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.fess.client.SearchClient;
import org.codelibs.fess.client.FessEsClient;
import org.codelibs.fess.crud.util.SAStrutsUtil;
import org.codelibs.fess.db.exentity.ScheduledJob;
import org.codelibs.fess.helper.JobHelper;
import org.codelibs.fess.helper.SystemHelper;
import org.codelibs.fess.helper.WebManagementHelper;
import org.codelibs.fess.service.ScheduledJobService;
import org.codelibs.sastruts.core.annotation.Token;
import org.seasar.struts.annotation.ActionForm;
@ -46,7 +45,7 @@ public class SystemAction implements Serializable {
protected SystemForm systemForm;
@Resource
protected SearchClient searchClient;
protected FessEsClient fessEsClient;
@Resource
protected SystemHelper systemHelper;

7
src/main/java/org/codelibs/fess/web/admin/WizardAction.java
View file

@ -265,12 +265,7 @@ public class WizardAction implements Serializable {
if (!jobHelper.isCrawlProcessRunning()) {
final List<ScheduledJob> scheduledJobList = scheduledJobService.getCrawloerJobList();
for (final ScheduledJob scheduledJob : scheduledJobList) {
new Thread(new Runnable() {
@Override
public void run() {
new TriggeredJob().execute(scheduledJob);
}
}).start();
new Thread(() -> new TriggeredJob().execute(scheduledJob)).start();
}
SAStrutsUtil.addSessionMessage("success.start_crawl_process");
} else {

2
src/main/resources/app.dicon
View file

@ -249,8 +249,6 @@
<property name="osddPath">"/WEB-INF/orig/open-search/osdd.xml"</property>
<property name="encoding">"UTF-8"</property>
</component>
<component name="documentHelper" class="org.codelibs.fess.helper.DocumentHelper">
</component>
<component name="queryResponseList" class="org.codelibs.fess.util.QueryResponseList" instance="prototype">
</component>
<component name="userAgentName" class="java.lang.String">

8
src/main/resources/fess_es.dicon
View file

@ -2,9 +2,15 @@
<!DOCTYPE components PUBLIC "-//SEASAR//DTD S2Container 2.4//EN"
"http://www.seasar.org/dtd/components24.dtd">
<components>
<component name="searchClient" class="org.codelibs.fess.client.SearchClient">
<component name="fessEsClient" class="org.codelibs.fess.client.FessEsClient">
<property name="settings">
#{"http.cors.enabled" : "true"}
</property>
<initMethod name="addIndexConfig">
<arg>"fess/doc"</arg>
</initMethod>
<initMethod name="addIndexConfig">
<arg>".fess_config/boost_document_rule"</arg>
</initMethod>
</component>
</components>

9
src/main/resources/fess_indices/.fess_config.json Normal file
View file

@ -0,0 +1,9 @@
{
"settings": {
"index": {
"refresh_interval": "1s",
"number_of_shards": 5,
"number_of_replicas": 0
}
}
}

9
src/main/resources/fess_indices/fess.json Normal file
View file

@ -0,0 +1,9 @@
{
"settings": {
"index": {
"refresh_interval": "1s",
"number_of_shards": 5,
"number_of_replicas": 0
}
}
}

19
src/main/resources/fess_indices/fess/doc.json Normal file
View file

@ -0,0 +1,19 @@
{
"doc": {
"_source": {
"enabled": true
},
"_all": {
"enabled": false
},
"_id" : {
"path" : "id"
},
"properties": {
"id": {
"type": "string",
"index": "not_analyzed"
}
}
}
}

56
src/main/resources/json/index.json
View file

@ -1,56 +0,0 @@
{
"mappings": {
"doc": {
"_all": {
"enabled": false
},
"_id": {
"path": "id"
},
"properties": {
"id": {
"type": "string",
"index": "not_analyzed",
"doc_values": true
},
"doc_id": {
"type": "string",
"index": "not_analyzed",
"doc_values": true
},
"parent_id": {
"type": "string",
"index": "not_analyzed",
"doc_values": true
},
"config_id": {
"type": "string",
"index": "not_analyzed",
"doc_values": true
},
"url": {
"type": "string",
"index": "not_analyzed",
"doc_values": true
},
"anchor": {
"type": "string",
"index": "not_analyzed",
"doc_values": true
},
"segment": {
"type": "string",
"index": "not_analyzed",
"doc_values": true
}
}
}
},
"settings": {
"index": {
"refresh_interval": "1s",
"number_of_shards": 5,
"number_of_replicas": 0
}
}
}

1
src/test/resources/fess.dicon
View file

@ -2,6 +2,7 @@
<!DOCTYPE components PUBLIC "-//SEASAR//DTD S2Container 2.4//EN"
"http://www.seasar.org/dtd/components24.dtd">
<components>
<include path="fess_es.dicon"/>
<component name="crawlerProperties" class="org.codelibs.core.misc.DynamicProperties">
<arg>