0ct0pu5/fess
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix crawling problem

Browse source
This commit is contained in:
Shinsuke Sugaya 2015-11-26 10:22:22 +09:00
parent c28af46610
commit f88b2cba12
3 changed files with 29 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
pom.xml
View file

@ -283,7 +283,7 @@
<url
url="${maven.snapshot.repo.url}/org/codelibs/elasticsearch-configsync/2.0.0-SNAPSHOT/elasticsearch-configsync-2.0.0-20151113.133045-4.zip" />
<url
url="${maven.snapshot.repo.url}/org/codelibs/elasticsearch-langfield/2.0.0-SNAPSHOT/elasticsearch-langfield-2.0.0-20151113.132901-3.zip" />
url="${maven.snapshot.repo.url}/org/codelibs/elasticsearch-langfield/2.0.0-SNAPSHOT/elasticsearch-langfield-2.0.0-20151126.010100-4.zip" />
<url
url="http://maven.codelibs.org/archive/elasticsearch/plugin/kopf/elasticsearch-kopf-2.0.0.0.zip" />
</get>
@ -296,7 +296,7 @@
<unzip dest="${basedir}/plugins/configsync"
src="${basedir}/target/plugins/elasticsearch-configsync-2.0.0-20151113.133045-4.zip" />
<unzip dest="${basedir}/plugins/langfield"
src="${basedir}/target/plugins/elasticsearch-langfield-2.0.0-20151113.132901-3.zip" />
src="${basedir}/target/plugins/elasticsearch-langfield-2.0.0-20151126.010100-4.zip" />
<unzip dest="${basedir}/plugins/kopf"
src="${basedir}/target/plugins/elasticsearch-kopf-2.0.0.0.zip" />
</tasks>

16
src/main/java/org/codelibs/fess/es/client/FessEsClient.java
View file

@ -70,6 +70,8 @@ import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
import org.elasticsearch.action.admin.indices.optimize.OptimizeResponse;
import org.elasticsearch.action.admin.indices.refresh.RefreshResponse;
import org.elasticsearch.action.bulk.BulkItemResponse;
import org.elasticsearch.action.bulk.BulkItemResponse.Failure;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
@ -748,6 +750,20 @@ public class FessEsClient implements Client {
}
final BulkResponse response = bulkRequestBuilder.execute().actionGet();
if (response.hasFailures()) {
if (logger.isDebugEnabled()) {
List<ActionRequest> requests = bulkRequestBuilder.request().requests();
BulkItemResponse[] items = response.getItems();
if (requests.size() == items.length) {
for (int i = 0; i < requests.size(); i++) {
BulkItemResponse resp = items[i];
if (resp.isFailed() && resp.getFailure() != null) {
ActionRequest req = requests.get(i);
Failure failure = resp.getFailure();
logger.debug("Failed Request: " + req + "\n=>" + failure.getMessage());
}
}
}
}
throw new FessEsClientException(response.buildFailureMessage());
}
}

19
src/main/java/org/codelibs/fess/helper/IndexingHelper.java
View file

@ -42,15 +42,18 @@ public class IndexingHelper {
if (logger.isDebugEnabled()) {
logger.debug("Sending " + docList.size() + " documents to a server.");
}
synchronized (fessEsClient) {
deleteOldDocuments(fessEsClient, docList);
final FessConfig fessConfig = ComponentUtil.getFessConfig();
fessEsClient.addAll(fessConfig.getIndexDocumentIndex(), fessConfig.getIndexDocumentType(), docList);
try {
synchronized (fessEsClient) {
deleteOldDocuments(fessEsClient, docList);
final FessConfig fessConfig = ComponentUtil.getFessConfig();
fessEsClient.addAll(fessConfig.getIndexDocumentIndex(), fessConfig.getIndexDocumentType(), docList);
}
if (logger.isInfoEnabled()) {
logger.info("Sent " + docList.size() + " docs (" + (System.currentTimeMillis() - execTime) + "ms)");
}
} finally {
docList.clear();
}
if (logger.isInfoEnabled()) {
logger.info("Sent " + docList.size() + " docs (" + (System.currentTimeMillis() - execTime) + "ms)");
}
docList.clear();
}
private void deleteOldDocuments(final FessEsClient fessEsClient, final List<Map<String, Object>> docList) {