0ct0pu5/fess
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix #1978 add config.pipeline

Browse source
This commit is contained in:
Shinsuke Sugaya 2019-01-22 23:02:23 +09:00
parent 372f9211f4
commit c010e0aed1
3 changed files with 23 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
src/main/java/org/codelibs/fess/es/client/FessEsClient.java
View file

@ -31,6 +31,7 @@ import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.BiConsumer;
import java.util.function.Function;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
@ -902,12 +903,15 @@ public class FessEsClient implements Client {
}
}
public void addAll(final String index, final String type, final List<Map<String, Object>> docList) {
public void addAll(final String index, final String type, final List<Map<String, Object>> docList,
final BiConsumer<Map<String, Object>, IndexRequestBuilder> options) {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final BulkRequestBuilder bulkRequestBuilder = client.prepareBulk();
for (final Map<String, Object> doc : docList) {
final Object id = doc.remove(fessConfig.getIndexFieldId());
bulkRequestBuilder.add(client.prepareIndex(index, type, id.toString()).setSource(new DocMap(doc)));
final IndexRequestBuilder builder = client.prepareIndex(index, type, id.toString()).setSource(new DocMap(doc));
options.accept(doc, builder);
bulkRequestBuilder.add(builder);
}
final BulkResponse response = bulkRequestBuilder.execute().actionGet(ComponentUtil.getFessConfig().getIndexBulkTimeout());
if (response.hasFailures()) {

11
src/main/java/org/codelibs/fess/helper/CrawlingConfigHelper.java
View file

@ -36,6 +36,7 @@ import org.codelibs.fess.es.config.exbhv.FailureUrlBhv;
import org.codelibs.fess.es.config.exbhv.FileConfigBhv;
import org.codelibs.fess.es.config.exbhv.WebConfigBhv;
import org.codelibs.fess.es.config.exentity.CrawlingConfig;
import org.codelibs.fess.es.config.exentity.CrawlingConfig.ConfigName;
import org.codelibs.fess.es.config.exentity.CrawlingConfig.ConfigType;
import org.codelibs.fess.es.config.exentity.DataConfig;
import org.codelibs.fess.es.config.exentity.FailureUrl;
@ -44,6 +45,7 @@ import org.codelibs.fess.es.config.exentity.WebConfig;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.util.ComponentUtil;
import org.dbflute.cbean.result.ListResultBean;
import org.dbflute.optional.OptionalThing;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -118,6 +120,15 @@ public class CrawlingConfigHelper {
}
}
public OptionalThing<String> getPipeline(final String configId) {
final CrawlingConfig config = getCrawlingConfig(configId);
if (config == null) {
return OptionalThing.empty();
}
final Map<String, String> paramMap = config.getConfigParameterMap(ConfigName.CONFIG);
return OptionalThing.of(paramMap.get("pipeline"));
}
public void refresh() {
crawlingConfigCache.invalidateAll();
}

7
src/main/java/org/codelibs/fess/helper/IndexingHelper.java
View file

@ -68,9 +68,14 @@ public class IndexingHelper {
}
});
}
final CrawlingConfigHelper crawlingConfigHelper = ComponentUtil.getCrawlingConfigHelper();
synchronized (fessEsClient) {
deleteOldDocuments(fessEsClient, docList);
fessEsClient.addAll(fessConfig.getIndexDocumentUpdateIndex(), fessConfig.getIndexDocumentType(), docList);
fessEsClient.addAll(fessConfig.getIndexDocumentUpdateIndex(), fessConfig.getIndexDocumentType(), docList,
(doc, builder) -> {
final String configId = (String) doc.get(fessConfig.getIndexFieldConfigId());
crawlingConfigHelper.getPipeline(configId).ifPresent(s -> builder.setPipeline(s));
});
}
if (logger.isInfoEnabled()) {
if (docList.getContentSize() > 0) {