fix #2798 Enhance IndexingHelper with additional JUnit tests
This commit is contained in:
Shinsuke Sugaya
parent
b68736b95e
commit
938f4a4db2
2 changed files with 604 additions and 23 deletions
|
|
@ -21,12 +21,14 @@ import java.util.Map;
|
|||
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.apache.lucene.search.TotalHits;
|
||||
import org.codelibs.fess.es.client.SearchEngineClient;
|
||||
import org.codelibs.fess.mylasta.direction.FessConfig;
|
||||
import org.codelibs.fess.thumbnail.ThumbnailManager;
|
||||
import org.codelibs.fess.util.ComponentUtil;
|
||||
import org.codelibs.fess.util.DocList;
|
||||
import org.codelibs.fess.util.MemoryUtil;
|
||||
import org.opensearch.action.admin.indices.refresh.RefreshResponse;
|
||||
import org.opensearch.action.search.SearchResponse;
|
||||
import org.opensearch.index.query.QueryBuilder;
|
||||
import org.opensearch.index.query.QueryBuilders;
|
||||
|
|
@ -52,19 +54,23 @@ public class IndexingHelper {
|
|||
try {
|
||||
if (fessConfig.isThumbnailCrawlerEnabled()) {
|
||||
final ThumbnailManager thumbnailManager = ComponentUtil.getThumbnailManager();
|
||||
final String thumbnailField = fessConfig.getIndexFieldThumbnail();
|
||||
docList.stream().forEach(doc -> {
|
||||
if (!thumbnailManager.offer(doc)) {
|
||||
if (logger.isDebugEnabled()) {
|
||||
logger.debug("Removing {}={} from doc[{}]", fessConfig.getIndexFieldThumbnail(),
|
||||
doc.get(fessConfig.getIndexFieldThumbnail()), doc.get(fessConfig.getIndexFieldUrl()));
|
||||
logger.debug("Removing {}={} from doc[{}]", thumbnailField, doc.get(thumbnailField),
|
||||
doc.get(fessConfig.getIndexFieldUrl()));
|
||||
}
|
||||
doc.remove(fessConfig.getIndexFieldThumbnail());
|
||||
doc.remove(thumbnailField);
|
||||
}
|
||||
});
|
||||
}
|
||||
final CrawlingConfigHelper crawlingConfigHelper = ComponentUtil.getCrawlingConfigHelper();
|
||||
synchronized (searchEngineClient) {
|
||||
deleteOldDocuments(searchEngineClient, docList);
|
||||
final long deletedDocCount = deleteOldDocuments(searchEngineClient, docList);
|
||||
if (logger.isDebugEnabled()) {
|
||||
logger.debug("Deleted {} old docs", deletedDocCount);
|
||||
}
|
||||
searchEngineClient.addAll(fessConfig.getIndexDocumentUpdateIndex(), docList, (doc, builder) -> {
|
||||
final String configId = (String) doc.get(fessConfig.getIndexFieldConfigId());
|
||||
crawlingConfigHelper.getPipeline(configId).ifPresent(s -> builder.setPipeline(s));
|
||||
|
|
@ -85,7 +91,7 @@ public class IndexingHelper {
|
|||
}
|
||||
}
|
||||
|
||||
private void deleteOldDocuments(final SearchEngineClient searchEngineClient, final DocList docList) {
|
||||
protected long deleteOldDocuments(final SearchEngineClient searchEngineClient, final DocList docList) {
|
||||
final FessConfig fessConfig = ComponentUtil.getFessConfig();
|
||||
|
||||
final List<String> docIdList = new ArrayList<>();
|
||||
|
|
@ -108,7 +114,7 @@ public class IndexingHelper {
|
|||
new String[] { fessConfig.getIndexFieldId(), fessConfig.getIndexFieldDocId() });
|
||||
for (final Map<String, Object> doc : docs) {
|
||||
final Object oldIdValue = doc.get(fessConfig.getIndexFieldId());
|
||||
if (!idValue.equals(oldIdValue) && oldIdValue != null) {
|
||||
if (oldIdValue != null && !idValue.equals(oldIdValue)) {
|
||||
final Object oldDocIdValue = doc.get(fessConfig.getIndexFieldDocId());
|
||||
if (oldDocIdValue != null) {
|
||||
docIdList.add(oldDocIdValue.toString());
|
||||
|
|
@ -120,9 +126,10 @@ public class IndexingHelper {
|
|||
}
|
||||
}
|
||||
if (!docIdList.isEmpty()) {
|
||||
searchEngineClient.deleteByQuery(fessConfig.getIndexDocumentUpdateIndex(),
|
||||
return deleteDocumentByQuery(searchEngineClient, fessConfig.getIndexDocumentUpdateIndex(),
|
||||
QueryBuilders.termsQuery(fessConfig.getIndexFieldDocId(), docIdList.stream().toArray(n -> new String[n])));
|
||||
}
|
||||
return 0L;
|
||||
}
|
||||
|
||||
public boolean updateDocument(final SearchEngineClient searchEngineClient, final String id, final String field, final Object value) {
|
||||
|
|
@ -137,19 +144,23 @@ public class IndexingHelper {
|
|||
|
||||
public long deleteDocumentByUrl(final SearchEngineClient searchEngineClient, final String url) {
|
||||
final FessConfig fessConfig = ComponentUtil.getFessConfig();
|
||||
return searchEngineClient.deleteByQuery(fessConfig.getIndexDocumentUpdateIndex(),
|
||||
return deleteDocumentByQuery(searchEngineClient, fessConfig.getIndexDocumentUpdateIndex(),
|
||||
QueryBuilders.termQuery(fessConfig.getIndexFieldUrl(), url));
|
||||
}
|
||||
|
||||
public long deleteDocumentsByDocId(final SearchEngineClient searchEngineClient, final List<String> docIdList) {
|
||||
final FessConfig fessConfig = ComponentUtil.getFessConfig();
|
||||
return searchEngineClient.deleteByQuery(fessConfig.getIndexDocumentUpdateIndex(),
|
||||
return deleteDocumentByQuery(searchEngineClient, fessConfig.getIndexDocumentUpdateIndex(),
|
||||
QueryBuilders.termsQuery(fessConfig.getIndexFieldDocId(), docIdList.stream().toArray(n -> new String[n])));
|
||||
}
|
||||
|
||||
public long deleteDocumentByQuery(final SearchEngineClient searchEngineClient, final QueryBuilder queryBuilder) {
|
||||
final FessConfig fessConfig = ComponentUtil.getFessConfig();
|
||||
return searchEngineClient.deleteByQuery(fessConfig.getIndexDocumentUpdateIndex(), queryBuilder);
|
||||
return deleteDocumentByQuery(searchEngineClient, fessConfig.getIndexDocumentUpdateIndex(), queryBuilder);
|
||||
}
|
||||
|
||||
protected long deleteDocumentByQuery(final SearchEngineClient searchEngineClient, final String index, final QueryBuilder queryBuilder) {
|
||||
return searchEngineClient.deleteByQuery(index, queryBuilder);
|
||||
}
|
||||
|
||||
public Map<String, Object> getDocument(final SearchEngineClient searchEngineClient, final String id, final String[] fields) {
|
||||
|
|
@ -168,9 +179,9 @@ public class IndexingHelper {
|
|||
return getDocumentListByQuery(searchEngineClient, queryBuilder, fields);
|
||||
}
|
||||
|
||||
public void deleteChildDocument(final SearchEngineClient searchEngineClient, final String id) {
|
||||
public long deleteChildDocument(final SearchEngineClient searchEngineClient, final String id) {
|
||||
final FessConfig fessConfig = ComponentUtil.getFessConfig();
|
||||
searchEngineClient.deleteByQuery(fessConfig.getIndexDocumentUpdateIndex(),
|
||||
return searchEngineClient.deleteByQuery(fessConfig.getIndexDocumentUpdateIndex(),
|
||||
QueryBuilders.termQuery(fessConfig.getIndexFieldParentId(), id));
|
||||
}
|
||||
|
||||
|
|
@ -185,9 +196,7 @@ public class IndexingHelper {
|
|||
final String[] fields) {
|
||||
final FessConfig fessConfig = ComponentUtil.getFessConfig();
|
||||
|
||||
final SearchResponse countResponse = searchEngineClient.prepareSearch(fessConfig.getIndexDocumentUpdateIndex())
|
||||
.setQuery(queryBuilder).setSize(0).execute().actionGet(fessConfig.getIndexSearchTimeout());
|
||||
final long numFound = countResponse.getHits().getTotalHits().value;
|
||||
final long numFound = getDocumentSizeByQuery(searchEngineClient, queryBuilder, fessConfig);
|
||||
final long maxSearchDocSize = fessConfig.getIndexerMaxSearchDocSizeAsInteger().longValue();
|
||||
final boolean exceeded = numFound > maxSearchDocSize;
|
||||
if (exceeded) {
|
||||
|
|
@ -217,35 +226,74 @@ public class IndexingHelper {
|
|||
});
|
||||
}
|
||||
|
||||
protected long getDocumentSizeByQuery(final SearchEngineClient searchEngineClient, final QueryBuilder queryBuilder,
|
||||
final FessConfig fessConfig) {
|
||||
final SearchResponse countResponse = searchEngineClient.prepareSearch(fessConfig.getIndexDocumentUpdateIndex())
|
||||
.setQuery(queryBuilder).setSize(0).setTrackTotalHits(true).execute().actionGet(fessConfig.getIndexSearchTimeout());
|
||||
final TotalHits totalHits = countResponse.getHits().getTotalHits();
|
||||
if (totalHits != null) {
|
||||
return totalHits.value;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
public long deleteBySessionId(final String sessionId) {
|
||||
final SearchEngineClient searchEngineClient = ComponentUtil.getSearchEngineClient();
|
||||
final FessConfig fessConfig = ComponentUtil.getFessConfig();
|
||||
final String index = fessConfig.getIndexDocumentUpdateIndex();
|
||||
return deleteBySessionId(searchEngineClient, index, sessionId);
|
||||
}
|
||||
|
||||
public long deleteBySessionId(final SearchEngineClient searchEngineClient, final String index, final String sessionId) {
|
||||
final FessConfig fessConfig = ComponentUtil.getFessConfig();
|
||||
final QueryBuilder queryBuilder = QueryBuilders.termQuery(fessConfig.getIndexFieldSegment(), sessionId);
|
||||
return deleteByQueryBuilder(index, queryBuilder);
|
||||
return deleteByQueryBuilder(searchEngineClient, index, queryBuilder);
|
||||
}
|
||||
|
||||
public long deleteByConfigId(final String configId) {
|
||||
final SearchEngineClient searchEngineClient = ComponentUtil.getSearchEngineClient();
|
||||
final FessConfig fessConfig = ComponentUtil.getFessConfig();
|
||||
final String index = fessConfig.getIndexDocumentUpdateIndex();
|
||||
return deleteByConfigId(searchEngineClient, index, configId);
|
||||
}
|
||||
|
||||
public long deleteByConfigId(final SearchEngineClient searchEngineClient, final String index, final String configId) {
|
||||
final FessConfig fessConfig = ComponentUtil.getFessConfig();
|
||||
final QueryBuilder queryBuilder = QueryBuilders.termQuery(fessConfig.getIndexFieldConfigId(), configId);
|
||||
return deleteByQueryBuilder(index, queryBuilder);
|
||||
return deleteByQueryBuilder(searchEngineClient, index, queryBuilder);
|
||||
}
|
||||
|
||||
public long deleteByVirtualHost(final String virtualHost) {
|
||||
final SearchEngineClient searchEngineClient = ComponentUtil.getSearchEngineClient();
|
||||
final FessConfig fessConfig = ComponentUtil.getFessConfig();
|
||||
final String index = fessConfig.getIndexDocumentUpdateIndex();
|
||||
final QueryBuilder queryBuilder = QueryBuilders.termQuery(fessConfig.getIndexFieldVirtualHost(), virtualHost);
|
||||
return deleteByQueryBuilder(index, queryBuilder);
|
||||
return deleteByVirtualHost(searchEngineClient, index, virtualHost);
|
||||
}
|
||||
|
||||
protected long deleteByQueryBuilder(final String index, final QueryBuilder queryBuilder) {
|
||||
final SearchEngineClient searchEngineClient = ComponentUtil.getSearchEngineClient();
|
||||
searchEngineClient.admin().indices().prepareRefresh(index).execute().actionGet();
|
||||
public long deleteByVirtualHost(final SearchEngineClient searchEngineClient, final String index, final String virtualHost) {
|
||||
final FessConfig fessConfig = ComponentUtil.getFessConfig();
|
||||
final QueryBuilder queryBuilder = QueryBuilders.termQuery(fessConfig.getIndexFieldVirtualHost(), virtualHost);
|
||||
return deleteByQueryBuilder(searchEngineClient, index, queryBuilder);
|
||||
}
|
||||
|
||||
protected long deleteByQueryBuilder(final SearchEngineClient searchEngineClient, final String index, final QueryBuilder queryBuilder) {
|
||||
refreshIndex(searchEngineClient, index);
|
||||
final long numOfDeleted = searchEngineClient.deleteByQuery(index, queryBuilder);
|
||||
logger.info("Deleted {} old docs.", numOfDeleted);
|
||||
if (logger.isDebugEnabled()) {
|
||||
logger.debug("Deleted {} old docs.", numOfDeleted);
|
||||
}
|
||||
return numOfDeleted;
|
||||
}
|
||||
|
||||
protected int refreshIndex(final SearchEngineClient searchEngineClient, final String index) {
|
||||
final RefreshResponse response = searchEngineClient.admin().indices().prepareRefresh(index).execute().actionGet();
|
||||
if (logger.isDebugEnabled()) {
|
||||
logger.debug("[{}] refresh status: {} ({}/{}/{})", index, response.getStatus(), response.getTotalShards(),
|
||||
response.getSuccessfulShards(), response.getFailedShards());
|
||||
}
|
||||
return response.getStatus().getStatus();
|
||||
}
|
||||
|
||||
public long calculateDocumentSize(final Map<String, Object> dataMap) {
|
||||
return MemoryUtil.sizeOf(dataMap);
|
||||
}
|
||||
|
|
|
|||
533
src/test/java/org/codelibs/fess/helper/IndexingHelperTest.java
Normal file
533
src/test/java/org/codelibs/fess/helper/IndexingHelperTest.java
Normal file
|
|
@ -0,0 +1,533 @@
|
|||
/*
|
||||
* Copyright 2012-2023 CodeLibs Project and the Others.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
|
||||
* either express or implied. See the License for the specific language
|
||||
* governing permissions and limitations under the License.
|
||||
*/
|
||||
package org.codelibs.fess.helper;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
import java.util.function.BiConsumer;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.codelibs.fess.app.service.WebConfigService;
|
||||
import org.codelibs.fess.es.client.SearchEngineClient;
|
||||
import org.codelibs.fess.es.config.exentity.WebConfig;
|
||||
import org.codelibs.fess.mylasta.direction.FessConfig;
|
||||
import org.codelibs.fess.thumbnail.ThumbnailManager;
|
||||
import org.codelibs.fess.unit.UnitFessTestCase;
|
||||
import org.codelibs.fess.util.ComponentUtil;
|
||||
import org.codelibs.fess.util.DocList;
|
||||
import org.dbflute.optional.OptionalEntity;
|
||||
import org.opensearch.action.index.IndexAction;
|
||||
import org.opensearch.action.index.IndexRequestBuilder;
|
||||
import org.opensearch.action.search.SearchAction;
|
||||
import org.opensearch.action.search.SearchRequestBuilder;
|
||||
import org.opensearch.index.query.QueryBuilder;
|
||||
import org.opensearch.index.query.QueryBuilders;
|
||||
import org.opensearch.index.query.TermQueryBuilder;
|
||||
import org.opensearch.index.query.TermsQueryBuilder;
|
||||
|
||||
public class IndexingHelperTest extends UnitFessTestCase {
|
||||
private IndexingHelper indexingHelper;
|
||||
|
||||
private long documentSizeByQuery = 0L;
|
||||
|
||||
@Override
|
||||
public void setUp() throws Exception {
|
||||
super.setUp();
|
||||
indexingHelper = new IndexingHelper() {
|
||||
@Override
|
||||
protected long getDocumentSizeByQuery(final SearchEngineClient searchEngineClient, final QueryBuilder queryBuilder,
|
||||
final FessConfig fessConfig) {
|
||||
return documentSizeByQuery;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected int refreshIndex(final SearchEngineClient searchEngineClient, final String index) {
|
||||
return 200;
|
||||
}
|
||||
};
|
||||
indexingHelper.setMaxRetryCount(5);
|
||||
indexingHelper.setDefaultRowSize(100);
|
||||
indexingHelper.setRequestInterval(500);
|
||||
ComponentUtil.register(indexingHelper, "indexingHelper");
|
||||
ComponentUtil.register(new ThumbnailManager(), "thumbnailManager");
|
||||
final CrawlingConfigHelper crawlingConfigHelper = new CrawlingConfigHelper();
|
||||
crawlingConfigHelper.init();
|
||||
ComponentUtil.register(crawlingConfigHelper, "crawlingConfigHelper");
|
||||
ComponentUtil.register(new WebConfigService() {
|
||||
@Override
|
||||
public OptionalEntity<WebConfig> getWebConfig(final String id) {
|
||||
final WebConfig webConfig = new WebConfig();
|
||||
webConfig.setId(id);
|
||||
return OptionalEntity.of(webConfig);
|
||||
}
|
||||
}, WebConfigService.class.getCanonicalName());
|
||||
}
|
||||
|
||||
public void test_sendDocuments() {
|
||||
documentSizeByQuery = 0L;
|
||||
final AtomicReference<String> sentIndex = new AtomicReference<>();
|
||||
final List<Map<String, Object>> sentDocList = new ArrayList<>();
|
||||
final SearchEngineClient client = new SearchEngineClient() {
|
||||
@Override
|
||||
public String[] addAll(final String index, final List<Map<String, Object>> docList,
|
||||
final BiConsumer<Map<String, Object>, IndexRequestBuilder> options) {
|
||||
sentIndex.set(index);
|
||||
docList.forEach(x -> options.accept(x, new IndexRequestBuilder(this, IndexAction.INSTANCE)));
|
||||
sentDocList.addAll(docList);
|
||||
return docList.stream().map(x -> (String) x.get("id")).toArray(n -> new String[n]);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Map<String, Object>> getDocumentList(final String index, final SearchCondition<SearchRequestBuilder> condition) {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
};
|
||||
ComponentUtil.register(client, "searchEngineClient");
|
||||
final DocList docList = new DocList();
|
||||
indexingHelper.sendDocuments(client, docList);
|
||||
assertEquals(0, docList.size());
|
||||
assertEquals(0, sentDocList.size());
|
||||
assertNull(sentIndex.get());
|
||||
|
||||
sentIndex.set(null);
|
||||
sentDocList.clear();
|
||||
docList.add(new HashMap<>(Map.of(//
|
||||
"_id", "001", //
|
||||
"config_id", "W01", //
|
||||
"url", "http://test.com/001"//
|
||||
)));
|
||||
docList.add(new HashMap<>(Map.of(//
|
||||
"_id", "002", //
|
||||
"thumbnail", "http://test.com/002", //
|
||||
"url", "http://test.com/002"//
|
||||
)));
|
||||
indexingHelper.sendDocuments(client, docList);
|
||||
assertEquals(0, docList.size());
|
||||
assertEquals(2, sentDocList.size());
|
||||
assertEquals("fess.update", sentIndex.get());
|
||||
}
|
||||
|
||||
public void test_deleteOldDocuments() {
|
||||
documentSizeByQuery = 0L;
|
||||
final List<String> deletedDocIdList = new ArrayList<>();
|
||||
final List<Map<String, Object>> oldDocList = new ArrayList<>();
|
||||
final SearchEngineClient client = new SearchEngineClient() {
|
||||
@Override
|
||||
public List<Map<String, Object>> getDocumentList(final String index, final SearchCondition<SearchRequestBuilder> condition) {
|
||||
return oldDocList;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long deleteByQuery(final String index, final QueryBuilder queryBuilder) {
|
||||
if (queryBuilder instanceof final TermsQueryBuilder termsQueryBuilder) {
|
||||
termsQueryBuilder.values().stream().forEach(o -> deletedDocIdList.add(o.toString()));
|
||||
}
|
||||
return deletedDocIdList.size();
|
||||
}
|
||||
};
|
||||
ComponentUtil.register(client, "searchEngineClient");
|
||||
final DocList docList = new DocList();
|
||||
assertEquals(0, indexingHelper.deleteOldDocuments(client, docList));
|
||||
assertEquals(0, docList.size());
|
||||
assertEquals(0, deletedDocIdList.size());
|
||||
|
||||
docList.clear();
|
||||
deletedDocIdList.clear();
|
||||
docList.add(new HashMap<>(Map.of(//
|
||||
"config_id", "W01", //
|
||||
"doc_id", "1", //
|
||||
"url", "http://test.com/001"//
|
||||
)));
|
||||
documentSizeByQuery = oldDocList.size();
|
||||
assertEquals(0, indexingHelper.deleteOldDocuments(client, docList));
|
||||
assertEquals(1, docList.size());
|
||||
assertEquals(0, deletedDocIdList.size());
|
||||
|
||||
docList.clear();
|
||||
deletedDocIdList.clear();
|
||||
docList.add(new HashMap<>(Map.of(//
|
||||
"_id", "001", //
|
||||
"doc_id", "1", //
|
||||
"url", "http://test.com/001"//
|
||||
)));
|
||||
documentSizeByQuery = oldDocList.size();
|
||||
assertEquals(0, indexingHelper.deleteOldDocuments(client, docList));
|
||||
assertEquals(1, docList.size());
|
||||
assertEquals(0, deletedDocIdList.size());
|
||||
|
||||
docList.clear();
|
||||
deletedDocIdList.clear();
|
||||
docList.add(new HashMap<>(Map.of(//
|
||||
"_id", "001", //
|
||||
"config_id", "W01", //
|
||||
"doc_id", "1", //
|
||||
"url", "http://test.com/001"//
|
||||
)));
|
||||
documentSizeByQuery = oldDocList.size();
|
||||
assertEquals(0, indexingHelper.deleteOldDocuments(client, docList));
|
||||
assertEquals(1, docList.size());
|
||||
assertEquals(0, deletedDocIdList.size());
|
||||
|
||||
deletedDocIdList.clear();
|
||||
oldDocList.add(new HashMap<>(Map.of(//
|
||||
"_id", "001", // same id
|
||||
"config_id", "W01", //
|
||||
"doc_id", "1", //
|
||||
"url", "http://test.com/001"//
|
||||
)));
|
||||
documentSizeByQuery = oldDocList.size();
|
||||
assertEquals(0, indexingHelper.deleteOldDocuments(client, docList));
|
||||
assertEquals(1, docList.size());
|
||||
assertEquals(0, deletedDocIdList.size());
|
||||
|
||||
deletedDocIdList.clear();
|
||||
oldDocList.clear();
|
||||
oldDocList.add(new HashMap<>(Map.of(//
|
||||
"_id", "001_OLD", // same id
|
||||
"config_id", "W01", //
|
||||
"doc_id", "1", //
|
||||
"url", "http://test.com/001"//
|
||||
)));
|
||||
documentSizeByQuery = oldDocList.size();
|
||||
assertEquals(1, indexingHelper.deleteOldDocuments(client, docList));
|
||||
assertEquals(1, docList.size());
|
||||
assertEquals(1, deletedDocIdList.size());
|
||||
assertEquals("1", deletedDocIdList.get(0));
|
||||
}
|
||||
|
||||
public void test_updateDocument() {
|
||||
final Map<String, String> resultMap = new HashMap<>();
|
||||
final SearchEngineClient client = new SearchEngineClient() {
|
||||
@Override
|
||||
public boolean update(final String index, final String id, final String field, final Object value) {
|
||||
resultMap.put("index", index);
|
||||
resultMap.put("id", id);
|
||||
resultMap.put("field", field);
|
||||
resultMap.put("value", value.toString());
|
||||
return true;
|
||||
}
|
||||
};
|
||||
ComponentUtil.register(client, "searchEngineClient");
|
||||
|
||||
final String id = "001";
|
||||
final String field = "content";
|
||||
final String value = "test";
|
||||
|
||||
assertTrue(indexingHelper.updateDocument(client, id, field, value));
|
||||
assertEquals("fess.update", resultMap.get("index"));
|
||||
assertEquals(id, resultMap.get("id"));
|
||||
assertEquals(field, resultMap.get("field"));
|
||||
assertEquals(value, resultMap.get("value"));
|
||||
}
|
||||
|
||||
public void test_deleteDocument() {
|
||||
final Map<String, String> resultMap = new HashMap<>();
|
||||
final SearchEngineClient client = new SearchEngineClient() {
|
||||
@Override
|
||||
public boolean delete(final String index, final String id) {
|
||||
resultMap.put("index", index);
|
||||
resultMap.put("id", id);
|
||||
return true;
|
||||
}
|
||||
};
|
||||
ComponentUtil.register(client, "searchEngineClient");
|
||||
|
||||
final String id = "001";
|
||||
|
||||
assertTrue(indexingHelper.deleteDocument(client, id));
|
||||
assertEquals("fess.update", resultMap.get("index"));
|
||||
assertEquals(id, resultMap.get("id"));
|
||||
}
|
||||
|
||||
public void test_deleteDocumentByUrl() {
|
||||
final Map<String, String> resultMap = new HashMap<>();
|
||||
final SearchEngineClient client = new SearchEngineClient() {
|
||||
@Override
|
||||
public long deleteByQuery(final String index, final QueryBuilder queryBuilder) {
|
||||
if (queryBuilder instanceof final TermQueryBuilder termQueryBuilder) {
|
||||
resultMap.put("index", index);
|
||||
resultMap.put("url", termQueryBuilder.value().toString());
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
};
|
||||
ComponentUtil.register(client, "searchEngineClient");
|
||||
|
||||
final String url = "http://test.com/";
|
||||
|
||||
assertEquals(1, indexingHelper.deleteDocumentByUrl(client, url));
|
||||
assertEquals("fess.update", resultMap.get("index"));
|
||||
assertEquals(url, resultMap.get("url"));
|
||||
}
|
||||
|
||||
public void test_deleteDocumentsByDocId() {
|
||||
final Map<String, Object> resultMap = new HashMap<>();
|
||||
final SearchEngineClient client = new SearchEngineClient() {
|
||||
@Override
|
||||
public long deleteByQuery(final String index, final QueryBuilder queryBuilder) {
|
||||
if (queryBuilder instanceof final TermsQueryBuilder termsQueryBuilder) {
|
||||
resultMap.put("index", index);
|
||||
resultMap.put("ids", termsQueryBuilder.values().stream().map(Object::toString).collect(Collectors.toList()));
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
};
|
||||
ComponentUtil.register(client, "searchEngineClient");
|
||||
|
||||
final List<String> docList = new ArrayList<>();
|
||||
|
||||
indexingHelper.deleteDocumentsByDocId(client, docList);
|
||||
assertEquals("fess.update", resultMap.get("index"));
|
||||
assertEquals(0, ((List<String>) resultMap.get("ids")).size());
|
||||
|
||||
docList.add("001");
|
||||
indexingHelper.deleteDocumentsByDocId(client, docList);
|
||||
assertEquals("fess.update", resultMap.get("index"));
|
||||
assertEquals(1, ((List<String>) resultMap.get("ids")).size());
|
||||
assertEquals("001", ((List<String>) resultMap.get("ids")).get(0));
|
||||
}
|
||||
|
||||
public void test_deleteDocumentByQuery() {
|
||||
final Map<String, String> resultMap = new HashMap<>();
|
||||
final SearchEngineClient client = new SearchEngineClient() {
|
||||
@Override
|
||||
public long deleteByQuery(final String index, final QueryBuilder queryBuilder) {
|
||||
if (queryBuilder instanceof final TermQueryBuilder termQueryBuilder) {
|
||||
resultMap.put("index", index);
|
||||
resultMap.put("url", termQueryBuilder.value().toString());
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
};
|
||||
ComponentUtil.register(client, "searchEngineClient");
|
||||
|
||||
final String url = "http://test.com/";
|
||||
|
||||
assertEquals(1, indexingHelper.deleteDocumentByQuery(client, QueryBuilders.termQuery("fess.update", url)));
|
||||
assertEquals("fess.update", resultMap.get("index"));
|
||||
assertEquals(url, resultMap.get("url"));
|
||||
}
|
||||
|
||||
public void test_getDocument() {
|
||||
final Map<String, String> resultMap = new HashMap<>();
|
||||
final SearchEngineClient client = new SearchEngineClient() {
|
||||
|
||||
@Override
|
||||
public OptionalEntity<Map<String, Object>> getDocument(final String index,
|
||||
final SearchCondition<SearchRequestBuilder> condition) {
|
||||
final SearchRequestBuilder builder = new SearchRequestBuilder(this, SearchAction.INSTANCE);
|
||||
condition.build(builder);
|
||||
resultMap.put("index", index);
|
||||
resultMap.put("query", builder.toString());
|
||||
return OptionalEntity.of(Map.of("_id", "001", "title", "test1", "content", "test2"));
|
||||
}
|
||||
};
|
||||
ComponentUtil.register(client, "searchEngineClient");
|
||||
|
||||
final Map<String, Object> document = indexingHelper.getDocument(client, "001", new String[] { "title", "content" });
|
||||
assertEquals("fess.update", resultMap.get("index"));
|
||||
assertEquals(
|
||||
"{\"query\":{\"ids\":{\"values\":[\"001\"],\"boost\":1.0}},\"_source\":{\"includes\":[\"title\",\"content\"],\"excludes\":[]}}",
|
||||
resultMap.get("query"));
|
||||
assertEquals(3, document.size());
|
||||
}
|
||||
|
||||
public void test_getDocumentListByPrefixId() {
|
||||
documentSizeByQuery = 1L;
|
||||
final Map<String, String> resultMap = new HashMap<>();
|
||||
final SearchEngineClient client = new SearchEngineClient() {
|
||||
@Override
|
||||
public List<Map<String, Object>> getDocumentList(final String index, final SearchCondition<SearchRequestBuilder> condition) {
|
||||
final SearchRequestBuilder builder = new SearchRequestBuilder(this, SearchAction.INSTANCE);
|
||||
condition.build(builder);
|
||||
resultMap.put("index", index);
|
||||
resultMap.put("query", builder.toString());
|
||||
final List<Map<String, Object>> docList = new ArrayList<>();
|
||||
docList.add(Map.of("_id", "001", "title", "test1", "content", "test2"));
|
||||
return docList;
|
||||
}
|
||||
};
|
||||
ComponentUtil.register(client, "searchEngineClient");
|
||||
|
||||
final List<Map<String, Object>> documents =
|
||||
indexingHelper.getDocumentListByPrefixId(client, "001", new String[] { "title", "content" });
|
||||
assertEquals("fess.update", resultMap.get("index"));
|
||||
assertEquals(
|
||||
"{\"size\":1,\"query\":{\"prefix\":{\"_id\":{\"value\":\"001\",\"boost\":1.0}}},\"_source\":{\"includes\":[\"title\",\"content\"],\"excludes\":[]}}",
|
||||
resultMap.get("query"));
|
||||
assertEquals(1, documents.size());
|
||||
}
|
||||
|
||||
public void test_deleteChildDocument() {
|
||||
final Map<String, String> resultMap = new HashMap<>();
|
||||
final SearchEngineClient client = new SearchEngineClient() {
|
||||
@Override
|
||||
public long deleteByQuery(final String index, final QueryBuilder queryBuilder) {
|
||||
if (queryBuilder instanceof final TermQueryBuilder termQueryBuilder) {
|
||||
resultMap.put("index", index);
|
||||
resultMap.put("id", termQueryBuilder.value().toString());
|
||||
resultMap.put("field", termQueryBuilder.fieldName());
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
};
|
||||
ComponentUtil.register(client, "searchEngineClient");
|
||||
|
||||
final String id = "001";
|
||||
|
||||
assertEquals(1, indexingHelper.deleteChildDocument(client, id));
|
||||
assertEquals("fess.update", resultMap.get("index"));
|
||||
assertEquals(id, resultMap.get("id"));
|
||||
assertEquals("parent_id", resultMap.get("field"));
|
||||
}
|
||||
|
||||
public void test_getChildDocumentList() {
|
||||
documentSizeByQuery = 1L;
|
||||
final Map<String, String> resultMap = new HashMap<>();
|
||||
final SearchEngineClient client = new SearchEngineClient() {
|
||||
@Override
|
||||
public List<Map<String, Object>> getDocumentList(final String index, final SearchCondition<SearchRequestBuilder> condition) {
|
||||
final SearchRequestBuilder builder = new SearchRequestBuilder(this, SearchAction.INSTANCE);
|
||||
condition.build(builder);
|
||||
resultMap.put("index", index);
|
||||
resultMap.put("query", builder.toString());
|
||||
final List<Map<String, Object>> docList = new ArrayList<>();
|
||||
docList.add(Map.of("_id", "001", "title", "test1", "content", "test2"));
|
||||
return docList;
|
||||
}
|
||||
};
|
||||
ComponentUtil.register(client, "searchEngineClient");
|
||||
|
||||
final List<Map<String, Object>> documents = indexingHelper.getChildDocumentList(client, "001", new String[] { "title", "content" });
|
||||
assertEquals("fess.update", resultMap.get("index"));
|
||||
assertEquals(
|
||||
"{\"size\":1,\"query\":{\"term\":{\"parent_id\":{\"value\":\"001\",\"boost\":1.0}}},\"_source\":{\"includes\":[\"title\",\"content\"],\"excludes\":[]}}",
|
||||
resultMap.get("query"));
|
||||
assertEquals(1, documents.size());
|
||||
}
|
||||
|
||||
public void test_getDocumentListByQuery() {
|
||||
documentSizeByQuery = 1L;
|
||||
final Map<String, String> resultMap = new HashMap<>();
|
||||
final SearchEngineClient client = new SearchEngineClient() {
|
||||
@Override
|
||||
public List<Map<String, Object>> getDocumentList(final String index, final SearchCondition<SearchRequestBuilder> condition) {
|
||||
final SearchRequestBuilder builder = new SearchRequestBuilder(this, SearchAction.INSTANCE);
|
||||
condition.build(builder);
|
||||
resultMap.put("index", index);
|
||||
resultMap.put("query", builder.toString());
|
||||
final List<Map<String, Object>> docList = new ArrayList<>();
|
||||
docList.add(Map.of("_id", "001", "title", "test1", "content", "test2"));
|
||||
return docList;
|
||||
}
|
||||
};
|
||||
ComponentUtil.register(client, "searchEngineClient");
|
||||
|
||||
final List<Map<String, Object>> documents =
|
||||
indexingHelper.getDocumentListByQuery(client, QueryBuilders.idsQuery().addIds("001"), new String[] { "title", "content" });
|
||||
assertEquals("fess.update", resultMap.get("index"));
|
||||
assertEquals(
|
||||
"{\"size\":1,\"query\":{\"ids\":{\"values\":[\"001\"],\"boost\":1.0}},\"_source\":{\"includes\":[\"title\",\"content\"],\"excludes\":[]}}",
|
||||
resultMap.get("query"));
|
||||
assertEquals(1, documents.size());
|
||||
}
|
||||
|
||||
public void test_deleteBySessionId() {
|
||||
final Map<String, String> resultMap = new HashMap<>();
|
||||
final SearchEngineClient client = new SearchEngineClient() {
|
||||
@Override
|
||||
public long deleteByQuery(final String index, final QueryBuilder queryBuilder) {
|
||||
if (queryBuilder instanceof final TermQueryBuilder termQueryBuilder) {
|
||||
resultMap.put("index", index);
|
||||
resultMap.put("field", termQueryBuilder.fieldName());
|
||||
resultMap.put("value", termQueryBuilder.value().toString());
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
};
|
||||
ComponentUtil.register(client, "searchEngineClient");
|
||||
|
||||
final String sessionId = "session1";
|
||||
|
||||
assertEquals(1, indexingHelper.deleteBySessionId(sessionId));
|
||||
assertEquals("fess.update", resultMap.get("index"));
|
||||
assertEquals("segment", resultMap.get("field"));
|
||||
assertEquals(sessionId, resultMap.get("value"));
|
||||
}
|
||||
|
||||
public void test_deleteByConfigId() {
|
||||
final Map<String, String> resultMap = new HashMap<>();
|
||||
final SearchEngineClient client = new SearchEngineClient() {
|
||||
@Override
|
||||
public long deleteByQuery(final String index, final QueryBuilder queryBuilder) {
|
||||
if (queryBuilder instanceof final TermQueryBuilder termQueryBuilder) {
|
||||
resultMap.put("index", index);
|
||||
resultMap.put("field", termQueryBuilder.fieldName());
|
||||
resultMap.put("value", termQueryBuilder.value().toString());
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
};
|
||||
ComponentUtil.register(client, "searchEngineClient");
|
||||
|
||||
final String configId = "W01";
|
||||
|
||||
assertEquals(1, indexingHelper.deleteByConfigId(configId));
|
||||
assertEquals("fess.update", resultMap.get("index"));
|
||||
assertEquals("config_id", resultMap.get("field"));
|
||||
assertEquals(configId, resultMap.get("value"));
|
||||
}
|
||||
|
||||
public void test_deleteByVirtualHost() {
|
||||
final Map<String, String> resultMap = new HashMap<>();
|
||||
final SearchEngineClient client = new SearchEngineClient() {
|
||||
@Override
|
||||
public long deleteByQuery(final String index, final QueryBuilder queryBuilder) {
|
||||
if (queryBuilder instanceof final TermQueryBuilder termQueryBuilder) {
|
||||
resultMap.put("index", index);
|
||||
resultMap.put("field", termQueryBuilder.fieldName());
|
||||
resultMap.put("value", termQueryBuilder.value().toString());
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
};
|
||||
ComponentUtil.register(client, "searchEngineClient");
|
||||
|
||||
final String virtualHost = "aaa";
|
||||
|
||||
assertEquals(1, indexingHelper.deleteByVirtualHost(virtualHost));
|
||||
assertEquals("fess.update", resultMap.get("index"));
|
||||
assertEquals("virtual_host", resultMap.get("field"));
|
||||
assertEquals(virtualHost, resultMap.get("value"));
|
||||
}
|
||||
|
||||
public void test_calculateDocumentSize() {
|
||||
assertEquals(0, indexingHelper.calculateDocumentSize(Collections.emptyMap()));
|
||||
assertEquals(118, indexingHelper.calculateDocumentSize(Map.of("id", "test")));
|
||||
assertEquals(249, indexingHelper.calculateDocumentSize(Map.of("id", "test", "url", "http://test.com/")));
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue