0ct0pu5/fess
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

replace with index.document.*.index

Browse source
This commit is contained in:
Shinsuke Sugaya 2015-12-18 06:29:30 +09:00
parent 8501a106d8
commit 1a394a16aa
14 changed files with 84 additions and 65 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
src/main/java/org/codelibs/fess/app/service/SearchService.java
View file

@ -107,7 +107,7 @@ public class SearchService {
final String sortField = params.getSort();
final List<Map<String, Object>> documentItems =
fessEsClient.search(
fessConfig.getIndexDocumentIndex(),
fessConfig.getIndexDocumentSearchIndex(),
fessConfig.getIndexDocumentType(),
searchRequestBuilder -> {
if (StringUtil.isNotBlank(sortField)) {
@ -188,7 +188,7 @@ public class SearchService {
final QueryContext queryContext = queryHelper.build(query, context -> {
context.skipRoleQuery();
});
return fessEsClient.deleteByQuery(fessConfig.getIndexDocumentIndex(), fessConfig.getIndexDocumentType(),
return fessEsClient.deleteByQuery(fessConfig.getIndexDocumentUpdateIndex(), fessConfig.getIndexDocumentType(),
queryContext.getQueryBuilder());
}
@ -286,7 +286,7 @@ public class SearchService {
}
public OptionalEntity<Map<String, Object>> getDocumentByDocId(final String docId, final String[] fields) {
return fessEsClient.getDocument(fessConfig.getIndexDocumentIndex(), fessConfig.getIndexDocumentType(), builder -> {
return fessEsClient.getDocument(fessConfig.getIndexDocumentSearchIndex(), fessConfig.getIndexDocumentType(), builder -> {
builder.setQuery(QueryBuilders.termQuery(fessConfig.getIndexFieldDocId(), docId));
builder.addFields(fields);
return true;
@ -294,7 +294,7 @@ public class SearchService {
}
public List<Map<String, Object>> getDocumentListByDocIds(final String[] docIds, final String[] fields) {
return fessEsClient.getDocumentList(fessConfig.getIndexDocumentIndex(), fessConfig.getIndexDocumentType(), builder -> {
return fessEsClient.getDocumentList(fessConfig.getIndexDocumentSearchIndex(), fessConfig.getIndexDocumentType(), builder -> {
builder.setQuery(QueryBuilders.termsQuery(fessConfig.getIndexFieldDocId(), docIds));
builder.setSize(fessConfig.getPagingSearchPageMaxSizeAsInteger().intValue());
builder.addFields(fields);
@ -303,13 +303,13 @@ public class SearchService {
}
public boolean update(final String id, final String field, final Object value) {
return fessEsClient.update(fessConfig.getIndexDocumentIndex(), fessConfig.getIndexDocumentType(), id, field, value);
return fessEsClient.update(fessConfig.getIndexDocumentUpdateIndex(), fessConfig.getIndexDocumentType(), id, field, value);
}
public boolean update(final String id, final Consumer<UpdateRequestBuilder> builderLambda) {
try {
final UpdateRequestBuilder builder =
fessEsClient.prepareUpdate(fessConfig.getIndexDocumentIndex(), fessConfig.getIndexDocumentType(), id);
fessEsClient.prepareUpdate(fessConfig.getIndexDocumentUpdateIndex(), fessConfig.getIndexDocumentType(), id);
builderLambda.accept(builder);
final UpdateResponse response = builder.execute().actionGet();
return response.isCreated();

2
src/main/java/org/codelibs/fess/app/web/admin/searchlist/AdminSearchlistAction.java
View file

@ -198,7 +198,7 @@ public class AdminSearchlistAction extends FessAdminAction {
}
try {
final QueryBuilder query = QueryBuilders.termQuery(fessConfig.getIndexFieldDocId(), docId);
fessEsClient.deleteByQuery(fessConfig.getIndexDocumentIndex(), fessConfig.getIndexDocumentType(), query);
fessEsClient.deleteByQuery(fessConfig.getIndexDocumentUpdateIndex(), fessConfig.getIndexDocumentType(), query);
saveInfo(messages -> messages.addSuccessDeleteDocFromIndex(GLOBAL));
} catch (final Exception e) {
throwValidationError(messages -> messages.addErrorsFailedToDeleteDocInAdmin(GLOBAL), () -> asListHtml());

14
src/main/java/org/codelibs/fess/app/web/cache/CacheAction.java vendored
View file

@ -51,12 +51,14 @@ public class CacheAction extends FessSearchAction {
Map<String, Object> doc = null;
try {
doc = fessEsClient.getDocument(fessConfig.getIndexDocumentIndex(), fessConfig.getIndexDocumentType(), queryRequestBuilder -> {
final TermQueryBuilder termQuery = QueryBuilders.termQuery(fessConfig.getIndexFieldDocId(), form.docId);
queryRequestBuilder.setQuery(termQuery);
queryRequestBuilder.addFields(queryHelper.getCacheResponseFields());
return true;
}).get();
doc =
fessEsClient.getDocument(fessConfig.getIndexDocumentSearchIndex(), fessConfig.getIndexDocumentType(),
queryRequestBuilder -> {
final TermQueryBuilder termQuery = QueryBuilders.termQuery(fessConfig.getIndexFieldDocId(), form.docId);
queryRequestBuilder.setQuery(termQuery);
queryRequestBuilder.addFields(queryHelper.getCacheResponseFields());
return true;
}).get();
} catch (final Exception e) {
logger.warn("Failed to request: " + form.docId, e);
}

14
src/main/java/org/codelibs/fess/app/web/go/GoAction.java
View file

@ -68,12 +68,14 @@ public class GoAction extends FessSearchAction {
Map<String, Object> doc = null;
try {
doc = fessEsClient.getDocument(fessConfig.getIndexDocumentIndex(), fessConfig.getIndexDocumentType(), queryRequestBuilder -> {
final TermQueryBuilder termQuery = QueryBuilders.termQuery(fessConfig.getIndexFieldDocId(), form.docId);
queryRequestBuilder.setQuery(termQuery);
queryRequestBuilder.addFields(fessConfig.getIndexFieldUrl());
return true;
}).get();
doc =
fessEsClient.getDocument(fessConfig.getIndexDocumentSearchIndex(), fessConfig.getIndexDocumentType(),
queryRequestBuilder -> {
final TermQueryBuilder termQuery = QueryBuilders.termQuery(fessConfig.getIndexFieldDocId(), form.docId);
queryRequestBuilder.setQuery(termQuery);
queryRequestBuilder.addFields(fessConfig.getIndexFieldUrl());
return true;
}).get();
} catch (final Exception e) {
logger.warn("Failed to request: " + form.docId, e);
}

2
src/main/java/org/codelibs/fess/app/web/screenshot/ScreenshotAction.java
View file

@ -65,7 +65,7 @@ public class ScreenshotAction extends FessSearchAction {
BufferedInputStream in = null;
try {
final Map<String, Object> doc =
fessEsClient.getDocument(fessConfig.getIndexDocumentIndex(), fessConfig.getIndexDocumentType(),
fessEsClient.getDocument(fessConfig.getIndexDocumentSearchIndex(), fessConfig.getIndexDocumentType(),
queryRequestBuilder -> {
final TermQueryBuilder termQuery = QueryBuilders.termQuery(fessConfig.getIndexFieldDocId(), form.docId);
queryRequestBuilder.setQuery(termQuery);

2
src/main/java/org/codelibs/fess/helper/CrawlingInfoHelper.java
View file

@ -161,7 +161,7 @@ public class CrawlingInfoHelper implements Serializable {
public List<Map<String, String>> getSessionIdList(final FessEsClient fessEsClient) {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
return fessEsClient.search(
fessConfig.getIndexDocumentIndex(),
fessConfig.getIndexDocumentSearchIndex(),
fessConfig.getIndexDocumentType(),
queryRequestBuilder -> {
queryRequestBuilder.setQuery(QueryBuilders.matchAllQuery());

4
src/main/java/org/codelibs/fess/helper/DataIndexHelper.java
View file

@ -258,8 +258,8 @@ public class DataIndexHelper implements Serializable {
QueryBuilders.boolQuery().must(QueryBuilders.termQuery(fessConfig.getIndexFieldConfigId(), dataConfig.getConfigId()))
.mustNot(QueryBuilders.termQuery(fessConfig.getIndexFieldSegment(), sessionId));
try {
ComponentUtil.getElasticsearchClient().deleteByQuery(fessConfig.getIndexDocumentIndex(), fessConfig.getIndexDocumentType(),
queryBuilder);
ComponentUtil.getElasticsearchClient().deleteByQuery(fessConfig.getIndexDocumentUpdateIndex(),
fessConfig.getIndexDocumentType(), queryBuilder);
} catch (final Exception e) {
logger.error("Could not delete old docs at " + dataConfig, e);
}

34
src/main/java/org/codelibs/fess/helper/IndexingHelper.java
View file

@ -46,7 +46,7 @@ public class IndexingHelper {
synchronized (fessEsClient) {
deleteOldDocuments(fessEsClient, docList);
final FessConfig fessConfig = ComponentUtil.getFessConfig();
fessEsClient.addAll(fessConfig.getIndexDocumentIndex(), fessConfig.getIndexDocumentType(), docList);
fessEsClient.addAll(fessConfig.getIndexDocumentUpdateIndex(), fessConfig.getIndexDocumentType(), docList);
}
if (logger.isInfoEnabled()) {
logger.info("Sent " + docList.size() + " docs (" + (System.currentTimeMillis() - execTime) + "ms)");
@ -93,7 +93,7 @@ public class IndexingHelper {
}
}
if (!docIdList.isEmpty()) {
fessEsClient.deleteByQuery(fessConfig.getIndexDocumentIndex(), fessConfig.getIndexDocumentType(),
fessEsClient.deleteByQuery(fessConfig.getIndexDocumentUpdateIndex(), fessConfig.getIndexDocumentType(),
QueryBuilders.idsQuery(fessConfig.getIndexDocumentType()).ids(docIdList.stream().toArray(n -> new String[n])));
}
@ -101,20 +101,21 @@ public class IndexingHelper {
public void deleteDocument(final FessEsClient fessEsClient, final String id) {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
fessEsClient.delete(fessConfig.getIndexDocumentIndex(), fessConfig.getIndexDocumentType(), id, 0);
fessEsClient.delete(fessConfig.getIndexDocumentUpdateIndex(), fessConfig.getIndexDocumentType(), id, 0);
}
public void deleteDocumentsByDocId(final FessEsClient fessEsClient, final List<String> docIdList) {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
fessEsClient.deleteByQuery(fessConfig.getIndexDocumentIndex(), fessConfig.getIndexDocumentType(),
fessEsClient.deleteByQuery(fessConfig.getIndexDocumentUpdateIndex(), fessConfig.getIndexDocumentType(),
QueryBuilders.idsQuery(fessConfig.getIndexDocumentType()).ids(docIdList.stream().toArray(n -> new String[n])));
}
public Map<String, Object> getDocument(final FessEsClient fessEsClient, final String id, final String[] fields) {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
return fessEsClient.getDocument(fessConfig.getIndexDocumentIndex(), fessConfig.getIndexDocumentType(), id, requestBuilder -> {
return true;
}).orElseGet(() -> null);
return fessEsClient.getDocument(fessConfig.getIndexDocumentSearchIndex(), fessConfig.getIndexDocumentType(), id,
requestBuilder -> {
return true;
}).orElseGet(() -> null);
}
public List<Map<String, Object>> getDocumentListByPrefixId(final FessEsClient fessEsClient, final String id, final String[] fields) {
@ -125,7 +126,7 @@ public class IndexingHelper {
public void deleteChildDocument(final FessEsClient fessEsClient, final String id) {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
fessEsClient.deleteByQuery(fessConfig.getIndexDocumentIndex(), fessConfig.getIndexDocumentType(),
fessEsClient.deleteByQuery(fessConfig.getIndexDocumentUpdateIndex(), fessConfig.getIndexDocumentType(),
QueryBuilders.termQuery(fessConfig.getIndexFieldParentId(), id));
}
@ -140,18 +141,19 @@ public class IndexingHelper {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final CountResponse countResponse =
fessEsClient.prepareCount(fessConfig.getIndexDocumentIndex()).setTypes(fessConfig.getIndexDocumentType())
fessEsClient.prepareCount(fessConfig.getIndexDocumentSearchIndex()).setTypes(fessConfig.getIndexDocumentType())
.setQuery(queryBuilder).execute().actionGet();
final long numFound = countResponse.getCount();
// TODO max threshold
return fessEsClient.getDocumentList(fessConfig.getIndexDocumentIndex(), fessConfig.getIndexDocumentType(), requestBuilder -> {
requestBuilder.setQuery(queryBuilder).setSize((int) numFound);
if (fields != null) {
requestBuilder.addFields(fields);
}
return true;
});
return fessEsClient.getDocumentList(fessConfig.getIndexDocumentSearchIndex(), fessConfig.getIndexDocumentType(),
requestBuilder -> {
requestBuilder.setQuery(queryBuilder).setSize((int) numFound);
if (fields != null) {
requestBuilder.addFields(fields);
}
return true;
});
}
}

2
src/main/java/org/codelibs/fess/helper/KeyMatchHelper.java
View file

@ -90,7 +90,7 @@ public class KeyMatchHelper {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final List<Map<String, Object>> documentList =
fessEsClient.getDocumentList(
fessConfig.getIndexDocumentIndex(),
fessConfig.getIndexDocumentSearchIndex(),
fessConfig.getIndexDocumentType(),
searchRequestBuilder -> {
return SearchConditionBuilder.builder(searchRequestBuilder).administrativeAccess(true)

30
src/main/java/org/codelibs/fess/helper/SearchLogHelper.java
View file

@ -244,22 +244,20 @@ public class SearchLogHelper {
searchService.bulkUpdate(builder -> {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
searchService.getDocumentListByDocIds(clickCountMap.keySet().toArray(new String[clickCountMap.size()]),
new String[] { fessConfig.getIndexFieldDocId() })
.forEach(
doc -> {
final String id = DocumentUtil.getValue(doc, fessConfig.getIndexFieldDocId(), String.class);
final String docId = DocumentUtil.getValue(doc, fessConfig.getIndexFieldDocId(), String.class);
if (id != null && docId != null && clickCountMap.containsKey(docId)) {
final Integer count = clickCountMap.get(docId);
final Script script =
new Script("ctx._source." + fessConfig.getIndexFieldClickCount() + "+="
+ count.toString());
final Map<String, Object> upsertMap = new HashMap<>();
upsertMap.put(fessConfig.getIndexFieldClickCount(), 1);
builder.add(new UpdateRequest(fessConfig.getIndexDocumentIndex(), fessConfig
.getIndexDocumentType(), id).script(script).upsert(upsertMap));
}
});
new String[] { fessConfig.getIndexFieldDocId() }).forEach(
doc -> {
final String id = DocumentUtil.getValue(doc, fessConfig.getIndexFieldDocId(), String.class);
final String docId = DocumentUtil.getValue(doc, fessConfig.getIndexFieldDocId(), String.class);
if (id != null && docId != null && clickCountMap.containsKey(docId)) {
final Integer count = clickCountMap.get(docId);
final Script script =
new Script("ctx._source." + fessConfig.getIndexFieldClickCount() + "+=" + count.toString());
final Map<String, Object> upsertMap = new HashMap<>();
upsertMap.put(fessConfig.getIndexFieldClickCount(), 1);
builder.add(new UpdateRequest(fessConfig.getIndexDocumentUpdateIndex(), fessConfig
.getIndexDocumentType(), id).script(script).upsert(upsertMap));
}
});
});
} catch (final Exception e) {
logger.warn("Failed to update clickCounts", e);

4
src/main/java/org/codelibs/fess/helper/SuggestHelper.java
View file

@ -83,7 +83,7 @@ public class SuggestHelper {
final Thread th = new Thread(() -> {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
fessEsClient.admin().cluster().prepareHealth().setWaitForYellowStatus().execute().actionGet();
suggester = Suggester.builder().build(fessEsClient, fessConfig.getIndexDocumentIndex());
suggester = Suggester.builder().build(fessEsClient, fessConfig.getIndexDocumentSearchIndex());
suggester.settings().array().delete(SuggestSettings.DefaultKeys.SUPPORTED_FIELDS);
for (final String field : contentsIndexFieldNames) {
suggester.settings().array().add(SuggestSettings.DefaultKeys.SUPPORTED_FIELDS, field);
@ -142,7 +142,7 @@ public class SuggestHelper {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final ESSourceReader reader =
new ESSourceReader(fessEsClient, suggester.settings(), fessConfig.getIndexDocumentIndex(),
new ESSourceReader(fessEsClient, suggester.settings(), fessConfig.getIndexDocumentSearchIndex(),
fessConfig.getIndexDocumentType());
reader.setScrollSize(sourceReaderScrollSize);
suggester.indexer().indexFromDocument(reader, 2, updateRequestIntervalMills).then(response -> {

2
src/main/java/org/codelibs/fess/job/PurgeDocJob.java
View file

@ -38,7 +38,7 @@ public class PurgeDocJob {
// clean up
final QueryBuilder queryBuilder = QueryBuilders.rangeQuery(fessConfig.getIndexFieldExpires()).to(new Date());
try {
fessEsClient.deleteByQuery(fessConfig.getIndexDocumentIndex(), fessConfig.getIndexDocumentType(), queryBuilder);
fessEsClient.deleteByQuery(fessConfig.getIndexDocumentUpdateIndex(), fessConfig.getIndexDocumentType(), queryBuilder);
} catch (final Exception e) {
logger.error("Could not delete expired documents: " + queryBuilder.toString(), e);

24
src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java
View file

@ -125,7 +125,10 @@ public interface FessConfig extends FessEnv {
String INDEX_FIELD_FILETYPE = "index.field.filetype";
/** The key of the configuration. e.g. fess */
String INDEX_DOCUMENT_INDEX = "index.document.index";
String INDEX_DOCUMENT_SEARCH_INDEX = "index.document.search.index";
/** The key of the configuration. e.g. fess */
String INDEX_DOCUMENT_UPDATE_INDEX = "index.document.update.index";
/** The key of the configuration. e.g. doc */
String INDEX_DOCUMENT_TYPE = "index.document.type";
@ -570,12 +573,19 @@ public interface FessConfig extends FessEnv {
String getIndexFieldFiletype();
/**
* Get the value for the key 'index.document.index'. <br>
* Get the value for the key 'index.document.search.index'. <br>
* The value is, e.g. fess <br>
* comment: document index
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getIndexDocumentIndex();
String getIndexDocumentSearchIndex();
/**
* Get the value for the key 'index.document.update.index'. <br>
* The value is, e.g. fess <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getIndexDocumentUpdateIndex();
/**
* Get the value for the key 'index.document.type'. <br>
@ -1322,8 +1332,12 @@ public interface FessConfig extends FessEnv {
return get(FessConfig.INDEX_FIELD_FILETYPE);
}
public String getIndexDocumentIndex() {
return get(FessConfig.INDEX_DOCUMENT_INDEX);
public String getIndexDocumentSearchIndex() {
return get(FessConfig.INDEX_DOCUMENT_SEARCH_INDEX);
}
public String getIndexDocumentUpdateIndex() {
return get(FessConfig.INDEX_DOCUMENT_UPDATE_INDEX);
}
public String getIndexDocumentType() {

3
src/main/resources/fess_config.properties
View file

@ -53,7 +53,8 @@ index.field.content_length=content_length
index.field.filetype=filetype
# document index
index.document.index=fess
index.document.search.index=fess
index.document.update.index=fess
index.document.type=doc
# query