0ct0pu5/fess
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix #448 : add indexer.data.max.document.request.size

Browse source
This commit is contained in:
Shinsuke Sugaya 2016-03-23 06:48:21 +09:00
parent f609d3abca
commit 7c1147aeac
3 changed files with 62 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

49
src/main/java/org/codelibs/fess/ds/impl/IndexUpdateCallbackImpl.java
View file

@ -18,6 +18,8 @@ package org.codelibs.fess.ds.impl;
import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
import javax.annotation.PostConstruct;
import org.codelibs.fess.ds.IndexUpdateCallback;
import org.codelibs.fess.es.client.FessEsClient;
import org.codelibs.fess.exception.FessSystemException;
@ -28,23 +30,31 @@ import org.codelibs.fess.helper.SystemHelper;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.util.ComponentUtil;
import org.codelibs.fess.util.DocList;
import org.codelibs.fess.util.DocumentUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class IndexUpdateCallbackImpl implements IndexUpdateCallback {
private static final Logger logger = LoggerFactory.getLogger(IndexUpdateCallbackImpl.class);
protected volatile AtomicLong documentSize = new AtomicLong(0);
protected AtomicLong documentSize = new AtomicLong(0);
protected volatile long executeTime = 0;
final DocList docList = new DocList();
protected final DocList docList = new DocList();
protected long maxDocumentRequestSize;
@PostConstruct
public void init() {
maxDocumentRequestSize = ComponentUtil.getFessConfig().getIndexerWebfsMaxDocumentRequestSizeAsInteger().longValue();
}
/* (non-Javadoc)
* @see org.codelibs.fess.ds.impl.IndexUpdateCallback#store(java.util.Map)
*/
@Override
public synchronized boolean store(final Map<String, String> paramMap, final Map<String, Object> dataMap) {
public boolean store(final Map<String, String> paramMap, final Map<String, Object> dataMap) {
final long startTime = System.currentTimeMillis();
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final FessEsClient fessEsClient = ComponentUtil.getElasticsearchClient();
@ -78,30 +88,41 @@ public class IndexUpdateCallbackImpl implements IndexUpdateCallback {
dataMap.put(fessConfig.getIndexFieldDocId(), systemHelper.generateDocId(dataMap));
}
docList.add(dataMap);
if (logger.isDebugEnabled()) {
logger.debug("Added the document. " + "The number of a document cache is " + docList.size() + ".");
synchronized (docList) {
docList.add(dataMap);
if (logger.isDebugEnabled()) {
logger.debug("Added the document. " + "The number of a document cache is " + docList.size() + ".");
}
final Long contentLength = DocumentUtil.getValue(dataMap, fessConfig.getIndexFieldContentLength(), Long.class);
if (contentLength != null) {
docList.addContentSize(contentLength.longValue());
if (docList.getContentSize() >= maxDocumentRequestSize) {
indexingHelper.sendDocuments(fessEsClient, docList);
}
} else if (docList.size() >= fessConfig.getIndexerDataMaxDocumentCacheSizeAsInteger().intValue()) {
indexingHelper.sendDocuments(fessEsClient, docList);
}
executeTime += System.currentTimeMillis() - startTime;
}
if (docList.size() >= fessConfig.getIndexerDataMaxDocumentCacheSizeAsInteger().intValue()) {
indexingHelper.sendDocuments(fessEsClient, docList);
}
documentSize.getAndIncrement();
if (logger.isDebugEnabled()) {
logger.debug("The number of an added document is " + documentSize.get() + ".");
}
executeTime += System.currentTimeMillis() - startTime;
return true;
}
@Override
public void commit() {
if (!docList.isEmpty()) {
final IndexingHelper indexingHelper = ComponentUtil.getIndexingHelper();
final FessEsClient fessEsClient = ComponentUtil.getElasticsearchClient();
indexingHelper.sendDocuments(fessEsClient, docList);
synchronized (docList) {
if (!docList.isEmpty()) {
final IndexingHelper indexingHelper = ComponentUtil.getIndexingHelper();
final FessEsClient fessEsClient = ComponentUtil.getElasticsearchClient();
indexingHelper.sendDocuments(fessEsClient, docList);
}
}
}

26
src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java
View file

@ -234,6 +234,9 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
/** The key of the configuration. e.g. 5 */
String INDEXER_DATA_MAX_DOCUMENT_CACHE_SIZE = "indexer.data.max.document.cache.size";
/** The key of the configuration. e.g. 10485760 */
String INDEXER_DATA_MAX_DOCUMENT_REQUEST_SIZE = "indexer.data.max.document.request.size";
/** The key of the configuration. e.g. favorite_count */
String INDEX_FIELD_favorite_count = "index.field.favorite_count";
@ -1468,6 +1471,21 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
*/
Integer getIndexerDataMaxDocumentCacheSizeAsInteger();
/**
* Get the value for the key 'indexer.data.max.document.request.size'. <br>
* The value is, e.g. 10485760 <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getIndexerDataMaxDocumentRequestSize();
/**
* Get the value for the key 'indexer.data.max.document.request.size' as {@link Integer}. <br>
* The value is, e.g. 10485760 <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
* @throws NumberFormatException When the property is not integer.
*/
Integer getIndexerDataMaxDocumentRequestSizeAsInteger();
/**
* Get the value for the key 'index.field.favorite_count'. <br>
* The value is, e.g. favorite_count <br>
@ -3529,6 +3547,14 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
return getAsInteger(FessConfig.INDEXER_DATA_MAX_DOCUMENT_CACHE_SIZE);
}
public String getIndexerDataMaxDocumentRequestSize() {
return get(FessConfig.INDEXER_DATA_MAX_DOCUMENT_REQUEST_SIZE);
}
public Integer getIndexerDataMaxDocumentRequestSizeAsInteger() {
return getAsInteger(FessConfig.INDEXER_DATA_MAX_DOCUMENT_REQUEST_SIZE);
}
public String getIndexFieldFavoriteCount() {
return get(FessConfig.INDEX_FIELD_favorite_count);
}

1
src/main/resources/fess_config.properties
View file

@ -124,6 +124,7 @@ indexer.webfs.update.interval=10000
indexer.webfs.max.document.cache.size=100
indexer.webfs.max.document.request.size=10485760
indexer.data.max.document.cache.size=5
indexer.data.max.document.request.size=10485760
# field names
index.field.favorite_count=favorite_count