0ct0pu5/fess
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix #2423 use sizeOf

Browse source
This commit is contained in:
Shinsuke Sugaya 2020-03-05 07:04:40 +09:00
parent 2e49c2db49
commit de79d8f17d
5 changed files with 26 additions and 30 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
src/main/java/org/codelibs/fess/ds/callback/IndexUpdateCallbackImpl.java
View file

@ -35,6 +35,7 @@ import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.util.ComponentUtil;
import org.codelibs.fess.util.DocList;
import org.codelibs.fess.util.DocumentUtil;
import org.codelibs.fess.util.MemoryUtil;
public class IndexUpdateCallbackImpl implements IndexUpdateCallback {
private static final Logger logger = LogManager.getLogger(IndexUpdateCallbackImpl.class);
@ -47,12 +48,15 @@ public class IndexUpdateCallbackImpl implements IndexUpdateCallback {
protected long maxDocumentRequestSize;
protected int maxDocumentCacheSize;
@PostConstruct
public void init() {
if (logger.isDebugEnabled()) {
logger.debug("Initialize {}", this.getClass().getSimpleName());
}
maxDocumentRequestSize = ComponentUtil.getFessConfig().getIndexerDataMaxDocumentRequestSizeAsInteger().longValue();
maxDocumentRequestSize = Long.parseLong(ComponentUtil.getFessConfig().getIndexerDataMaxDocumentRequestSize());
maxDocumentCacheSize = ComponentUtil.getFessConfig().getIndexerDataMaxDocumentCacheSizeAsInteger();
}
/* (non-Javadoc)
@ -106,20 +110,19 @@ public class IndexUpdateCallbackImpl implements IndexUpdateCallback {
synchronized (docList) {
docList.add(dataMap);
final long contentSize = indexingHelper.calculateDocumentSize(dataMap);
docList.addContentSize(contentSize);
final long processingTime = System.currentTimeMillis() - startTime;
docList.addProcessingTime(processingTime);
if (logger.isDebugEnabled()) {
logger.debug("Added the document. The number of a document cache is {}.", docList.size());
logger.debug("Added the document({}, {}ms). The number of a document cache is {}.",
MemoryUtil.byteCountToDisplaySize(contentSize), processingTime, docList.size());
}
final Long contentLength = DocumentUtil.getValue(dataMap, fessConfig.getIndexFieldContentLength(), Long.class);
if (contentLength != null) {
docList.addContentSize(indexingHelper.calculateDocumentSize(dataMap, contentLength.longValue()));
if (docList.getContentSize() >= maxDocumentRequestSize) {
indexingHelper.sendDocuments(fessEsClient, docList);
}
} else if (docList.size() >= fessConfig.getIndexerDataMaxDocumentCacheSizeAsInteger().intValue()) {
if (docList.getContentSize() >= maxDocumentRequestSize || docList.size() >= maxDocumentCacheSize) {
indexingHelper.sendDocuments(fessEsClient, docList);
}
executeTime += System.currentTimeMillis() - startTime;
executeTime += processingTime;
}
documentSize.getAndIncrement();

8
src/main/java/org/codelibs/fess/helper/IndexingHelper.java
View file

@ -203,12 +203,8 @@ public class IndexingHelper {
}
public long calculateDocumentSize(final Map<String, Object> dataMap, final long size) {
final long objSize = MemoryUtil.sizeOf(dataMap);
if (objSize > size) {
return objSize;
}
return size;
public long calculateDocumentSize(final Map<String, Object> dataMap) {
return MemoryUtil.sizeOf(dataMap);
}
public void setMaxRetryCount(final int maxRetryCount) {

15
src/main/java/org/codelibs/fess/indexer/IndexUpdater.java
View file

@ -308,7 +308,7 @@ public class IndexUpdater extends Thread {
private void processAccessResults(final DocList docList, final List<EsAccessResult> accessResultList, final List<EsAccessResult> arList) {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final long maxDocumentRequestSize = fessConfig.getIndexerWebfsMaxDocumentRequestSizeAsInteger().longValue();
final long maxDocumentRequestSize = Long.parseLong(fessConfig.getIndexerWebfsMaxDocumentRequestSize());
for (final EsAccessResult accessResult : arList) {
if (logger.isDebugEnabled()) {
logger.debug("Indexing {}", accessResult.getUrl());
@ -355,20 +355,17 @@ public class IndexUpdater extends Thread {
updateDocument(map);
docList.add(map);
final long contentSize = indexingHelper.calculateDocumentSize(map);
docList.addContentSize(contentSize);
final long processingTime = System.currentTimeMillis() - startTime;
docList.addProcessingTime(processingTime);
if (logger.isDebugEnabled()) {
logger.debug("Added the document({}, {}ms). " + "The number of a document cache is {}.",
MemoryUtil.byteCountToDisplaySize(docList.getContentSize()), processingTime, docList.size());
logger.debug("Added the document({}, {}ms). The number of a document cache is {}.",
MemoryUtil.byteCountToDisplaySize(contentSize), processingTime, docList.size());
}
if (accessResult.getContentLength() == null) {
if (docList.getContentSize() >= maxDocumentRequestSize) {
indexingHelper.sendDocuments(fessEsClient, docList);
} else {
docList.addContentSize(accessResult.getContentLength().longValue());
if (docList.getContentSize() >= maxDocumentRequestSize) {
indexingHelper.sendDocuments(fessEsClient, docList);
}
}
documentSize++;
if (logger.isDebugEnabled()) {

8
src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java
View file

@ -417,7 +417,7 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
/** The key of the configuration. e.g. 1048576 */
String INDEXER_WEBFS_MAX_DOCUMENT_REQUEST_SIZE = "indexer.webfs.max.document.request.size";
/** The key of the configuration. e.g. 5 */
/** The key of the configuration. e.g. 10000 */
String INDEXER_DATA_MAX_DOCUMENT_CACHE_SIZE = "indexer.data.max.document.cache.size";
/** The key of the configuration. e.g. 1048576 */
@ -2809,14 +2809,14 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
/**
* Get the value for the key 'indexer.data.max.document.cache.size'. <br>
* The value is, e.g. 5 <br>
* The value is, e.g. 10000 <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getIndexerDataMaxDocumentCacheSize();
/**
* Get the value for the key 'indexer.data.max.document.cache.size' as {@link Integer}. <br>
* The value is, e.g. 5 <br>
* The value is, e.g. 10000 <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
* @throws NumberFormatException When the property is not integer.
*/
@ -9167,7 +9167,7 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
defaultMap.put(FessConfig.INDEXER_WEBFS_UPDATE_INTERVAL, "10000");
defaultMap.put(FessConfig.INDEXER_WEBFS_MAX_DOCUMENT_CACHE_SIZE, "10");
defaultMap.put(FessConfig.INDEXER_WEBFS_MAX_DOCUMENT_REQUEST_SIZE, "1048576");
defaultMap.put(FessConfig.INDEXER_DATA_MAX_DOCUMENT_CACHE_SIZE, "5");
defaultMap.put(FessConfig.INDEXER_DATA_MAX_DOCUMENT_CACHE_SIZE, "10000");
defaultMap.put(FessConfig.INDEXER_DATA_MAX_DOCUMENT_REQUEST_SIZE, "1048576");
defaultMap.put(FessConfig.INDEXER_LANGUAGE_FIELDS, "content,important_content,title");
defaultMap.put(FessConfig.INDEXER_LANGUAGE_DETECT_LENGTH, "1000");

2
src/main/resources/fess_config.properties
View file

@ -246,7 +246,7 @@ indexer.webfs.max.empty.list.count=3600
indexer.webfs.update.interval=10000
indexer.webfs.max.document.cache.size=10
indexer.webfs.max.document.request.size=1048576
indexer.data.max.document.cache.size=5
indexer.data.max.document.cache.size=10000
indexer.data.max.document.request.size=1048576
indexer.language.fields=content,important_content,title
indexer.language.detect.length=1000