diff --git a/src/main/java/org/codelibs/fess/ds/callback/IndexUpdateCallbackImpl.java b/src/main/java/org/codelibs/fess/ds/callback/IndexUpdateCallbackImpl.java index 85f3e2105..3c2241b3c 100644 --- a/src/main/java/org/codelibs/fess/ds/callback/IndexUpdateCallbackImpl.java +++ b/src/main/java/org/codelibs/fess/ds/callback/IndexUpdateCallbackImpl.java @@ -112,7 +112,7 @@ public class IndexUpdateCallbackImpl implements IndexUpdateCallback { final Long contentLength = DocumentUtil.getValue(dataMap, fessConfig.getIndexFieldContentLength(), Long.class); if (contentLength != null) { - docList.addContentSize(contentLength.longValue()); + docList.addContentSize(indexingHelper.calculateDocumentSize(dataMap, contentLength.longValue())); if (docList.getContentSize() >= maxDocumentRequestSize) { indexingHelper.sendDocuments(fessEsClient, docList); } diff --git a/src/main/java/org/codelibs/fess/helper/IndexingHelper.java b/src/main/java/org/codelibs/fess/helper/IndexingHelper.java index b554d640c..3b8943358 100644 --- a/src/main/java/org/codelibs/fess/helper/IndexingHelper.java +++ b/src/main/java/org/codelibs/fess/helper/IndexingHelper.java @@ -203,6 +203,14 @@ public class IndexingHelper { } + public long calculateDocumentSize(final Map dataMap, final long size) { + final long objSize = MemoryUtil.sizeOf(dataMap); + if (objSize > size) { + return objSize; + } + return size; + } + public void setMaxRetryCount(final int maxRetryCount) { this.maxRetryCount = maxRetryCount; } @@ -214,4 +222,5 @@ public class IndexingHelper { public void setRequestInterval(final long requestInterval) { this.requestInterval = requestInterval; } + } diff --git a/src/main/java/org/codelibs/fess/util/MemoryUtil.java b/src/main/java/org/codelibs/fess/util/MemoryUtil.java index 6562169fe..a7e7aaaf1 100644 --- a/src/main/java/org/codelibs/fess/util/MemoryUtil.java +++ b/src/main/java/org/codelibs/fess/util/MemoryUtil.java @@ -15,6 +15,12 @@ */ package org.codelibs.fess.util; +import java.time.LocalDateTime; +import java.time.ZonedDateTime; +import java.util.Collection; +import java.util.Date; +import java.util.Map; + import org.apache.commons.io.FileUtils; import org.codelibs.core.lang.StringUtil; @@ -42,4 +48,40 @@ public final class MemoryUtil { final long totalBytes = runtime.totalMemory(); return totalBytes - freeBytes; } + + public static long sizeOf(final Object obj) { + if (obj == null) { + return 0L; + } else if (obj instanceof String) { + return ((String) obj).length() + 56L; + } else if (obj instanceof Number) { + return 24L; + } else if (obj instanceof Date) { + return 32L; + } else if (obj instanceof LocalDateTime) { + return 80L; + } else if (obj instanceof ZonedDateTime) { + return 2128L; + } else if (obj instanceof Object[]) { + long size = 0; + for (final Object value : (Object[]) obj) { + size += sizeOf(value); + } + return size; + } else if (obj instanceof Collection) { + long size = 0; + for (final Object value : (Collection) obj) { + size += sizeOf(value); + } + return size; + } else if (obj instanceof Map) { + long size = 0; + for (final Map.Entry entry : ((Map) obj).entrySet()) { + size += sizeOf(entry.getKey()); + size += sizeOf(entry.getValue()); + } + return size; + } + return 16L; + } } diff --git a/src/test/java/org/codelibs/fess/util/MemoryUtilTest.java b/src/test/java/org/codelibs/fess/util/MemoryUtilTest.java index a5a09eb4e..404d9efb5 100644 --- a/src/test/java/org/codelibs/fess/util/MemoryUtilTest.java +++ b/src/test/java/org/codelibs/fess/util/MemoryUtilTest.java @@ -15,7 +15,15 @@ */ package org.codelibs.fess.util; +import java.time.LocalDateTime; +import java.time.ZonedDateTime; +import java.util.Date; + +import org.codelibs.core.collection.Maps; import org.codelibs.fess.unit.UnitFessTestCase; +import org.codelibs.sai.internal.ir.debug.ObjectSizeCalculator; + +import com.google.common.collect.Lists; public class MemoryUtilTest extends UnitFessTestCase { @@ -23,4 +31,28 @@ public class MemoryUtilTest extends UnitFessTestCase { assertTrue(MemoryUtil.getUsedMemory() >= 0); } + public void test_sizeOf() throws Exception { + // System.out.println("size: " + getObjectSize("")); + assertEquals(24L, MemoryUtil.sizeOf(Integer.MAX_VALUE)); + assertEquals(24L, MemoryUtil.sizeOf(Long.MAX_VALUE)); + assertEquals(24L, MemoryUtil.sizeOf(Short.MAX_VALUE)); + assertEquals(24L, MemoryUtil.sizeOf(Float.MAX_VALUE)); + assertEquals(24L, MemoryUtil.sizeOf(Double.MAX_VALUE)); + assertEquals(24L, MemoryUtil.sizeOf(Byte.MAX_VALUE)); + assertEquals(16L, MemoryUtil.sizeOf(Boolean.TRUE)); + assertEquals(32L, MemoryUtil.sizeOf(new Date())); + assertEquals(80L, MemoryUtil.sizeOf(LocalDateTime.now())); + assertEquals(2128L, MemoryUtil.sizeOf(ZonedDateTime.now())); + assertEquals(66L, MemoryUtil.sizeOf("1234567890")); + assertEquals(76L, MemoryUtil.sizeOf("12345678901234567890")); + assertEquals(66L, MemoryUtil.sizeOf(new String[] { "1234567890" })); + assertEquals(132L, MemoryUtil.sizeOf(new String[] { "1234567890", "1234567890" })); + assertEquals(132L, MemoryUtil.sizeOf(Lists.asList("1234567890", new String[] { "1234567890" }))); + assertEquals(132L, MemoryUtil.sizeOf(Maps.map("1234567890", "1234567890").$())); + } + + private long getObjectSize(Object value) { + System.setProperty("java.vm.name", "Java HotSpot(TM) "); + return ObjectSizeCalculator.getObjectSize(value); + } } \ No newline at end of file