diff --git a/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java b/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java
index 52442ed69..0d5bf0af8 100644
--- a/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java
+++ b/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java
@@ -743,6 +743,9 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
/** The key of the configuration. e.g. 100 */
String PAGE_THUMBNAIL_QUEUE_MAX_FETCH_SIZE = "page.thumbnail.queue.max.fetch.size";
+ /** The key of the configuration. e.g. 100 */
+ String PAGE_THUMBNAIL_PURGE_MAX_FETCH_SIZE = "page.thumbnail.purge.max.fetch.size";
+
/** The key of the configuration. e.g. 0 */
String PAGING_SEARCH_PAGE_START = "paging.search.page.start";
@@ -3631,6 +3634,21 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
*/
Integer getPageThumbnailQueueMaxFetchSizeAsInteger();
+ /**
+ * Get the value for the key 'page.thumbnail.purge.max.fetch.size'.
+ * The value is, e.g. 100
+ * @return The value of found property. (NotNull: if not found, exception but basically no way)
+ */
+ String getPageThumbnailPurgeMaxFetchSize();
+
+ /**
+ * Get the value for the key 'page.thumbnail.purge.max.fetch.size' as {@link Integer}.
+ * The value is, e.g. 100
+ * @return The value of found property. (NotNull: if not found, exception but basically no way)
+ * @throws NumberFormatException When the property is not integer.
+ */
+ Integer getPageThumbnailPurgeMaxFetchSizeAsInteger();
+
/**
* Get the value for the key 'paging.search.page.start'.
* The value is, e.g. 0
@@ -6405,6 +6423,14 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
return getAsInteger(FessConfig.PAGE_THUMBNAIL_QUEUE_MAX_FETCH_SIZE);
}
+ public String getPageThumbnailPurgeMaxFetchSize() {
+ return get(FessConfig.PAGE_THUMBNAIL_PURGE_MAX_FETCH_SIZE);
+ }
+
+ public Integer getPageThumbnailPurgeMaxFetchSizeAsInteger() {
+ return getAsInteger(FessConfig.PAGE_THUMBNAIL_PURGE_MAX_FETCH_SIZE);
+ }
+
public String getPagingSearchPageStart() {
return get(FessConfig.PAGING_SEARCH_PAGE_START);
}
diff --git a/src/main/java/org/codelibs/fess/thumbnail/ThumbnailManager.java b/src/main/java/org/codelibs/fess/thumbnail/ThumbnailManager.java
index 9c03561a7..275f83011 100644
--- a/src/main/java/org/codelibs/fess/thumbnail/ThumbnailManager.java
+++ b/src/main/java/org/codelibs/fess/thumbnail/ThumbnailManager.java
@@ -38,6 +38,7 @@ import org.codelibs.core.collection.LruHashMap;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.core.misc.Tuple4;
import org.codelibs.fess.Constants;
+import org.codelibs.fess.es.client.FessEsClient;
import org.codelibs.fess.es.config.exbhv.ThumbnailQueueBhv;
import org.codelibs.fess.es.config.exentity.ThumbnailQueue;
import org.codelibs.fess.exception.FessSystemException;
@@ -47,6 +48,7 @@ import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.util.ComponentUtil;
import org.codelibs.fess.util.DocumentUtil;
import org.codelibs.fess.util.ResourceUtil;
+import org.elasticsearch.index.query.QueryBuilders;
import org.lastaflute.web.util.LaRequestUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -60,6 +62,8 @@ public class ThumbnailManager {
private static final String NOIMAGE_FILE_SUFFIX = ".txt";
+ protected static final String THUMBNAILS_DIR_NAME = "thumbnails";
+
private static final Logger logger = LoggerFactory.getLogger(ThumbnailManager.class);
protected File baseDir;
@@ -94,7 +98,7 @@ public class ThumbnailManager {
} else {
final String varPath = System.getProperty(FESS_VAR_PATH);
if (varPath != null) {
- baseDir = new File(varPath, "thumbnails");
+ baseDir = new File(varPath, THUMBNAILS_DIR_NAME);
} else {
baseDir = ResourceUtil.getThumbnailPath().toFile();
}
@@ -331,7 +335,7 @@ public class ThumbnailManager {
return 0;
}
try {
- final FilePurgeVisitor visitor = new FilePurgeVisitor(expiry);
+ final FilePurgeVisitor visitor = new FilePurgeVisitor(baseDir.toPath(), imageExtention, expiry);
Files.walkFileTree(baseDir.toPath(), visitor);
return visitor.getCount();
} catch (final Exception e) {
@@ -339,17 +343,95 @@ public class ThumbnailManager {
}
}
- private static class FilePurgeVisitor implements FileVisitor {
+ protected static class FilePurgeVisitor implements FileVisitor {
- private final long expiry;
+ protected final long expiry;
- private long count;
+ protected long count;
- FilePurgeVisitor(final long expiry) {
+ protected final int maxPurgeSize;
+
+ protected final List deletedFileList = new ArrayList<>();
+
+ protected final Path basePath;
+
+ protected final String imageExtention;
+
+ protected final FessEsClient fessEsClient;
+
+ protected final FessConfig fessConfig;
+
+ FilePurgeVisitor(final Path basePath, final String imageExtention, final long expiry) {
+ this.basePath = basePath;
+ this.imageExtention = imageExtention;
this.expiry = expiry;
+ this.fessConfig = ComponentUtil.getFessConfig();
+ this.maxPurgeSize = fessConfig.getPageThumbnailPurgeMaxFetchSizeAsInteger();
+ this.fessEsClient = ComponentUtil.getFessEsClient();
+ }
+
+ protected void deleteFiles() {
+ final Map deleteFileMap = new HashMap<>();
+ for (final Path path : deletedFileList) {
+ final String docId = getDocId(path);
+ if (StringUtil.isBlank(docId) || deleteFileMap.containsKey(docId)) {
+ deleteFile(path);
+ } else {
+ deleteFileMap.put(docId, path);
+ }
+ }
+ deletedFileList.clear();
+
+ if (!deleteFileMap.isEmpty()) {
+ final String docIdField = fessConfig.getIndexFieldDocId();
+ fessEsClient.getDocumentList(
+ fessConfig.getIndexDocumentSearchIndex(),
+ fessConfig.getIndexDocumentType(),
+ searchRequestBuilder -> {
+ searchRequestBuilder.setQuery(QueryBuilders.termsQuery(docIdField,
+ deleteFileMap.keySet().toArray(new String[deleteFileMap.size()])));
+ searchRequestBuilder.setFetchSource(new String[] { docIdField }, StringUtil.EMPTY_STRINGS);
+ return true;
+ }).forEach(m -> {
+ final Object docId = m.get(docIdField);
+ if (docId != null) {
+ deleteFileMap.remove(docId);
+ if (logger.isDebugEnabled()) {
+ logger.debug("Keep thumbnail: " + docId);
+ }
+ }
+ });
+ ;
+ deleteFileMap.values().forEach(v -> deleteFile(v));
+ count += deleteFileMap.size();
+ }
+ }
+
+ protected void deleteFile(final Path path) {
+ try {
+ Files.delete(path);
+ if (logger.isDebugEnabled()) {
+ logger.debug("Delete " + path);
+ }
+ } catch (IOException e) {
+ logger.warn("Failed to delete " + path, e);
+ }
+ }
+
+ protected String getDocId(final Path file) {
+ final String s = file.toUri().toString();
+ final String b = basePath.toUri().toString();
+ final String id = s.replace(b, StringUtil.EMPTY).replace("." + imageExtention, StringUtil.EMPTY).replace("/", StringUtil.EMPTY);
+ if (logger.isDebugEnabled()) {
+ logger.debug("Base: " + b + " File: " + s + " DocId: " + id);
+ }
+ return id;
}
public long getCount() {
+ if (!deletedFileList.isEmpty()) {
+ deleteFiles();
+ }
return count;
}
@@ -361,8 +443,10 @@ public class ThumbnailManager {
@Override
public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException {
if (System.currentTimeMillis() - Files.getLastModifiedTime(file).toMillis() > expiry) {
- Files.delete(file);
- count++;
+ deletedFileList.add(file);
+ if (deletedFileList.size() > maxPurgeSize) {
+ deleteFiles();
+ }
}
return FileVisitResult.CONTINUE;
}
@@ -380,7 +464,7 @@ public class ThumbnailManager {
if (e != null) {
logger.warn("I/O exception on " + dir, e);
}
- if (dir.toFile().list().length == 0) {
+ if (dir.toFile().list().length == 0 && !dir.toFile().getName().equals(THUMBNAILS_DIR_NAME)) {
Files.delete(dir);
}
return FileVisitResult.CONTINUE;
diff --git a/src/main/resources/fess_config.properties b/src/main/resources/fess_config.properties
index 74049c605..271387253 100644
--- a/src/main/resources/fess_config.properties
+++ b/src/main/resources/fess_config.properties
@@ -388,6 +388,7 @@ page.elevate.word.max.fetch.size=1000
page.bad.word.max.fetch.size=1000
page.dictionary.max.fetch.size=1000
page.thumbnail.queue.max.fetch.size=100
+page.thumbnail.purge.max.fetch.size=100
# search page
paging.search.page.start=0