diff --git a/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java b/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java index 52442ed69..0d5bf0af8 100644 --- a/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java +++ b/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java @@ -743,6 +743,9 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction /** The key of the configuration. e.g. 100 */ String PAGE_THUMBNAIL_QUEUE_MAX_FETCH_SIZE = "page.thumbnail.queue.max.fetch.size"; + /** The key of the configuration. e.g. 100 */ + String PAGE_THUMBNAIL_PURGE_MAX_FETCH_SIZE = "page.thumbnail.purge.max.fetch.size"; + /** The key of the configuration. e.g. 0 */ String PAGING_SEARCH_PAGE_START = "paging.search.page.start"; @@ -3631,6 +3634,21 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction */ Integer getPageThumbnailQueueMaxFetchSizeAsInteger(); + /** + * Get the value for the key 'page.thumbnail.purge.max.fetch.size'.
+ * The value is, e.g. 100
+ * @return The value of found property. (NotNull: if not found, exception but basically no way) + */ + String getPageThumbnailPurgeMaxFetchSize(); + + /** + * Get the value for the key 'page.thumbnail.purge.max.fetch.size' as {@link Integer}.
+ * The value is, e.g. 100
+ * @return The value of found property. (NotNull: if not found, exception but basically no way) + * @throws NumberFormatException When the property is not integer. + */ + Integer getPageThumbnailPurgeMaxFetchSizeAsInteger(); + /** * Get the value for the key 'paging.search.page.start'.
* The value is, e.g. 0
@@ -6405,6 +6423,14 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction return getAsInteger(FessConfig.PAGE_THUMBNAIL_QUEUE_MAX_FETCH_SIZE); } + public String getPageThumbnailPurgeMaxFetchSize() { + return get(FessConfig.PAGE_THUMBNAIL_PURGE_MAX_FETCH_SIZE); + } + + public Integer getPageThumbnailPurgeMaxFetchSizeAsInteger() { + return getAsInteger(FessConfig.PAGE_THUMBNAIL_PURGE_MAX_FETCH_SIZE); + } + public String getPagingSearchPageStart() { return get(FessConfig.PAGING_SEARCH_PAGE_START); } diff --git a/src/main/java/org/codelibs/fess/thumbnail/ThumbnailManager.java b/src/main/java/org/codelibs/fess/thumbnail/ThumbnailManager.java index 9c03561a7..275f83011 100644 --- a/src/main/java/org/codelibs/fess/thumbnail/ThumbnailManager.java +++ b/src/main/java/org/codelibs/fess/thumbnail/ThumbnailManager.java @@ -38,6 +38,7 @@ import org.codelibs.core.collection.LruHashMap; import org.codelibs.core.lang.StringUtil; import org.codelibs.core.misc.Tuple4; import org.codelibs.fess.Constants; +import org.codelibs.fess.es.client.FessEsClient; import org.codelibs.fess.es.config.exbhv.ThumbnailQueueBhv; import org.codelibs.fess.es.config.exentity.ThumbnailQueue; import org.codelibs.fess.exception.FessSystemException; @@ -47,6 +48,7 @@ import org.codelibs.fess.mylasta.direction.FessConfig; import org.codelibs.fess.util.ComponentUtil; import org.codelibs.fess.util.DocumentUtil; import org.codelibs.fess.util.ResourceUtil; +import org.elasticsearch.index.query.QueryBuilders; import org.lastaflute.web.util.LaRequestUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -60,6 +62,8 @@ public class ThumbnailManager { private static final String NOIMAGE_FILE_SUFFIX = ".txt"; + protected static final String THUMBNAILS_DIR_NAME = "thumbnails"; + private static final Logger logger = LoggerFactory.getLogger(ThumbnailManager.class); protected File baseDir; @@ -94,7 +98,7 @@ public class ThumbnailManager { } else { final String varPath = System.getProperty(FESS_VAR_PATH); if (varPath != null) { - baseDir = new File(varPath, "thumbnails"); + baseDir = new File(varPath, THUMBNAILS_DIR_NAME); } else { baseDir = ResourceUtil.getThumbnailPath().toFile(); } @@ -331,7 +335,7 @@ public class ThumbnailManager { return 0; } try { - final FilePurgeVisitor visitor = new FilePurgeVisitor(expiry); + final FilePurgeVisitor visitor = new FilePurgeVisitor(baseDir.toPath(), imageExtention, expiry); Files.walkFileTree(baseDir.toPath(), visitor); return visitor.getCount(); } catch (final Exception e) { @@ -339,17 +343,95 @@ public class ThumbnailManager { } } - private static class FilePurgeVisitor implements FileVisitor { + protected static class FilePurgeVisitor implements FileVisitor { - private final long expiry; + protected final long expiry; - private long count; + protected long count; - FilePurgeVisitor(final long expiry) { + protected final int maxPurgeSize; + + protected final List deletedFileList = new ArrayList<>(); + + protected final Path basePath; + + protected final String imageExtention; + + protected final FessEsClient fessEsClient; + + protected final FessConfig fessConfig; + + FilePurgeVisitor(final Path basePath, final String imageExtention, final long expiry) { + this.basePath = basePath; + this.imageExtention = imageExtention; this.expiry = expiry; + this.fessConfig = ComponentUtil.getFessConfig(); + this.maxPurgeSize = fessConfig.getPageThumbnailPurgeMaxFetchSizeAsInteger(); + this.fessEsClient = ComponentUtil.getFessEsClient(); + } + + protected void deleteFiles() { + final Map deleteFileMap = new HashMap<>(); + for (final Path path : deletedFileList) { + final String docId = getDocId(path); + if (StringUtil.isBlank(docId) || deleteFileMap.containsKey(docId)) { + deleteFile(path); + } else { + deleteFileMap.put(docId, path); + } + } + deletedFileList.clear(); + + if (!deleteFileMap.isEmpty()) { + final String docIdField = fessConfig.getIndexFieldDocId(); + fessEsClient.getDocumentList( + fessConfig.getIndexDocumentSearchIndex(), + fessConfig.getIndexDocumentType(), + searchRequestBuilder -> { + searchRequestBuilder.setQuery(QueryBuilders.termsQuery(docIdField, + deleteFileMap.keySet().toArray(new String[deleteFileMap.size()]))); + searchRequestBuilder.setFetchSource(new String[] { docIdField }, StringUtil.EMPTY_STRINGS); + return true; + }).forEach(m -> { + final Object docId = m.get(docIdField); + if (docId != null) { + deleteFileMap.remove(docId); + if (logger.isDebugEnabled()) { + logger.debug("Keep thumbnail: " + docId); + } + } + }); + ; + deleteFileMap.values().forEach(v -> deleteFile(v)); + count += deleteFileMap.size(); + } + } + + protected void deleteFile(final Path path) { + try { + Files.delete(path); + if (logger.isDebugEnabled()) { + logger.debug("Delete " + path); + } + } catch (IOException e) { + logger.warn("Failed to delete " + path, e); + } + } + + protected String getDocId(final Path file) { + final String s = file.toUri().toString(); + final String b = basePath.toUri().toString(); + final String id = s.replace(b, StringUtil.EMPTY).replace("." + imageExtention, StringUtil.EMPTY).replace("/", StringUtil.EMPTY); + if (logger.isDebugEnabled()) { + logger.debug("Base: " + b + " File: " + s + " DocId: " + id); + } + return id; } public long getCount() { + if (!deletedFileList.isEmpty()) { + deleteFiles(); + } return count; } @@ -361,8 +443,10 @@ public class ThumbnailManager { @Override public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException { if (System.currentTimeMillis() - Files.getLastModifiedTime(file).toMillis() > expiry) { - Files.delete(file); - count++; + deletedFileList.add(file); + if (deletedFileList.size() > maxPurgeSize) { + deleteFiles(); + } } return FileVisitResult.CONTINUE; } @@ -380,7 +464,7 @@ public class ThumbnailManager { if (e != null) { logger.warn("I/O exception on " + dir, e); } - if (dir.toFile().list().length == 0) { + if (dir.toFile().list().length == 0 && !dir.toFile().getName().equals(THUMBNAILS_DIR_NAME)) { Files.delete(dir); } return FileVisitResult.CONTINUE; diff --git a/src/main/resources/fess_config.properties b/src/main/resources/fess_config.properties index 74049c605..271387253 100644 --- a/src/main/resources/fess_config.properties +++ b/src/main/resources/fess_config.properties @@ -388,6 +388,7 @@ page.elevate.word.max.fetch.size=1000 page.bad.word.max.fetch.size=1000 page.dictionary.max.fetch.size=1000 page.thumbnail.queue.max.fetch.size=100 +page.thumbnail.purge.max.fetch.size=100 # search page paging.search.page.start=0