diff --git a/src/main/java/org/codelibs/fess/helper/IndexingHelper.java b/src/main/java/org/codelibs/fess/helper/IndexingHelper.java index 422ae4c8f..21f9481a8 100644 --- a/src/main/java/org/codelibs/fess/helper/IndexingHelper.java +++ b/src/main/java/org/codelibs/fess/helper/IndexingHelper.java @@ -21,6 +21,7 @@ import java.util.Map; import org.codelibs.fess.es.client.FessEsClient; import org.codelibs.fess.mylasta.direction.FessConfig; +import org.codelibs.fess.thumbnail.ThumbnailManager; import org.codelibs.fess.util.ComponentUtil; import org.codelibs.fess.util.DocList; import org.codelibs.fess.util.MemoryUtil; @@ -43,6 +44,7 @@ public class IndexingHelper { if (docList.isEmpty()) { return; } + final FessConfig fessConfig = ComponentUtil.getFessConfig(); final long execTime = System.currentTimeMillis(); if (logger.isDebugEnabled()) { logger.debug("Sending " + docList.size() + " documents to a server."); @@ -50,9 +52,12 @@ public class IndexingHelper { try { synchronized (fessEsClient) { deleteOldDocuments(fessEsClient, docList); - final FessConfig fessConfig = ComponentUtil.getFessConfig(); fessEsClient.addAll(fessConfig.getIndexDocumentUpdateIndex(), fessConfig.getIndexDocumentType(), docList); } + if (fessConfig.isThumbnailCrawlerEnabled()) { + final ThumbnailManager thumbnailManager = ComponentUtil.getThumbnailManager(); + docList.stream().forEach(doc -> thumbnailManager.offer(doc)); + } if (logger.isInfoEnabled()) { if (docList.getContentSize() > 0) { logger.info("Sent " + docList.size() + " docs (Doc:{process " + docList.getProcessingTime() + "ms, send " diff --git a/src/main/java/org/codelibs/fess/job/CrawlJob.java b/src/main/java/org/codelibs/fess/job/CrawlJob.java index 5818e98f9..43856b432 100644 --- a/src/main/java/org/codelibs/fess/job/CrawlJob.java +++ b/src/main/java/org/codelibs/fess/job/CrawlJob.java @@ -331,6 +331,8 @@ public class CrawlJob { } } + cmdList.add(ComponentUtil.getThumbnailManager().getThumbnailPathOption()); + if (StringUtil.isNotBlank(jvmOptions)) { split(jvmOptions, " ").of(stream -> stream.filter(StringUtil::isNotBlank).forEach(s -> cmdList.add(s))); } diff --git a/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java b/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java index 9430698c4..df92d113d 100644 --- a/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java +++ b/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java @@ -658,6 +658,9 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction /** The key of the configuration. e.g. all */ String THUMBNAIL_GENERATOR_TARGETS = "thumbnail.generator.targets"; + /** The key of the configuration. e.g. false */ + String THUMBNAIL_CRAWLER_ENABLED = "thumbnail.crawler.enabled"; + /** The key of the configuration. e.g. Administrator */ String MAIL_FROM_NAME = "mail.from.name"; @@ -3190,6 +3193,20 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction */ String getThumbnailGeneratorTargets(); + /** + * Get the value for the key 'thumbnail.crawler.enabled'.
+ * The value is, e.g. false
+ * @return The value of found property. (NotNull: if not found, exception but basically no way) + */ + String getThumbnailCrawlerEnabled(); + + /** + * Is the property for the key 'thumbnail.crawler.enabled' true?
+ * The value is, e.g. false
+ * @return The determination, true or false. (if not found, exception but basically no way) + */ + boolean isThumbnailCrawlerEnabled(); + /** * Get the value for the key 'mail.from.name'.
* The value is, e.g. Administrator
@@ -5437,6 +5454,14 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction return get(FessConfig.THUMBNAIL_GENERATOR_TARGETS); } + public String getThumbnailCrawlerEnabled() { + return get(FessConfig.THUMBNAIL_CRAWLER_ENABLED); + } + + public boolean isThumbnailCrawlerEnabled() { + return is(FessConfig.THUMBNAIL_CRAWLER_ENABLED); + } + public String getMailFromName() { return get(FessConfig.MAIL_FROM_NAME); } diff --git a/src/main/java/org/codelibs/fess/thumbnail/ThumbnailManager.java b/src/main/java/org/codelibs/fess/thumbnail/ThumbnailManager.java index 4f8ebfd3d..f0f5d10a1 100644 --- a/src/main/java/org/codelibs/fess/thumbnail/ThumbnailManager.java +++ b/src/main/java/org/codelibs/fess/thumbnail/ThumbnailManager.java @@ -32,8 +32,6 @@ import java.util.concurrent.TimeUnit; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; -import javax.annotation.Resource; -import javax.servlet.ServletContext; import javax.servlet.http.HttpSession; import org.codelibs.core.collection.LruHashMap; @@ -48,6 +46,7 @@ import org.codelibs.fess.helper.SystemHelper; import org.codelibs.fess.mylasta.direction.FessConfig; import org.codelibs.fess.util.ComponentUtil; import org.codelibs.fess.util.DocumentUtil; +import org.codelibs.fess.util.ResourceUtil; import org.lastaflute.web.util.LaRequestUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -55,15 +54,14 @@ import org.slf4j.LoggerFactory; import com.google.common.collect.Lists; public class ThumbnailManager { - private static final String DEFAULT_SCREENSHOT_DIR = "/WEB-INF/thumbnails"; + private static final String FESS_THUMBNAIL_PATH = "fess.thumbnail.path"; + + private static final String FESS_VAR_PATH = "fess.var.path"; private static final String NOIMAGE_FILE_SUFFIX = ".txt"; private static final Logger logger = LoggerFactory.getLogger(ThumbnailManager.class); - @Resource - protected ServletContext application; - protected File baseDir; private final List generatorList = new ArrayList<>(); @@ -72,7 +70,7 @@ public class ThumbnailManager { private volatile boolean generating; - private Thread thumbnailGeneratorThread; + private Thread thumbnailQueueThread; protected int thumbnailPathCacheSize = 10; @@ -90,15 +88,15 @@ public class ThumbnailManager { @PostConstruct public void init() { - final String varPath = System.getProperty("fess.var.path"); - if (varPath != null) { - baseDir = new File(varPath, "thumbnails"); + final String thumbnailPath = System.getProperty(FESS_THUMBNAIL_PATH); + if (thumbnailPath != null) { + baseDir = new File(thumbnailPath); } else { - final String path = application.getRealPath(DEFAULT_SCREENSHOT_DIR); - if (StringUtil.isNotBlank(path)) { - baseDir = new File(path); + final String varPath = System.getProperty(FESS_VAR_PATH); + if (varPath != null) { + baseDir = new File(varPath, "thumbnails"); } else { - baseDir = new File("." + DEFAULT_SCREENSHOT_DIR); + baseDir = ResourceUtil.getThumbnailPath().toFile(); } } if (baseDir.mkdirs()) { @@ -114,7 +112,7 @@ public class ThumbnailManager { thumbnailTaskQueue = new LinkedBlockingQueue<>(thumbnailTaskQueueSize); generating = true; - thumbnailGeneratorThread = new Thread((Runnable) () -> { + thumbnailQueueThread = new Thread((Runnable) () -> { final List> taskList = new ArrayList<>(); while (generating) { try { @@ -138,15 +136,15 @@ public class ThumbnailManager { } } }, "ThumbnailGenerator"); - thumbnailGeneratorThread.start(); + thumbnailQueueThread.start(); } @PreDestroy public void destroy() { generating = false; - thumbnailGeneratorThread.interrupt(); + thumbnailQueueThread.interrupt(); try { - thumbnailGeneratorThread.join(10000); + thumbnailQueueThread.join(10000); } catch (final InterruptedException e) { logger.warn("Thumbnail thread is timeouted.", e); } @@ -159,6 +157,10 @@ public class ThumbnailManager { }); } + public String getThumbnailPathOption() { + return "-D" + FESS_THUMBNAIL_PATH + "=" + baseDir.getAbsolutePath(); + } + protected void storeQueue(final List> taskList) { final FessConfig fessConfig = ComponentUtil.getFessConfig(); final SystemHelper systemHelper = ComponentUtil.getSystemHelper(); diff --git a/src/main/java/org/codelibs/fess/thumbnail/impl/BaseThumbnailGenerator.java b/src/main/java/org/codelibs/fess/thumbnail/impl/BaseThumbnailGenerator.java index 2f05e8b51..d56634e3f 100644 --- a/src/main/java/org/codelibs/fess/thumbnail/impl/BaseThumbnailGenerator.java +++ b/src/main/java/org/codelibs/fess/thumbnail/impl/BaseThumbnailGenerator.java @@ -23,16 +23,10 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import javax.annotation.Resource; -import javax.servlet.ServletContext; - import org.codelibs.fess.thumbnail.ThumbnailGenerator; public abstract class BaseThumbnailGenerator implements ThumbnailGenerator { - @Resource - protected ServletContext application; - protected final Map conditionMap = new HashMap<>(); protected int directoryNameLength = 5; diff --git a/src/main/java/org/codelibs/fess/thumbnail/impl/CommandGenerator.java b/src/main/java/org/codelibs/fess/thumbnail/impl/CommandGenerator.java index 17c8cd35d..7228bfd0c 100644 --- a/src/main/java/org/codelibs/fess/thumbnail/impl/CommandGenerator.java +++ b/src/main/java/org/codelibs/fess/thumbnail/impl/CommandGenerator.java @@ -25,6 +25,8 @@ import java.util.Timer; import java.util.TimerTask; import javax.annotation.PostConstruct; +import javax.annotation.Resource; +import javax.servlet.ServletContext; import org.apache.commons.io.IOUtils; import org.slf4j.Logger; @@ -33,6 +35,9 @@ import org.slf4j.LoggerFactory; public class CommandGenerator extends BaseThumbnailGenerator { private static final Logger logger = LoggerFactory.getLogger(CommandGenerator.class); + @Resource + protected ServletContext application; + public List commandList; public long commandTimeout = 10 * 1000L;// 10sec diff --git a/src/main/java/org/codelibs/fess/thumbnail/impl/EmptyGenerator.java b/src/main/java/org/codelibs/fess/thumbnail/impl/EmptyGenerator.java new file mode 100644 index 000000000..1ae680dee --- /dev/null +++ b/src/main/java/org/codelibs/fess/thumbnail/impl/EmptyGenerator.java @@ -0,0 +1,36 @@ +/* + * Copyright 2012-2016 CodeLibs Project and the Others. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ +package org.codelibs.fess.thumbnail.impl; + +import java.io.File; + +public class EmptyGenerator extends BaseThumbnailGenerator { + + @Override + public boolean generate(String url, File outputFile) { + return false; + } + + @Override + public void destroy() { + // nothing + } + + @Override + public boolean isAvailable() { + return true; + } +} diff --git a/src/main/java/org/codelibs/fess/util/ResourceUtil.java b/src/main/java/org/codelibs/fess/util/ResourceUtil.java index cc54adeb9..78c2f8bd1 100644 --- a/src/main/java/org/codelibs/fess/util/ResourceUtil.java +++ b/src/main/java/org/codelibs/fess/util/ResourceUtil.java @@ -72,6 +72,10 @@ public class ResourceUtil { return getPath("dict", names); } + public static Path getThumbnailPath(final String... names) { + return getPath("thumbnails", names); + } + protected static Path getPath(final String base, final String... names) { try { diff --git a/src/main/resources/fess_config.properties b/src/main/resources/fess_config.properties index 2f424c377..bb43d8725 100644 --- a/src/main/resources/fess_config.properties +++ b/src/main/resources/fess_config.properties @@ -351,6 +351,7 @@ paging.search.page.max.size=100 thumbnail.html.phantomjs.enabled=true thumbnail.generator.targets=all +thumbnail.crawler.enabled=false # ---------------------------------------------------------- # Mail diff --git a/src/main/webapp/WEB-INF/crawler/resources/app.xml b/src/main/webapp/WEB-INF/crawler/resources/app.xml index 7c5fe8693..cbb06da17 100644 --- a/src/main/webapp/WEB-INF/crawler/resources/app.xml +++ b/src/main/webapp/WEB-INF/crawler/resources/app.xml @@ -7,6 +7,7 @@ + diff --git a/src/main/webapp/WEB-INF/crawler/resources/crawler_thumbnail.xml b/src/main/webapp/WEB-INF/crawler/resources/crawler_thumbnail.xml new file mode 100644 index 000000000..e7dbbe2a6 --- /dev/null +++ b/src/main/webapp/WEB-INF/crawler/resources/crawler_thumbnail.xml @@ -0,0 +1,62 @@ + + + + + + htmlThumbnailGenerator + + + msofficeThumbnailGenerator + + + + "htmlThumbnailGenerator" + + "mimetype" + "text/html" + + + + "msofficeThumbnailGenerator" + + "mimetype" + "text/html" + + + "mimetype" + "application/vnd.openxmlformats-officedocument.wordprocessingml.document" + + + + "mimetype" + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + + + + "mimetype" + "application/vnd.openxmlformats-officedocument.presentationml.presentation" + + + + "mimetype" + "application/msword" + + + + "mimetype" + "application/vnd.ms-excel" + + + + "mimetype" + "application/vnd.ms-powerpoint" + + + + "mimetype" + "application/rtf" + + + +