0ct0pu5/fess
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix #773 generate thumbnailes on crawler

Browse source
This commit is contained in:
Shinsuke Sugaya 2016-11-05 23:57:41 +09:00
parent f96312da20
commit bbc7e0f488
11 changed files with 162 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
src/main/java/org/codelibs/fess/helper/IndexingHelper.java
View file

@ -21,6 +21,7 @@ import java.util.Map;
import org.codelibs.fess.es.client.FessEsClient;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.thumbnail.ThumbnailManager;
import org.codelibs.fess.util.ComponentUtil;
import org.codelibs.fess.util.DocList;
import org.codelibs.fess.util.MemoryUtil;
@ -43,6 +44,7 @@ public class IndexingHelper {
if (docList.isEmpty()) {
return;
}
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final long execTime = System.currentTimeMillis();
if (logger.isDebugEnabled()) {
logger.debug("Sending " + docList.size() + " documents to a server.");
@ -50,9 +52,12 @@ public class IndexingHelper {
try {
synchronized (fessEsClient) {
deleteOldDocuments(fessEsClient, docList);
final FessConfig fessConfig = ComponentUtil.getFessConfig();
fessEsClient.addAll(fessConfig.getIndexDocumentUpdateIndex(), fessConfig.getIndexDocumentType(), docList);
}
if (fessConfig.isThumbnailCrawlerEnabled()) {
final ThumbnailManager thumbnailManager = ComponentUtil.getThumbnailManager();
docList.stream().forEach(doc -> thumbnailManager.offer(doc));
}
if (logger.isInfoEnabled()) {
if (docList.getContentSize() > 0) {
logger.info("Sent " + docList.size() + " docs (Doc:{process " + docList.getProcessingTime() + "ms, send "

2
src/main/java/org/codelibs/fess/job/CrawlJob.java
View file

@ -331,6 +331,8 @@ public class CrawlJob {
}
}
cmdList.add(ComponentUtil.getThumbnailManager().getThumbnailPathOption());
if (StringUtil.isNotBlank(jvmOptions)) {
split(jvmOptions, " ").of(stream -> stream.filter(StringUtil::isNotBlank).forEach(s -> cmdList.add(s)));
}

25
src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java
View file

@ -658,6 +658,9 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
/** The key of the configuration. e.g. all */
String THUMBNAIL_GENERATOR_TARGETS = "thumbnail.generator.targets";
/** The key of the configuration. e.g. false */
String THUMBNAIL_CRAWLER_ENABLED = "thumbnail.crawler.enabled";
/** The key of the configuration. e.g. Administrator */
String MAIL_FROM_NAME = "mail.from.name";
@ -3190,6 +3193,20 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
*/
String getThumbnailGeneratorTargets();
/**
* Get the value for the key 'thumbnail.crawler.enabled'. <br>
* The value is, e.g. false <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getThumbnailCrawlerEnabled();
/**
* Is the property for the key 'thumbnail.crawler.enabled' true? <br>
* The value is, e.g. false <br>
* @return The determination, true or false. (if not found, exception but basically no way)
*/
boolean isThumbnailCrawlerEnabled();
/**
* Get the value for the key 'mail.from.name'. <br>
* The value is, e.g. Administrator <br>
@ -5437,6 +5454,14 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
return get(FessConfig.THUMBNAIL_GENERATOR_TARGETS);
}
public String getThumbnailCrawlerEnabled() {
return get(FessConfig.THUMBNAIL_CRAWLER_ENABLED);
}
public boolean isThumbnailCrawlerEnabled() {
return is(FessConfig.THUMBNAIL_CRAWLER_ENABLED);
}
public String getMailFromName() {
return get(FessConfig.MAIL_FROM_NAME);
}

38
src/main/java/org/codelibs/fess/thumbnail/ThumbnailManager.java
View file

@ -32,8 +32,6 @@ import java.util.concurrent.TimeUnit;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import javax.annotation.Resource;
import javax.servlet.ServletContext;
import javax.servlet.http.HttpSession;
import org.codelibs.core.collection.LruHashMap;
@ -48,6 +46,7 @@ import org.codelibs.fess.helper.SystemHelper;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.util.ComponentUtil;
import org.codelibs.fess.util.DocumentUtil;
import org.codelibs.fess.util.ResourceUtil;
import org.lastaflute.web.util.LaRequestUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -55,15 +54,14 @@ import org.slf4j.LoggerFactory;
import com.google.common.collect.Lists;
public class ThumbnailManager {
private static final String DEFAULT_SCREENSHOT_DIR = "/WEB-INF/thumbnails";
private static final String FESS_THUMBNAIL_PATH = "fess.thumbnail.path";
private static final String FESS_VAR_PATH = "fess.var.path";
private static final String NOIMAGE_FILE_SUFFIX = ".txt";
private static final Logger logger = LoggerFactory.getLogger(ThumbnailManager.class);
@Resource
protected ServletContext application;
protected File baseDir;
private final List<ThumbnailGenerator> generatorList = new ArrayList<>();
@ -72,7 +70,7 @@ public class ThumbnailManager {
private volatile boolean generating;
private Thread thumbnailGeneratorThread;
private Thread thumbnailQueueThread;
protected int thumbnailPathCacheSize = 10;
@ -90,15 +88,15 @@ public class ThumbnailManager {
@PostConstruct
public void init() {
final String varPath = System.getProperty("fess.var.path");
if (varPath != null) {
baseDir = new File(varPath, "thumbnails");
final String thumbnailPath = System.getProperty(FESS_THUMBNAIL_PATH);
if (thumbnailPath != null) {
baseDir = new File(thumbnailPath);
} else {
final String path = application.getRealPath(DEFAULT_SCREENSHOT_DIR);
if (StringUtil.isNotBlank(path)) {
baseDir = new File(path);
final String varPath = System.getProperty(FESS_VAR_PATH);
if (varPath != null) {
baseDir = new File(varPath, "thumbnails");
} else {
baseDir = new File("." + DEFAULT_SCREENSHOT_DIR);
baseDir = ResourceUtil.getThumbnailPath().toFile();
}
}
if (baseDir.mkdirs()) {
@ -114,7 +112,7 @@ public class ThumbnailManager {
thumbnailTaskQueue = new LinkedBlockingQueue<>(thumbnailTaskQueueSize);
generating = true;
thumbnailGeneratorThread = new Thread((Runnable) () -> {
thumbnailQueueThread = new Thread((Runnable) () -> {
final List<Tuple3<String, String, String>> taskList = new ArrayList<>();
while (generating) {
try {
@ -138,15 +136,15 @@ public class ThumbnailManager {
}
}
}, "ThumbnailGenerator");
thumbnailGeneratorThread.start();
thumbnailQueueThread.start();
}
@PreDestroy
public void destroy() {
generating = false;
thumbnailGeneratorThread.interrupt();
thumbnailQueueThread.interrupt();
try {
thumbnailGeneratorThread.join(10000);
thumbnailQueueThread.join(10000);
} catch (final InterruptedException e) {
logger.warn("Thumbnail thread is timeouted.", e);
}
@ -159,6 +157,10 @@ public class ThumbnailManager {
});
}
public String getThumbnailPathOption() {
return "-D" + FESS_THUMBNAIL_PATH + "=" + baseDir.getAbsolutePath();
}
protected void storeQueue(final List<Tuple3<String, String, String>> taskList) {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final SystemHelper systemHelper = ComponentUtil.getSystemHelper();

6
src/main/java/org/codelibs/fess/thumbnail/impl/BaseThumbnailGenerator.java
View file

@ -23,16 +23,10 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.annotation.Resource;
import javax.servlet.ServletContext;
import org.codelibs.fess.thumbnail.ThumbnailGenerator;
public abstract class BaseThumbnailGenerator implements ThumbnailGenerator {
@Resource
protected ServletContext application;
protected final Map<String, String> conditionMap = new HashMap<>();
protected int directoryNameLength = 5;

5
src/main/java/org/codelibs/fess/thumbnail/impl/CommandGenerator.java
View file

@ -25,6 +25,8 @@ import java.util.Timer;
import java.util.TimerTask;
import javax.annotation.PostConstruct;
import javax.annotation.Resource;
import javax.servlet.ServletContext;
import org.apache.commons.io.IOUtils;
import org.slf4j.Logger;
@ -33,6 +35,9 @@ import org.slf4j.LoggerFactory;
public class CommandGenerator extends BaseThumbnailGenerator {
private static final Logger logger = LoggerFactory.getLogger(CommandGenerator.class);
@Resource
protected ServletContext application;
public List<String> commandList;
public long commandTimeout = 10 * 1000L;// 10sec

36
src/main/java/org/codelibs/fess/thumbnail/impl/EmptyGenerator.java Normal file
View file

@ -0,0 +1,36 @@
/*
* Copyright 2012-2016 CodeLibs Project and the Others.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
* either express or implied. See the License for the specific language
* governing permissions and limitations under the License.
*/
package org.codelibs.fess.thumbnail.impl;
import java.io.File;
public class EmptyGenerator extends BaseThumbnailGenerator {
@Override
public boolean generate(String url, File outputFile) {
return false;
}
@Override
public void destroy() {
// nothing
}
@Override
public boolean isAvailable() {
return true;
}
}

4
src/main/java/org/codelibs/fess/util/ResourceUtil.java
View file

@ -72,6 +72,10 @@ public class ResourceUtil {
return getPath("dict", names);
}
public static Path getThumbnailPath(final String... names) {
return getPath("thumbnails", names);
}
protected static Path getPath(final String base, final String... names) {
try {

1
src/main/resources/fess_config.properties
View file

@ -351,6 +351,7 @@ paging.search.page.max.size=100
thumbnail.html.phantomjs.enabled=true
thumbnail.generator.targets=all
thumbnail.crawler.enabled=false
# ----------------------------------------------------------
# Mail

1
src/main/webapp/WEB-INF/crawler/resources/app.xml
View file

@ -7,6 +7,7 @@
<include path="fess.xml" />
<include path="crawler_es.xml" />
<include path="crawler_thumbnail.xml" />
<component name="indexingHelper" class="org.codelibs.fess.helper.IndexingHelper">
</component>

62
src/main/webapp/WEB-INF/crawler/resources/crawler_thumbnail.xml Normal file
View file

@ -0,0 +1,62 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE components PUBLIC "-//DBFLUTE//DTD LastaDi 1.0//EN"
"http://dbflute.org/meta/lastadi10.dtd">
<components>
<component name="thumbnailManager" class="org.codelibs.fess.thumbnail.ThumbnailManager">
<postConstruct name="add">
<arg>htmlThumbnailGenerator</arg>
</postConstruct>
<postConstruct name="add">
<arg>msofficeThumbnailGenerator</arg>
</postConstruct>
</component>
<component name="htmlThumbnailGenerator" class="org.codelibs.fess.thumbnail.impl.EmptyGenerator">
<property name="name">"htmlThumbnailGenerator"</property>
<postConstruct name="addCondition">
<arg>"mimetype"</arg>
<arg>"text/html"</arg>
</postConstruct>
</component>
<component name="msofficeThumbnailGenerator" class="org.codelibs.fess.thumbnail.impl.EmptyGenerator">
<property name="name">"msofficeThumbnailGenerator"</property>
<postConstruct name="addCondition">
<arg>"mimetype"</arg>
<arg>"text/html"</arg>
</postConstruct>
<postConstruct name="addCondition">
<arg>"mimetype"</arg>
<arg>"application/vnd.openxmlformats-officedocument.wordprocessingml.document"
</arg>
</postConstruct>
<postConstruct name="addCondition">
<arg>"mimetype"</arg>
<arg>"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
</arg>
</postConstruct>
<postConstruct name="addCondition">
<arg>"mimetype"</arg>
<arg>"application/vnd.openxmlformats-officedocument.presentationml.presentation"
</arg>
</postConstruct>
<postConstruct name="addCondition">
<arg>"mimetype"</arg>
<arg>"application/msword"
</arg>
</postConstruct>
<postConstruct name="addCondition">
<arg>"mimetype"</arg>
<arg>"application/vnd.ms-excel"
</arg>
</postConstruct>
<postConstruct name="addCondition">
<arg>"mimetype"</arg>
<arg>"application/vnd.ms-powerpoint"
</arg>
</postConstruct>
<postConstruct name="addCondition">
<arg>"mimetype"</arg>
<arg>"application/rtf"
</arg>
</postConstruct>
</component>
</components>