#589 replace screenshot with thumbnail

This commit is contained in:
Shinsuke Sugaya 2016-07-21 23:29:08 +09:00
parent 93228789a3
commit 91cb97ff04
15 changed files with 142 additions and 116 deletions

View file

@ -261,7 +261,7 @@ public class Constants extends CoreLibConstants {
public static final String RESULT_DOC_ID_CACHE = "resultDocIds";
public static final String SCREEN_SHOT_PATH_CACHE = "screenShotPaths";
public static final String SCREEN_SHOT_PATH_CACHE = "thumbnailPaths";
public static final String CRAWLING_INFO_SYSTEM_NAME = "system";

View file

@ -39,7 +39,7 @@ import org.codelibs.fess.helper.RoleQueryHelper;
import org.codelibs.fess.helper.SystemHelper;
import org.codelibs.fess.helper.UserInfoHelper;
import org.codelibs.fess.helper.ViewHelper;
import org.codelibs.fess.screenshot.ScreenShotManager;
import org.codelibs.fess.thumbnail.ThumbnailManager;
import org.dbflute.optional.OptionalThing;
import org.lastaflute.web.login.LoginManager;
import org.lastaflute.web.response.ActionResponse;
@ -54,7 +54,7 @@ public abstract class FessSearchAction extends FessBaseAction {
protected FessEsClient fessEsClient;
@Resource
protected ScreenShotManager screenShotManager;
protected ThumbnailManager thumbnailManager;
@Resource
protected LabelTypeHelper labelTypeHelper;

View file

@ -126,14 +126,13 @@ public class SearchAction extends FessSearchAction {
searchService.search(form, renderData, getUserBean());
return asHtml(path_SearchJsp).renderWith(data -> {
renderData.register(data);
// favorite or screenshot
// favorite or thumbnail
if (favoriteSupport || thumbnailSupport) {
final String queryId = renderData.getQueryId();
final List<Map<String, Object>> documentItems = renderData.getDocumentItems();
userInfoHelper.storeQueryId(queryId, documentItems);
if (thumbnailSupport) {
screenShotManager.storeRequest(queryId, documentItems);
RenderDataUtil.register(data, "screenShotSupport", true);
thumbnailManager.storeRequest(queryId, documentItems);
}
}
RenderDataUtil.register(data, "displayQuery", getDisplayQuery(form, labelTypeHelper.getLabelTypeItemList()));

View file

@ -13,7 +13,7 @@
* either express or implied. See the License for the specific language
* governing permissions and limitations under the License.
*/
package org.codelibs.fess.app.web.screenshot;
package org.codelibs.fess.app.web.thumbnail;
import java.io.BufferedInputStream;
import java.io.File;
@ -31,7 +31,7 @@ import org.elasticsearch.index.query.TermQueryBuilder;
import org.lastaflute.web.Execute;
import org.lastaflute.web.response.ActionResponse;
public class ScreenshotAction extends FessSearchAction {
public class ThumbnailAction extends FessSearchAction {
// ===================================================================================
// Attribute
@ -47,7 +47,7 @@ public class ScreenshotAction extends FessSearchAction {
// Search Execute
// ==============
@Execute
public ActionResponse index(final ScreenshotForm form) {
public ActionResponse index(final ThumbnailForm form) {
validate(form, messages -> {}, () -> asHtml(path_Error_ErrorJsp));
if (isLoginRequired()) {
return redirectToLogin();
@ -65,20 +65,20 @@ public class ScreenshotAction extends FessSearchAction {
final String url = DocumentUtil.getValue(doc, fessConfig.getIndexFieldUrl(), String.class);
if (StringUtil.isBlank(form.queryId) || StringUtil.isBlank(url) || !thumbnailSupport) {
// 404
throw404("Screenshot for " + form.docId + " is not found.");
throw404("Thumbnail for " + form.docId + " is not found.");
return null;
}
final File screenShotFile = screenShotManager.getScreenShotFile(form.queryId, form.docId);
if (screenShotFile == null) {
final File thumbnailFile = thumbnailManager.getThumbnailFile(form.queryId, form.docId);
if (thumbnailFile == null) {
// 404
throw404("Screenshot for " + form.docId + " is under generating.");
screenShotManager.generate(doc);
throw404("Thumbnail for " + form.docId + " is under generating.");
thumbnailManager.generate(doc);
return null;
}
return asStream(form.docId).contentType(getImageMimeType(screenShotFile)).stream(out -> {
try (BufferedInputStream in = new BufferedInputStream(new FileInputStream(screenShotFile))) {
return asStream(form.docId).contentType(getImageMimeType(thumbnailFile)).stream(out -> {
try (BufferedInputStream in = new BufferedInputStream(new FileInputStream(thumbnailFile))) {
out.write(in);
}
});

View file

@ -13,7 +13,7 @@
* either express or implied. See the License for the specific language
* governing permissions and limitations under the License.
*/
package org.codelibs.fess.app.web.screenshot;
package org.codelibs.fess.app.web.thumbnail;
import java.util.HashMap;
import java.util.Map;
@ -22,7 +22,7 @@ import javax.validation.constraints.Size;
import org.lastaflute.web.validation.Required;
public class ScreenshotForm {
public class ThumbnailForm {
@Required
@Size(max = 100)

View file

@ -13,15 +13,17 @@
* either express or implied. See the License for the specific language
* governing permissions and limitations under the License.
*/
package org.codelibs.fess.screenshot;
package org.codelibs.fess.thumbnail;
import java.io.File;
import java.util.Map;
public interface ScreenShotGenerator {
public interface ThumbnailGenerator {
void generate(String url, File outputFile);
boolean isTarget(Map<String, Object> docMap);
boolean isAvailable();
}

View file

@ -13,7 +13,7 @@
* either express or implied. See the License for the specific language
* governing permissions and limitations under the License.
*/
package org.codelibs.fess.screenshot;
package org.codelibs.fess.thumbnail;
import java.io.File;
import java.util.ArrayList;
@ -40,10 +40,10 @@ import org.lastaflute.web.util.LaRequestUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class ScreenShotManager {
private static final String DEFAULT_SCREENSHOT_DIR = "/WEB-INF/screenshots";
public class ThumbnailManager {
private static final String DEFAULT_SCREENSHOT_DIR = "/WEB-INF/thumbnails";
private static final Logger logger = LoggerFactory.getLogger(ScreenShotManager.class);
private static final Logger logger = LoggerFactory.getLogger(ThumbnailManager.class);
@Resource
protected ServletContext application;
@ -52,25 +52,25 @@ public class ScreenShotManager {
public long shutdownTimeout = 5 * 60 * 1000L; // 5min
public int screenShotPathCacheSize = 10;
public int thumbnailPathCacheSize = 10;
private final List<ScreenShotGenerator> generatorList = new ArrayList<>();
private final List<ThumbnailGenerator> generatorList = new ArrayList<>();
public String imageExtention = "png";
public int splitSize = 5;
private final BlockingQueue<ScreenShotTask> screenShotTaskQueue = new LinkedBlockingQueue<>();
private final BlockingQueue<ThumbnailTask> thumbnailTaskQueue = new LinkedBlockingQueue<>();
private boolean generating;
private Thread screenshotGeneratorThread;
private Thread thumbnailGeneratorThread;
@PostConstruct
public void init() {
final String varPath = System.getProperty("fess.var.path");
if (varPath != null) {
baseDir = new File(varPath, "screenshots");
baseDir = new File(varPath, "thumbnails");
} else {
final String path = application.getRealPath(DEFAULT_SCREENSHOT_DIR);
if (StringUtil.isNotBlank(path)) {
@ -87,38 +87,38 @@ public class ScreenShotManager {
}
if (logger.isDebugEnabled()) {
logger.debug("ScreenShot Directory: " + baseDir.getAbsolutePath());
logger.debug("Thumbnail Directory: " + baseDir.getAbsolutePath());
}
generating = true;
screenshotGeneratorThread = new Thread((Runnable) () -> {
thumbnailGeneratorThread = new Thread((Runnable) () -> {
while (generating) {
try {
screenShotTaskQueue.take().generate();
thumbnailTaskQueue.take().generate();
} catch (final InterruptedException e1) {
logger.debug("Interupted task.", e1);
} catch (final Exception e2) {
logger.warn("Failed to generage a screenshot.", e2);
logger.warn("Failed to generage a thumbnail.", e2);
}
}
}, "ScreenShotGenerator");
screenshotGeneratorThread.start();
}, "ThumbnailGenerator");
thumbnailGeneratorThread.start();
}
@PreDestroy
public void destroy() {
generating = false;
screenshotGeneratorThread.interrupt();
thumbnailGeneratorThread.interrupt();
}
public void generate(final Map<String, Object> docMap) {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
for (final ScreenShotGenerator generator : generatorList) {
for (final ThumbnailGenerator generator : generatorList) {
if (generator.isTarget(docMap)) {
final String url = DocumentUtil.getValue(docMap, fessConfig.getIndexFieldUrl(), String.class);
final String path = getImageFilename(docMap);
if (!screenShotTaskQueue.offer(new ScreenShotTask(url, new File(baseDir, path), generator))) {
logger.warn("Failed to offer a screenshot task: " + url + " -> " + path);
if (!thumbnailTaskQueue.offer(new ThumbnailTask(url, new File(baseDir, path), generator))) {
logger.warn("Failed to offer a thumbnail task: " + url + " -> " + path);
}
break;
}
@ -144,20 +144,20 @@ public class ScreenShotManager {
final Map<String, String> dataMap = new HashMap<>(documentItems.size());
for (final Map<String, Object> docMap : documentItems) {
final String docid = (String) docMap.get(fessConfig.getIndexFieldDocId());
final String screenShotPath = getImageFilename(docMap);
if (StringUtil.isNotBlank(docid) && StringUtil.isNotBlank(screenShotPath)) {
dataMap.put(docid, screenShotPath);
final String thumbnailPath = getImageFilename(docMap);
if (StringUtil.isNotBlank(docid) && StringUtil.isNotBlank(thumbnailPath)) {
dataMap.put(docid, thumbnailPath);
}
}
final Map<String, Map<String, String>> screenShotPathCache = getScreenShotPathCache(LaRequestUtil.getRequest().getSession());
screenShotPathCache.put(queryId, dataMap);
final Map<String, Map<String, String>> thumbnailPathCache = getThumbnailPathCache(LaRequestUtil.getRequest().getSession());
thumbnailPathCache.put(queryId, dataMap);
}
public File getScreenShotFile(final String queryId, final String docId) {
public File getThumbnailFile(final String queryId, final String docId) {
final HttpSession session = LaRequestUtil.getRequest().getSession(false);
if (session != null) {
final Map<String, Map<String, String>> screenShotPathCache = getScreenShotPathCache(session);
final Map<String, String> dataMap = screenShotPathCache.get(queryId);
final Map<String, Map<String, String>> thumbnailPathCache = getThumbnailPathCache(session);
final Map<String, String> dataMap = thumbnailPathCache.get(queryId);
if (dataMap != null) {
final String path = dataMap.get(docId);
final File file = new File(baseDir, path);
@ -169,29 +169,31 @@ public class ScreenShotManager {
return null;
}
private Map<String, Map<String, String>> getScreenShotPathCache(final HttpSession session) {
private Map<String, Map<String, String>> getThumbnailPathCache(final HttpSession session) {
@SuppressWarnings("unchecked")
Map<String, Map<String, String>> screenShotPathCache =
Map<String, Map<String, String>> thumbnailPathCache =
(Map<String, Map<String, String>>) session.getAttribute(Constants.SCREEN_SHOT_PATH_CACHE);
if (screenShotPathCache == null) {
screenShotPathCache = new LruHashMap<>(screenShotPathCacheSize);
session.setAttribute(Constants.SCREEN_SHOT_PATH_CACHE, screenShotPathCache);
if (thumbnailPathCache == null) {
thumbnailPathCache = new LruHashMap<>(thumbnailPathCacheSize);
session.setAttribute(Constants.SCREEN_SHOT_PATH_CACHE, thumbnailPathCache);
}
return screenShotPathCache;
return thumbnailPathCache;
}
public void add(final ScreenShotGenerator generator) {
generatorList.add(generator);
public void add(final ThumbnailGenerator generator) {
if (generator.isAvailable()) {
generatorList.add(generator);
}
}
protected static class ScreenShotTask {
protected static class ThumbnailTask {
String url;
File outputFile;
ScreenShotGenerator generator;
ThumbnailGenerator generator;
protected ScreenShotTask(final String url, final File outputFile, final ScreenShotGenerator generator) {
protected ThumbnailTask(final String url, final File outputFile, final ThumbnailGenerator generator) {
this.url = url;
this.outputFile = outputFile;
this.generator = generator;
@ -221,7 +223,7 @@ public class ScreenShotManager {
if (getClass() != obj.getClass()) {
return false;
}
final ScreenShotTask other = (ScreenShotTask) obj;
final ThumbnailTask other = (ThumbnailTask) obj;
if (outputFile == null) {
if (other.outputFile != null) {
return false;

View file

@ -13,24 +13,28 @@
* either express or implied. See the License for the specific language
* governing permissions and limitations under the License.
*/
package org.codelibs.fess.screenshot.impl;
package org.codelibs.fess.thumbnail.impl;
import java.io.File;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.annotation.Resource;
import javax.servlet.ServletContext;
import org.codelibs.fess.screenshot.ScreenShotGenerator;
import org.codelibs.fess.thumbnail.ThumbnailGenerator;
public abstract class BaseScreenShotGenerator implements ScreenShotGenerator {
public abstract class BaseThumbnailGenerator implements ThumbnailGenerator {
@Resource
protected ServletContext application;
protected final Map<String, String> conditionMap = new HashMap<>();
public int directoryNameLength = 5;
protected int directoryNameLength = 5;
public List<String> generatorList;
public void addCondition(final String key, final String regex) {
conditionMap.put(key, regex);
@ -47,4 +51,16 @@ public abstract class BaseScreenShotGenerator implements ScreenShotGenerator {
return true;
}
@Override
public boolean isAvailable() {
if (generatorList != null && !generatorList.isEmpty()) {
return generatorList.stream().allMatch(s -> new File(s).isFile());
}
return true;
}
public void setDirectoryNameLength(int directoryNameLength) {
this.directoryNameLength = directoryNameLength;
}
}

View file

@ -13,7 +13,7 @@
* either express or implied. See the License for the specific language
* governing permissions and limitations under the License.
*/
package org.codelibs.fess.screenshot.impl;
package org.codelibs.fess.thumbnail.impl;
import java.io.BufferedReader;
import java.io.File;
@ -31,7 +31,7 @@ import org.apache.commons.io.IOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class CommandGenerator extends BaseScreenShotGenerator {
public class CommandGenerator extends BaseThumbnailGenerator {
private static final Logger logger = LoggerFactory.getLogger(CommandGenerator.class);
public List<String> commandList;
@ -59,12 +59,12 @@ public class CommandGenerator extends BaseScreenShotGenerator {
@Override
public void generate(final String url, final File outputFile) {
if (logger.isDebugEnabled()) {
logger.debug("Generate ScreenShot: " + url);
logger.debug("Generate Thumbnail: " + url);
}
if (outputFile.exists()) {
if (logger.isDebugEnabled()) {
logger.debug("The screenshot file exists: " + outputFile.getAbsolutePath());
logger.debug("The thumbnail file exists: " + outputFile.getAbsolutePath());
}
return;
}
@ -88,7 +88,7 @@ public class CommandGenerator extends BaseScreenShotGenerator {
Process p = null;
if (logger.isDebugEnabled()) {
logger.debug("ScreenShot Command: " + cmdList);
logger.debug("Thumbnail Command: " + cmdList);
}
TimerTask task = null;
@ -121,7 +121,7 @@ public class CommandGenerator extends BaseScreenShotGenerator {
p.destroy();
}
} catch (final Exception e) {
logger.warn("Failed to generate a screenshot of " + url, e);
logger.warn("Failed to generate a thumbnail of " + url, e);
}
if (task != null) {
task.cancel();
@ -129,14 +129,14 @@ public class CommandGenerator extends BaseScreenShotGenerator {
}
if (outputFile.isFile() && outputFile.length() == 0) {
logger.warn("ScreenShot File is empty. URL is " + url);
logger.warn("Thumbnail File is empty. URL is " + url);
if (outputFile.delete()) {
logger.info("Deleted: " + outputFile.getAbsolutePath());
}
}
if (logger.isDebugEnabled()) {
logger.debug("ScreenShot File: " + outputPath);
logger.debug("Thumbnail File: " + outputPath);
}
}

View file

@ -13,7 +13,7 @@
* either express or implied. See the License for the specific language
* governing permissions and limitations under the License.
*/
package org.codelibs.fess.screenshot.impl;
package org.codelibs.fess.thumbnail.impl;
import java.awt.Image;
import java.awt.image.BufferedImage;
@ -25,6 +25,7 @@ import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import javax.imageio.ImageIO;
import org.openqa.selenium.Capabilities;
import org.openqa.selenium.Dimension;
import org.openqa.selenium.OutputType;
import org.openqa.selenium.TakesScreenshot;
@ -33,42 +34,54 @@ import org.openqa.selenium.phantomjs.PhantomJSDriver;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class WebDriverGenerator extends BaseScreenShotGenerator {
public class WebDriverGenerator extends BaseThumbnailGenerator {
private static final Logger logger = LoggerFactory.getLogger(WebDriverGenerator.class);
public WebDriver webDriver;
public Capabilities webDriverCapabilities;
public int windowWidth = 1200;
public int windowHeight = 800;
public int screenShotWidth = 400;
public int thumbnailWidth = 400;
public String imageFormatName = "png";
@PostConstruct
public void init() {
if (webDriver == null) {
webDriver = new PhantomJSDriver();
try {
if (webDriver == null) {
webDriver = webDriverCapabilities == null ? new PhantomJSDriver() : new PhantomJSDriver(webDriverCapabilities);
}
webDriver.manage().window().setSize(new Dimension(windowWidth, windowHeight));
} catch (final Exception e) {
if (logger.isDebugEnabled()) {
logger.debug("WebDriver is not available for generating thumbnails.", e);
} else {
logger.info("WebDriver is not available for generating thumbnails.");
}
}
webDriver.manage().window().setSize(new Dimension(windowWidth, windowHeight));
}
@PreDestroy
public void destroy() {
webDriver.close();
if (webDriver != null) {
webDriver.quit();
}
}
@Override
public void generate(final String url, final File outputFile) {
if (logger.isDebugEnabled()) {
logger.debug("Generate ScreenShot: " + url);
logger.debug("Generate Thumbnail: " + url);
}
if (outputFile.exists()) {
if (logger.isDebugEnabled()) {
logger.debug("The screenshot file exists: " + outputFile.getAbsolutePath());
logger.debug("The thumbnail file exists: " + outputFile.getAbsolutePath());
}
return;
}
@ -84,22 +97,30 @@ public class WebDriverGenerator extends BaseScreenShotGenerator {
if (webDriver instanceof TakesScreenshot) {
webDriver.get(url);
final File screenshot = ((TakesScreenshot) webDriver).getScreenshotAs(OutputType.FILE);
convert(screenshot, outputFile);
final File thumbnail = ((TakesScreenshot) webDriver).getScreenshotAs(OutputType.FILE);
convert(thumbnail, outputFile);
} else {
logger.warn("WebDriver is not instance of TakesScreenshot: " + webDriver);
}
}
@Override
public boolean isAvailable() {
if (webDriver == null) {
return false;
}
return super.isAvailable();
}
protected void convert(final File inputFile, final File outputFile) {
try {
final BufferedImage image = loadImage(inputFile);
final int screenShotHeight = screenShotWidth * image.getHeight() / windowWidth;
final BufferedImage screenShotImage = new BufferedImage(screenShotWidth, screenShotHeight, image.getType());
screenShotImage.getGraphics().drawImage(image.getScaledInstance(screenShotWidth, screenShotHeight, Image.SCALE_AREA_AVERAGING),
0, 0, screenShotWidth, screenShotHeight, null);
final int thumbnailHeight = thumbnailWidth * image.getHeight() / windowWidth;
final BufferedImage thumbnailImage = new BufferedImage(thumbnailWidth, thumbnailHeight, image.getType());
thumbnailImage.getGraphics().drawImage(image.getScaledInstance(thumbnailWidth, thumbnailHeight, Image.SCALE_AREA_AVERAGING), 0,
0, thumbnailWidth, thumbnailHeight, null);
ImageIO.write(screenShotImage, imageFormatName, outputFile);
ImageIO.write(thumbnailImage, imageFormatName, outputFile);
} catch (final Exception e) {
logger.warn("Failed to convert " + inputFile.getAbsolutePath(), e);
inputFile.renameTo(outputFile);

View file

@ -11,7 +11,7 @@
<include path="fess_api.xml"/>
<include path="fess_dict.xml"/>
<include path="fess_job.xml"/>
<include path="fess_screenshot.xml"/>
<include path="fess_thumbnail.xml"/>
<include path="fess_sso.xml"/>
<include path="crawler/client.xml" />

View file

@ -2,40 +2,36 @@
<!DOCTYPE components PUBLIC "-//DBFLUTE//DTD LastaDi 1.0//EN"
"http://dbflute.org/meta/lastadi10.dtd">
<components>
<component name="screenShotManager" class="org.codelibs.fess.screenshot.ScreenShotManager">
<!--
<component name="thumbnailManager" class="org.codelibs.fess.thumbnail.ThumbnailManager">
<postConstruct name="add">
<arg>htmlScreenShotGenerator</arg>
<arg>htmlThumbnailGenerator</arg>
</postConstruct>
-->
</component>
<!--
<component name="webDriver" class="org.openqa.selenium.phantomjs.PhantomJSDriver">
<arg>
<component name="htmlThumbnailGenerator" class="org.codelibs.fess.thumbnail.impl.WebDriverGenerator">
<property name="webDriverCapabilities">
<component class="org.openqa.selenium.remote.DesiredCapabilities">
<postConstruct name="setCapability">
<arg>"phantomjs.binary.path"</arg>
<arg>"/usr/bin/phantomjs"</arg>
</postConstruct>
</component>
</arg>
<preDestroy name="quit"></preDestroy>
</component>
<component name="htmlScreenShotGenerator" class="org.codelibs.fess.screenshot.impl.WebDriverGenerator">
<property name="webDriver">webDriver</property>
</property>
<postConstruct name="addCondition">
<arg>"mimetype"</arg>
<arg>"text/html"</arg>
</postConstruct>
</component>
-->
<!--
<component name="htmlScreenShotGenerator" class="org.codelibs.fess.screenshot.impl.CommandGenerator">
<component name="htmlThumbnailGenerator" class="org.codelibs.fess.thumbnail.impl.CommandGenerator">
<property name="commandList">
{"bash",
"/opt/fess/bin/html-screenshot.sh",
["bash",
"/usr/share/fess/bin/generate-thumbnail",
"html",
"${url}",
"${outputFile}"}
"${outputFile}"]
</property>
<property name="generatorList">
["/usr/share/fess/bin/generate-thumbnail"]
</property>
<postConstruct name="addCondition">
<arg>"mimetype"</arg>

View file

@ -104,9 +104,6 @@
</div>
<aside class="col-md-4 hidden-sm-down">
<%-- Side Content --%>
<c:if test="${screenShotSupport}">
<div id="screenshot"></div>
</c:if>
<c:if test="${facetResponse != null}">
<c:forEach var="fieldData" items="${facetResponse.fieldList}">
<c:if

View file

@ -104,9 +104,6 @@
</div>
<aside class="col-md-4 hidden-sm-down">
<%-- Side Content --%>
<c:if test="${screenShotSupport}">
<div id="screenshot"></div>
</c:if>
<c:if test="${facetResponse != null}">
<c:forEach var="fieldData" items="${facetResponse.fieldList}">
<c:if

View file

@ -129,10 +129,6 @@ h1.mainLogo {
display: none;
}
#screenshot {
position: fixed;
}
.searchFormBox {
text-align: center;
margin-top: 160px;
@ -372,4 +368,4 @@ body.search #searchOptions.active, body.help #searchOptions.active, body.error
input#query.form-control {
padding-bottom: 0.75rem;
}
}
}