diff --git a/pom.xml b/pom.xml index 09ae23d0a..4c701e8b4 100644 --- a/pom.xml +++ b/pom.xml @@ -796,6 +796,11 @@ groovy-all 2.2.2 + + com.github.detro + phantomjsdriver + 1.2.0 + junit diff --git a/src/main/java/jp/sf/fess/action/IndexAction.java b/src/main/java/jp/sf/fess/action/IndexAction.java index e9d5fa915..6aa6f76ac 100644 --- a/src/main/java/jp/sf/fess/action/IndexAction.java +++ b/src/main/java/jp/sf/fess/action/IndexAction.java @@ -531,10 +531,11 @@ public class IndexAction { } final File screenShotFile = screenShotManager.getScreenShotFile( - indexForm.queryId, url); + indexForm.queryId, indexForm.docId); if (screenShotFile == null) { // 404 response.sendError(HttpServletResponse.SC_NOT_FOUND); + screenShotManager.generate(doc); return null; } diff --git a/src/main/java/jp/sf/fess/helper/QueryHelper.java b/src/main/java/jp/sf/fess/helper/QueryHelper.java index 0c8e01d4a..3d02338be 100644 --- a/src/main/java/jp/sf/fess/helper/QueryHelper.java +++ b/src/main/java/jp/sf/fess/helper/QueryHelper.java @@ -103,14 +103,14 @@ public class QueryHelper implements Serializable { protected String[] responseFields = new String[] { "id", "docId", "score", "boost", "contentLength", "host", "site", "lastModified", "mimetype", "filetype_s", "created", TITLE_FIELD, "digest", "url", - "clickCount_l_x_dv", "favoriteCount_l_x_dv", "screenshot_s_s", - "cid_s_s", "lang_s", "hasCache_s_s" }; + "clickCount_l_x_dv", "favoriteCount_l_x_dv", "cid_s_s", "lang_s", + "hasCache_s_s" }; protected String[] cacheResponseFields = new String[] { "id", "docId", "score", "boost", "contentLength", "host", "site", "lastModified", "mimetype", "filetype_s", "created", TITLE_FIELD, "digest", "url", - "clickCount_l_x_dv", "favoriteCount_l_x_dv", "screenshot_s_s", - "cid_s_s", "lang_s", "cache" }; + "clickCount_l_x_dv", "favoriteCount_l_x_dv", "cid_s_s", "lang_s", + "cache" }; protected String[] responseDocValuesFields = new String[] { "clickCount_l_x_dv", "favoriteCount_l_x_dv" }; diff --git a/src/main/java/jp/sf/fess/helper/SystemHelper.java b/src/main/java/jp/sf/fess/helper/SystemHelper.java index 10d82b857..5a51c33b2 100644 --- a/src/main/java/jp/sf/fess/helper/SystemHelper.java +++ b/src/main/java/jp/sf/fess/helper/SystemHelper.java @@ -121,8 +121,6 @@ public class SystemHelper implements Serializable { public String clickCountField = "clickCount_l_x_dv"; - public String screenshotField = "screenshot_s_s"; - public String configIdField = "cid_s_s"; public String expiresField = "expires_dt"; diff --git a/src/main/java/jp/sf/fess/screenshot/ScreenShotGenerator.java b/src/main/java/jp/sf/fess/screenshot/ScreenShotGenerator.java index 8f12803f3..a1f482cbb 100644 --- a/src/main/java/jp/sf/fess/screenshot/ScreenShotGenerator.java +++ b/src/main/java/jp/sf/fess/screenshot/ScreenShotGenerator.java @@ -21,8 +21,6 @@ import java.util.Map; public interface ScreenShotGenerator { - String getPath(Map docMap); - void generate(String url, File outputFile); boolean isTarget(Map docMap); diff --git a/src/main/java/jp/sf/fess/screenshot/ScreenShotManager.java b/src/main/java/jp/sf/fess/screenshot/ScreenShotManager.java index cb45f42f8..79ef38133 100644 --- a/src/main/java/jp/sf/fess/screenshot/ScreenShotManager.java +++ b/src/main/java/jp/sf/fess/screenshot/ScreenShotManager.java @@ -17,14 +17,12 @@ package jp.sf.fess.screenshot; import java.io.File; -import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; import javax.annotation.Resource; import javax.servlet.ServletContext; @@ -35,8 +33,8 @@ import jp.sf.fess.FessSystemException; import jp.sf.fess.helper.SystemHelper; import jp.sf.fess.util.ComponentUtil; -import org.apache.commons.io.FileUtils; import org.codelibs.core.util.StringUtil; +import org.seasar.framework.container.annotation.tiger.DestroyMethod; import org.seasar.framework.container.annotation.tiger.InitMethod; import org.seasar.robot.util.LruHashMap; import org.seasar.struts.util.RequestUtil; @@ -54,20 +52,24 @@ public class ScreenShotManager { public File baseDir; - public int threadNum = 10; - public long shutdownTimeout = 5 * 60 * 1000; // 5min public int screenShotPathCacheSize = 10; private final List generatorList = new ArrayList(); - private ExecutorService executorService; + public String imageExtention = "png"; + + public int splitSize = 5; + + private BlockingQueue screenShotTaskQueue = new LinkedBlockingQueue(); + + private boolean generating; + + private Thread screenshotGeneratorThread; @InitMethod public void init() { - executorService = Executors.newFixedThreadPool(threadNum); - if (baseDir == null) { final String path = application.getRealPath(DEFAULT_SCREENSHOT_DIR); if (StringUtil.isNotBlank(path)) { @@ -87,35 +89,71 @@ public class ScreenShotManager { if (logger.isDebugEnabled()) { logger.debug("ScreenShot Directory: " + baseDir.getAbsolutePath()); } + + generating = true; + screenshotGeneratorThread = new Thread(new Runnable() { + @Override + public void run() { + while (generating) { + try { + screenShotTaskQueue.take().generate(); + } catch (InterruptedException e) { + logger.debug("Interupted task.", e); + } catch (Exception e) { + logger.warn("Failed to generage a screenshot.", e); + } + } + } + }, "ScreenShotGenerator"); + screenshotGeneratorThread.start(); + } + + @DestroyMethod + public void destroy() { + generating = false; + screenshotGeneratorThread.interrupt(); } public void generate(final Map docMap) { - final SystemHelper systemHelper = ComponentUtil.getSystemHelper(); for (final ScreenShotGenerator generator : generatorList) { if (generator.isTarget(docMap)) { - final String segment = (String) docMap.get("segment"); final String url = (String) docMap.get("url"); - final String path = segment + "/" + generator.getPath(docMap); - docMap.put(systemHelper.screenshotField, path); - executorService.execute(new GenerateTask(url, new File(baseDir, - path), generator)); + final String path = getImageFilename(docMap); + if (!screenShotTaskQueue.offer((new ScreenShotTask(url, + new File(baseDir, path), generator)))) { + logger.warn("Failed to offer a screenshot task: " + url + + " -> " + path); + } break; } } } + protected String getImageFilename(final Map docMap) { + StringBuilder buf = new StringBuilder(50); + final SystemHelper systemHelper = ComponentUtil.getSystemHelper(); + final String docid = (String) docMap.get(systemHelper.docIdField); + for (int i = 0; i < docid.length(); i++) { + if (i > 0 && i % splitSize == 0) { + buf.append('/'); + } + buf.append(docid.charAt(i)); + } + buf.append('.').append(imageExtention); + return buf.toString(); + } + public void storeRequest(final String queryId, final List> documentItems) { final SystemHelper systemHelper = ComponentUtil.getSystemHelper(); final Map dataMap = new HashMap( documentItems.size()); for (final Map docMap : documentItems) { - final String url = (String) docMap.get("url"); - final String screenShotPath = (String) docMap - .get(systemHelper.screenshotField); - if (StringUtil.isNotBlank(url) + final String docid = (String) docMap.get(systemHelper.docIdField); + final String screenShotPath = getImageFilename(docMap); + if (StringUtil.isNotBlank(docid) && StringUtil.isNotBlank(screenShotPath)) { - dataMap.put(url, screenShotPath); + dataMap.put(docid, screenShotPath); } } final Map> screenShotPathCache = getScreenShotPathCache(RequestUtil @@ -123,14 +161,14 @@ public class ScreenShotManager { screenShotPathCache.put(queryId, dataMap); } - public File getScreenShotFile(final String queryId, final String url) { + public File getScreenShotFile(final String queryId, final String docId) { final HttpSession session = RequestUtil.getRequest().getSession(false); if (session != null) { final Map> screenShotPathCache = getScreenShotPathCache(session); final Map dataMap = screenShotPathCache .get(queryId); if (dataMap != null) { - final String path = dataMap.get(url); + final String path = dataMap.get(docId); final File file = new File(baseDir, path); if (file.isFile()) { return file; @@ -142,6 +180,7 @@ public class ScreenShotManager { private Map> getScreenShotPathCache( final HttpSession session) { + @SuppressWarnings("unchecked") Map> screenShotPathCache = (Map>) session .getAttribute(Constants.SCREEN_SHOT_PATH_CACHE); if (screenShotPathCache == null) { @@ -153,56 +192,60 @@ public class ScreenShotManager { return screenShotPathCache; } - public void delete(final String expiredSessionId) { - final File screenShotDir = new File(baseDir, expiredSessionId); - if (screenShotDir.isDirectory()) { - logger.info("Deleted: " + screenShotDir.getAbsolutePath()); - try { - FileUtils.deleteDirectory(screenShotDir); - } catch (final IOException e) { - logger.warn( - "Failed to delete " + screenShotDir.getAbsolutePath(), - e); - } - } - } - - public void shutdown() { - executorService.shutdown(); - try { - if (!executorService.awaitTermination(shutdownTimeout, - TimeUnit.MILLISECONDS)) { - logger.warn("Forced to shutdown processes. Modify shutdownTimeout if needed."); - executorService.shutdownNow(); - } - } catch (final InterruptedException e) { - logger.warn("Interrupted shutdown processes.", e); - executorService.shutdownNow(); - } - } - public void add(final ScreenShotGenerator generator) { generatorList.add(generator); } - protected static class GenerateTask implements Runnable { + protected static class ScreenShotTask { String url; File outputFile; ScreenShotGenerator generator; - protected GenerateTask(final String url, final File outputFile, + protected ScreenShotTask(final String url, final File outputFile, final ScreenShotGenerator generator) { this.url = url; this.outputFile = outputFile; this.generator = generator; } - @Override - public void run() { + public void generate() { generator.generate(url, outputFile); } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + + ((outputFile == null) ? 0 : outputFile.hashCode()); + result = prime * result + ((url == null) ? 0 : url.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + ScreenShotTask other = (ScreenShotTask) obj; + if (outputFile == null) { + if (other.outputFile != null) + return false; + } else if (!outputFile.equals(other.outputFile)) + return false; + if (url == null) { + if (other.url != null) + return false; + } else if (!url.equals(other.url)) + return false; + return true; + } + } } diff --git a/src/main/java/jp/sf/fess/screenshot/impl/BaseScreenShotGenerator.java b/src/main/java/jp/sf/fess/screenshot/impl/BaseScreenShotGenerator.java new file mode 100644 index 000000000..bd2abf4c1 --- /dev/null +++ b/src/main/java/jp/sf/fess/screenshot/impl/BaseScreenShotGenerator.java @@ -0,0 +1,36 @@ +package jp.sf.fess.screenshot.impl; + +import java.util.HashMap; +import java.util.Map; + +import javax.annotation.Resource; +import javax.servlet.ServletContext; + +import jp.sf.fess.screenshot.ScreenShotGenerator; + +public abstract class BaseScreenShotGenerator implements ScreenShotGenerator { + + @Resource + protected ServletContext application; + + protected final Map conditionMap = new HashMap(); + + public int directoryNameLength = 5; + + public void addCondition(final String key, final String regex) { + conditionMap.put(key, regex); + } + + @Override + public boolean isTarget(final Map docMap) { + for (final Map.Entry entry : conditionMap.entrySet()) { + final Object value = docMap.get(entry.getKey()); + if (value instanceof String + && !((String) value).matches(entry.getValue())) { + return false; + } + } + return true; + } + +} \ No newline at end of file diff --git a/src/main/java/jp/sf/fess/screenshot/impl/CommandGenerator.java b/src/main/java/jp/sf/fess/screenshot/impl/CommandGenerator.java index e9125a968..d3ba7a661 100644 --- a/src/main/java/jp/sf/fess/screenshot/impl/CommandGenerator.java +++ b/src/main/java/jp/sf/fess/screenshot/impl/CommandGenerator.java @@ -21,46 +21,28 @@ import java.io.File; import java.io.InputStreamReader; import java.nio.charset.Charset; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Timer; import java.util.TimerTask; -import javax.annotation.Resource; -import javax.servlet.ServletContext; - -import jp.sf.fess.screenshot.ScreenShotGenerator; - import org.apache.commons.io.IOUtils; -import org.apache.commons.lang.RandomStringUtils; import org.seasar.framework.container.annotation.tiger.Binding; import org.seasar.framework.container.annotation.tiger.BindingType; import org.seasar.framework.container.annotation.tiger.DestroyMethod; import org.seasar.framework.container.annotation.tiger.InitMethod; -import org.seasar.framework.util.Base64Util; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class CommandGenerator implements ScreenShotGenerator { +public class CommandGenerator extends BaseScreenShotGenerator { private static final Logger logger = LoggerFactory .getLogger(CommandGenerator.class); - @Resource - protected ServletContext application; - - public String imageExtention = "png"; - - public int directoryNameLength = 5; - @Binding(bindingType = BindingType.MUST) public List commandList; - public File baseDir; - public long commandTimeout = 10 * 1000;// 10sec - private final Map conditionMap = new HashMap(); + public File baseDir; private volatile Timer destoryTimer; @@ -79,42 +61,20 @@ public class CommandGenerator implements ScreenShotGenerator { destoryTimer = null; } - @Override - public String getPath(final Map docMap) { - final String url = (String) docMap.get("url"); - - final StringBuilder buf = new StringBuilder(50); - buf.append(RandomStringUtils.randomNumeric(directoryNameLength)); - buf.append('/'); - buf.append(Base64Util.encode(Integer.toString(url.hashCode()).getBytes( - Charset.defaultCharset()))); - buf.append('.'); - buf.append(imageExtention); - return buf.toString(); - } - - @Override - public boolean isTarget(final Map docMap) { - for (final Map.Entry entry : conditionMap.entrySet()) { - final Object value = docMap.get(entry.getKey()); - if (value instanceof String - && !((String) value).matches(entry.getValue())) { - return false; - } - } - return true; - } - - public void addCondition(final String key, final String regex) { - conditionMap.put(key, regex); - } - @Override public void generate(final String url, final File outputFile) { if (logger.isDebugEnabled()) { logger.debug("Generate ScreenShot: " + url); } + if (outputFile.exists()) { + if (logger.isDebugEnabled()) { + logger.debug("The screenshot file exists: " + + outputFile.getAbsolutePath()); + } + return; + } + final File parentFile = outputFile.getParentFile(); if (!parentFile.exists()) { parentFile.mkdirs(); diff --git a/src/main/java/jp/sf/fess/screenshot/impl/WebDriverGenerator.java b/src/main/java/jp/sf/fess/screenshot/impl/WebDriverGenerator.java new file mode 100644 index 000000000..09b773207 --- /dev/null +++ b/src/main/java/jp/sf/fess/screenshot/impl/WebDriverGenerator.java @@ -0,0 +1,109 @@ +package jp.sf.fess.screenshot.impl; + +import java.awt.Image; +import java.awt.image.BufferedImage; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; + +import javax.imageio.ImageIO; + +import org.openqa.selenium.Dimension; +import org.openqa.selenium.OutputType; +import org.openqa.selenium.TakesScreenshot; +import org.openqa.selenium.WebDriver; +import org.openqa.selenium.phantomjs.PhantomJSDriver; +import org.seasar.framework.container.annotation.tiger.DestroyMethod; +import org.seasar.framework.container.annotation.tiger.InitMethod; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class WebDriverGenerator extends BaseScreenShotGenerator { + + private static final Logger logger = LoggerFactory + .getLogger(WebDriverGenerator.class); + + public WebDriver webDriver; + + public int windowWidth = 1200; + + public int windowHeight = 800; + + public int screenShotWidth = 400; + + public String imageFormatName = "png"; + + @InitMethod + public void init() { + if (webDriver == null) { + webDriver = new PhantomJSDriver(); + } + webDriver.manage().window() + .setSize(new Dimension(windowWidth, windowHeight)); + } + + @DestroyMethod + public void destroy() { + webDriver.close(); + } + + @Override + public void generate(String url, File outputFile) { + if (logger.isDebugEnabled()) { + logger.debug("Generate ScreenShot: " + url); + } + + if (outputFile.exists()) { + if (logger.isDebugEnabled()) { + logger.debug("The screenshot file exists: " + + outputFile.getAbsolutePath()); + } + return; + } + + final File parentFile = outputFile.getParentFile(); + if (!parentFile.exists()) { + parentFile.mkdirs(); + } + if (!parentFile.isDirectory()) { + logger.warn("Not found: " + parentFile.getAbsolutePath()); + return; + } + + if (webDriver instanceof TakesScreenshot) { + webDriver.get(url); + File screenshot = ((TakesScreenshot) webDriver) + .getScreenshotAs(OutputType.FILE); + convert(screenshot, outputFile); + } else { + logger.warn("WebDriver is not instance of TakesScreenshot: " + + webDriver); + } + } + + protected void convert(File inputFile, File outputFile) { + try { + BufferedImage image = loadImage(inputFile); + int screenShotHeight = screenShotWidth * image.getHeight() + / windowWidth; + BufferedImage screenShotImage = new BufferedImage(screenShotWidth, + screenShotHeight, image.getType()); + screenShotImage.getGraphics().drawImage( + image.getScaledInstance(screenShotWidth, screenShotHeight, + Image.SCALE_AREA_AVERAGING), 0, 0, screenShotWidth, + screenShotHeight, null); + + ImageIO.write(screenShotImage, imageFormatName, outputFile); + } catch (Exception e) { + logger.warn("Failed to convert " + inputFile.getAbsolutePath(), e); + inputFile.renameTo(outputFile); + } + } + + protected BufferedImage loadImage(File file) throws IOException { + try (FileInputStream in = new FileInputStream(file)) { + return ImageIO.read(in); + } + } + +} diff --git a/src/main/java/jp/sf/fess/solr/IndexUpdater.java b/src/main/java/jp/sf/fess/solr/IndexUpdater.java index 0d519d5d9..ed2fa2c14 100644 --- a/src/main/java/jp/sf/fess/solr/IndexUpdater.java +++ b/src/main/java/jp/sf/fess/solr/IndexUpdater.java @@ -33,7 +33,6 @@ import jp.sf.fess.db.exbhv.pmbean.FavoriteUrlCountPmb; import jp.sf.fess.db.exentity.customize.FavoriteUrlCount; import jp.sf.fess.helper.IntervalControlHelper; import jp.sf.fess.helper.SystemHelper; -import jp.sf.fess.screenshot.ScreenShotManager; import jp.sf.fess.util.ComponentUtil; import org.apache.solr.common.SolrInputDocument; @@ -90,10 +89,6 @@ public class IndexUpdater extends Thread { @Resource protected SystemHelper systemHelper; - @Binding(bindingType = BindingType.MAY) - @Resource - protected ScreenShotManager screenShotManager; - public int maxDocumentCacheSize = 5; protected boolean finishCrawling = false; @@ -309,9 +304,6 @@ public class IndexUpdater extends Thread { forceStop(); } finally { intervalControlHelper.setCrawlerRunning(true); - if (screenShotManager != null) { - screenShotManager.shutdown(); - } } if (logger.isInfoEnabled()) { @@ -387,10 +379,6 @@ public class IndexUpdater extends Thread { map.remove(Constants.INDEXING_TARGET); } - if (screenShotManager != null) { - screenShotManager.generate(map); - } - final SolrInputDocument doc = createSolrDocument(map); docList.add(doc); diff --git a/src/main/resources/app.dicon b/src/main/resources/app.dicon index 39bd0fa7d..f055fa920 100644 --- a/src/main/resources/app.dicon +++ b/src/main/resources/app.dicon @@ -79,7 +79,7 @@ new String[]{ "id", "docId", "score", "boost", "contentLength", "host", "site", "lastModified", "mimetype", "filetype_s", "created", TITLE_FIELD, "digest", "url", - "clickCount_l_x_dv", "favoriteCount_l_x_dv", "screenshot_s_s", + "clickCount_l_x_dv", "favoriteCount_l_x_dv", "cid_s_s", "lang_s", "hasCache_s_s" } new String[]{ "clickCount_l_x_dv", "favoriteCount_l_x_dv"} diff --git a/src/main/resources/fess.dicon b/src/main/resources/fess.dicon index a1ec250b6..0f8a64e21 100644 --- a/src/main/resources/fess.dicon +++ b/src/main/resources/fess.dicon @@ -178,6 +178,25 @@ new String[] { htmlScreenShotGenerator + + + + + "phantomjs.binary.path" + "/usr/bin/phantomjs" + + + + + + webDriver + + "mimetype" + "text/html" + + +--> +