fix #167

pom.xml

@@ -796,6 +796,11 @@
       <artifactId>groovy-all</artifactId>
       <version>2.2.2</version>
     </dependency>
+    <dependency>
+      <groupId>com.github.detro</groupId>
+      <artifactId>phantomjsdriver</artifactId>
+      <version>1.2.0</version>
+    </dependency>
     <!-- Removed from war -->
     <dependency>
       <groupId>junit</groupId>

src/main/java/jp/sf/fess/action/IndexAction.java

@@ -531,10 +531,11 @@ public class IndexAction {
             }
 
             final File screenShotFile = screenShotManager.getScreenShotFile(
-                    indexForm.queryId, url);
+                    indexForm.queryId, indexForm.docId);
             if (screenShotFile == null) {
                 // 404
                 response.sendError(HttpServletResponse.SC_NOT_FOUND);
+                screenShotManager.generate(doc);
                 return null;
             }
 

src/main/java/jp/sf/fess/helper/QueryHelper.java

@@ -103,14 +103,14 @@ public class QueryHelper implements Serializable {
     protected String[] responseFields = new String[] { "id", "docId", "score",
             "boost", "contentLength", "host", "site", "lastModified",
             "mimetype", "filetype_s", "created", TITLE_FIELD, "digest", "url",
-            "clickCount_l_x_dv", "favoriteCount_l_x_dv", "screenshot_s_s",
-            "cid_s_s", "lang_s", "hasCache_s_s" };
+            "clickCount_l_x_dv", "favoriteCount_l_x_dv", "cid_s_s", "lang_s",
+            "hasCache_s_s" };
 
     protected String[] cacheResponseFields = new String[] { "id", "docId",
             "score", "boost", "contentLength", "host", "site", "lastModified",
             "mimetype", "filetype_s", "created", TITLE_FIELD, "digest", "url",
-            "clickCount_l_x_dv", "favoriteCount_l_x_dv", "screenshot_s_s",
-            "cid_s_s", "lang_s", "cache" };
+            "clickCount_l_x_dv", "favoriteCount_l_x_dv", "cid_s_s", "lang_s",
+            "cache" };
 
     protected String[] responseDocValuesFields = new String[] {
             "clickCount_l_x_dv", "favoriteCount_l_x_dv" };

src/main/java/jp/sf/fess/helper/SystemHelper.java

@@ -121,8 +121,6 @@ public class SystemHelper implements Serializable {
 
     public String clickCountField = "clickCount_l_x_dv";
 
-    public String screenshotField = "screenshot_s_s";
-
     public String configIdField = "cid_s_s";
 
     public String expiresField = "expires_dt";

src/main/java/jp/sf/fess/screenshot/ScreenShotGenerator.java

@@ -21,8 +21,6 @@ import java.util.Map;
 
 public interface ScreenShotGenerator {
 
-    String getPath(Map<String, Object> docMap);
-
     void generate(String url, File outputFile);
 
     boolean isTarget(Map<String, Object> docMap);

src/main/java/jp/sf/fess/screenshot/ScreenShotManager.java

@@ -17,14 +17,12 @@
 package jp.sf.fess.screenshot;
 
 import java.io.File;
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
 
 import javax.annotation.Resource;
 import javax.servlet.ServletContext;
@@ -35,8 +33,8 @@ import jp.sf.fess.FessSystemException;
 import jp.sf.fess.helper.SystemHelper;
 import jp.sf.fess.util.ComponentUtil;
 
-import org.apache.commons.io.FileUtils;
 import org.codelibs.core.util.StringUtil;
+import org.seasar.framework.container.annotation.tiger.DestroyMethod;
 import org.seasar.framework.container.annotation.tiger.InitMethod;
 import org.seasar.robot.util.LruHashMap;
 import org.seasar.struts.util.RequestUtil;
@@ -54,20 +52,24 @@ public class ScreenShotManager {
 
     public File baseDir;
 
-    public int threadNum = 10;
-
     public long shutdownTimeout = 5 * 60 * 1000; // 5min
 
     public int screenShotPathCacheSize = 10;
 
     private final List<ScreenShotGenerator> generatorList = new ArrayList<ScreenShotGenerator>();
 
-    private ExecutorService executorService;
+    public String imageExtention = "png";
+
+    public int splitSize = 5;
+
+    private BlockingQueue<ScreenShotTask> screenShotTaskQueue = new LinkedBlockingQueue<ScreenShotTask>();
+
+    private boolean generating;
+
+    private Thread screenshotGeneratorThread;
 
     @InitMethod
     public void init() {
-        executorService = Executors.newFixedThreadPool(threadNum);
-
         if (baseDir == null) {
             final String path = application.getRealPath(DEFAULT_SCREENSHOT_DIR);
             if (StringUtil.isNotBlank(path)) {
@@ -87,35 +89,71 @@ public class ScreenShotManager {
         if (logger.isDebugEnabled()) {
             logger.debug("ScreenShot Directory: " + baseDir.getAbsolutePath());
         }
+
+        generating = true;
+        screenshotGeneratorThread = new Thread(new Runnable() {
+            @Override
+            public void run() {
+                while (generating) {
+                    try {
+                        screenShotTaskQueue.take().generate();
+                    } catch (InterruptedException e) {
+                        logger.debug("Interupted task.", e);
+                    } catch (Exception e) {
+                        logger.warn("Failed to generage a screenshot.", e);
+                    }
+                }
+            }
+        }, "ScreenShotGenerator");
+        screenshotGeneratorThread.start();
+    }
+
+    @DestroyMethod
+    public void destroy() {
+        generating = false;
+        screenshotGeneratorThread.interrupt();
     }
 
     public void generate(final Map<String, Object> docMap) {
-        final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
         for (final ScreenShotGenerator generator : generatorList) {
             if (generator.isTarget(docMap)) {
-                final String segment = (String) docMap.get("segment");
                 final String url = (String) docMap.get("url");
-                final String path = segment + "/" + generator.getPath(docMap);
-                docMap.put(systemHelper.screenshotField, path);
-                executorService.execute(new GenerateTask(url, new File(baseDir,
-                        path), generator));
+                final String path = getImageFilename(docMap);
+                if (!screenShotTaskQueue.offer((new ScreenShotTask(url,
+                        new File(baseDir, path), generator)))) {
+                    logger.warn("Failed to offer a screenshot task: " + url
+                            + " -> " + path);
+                }
                 break;
             }
         }
     }
 
+    protected String getImageFilename(final Map<String, Object> docMap) {
+        StringBuilder buf = new StringBuilder(50);
+        final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
+        final String docid = (String) docMap.get(systemHelper.docIdField);
+        for (int i = 0; i < docid.length(); i++) {
+            if (i > 0 && i % splitSize == 0) {
+                buf.append('/');
+            }
+            buf.append(docid.charAt(i));
+        }
+        buf.append('.').append(imageExtention);
+        return buf.toString();
+    }
+
     public void storeRequest(final String queryId,
             final List<Map<String, Object>> documentItems) {
         final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
         final Map<String, String> dataMap = new HashMap<String, String>(
                 documentItems.size());
         for (final Map<String, Object> docMap : documentItems) {
-            final String url = (String) docMap.get("url");
-            final String screenShotPath = (String) docMap
-                    .get(systemHelper.screenshotField);
-            if (StringUtil.isNotBlank(url)
+            final String docid = (String) docMap.get(systemHelper.docIdField);
+            final String screenShotPath = getImageFilename(docMap);
+            if (StringUtil.isNotBlank(docid)
                     && StringUtil.isNotBlank(screenShotPath)) {
-                dataMap.put(url, screenShotPath);
+                dataMap.put(docid, screenShotPath);
             }
         }
         final Map<String, Map<String, String>> screenShotPathCache = getScreenShotPathCache(RequestUtil
@@ -123,14 +161,14 @@ public class ScreenShotManager {
         screenShotPathCache.put(queryId, dataMap);
     }
 
-    public File getScreenShotFile(final String queryId, final String url) {
+    public File getScreenShotFile(final String queryId, final String docId) {
         final HttpSession session = RequestUtil.getRequest().getSession(false);
         if (session != null) {
             final Map<String, Map<String, String>> screenShotPathCache = getScreenShotPathCache(session);
             final Map<String, String> dataMap = screenShotPathCache
                     .get(queryId);
             if (dataMap != null) {
-                final String path = dataMap.get(url);
+                final String path = dataMap.get(docId);
                 final File file = new File(baseDir, path);
                 if (file.isFile()) {
                     return file;
@@ -142,6 +180,7 @@ public class ScreenShotManager {
 
     private Map<String, Map<String, String>> getScreenShotPathCache(
             final HttpSession session) {
+        @SuppressWarnings("unchecked")
         Map<String, Map<String, String>> screenShotPathCache = (Map<String, Map<String, String>>) session
                 .getAttribute(Constants.SCREEN_SHOT_PATH_CACHE);
         if (screenShotPathCache == null) {
@@ -153,56 +192,60 @@ public class ScreenShotManager {
         return screenShotPathCache;
     }
 
-    public void delete(final String expiredSessionId) {
-        final File screenShotDir = new File(baseDir, expiredSessionId);
-        if (screenShotDir.isDirectory()) {
-            logger.info("Deleted: " + screenShotDir.getAbsolutePath());
-            try {
-                FileUtils.deleteDirectory(screenShotDir);
-            } catch (final IOException e) {
-                logger.warn(
-                        "Failed to delete " + screenShotDir.getAbsolutePath(),
-                        e);
-            }
-        }
-    }
-
-    public void shutdown() {
-        executorService.shutdown();
-        try {
-            if (!executorService.awaitTermination(shutdownTimeout,
-                    TimeUnit.MILLISECONDS)) {
-                logger.warn("Forced to  shutdown processes. Modify shutdownTimeout if needed.");
-                executorService.shutdownNow();
-            }
-        } catch (final InterruptedException e) {
-            logger.warn("Interrupted shutdown processes.", e);
-            executorService.shutdownNow();
-        }
-    }
-
     public void add(final ScreenShotGenerator generator) {
         generatorList.add(generator);
     }
 
-    protected static class GenerateTask implements Runnable {
+    protected static class ScreenShotTask {
         String url;
 
         File outputFile;
 
         ScreenShotGenerator generator;
 
-        protected GenerateTask(final String url, final File outputFile,
+        protected ScreenShotTask(final String url, final File outputFile,
                 final ScreenShotGenerator generator) {
             this.url = url;
             this.outputFile = outputFile;
             this.generator = generator;
         }
 
-        @Override
-        public void run() {
+        public void generate() {
             generator.generate(url, outputFile);
         }
+
+        @Override
+        public int hashCode() {
+            final int prime = 31;
+            int result = 1;
+            result = prime * result
+                    + ((outputFile == null) ? 0 : outputFile.hashCode());
+            result = prime * result + ((url == null) ? 0 : url.hashCode());
+            return result;
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (this == obj)
+                return true;
+            if (obj == null)
+                return false;
+            if (getClass() != obj.getClass())
+                return false;
+            ScreenShotTask other = (ScreenShotTask) obj;
+            if (outputFile == null) {
+                if (other.outputFile != null)
+                    return false;
+            } else if (!outputFile.equals(other.outputFile))
+                return false;
+            if (url == null) {
+                if (other.url != null)
+                    return false;
+            } else if (!url.equals(other.url))
+                return false;
+            return true;
+        }
+
     }
 
 }

src/main/java/jp/sf/fess/screenshot/impl/BaseScreenShotGenerator.java

@@ -0,0 +1,36 @@
+package jp.sf.fess.screenshot.impl;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.annotation.Resource;
+import javax.servlet.ServletContext;
+
+import jp.sf.fess.screenshot.ScreenShotGenerator;
+
+public abstract class BaseScreenShotGenerator implements ScreenShotGenerator {
+
+    @Resource
+    protected ServletContext application;
+
+    protected final Map<String, String> conditionMap = new HashMap<String, String>();
+
+    public int directoryNameLength = 5;
+
+    public void addCondition(final String key, final String regex) {
+        conditionMap.put(key, regex);
+    }
+
+    @Override
+    public boolean isTarget(final Map<String, Object> docMap) {
+        for (final Map.Entry<String, String> entry : conditionMap.entrySet()) {
+            final Object value = docMap.get(entry.getKey());
+            if (value instanceof String
+                    && !((String) value).matches(entry.getValue())) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+}

src/main/java/jp/sf/fess/screenshot/impl/CommandGenerator.java

@@ -21,46 +21,28 @@ import java.io.File;
 import java.io.InputStreamReader;
 import java.nio.charset.Charset;
 import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
 import java.util.Timer;
 import java.util.TimerTask;
 
-import javax.annotation.Resource;
-import javax.servlet.ServletContext;
-
-import jp.sf.fess.screenshot.ScreenShotGenerator;
-
 import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.RandomStringUtils;
 import org.seasar.framework.container.annotation.tiger.Binding;
 import org.seasar.framework.container.annotation.tiger.BindingType;
 import org.seasar.framework.container.annotation.tiger.DestroyMethod;
 import org.seasar.framework.container.annotation.tiger.InitMethod;
-import org.seasar.framework.util.Base64Util;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-public class CommandGenerator implements ScreenShotGenerator {
+public class CommandGenerator extends BaseScreenShotGenerator {
     private static final Logger logger = LoggerFactory
             .getLogger(CommandGenerator.class);
 
-    @Resource
-    protected ServletContext application;
-
-    public String imageExtention = "png";
-
-    public int directoryNameLength = 5;
-
     @Binding(bindingType = BindingType.MUST)
     public List<String> commandList;
 
-    public File baseDir;
-
     public long commandTimeout = 10 * 1000;// 10sec
 
-    private final Map<String, String> conditionMap = new HashMap<String, String>();
+    public File baseDir;
 
     private volatile Timer destoryTimer;
 
@@ -79,42 +61,20 @@ public class CommandGenerator implements ScreenShotGenerator {
         destoryTimer = null;
     }
 
-    @Override
-    public String getPath(final Map<String, Object> docMap) {
-        final String url = (String) docMap.get("url");
-
-        final StringBuilder buf = new StringBuilder(50);
-        buf.append(RandomStringUtils.randomNumeric(directoryNameLength));
-        buf.append('/');
-        buf.append(Base64Util.encode(Integer.toString(url.hashCode()).getBytes(
-                Charset.defaultCharset())));
-        buf.append('.');
-        buf.append(imageExtention);
-        return buf.toString();
-    }
-
-    @Override
-    public boolean isTarget(final Map<String, Object> docMap) {
-        for (final Map.Entry<String, String> entry : conditionMap.entrySet()) {
-            final Object value = docMap.get(entry.getKey());
-            if (value instanceof String
-                    && !((String) value).matches(entry.getValue())) {
-                return false;
-            }
-        }
-        return true;
-    }
-
-    public void addCondition(final String key, final String regex) {
-        conditionMap.put(key, regex);
-    }
-
     @Override
     public void generate(final String url, final File outputFile) {
         if (logger.isDebugEnabled()) {
             logger.debug("Generate ScreenShot: " + url);
         }
 
+        if (outputFile.exists()) {
+            if (logger.isDebugEnabled()) {
+                logger.debug("The screenshot file exists: "
+                        + outputFile.getAbsolutePath());
+            }
+            return;
+        }
+
         final File parentFile = outputFile.getParentFile();
         if (!parentFile.exists()) {
             parentFile.mkdirs();

src/main/java/jp/sf/fess/screenshot/impl/WebDriverGenerator.java

@@ -0,0 +1,109 @@
+package jp.sf.fess.screenshot.impl;
+
+import java.awt.Image;
+import java.awt.image.BufferedImage;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+
+import javax.imageio.ImageIO;
+
+import org.openqa.selenium.Dimension;
+import org.openqa.selenium.OutputType;
+import org.openqa.selenium.TakesScreenshot;
+import org.openqa.selenium.WebDriver;
+import org.openqa.selenium.phantomjs.PhantomJSDriver;
+import org.seasar.framework.container.annotation.tiger.DestroyMethod;
+import org.seasar.framework.container.annotation.tiger.InitMethod;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class WebDriverGenerator extends BaseScreenShotGenerator {
+
+    private static final Logger logger = LoggerFactory
+            .getLogger(WebDriverGenerator.class);
+
+    public WebDriver webDriver;
+
+    public int windowWidth = 1200;
+
+    public int windowHeight = 800;
+
+    public int screenShotWidth = 400;
+
+    public String imageFormatName = "png";
+
+    @InitMethod
+    public void init() {
+        if (webDriver == null) {
+            webDriver = new PhantomJSDriver();
+        }
+        webDriver.manage().window()
+                .setSize(new Dimension(windowWidth, windowHeight));
+    }
+
+    @DestroyMethod
+    public void destroy() {
+        webDriver.close();
+    }
+
+    @Override
+    public void generate(String url, File outputFile) {
+        if (logger.isDebugEnabled()) {
+            logger.debug("Generate ScreenShot: " + url);
+        }
+
+        if (outputFile.exists()) {
+            if (logger.isDebugEnabled()) {
+                logger.debug("The screenshot file exists: "
+                        + outputFile.getAbsolutePath());
+            }
+            return;
+        }
+
+        final File parentFile = outputFile.getParentFile();
+        if (!parentFile.exists()) {
+            parentFile.mkdirs();
+        }
+        if (!parentFile.isDirectory()) {
+            logger.warn("Not found: " + parentFile.getAbsolutePath());
+            return;
+        }
+
+        if (webDriver instanceof TakesScreenshot) {
+            webDriver.get(url);
+            File screenshot = ((TakesScreenshot) webDriver)
+                    .getScreenshotAs(OutputType.FILE);
+            convert(screenshot, outputFile);
+        } else {
+            logger.warn("WebDriver is not instance of TakesScreenshot: "
+                    + webDriver);
+        }
+    }
+
+    protected void convert(File inputFile, File outputFile) {
+        try {
+            BufferedImage image = loadImage(inputFile);
+            int screenShotHeight = screenShotWidth * image.getHeight()
+                    / windowWidth;
+            BufferedImage screenShotImage = new BufferedImage(screenShotWidth,
+                    screenShotHeight, image.getType());
+            screenShotImage.getGraphics().drawImage(
+                    image.getScaledInstance(screenShotWidth, screenShotHeight,
+                            Image.SCALE_AREA_AVERAGING), 0, 0, screenShotWidth,
+                    screenShotHeight, null);
+
+            ImageIO.write(screenShotImage, imageFormatName, outputFile);
+        } catch (Exception e) {
+            logger.warn("Failed to convert " + inputFile.getAbsolutePath(), e);
+            inputFile.renameTo(outputFile);
+        }
+    }
+
+    protected BufferedImage loadImage(File file) throws IOException {
+        try (FileInputStream in = new FileInputStream(file)) {
+            return ImageIO.read(in);
+        }
+    }
+
+}

src/main/java/jp/sf/fess/solr/IndexUpdater.java

@@ -33,7 +33,6 @@ import jp.sf.fess.db.exbhv.pmbean.FavoriteUrlCountPmb;
 import jp.sf.fess.db.exentity.customize.FavoriteUrlCount;
 import jp.sf.fess.helper.IntervalControlHelper;
 import jp.sf.fess.helper.SystemHelper;
-import jp.sf.fess.screenshot.ScreenShotManager;
 import jp.sf.fess.util.ComponentUtil;
 
 import org.apache.solr.common.SolrInputDocument;
@@ -90,10 +89,6 @@ public class IndexUpdater extends Thread {
     @Resource
     protected SystemHelper systemHelper;
 
-    @Binding(bindingType = BindingType.MAY)
-    @Resource
-    protected ScreenShotManager screenShotManager;
-
     public int maxDocumentCacheSize = 5;
 
     protected boolean finishCrawling = false;
@@ -309,9 +304,6 @@ public class IndexUpdater extends Thread {
             forceStop();
         } finally {
             intervalControlHelper.setCrawlerRunning(true);
-            if (screenShotManager != null) {
-                screenShotManager.shutdown();
-            }
         }
 
         if (logger.isInfoEnabled()) {
@@ -387,10 +379,6 @@ public class IndexUpdater extends Thread {
                         map.remove(Constants.INDEXING_TARGET);
                     }
 
-                    if (screenShotManager != null) {
-                        screenShotManager.generate(map);
-                    }
-
                     final SolrInputDocument doc = createSolrDocument(map);
 
                     docList.add(doc);

src/main/resources/app.dicon

@@ -79,7 +79,7 @@
 		<property name="responseFields">new String[]{ "id", "docId", "score",
             "boost", "contentLength", "host", "site", "lastModified",
             "mimetype", "filetype_s", "created", TITLE_FIELD, "digest", "url",
-            "clickCount_l_x_dv", "favoriteCount_l_x_dv", "screenshot_s_s",
+            "clickCount_l_x_dv", "favoriteCount_l_x_dv",
             "cid_s_s", "lang_s", "hasCache_s_s" }</property>
 		<property name="responseDocValuesFields">new String[]{
             "clickCount_l_x_dv", "favoriteCount_l_x_dv"}</property>

src/main/resources/fess.dicon

@@ -178,6 +178,25 @@ new String[] {
 			<arg>htmlScreenShotGenerator</arg>
 		</initMethod>
 	</component>
+	<component name="webDriver" class="org.openqa.selenium.phantomjs.PhantomJSDriver">
+		<arg>
+			<component class="org.openqa.selenium.remote.DesiredCapabilities">
+				<initMethod name="setCapability">
+					<arg>"phantomjs.binary.path"</arg>
+					<arg>"/usr/bin/phantomjs"</arg>
+				</initMethod>
+			</component>
+		</arg>
+	</component>
+	<component name="htmlScreenShotGenerator" class="jp.sf.fess.screenshot.impl.WebDriverGenerator">
+		<property name="webDriver">webDriver</property>
+		<initMethod name="addCondition">
+			<arg>"mimetype"</arg>
+			<arg>"text/html"</arg>
+		</initMethod>
+	</component>
+-->
+<!--
 	<component name="htmlScreenShotGenerator" class="jp.sf.fess.screenshot.impl.CommandGenerator">
 		<property name="commandList">
 			{"bash",