0ct0pu5/fess
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix #167

Browse source
This commit is contained in:
Shinsuke Sugaya 2014-05-30 16:21:38 +09:00
parent 444145c0d5
commit 2a6876324a
12 changed files with 284 additions and 127 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
pom.xml
View file

@ -796,6 +796,11 @@
<artifactId>groovy-all</artifactId>
<version>2.2.2</version>
</dependency>
<dependency>
<groupId>com.github.detro</groupId>
<artifactId>phantomjsdriver</artifactId>
<version>1.2.0</version>
</dependency>
<!-- Removed from war -->
<dependency>
<groupId>junit</groupId>

3
src/main/java/jp/sf/fess/action/IndexAction.java
View file

@ -531,10 +531,11 @@ public class IndexAction {
}
final File screenShotFile = screenShotManager.getScreenShotFile(
indexForm.queryId, url);
indexForm.queryId, indexForm.docId);
if (screenShotFile == null) {
// 404
response.sendError(HttpServletResponse.SC_NOT_FOUND);
screenShotManager.generate(doc);
return null;
}

8
src/main/java/jp/sf/fess/helper/QueryHelper.java
View file

@ -103,14 +103,14 @@ public class QueryHelper implements Serializable {
protected String[] responseFields = new String[] { "id", "docId", "score",
"boost", "contentLength", "host", "site", "lastModified",
"mimetype", "filetype_s", "created", TITLE_FIELD, "digest", "url",
"clickCount_l_x_dv", "favoriteCount_l_x_dv", "screenshot_s_s",
"cid_s_s", "lang_s", "hasCache_s_s" };
"clickCount_l_x_dv", "favoriteCount_l_x_dv", "cid_s_s", "lang_s",
"hasCache_s_s" };
protected String[] cacheResponseFields = new String[] { "id", "docId",
"score", "boost", "contentLength", "host", "site", "lastModified",
"mimetype", "filetype_s", "created", TITLE_FIELD, "digest", "url",
"clickCount_l_x_dv", "favoriteCount_l_x_dv", "screenshot_s_s",
"cid_s_s", "lang_s", "cache" };
"clickCount_l_x_dv", "favoriteCount_l_x_dv", "cid_s_s", "lang_s",
"cache" };
protected String[] responseDocValuesFields = new String[] {
"clickCount_l_x_dv", "favoriteCount_l_x_dv" };

2
src/main/java/jp/sf/fess/helper/SystemHelper.java
View file

@ -121,8 +121,6 @@ public class SystemHelper implements Serializable {
public String clickCountField = "clickCount_l_x_dv";
public String screenshotField = "screenshot_s_s";
public String configIdField = "cid_s_s";
public String expiresField = "expires_dt";

2
src/main/java/jp/sf/fess/screenshot/ScreenShotGenerator.java
View file

@ -21,8 +21,6 @@ import java.util.Map;
public interface ScreenShotGenerator {
String getPath(Map<String, Object> docMap);
void generate(String url, File outputFile);
boolean isTarget(Map<String, Object> docMap);

153
src/main/java/jp/sf/fess/screenshot/ScreenShotManager.java
View file

@ -17,14 +17,12 @@
package jp.sf.fess.screenshot;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import javax.annotation.Resource;
import javax.servlet.ServletContext;
@ -35,8 +33,8 @@ import jp.sf.fess.FessSystemException;
import jp.sf.fess.helper.SystemHelper;
import jp.sf.fess.util.ComponentUtil;
import org.apache.commons.io.FileUtils;
import org.codelibs.core.util.StringUtil;
import org.seasar.framework.container.annotation.tiger.DestroyMethod;
import org.seasar.framework.container.annotation.tiger.InitMethod;
import org.seasar.robot.util.LruHashMap;
import org.seasar.struts.util.RequestUtil;
@ -54,20 +52,24 @@ public class ScreenShotManager {
public File baseDir;
public int threadNum = 10;
public long shutdownTimeout = 5 * 60 * 1000; // 5min
public int screenShotPathCacheSize = 10;
private final List<ScreenShotGenerator> generatorList = new ArrayList<ScreenShotGenerator>();
private ExecutorService executorService;
public String imageExtention = "png";
public int splitSize = 5;
private BlockingQueue<ScreenShotTask> screenShotTaskQueue = new LinkedBlockingQueue<ScreenShotTask>();
private boolean generating;
private Thread screenshotGeneratorThread;
@InitMethod
public void init() {
executorService = Executors.newFixedThreadPool(threadNum);
if (baseDir == null) {
final String path = application.getRealPath(DEFAULT_SCREENSHOT_DIR);
if (StringUtil.isNotBlank(path)) {
@ -87,35 +89,71 @@ public class ScreenShotManager {
if (logger.isDebugEnabled()) {
logger.debug("ScreenShot Directory: " + baseDir.getAbsolutePath());
}
generating = true;
screenshotGeneratorThread = new Thread(new Runnable() {
@Override
public void run() {
while (generating) {
try {
screenShotTaskQueue.take().generate();
} catch (InterruptedException e) {
logger.debug("Interupted task.", e);
} catch (Exception e) {
logger.warn("Failed to generage a screenshot.", e);
}
}
}
}, "ScreenShotGenerator");
screenshotGeneratorThread.start();
}
@DestroyMethod
public void destroy() {
generating = false;
screenshotGeneratorThread.interrupt();
}
public void generate(final Map<String, Object> docMap) {
final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
for (final ScreenShotGenerator generator : generatorList) {
if (generator.isTarget(docMap)) {
final String segment = (String) docMap.get("segment");
final String url = (String) docMap.get("url");
final String path = segment + "/" + generator.getPath(docMap);
docMap.put(systemHelper.screenshotField, path);
executorService.execute(new GenerateTask(url, new File(baseDir,
path), generator));
final String path = getImageFilename(docMap);
if (!screenShotTaskQueue.offer((new ScreenShotTask(url,
new File(baseDir, path), generator)))) {
logger.warn("Failed to offer a screenshot task: " + url
+ " -> " + path);
}
break;
}
}
}
protected String getImageFilename(final Map<String, Object> docMap) {
StringBuilder buf = new StringBuilder(50);
final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
final String docid = (String) docMap.get(systemHelper.docIdField);
for (int i = 0; i < docid.length(); i++) {
if (i > 0 && i % splitSize == 0) {
buf.append('/');
}
buf.append(docid.charAt(i));
}
buf.append('.').append(imageExtention);
return buf.toString();
}
public void storeRequest(final String queryId,
final List<Map<String, Object>> documentItems) {
final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
final Map<String, String> dataMap = new HashMap<String, String>(
documentItems.size());
for (final Map<String, Object> docMap : documentItems) {
final String url = (String) docMap.get("url");
final String screenShotPath = (String) docMap
.get(systemHelper.screenshotField);
if (StringUtil.isNotBlank(url)
final String docid = (String) docMap.get(systemHelper.docIdField);
final String screenShotPath = getImageFilename(docMap);
if (StringUtil.isNotBlank(docid)
&& StringUtil.isNotBlank(screenShotPath)) {
dataMap.put(url, screenShotPath);
dataMap.put(docid, screenShotPath);
}
}
final Map<String, Map<String, String>> screenShotPathCache = getScreenShotPathCache(RequestUtil
@ -123,14 +161,14 @@ public class ScreenShotManager {
screenShotPathCache.put(queryId, dataMap);
}
public File getScreenShotFile(final String queryId, final String url) {
public File getScreenShotFile(final String queryId, final String docId) {
final HttpSession session = RequestUtil.getRequest().getSession(false);
if (session != null) {
final Map<String, Map<String, String>> screenShotPathCache = getScreenShotPathCache(session);
final Map<String, String> dataMap = screenShotPathCache
.get(queryId);
if (dataMap != null) {
final String path = dataMap.get(url);
final String path = dataMap.get(docId);
final File file = new File(baseDir, path);
if (file.isFile()) {
return file;
@ -142,6 +180,7 @@ public class ScreenShotManager {
private Map<String, Map<String, String>> getScreenShotPathCache(
final HttpSession session) {
@SuppressWarnings("unchecked")
Map<String, Map<String, String>> screenShotPathCache = (Map<String, Map<String, String>>) session
.getAttribute(Constants.SCREEN_SHOT_PATH_CACHE);
if (screenShotPathCache == null) {
@ -153,56 +192,60 @@ public class ScreenShotManager {
return screenShotPathCache;
}
public void delete(final String expiredSessionId) {
final File screenShotDir = new File(baseDir, expiredSessionId);
if (screenShotDir.isDirectory()) {
logger.info("Deleted: " + screenShotDir.getAbsolutePath());
try {
FileUtils.deleteDirectory(screenShotDir);
} catch (final IOException e) {
logger.warn(
"Failed to delete " + screenShotDir.getAbsolutePath(),
e);
}
}
}
public void shutdown() {
executorService.shutdown();
try {
if (!executorService.awaitTermination(shutdownTimeout,
TimeUnit.MILLISECONDS)) {
logger.warn("Forced to shutdown processes. Modify shutdownTimeout if needed.");
executorService.shutdownNow();
}
} catch (final InterruptedException e) {
logger.warn("Interrupted shutdown processes.", e);
executorService.shutdownNow();
}
}
public void add(final ScreenShotGenerator generator) {
generatorList.add(generator);
}
protected static class GenerateTask implements Runnable {
protected static class ScreenShotTask {
String url;
File outputFile;
ScreenShotGenerator generator;
protected GenerateTask(final String url, final File outputFile,
protected ScreenShotTask(final String url, final File outputFile,
final ScreenShotGenerator generator) {
this.url = url;
this.outputFile = outputFile;
this.generator = generator;
}
@Override
public void run() {
public void generate() {
generator.generate(url, outputFile);
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result
+ ((outputFile == null) ? 0 : outputFile.hashCode());
result = prime * result + ((url == null) ? 0 : url.hashCode());
return result;
}
@Override
public boolean equals(Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
ScreenShotTask other = (ScreenShotTask) obj;
if (outputFile == null) {
if (other.outputFile != null)
return false;
} else if (!outputFile.equals(other.outputFile))
return false;
if (url == null) {
if (other.url != null)
return false;
} else if (!url.equals(other.url))
return false;
return true;
}
}
}

36
src/main/java/jp/sf/fess/screenshot/impl/BaseScreenShotGenerator.java Normal file
View file

@ -0,0 +1,36 @@
package jp.sf.fess.screenshot.impl;
import java.util.HashMap;
import java.util.Map;
import javax.annotation.Resource;
import javax.servlet.ServletContext;
import jp.sf.fess.screenshot.ScreenShotGenerator;
public abstract class BaseScreenShotGenerator implements ScreenShotGenerator {
@Resource
protected ServletContext application;
protected final Map<String, String> conditionMap = new HashMap<String, String>();
public int directoryNameLength = 5;
public void addCondition(final String key, final String regex) {
conditionMap.put(key, regex);
}
@Override
public boolean isTarget(final Map<String, Object> docMap) {
for (final Map.Entry<String, String> entry : conditionMap.entrySet()) {
final Object value = docMap.get(entry.getKey());
if (value instanceof String
&& !((String) value).matches(entry.getValue())) {
return false;
}
}
return true;
}
}

60
src/main/java/jp/sf/fess/screenshot/impl/CommandGenerator.java
View file

@ -21,46 +21,28 @@ import java.io.File;
import java.io.InputStreamReader;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Timer;
import java.util.TimerTask;
import javax.annotation.Resource;
import javax.servlet.ServletContext;
import jp.sf.fess.screenshot.ScreenShotGenerator;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.RandomStringUtils;
import org.seasar.framework.container.annotation.tiger.Binding;
import org.seasar.framework.container.annotation.tiger.BindingType;
import org.seasar.framework.container.annotation.tiger.DestroyMethod;
import org.seasar.framework.container.annotation.tiger.InitMethod;
import org.seasar.framework.util.Base64Util;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class CommandGenerator implements ScreenShotGenerator {
public class CommandGenerator extends BaseScreenShotGenerator {
private static final Logger logger = LoggerFactory
.getLogger(CommandGenerator.class);
@Resource
protected ServletContext application;
public String imageExtention = "png";
public int directoryNameLength = 5;
@Binding(bindingType = BindingType.MUST)
public List<String> commandList;
public File baseDir;
public long commandTimeout = 10 * 1000;// 10sec
private final Map<String, String> conditionMap = new HashMap<String, String>();
public File baseDir;
private volatile Timer destoryTimer;
@ -79,42 +61,20 @@ public class CommandGenerator implements ScreenShotGenerator {
destoryTimer = null;
}
@Override
public String getPath(final Map<String, Object> docMap) {
final String url = (String) docMap.get("url");
final StringBuilder buf = new StringBuilder(50);
buf.append(RandomStringUtils.randomNumeric(directoryNameLength));
buf.append('/');
buf.append(Base64Util.encode(Integer.toString(url.hashCode()).getBytes(
Charset.defaultCharset())));
buf.append('.');
buf.append(imageExtention);
return buf.toString();
}
@Override
public boolean isTarget(final Map<String, Object> docMap) {
for (final Map.Entry<String, String> entry : conditionMap.entrySet()) {
final Object value = docMap.get(entry.getKey());
if (value instanceof String
&& !((String) value).matches(entry.getValue())) {
return false;
}
}
return true;
}
public void addCondition(final String key, final String regex) {
conditionMap.put(key, regex);
}
@Override
public void generate(final String url, final File outputFile) {
if (logger.isDebugEnabled()) {
logger.debug("Generate ScreenShot: " + url);
}
if (outputFile.exists()) {
if (logger.isDebugEnabled()) {
logger.debug("The screenshot file exists: "
+ outputFile.getAbsolutePath());
}
return;
}
final File parentFile = outputFile.getParentFile();
if (!parentFile.exists()) {
parentFile.mkdirs();

109
src/main/java/jp/sf/fess/screenshot/impl/WebDriverGenerator.java Normal file
View file

@ -0,0 +1,109 @@
package jp.sf.fess.screenshot.impl;
import java.awt.Image;
import java.awt.image.BufferedImage;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import javax.imageio.ImageIO;
import org.openqa.selenium.Dimension;
import org.openqa.selenium.OutputType;
import org.openqa.selenium.TakesScreenshot;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.phantomjs.PhantomJSDriver;
import org.seasar.framework.container.annotation.tiger.DestroyMethod;
import org.seasar.framework.container.annotation.tiger.InitMethod;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class WebDriverGenerator extends BaseScreenShotGenerator {
private static final Logger logger = LoggerFactory
.getLogger(WebDriverGenerator.class);
public WebDriver webDriver;
public int windowWidth = 1200;
public int windowHeight = 800;
public int screenShotWidth = 400;
public String imageFormatName = "png";
@InitMethod
public void init() {
if (webDriver == null) {
webDriver = new PhantomJSDriver();
}
webDriver.manage().window()
.setSize(new Dimension(windowWidth, windowHeight));
}
@DestroyMethod
public void destroy() {
webDriver.close();
}
@Override
public void generate(String url, File outputFile) {
if (logger.isDebugEnabled()) {
logger.debug("Generate ScreenShot: " + url);
}
if (outputFile.exists()) {
if (logger.isDebugEnabled()) {
logger.debug("The screenshot file exists: "
+ outputFile.getAbsolutePath());
}
return;
}
final File parentFile = outputFile.getParentFile();
if (!parentFile.exists()) {
parentFile.mkdirs();
}
if (!parentFile.isDirectory()) {
logger.warn("Not found: " + parentFile.getAbsolutePath());
return;
}
if (webDriver instanceof TakesScreenshot) {
webDriver.get(url);
File screenshot = ((TakesScreenshot) webDriver)
.getScreenshotAs(OutputType.FILE);
convert(screenshot, outputFile);
} else {
logger.warn("WebDriver is not instance of TakesScreenshot: "
+ webDriver);
}
}
protected void convert(File inputFile, File outputFile) {
try {
BufferedImage image = loadImage(inputFile);
int screenShotHeight = screenShotWidth * image.getHeight()
/ windowWidth;
BufferedImage screenShotImage = new BufferedImage(screenShotWidth,
screenShotHeight, image.getType());
screenShotImage.getGraphics().drawImage(
image.getScaledInstance(screenShotWidth, screenShotHeight,
Image.SCALE_AREA_AVERAGING), 0, 0, screenShotWidth,
screenShotHeight, null);
ImageIO.write(screenShotImage, imageFormatName, outputFile);
} catch (Exception e) {
logger.warn("Failed to convert " + inputFile.getAbsolutePath(), e);
inputFile.renameTo(outputFile);
}
}
protected BufferedImage loadImage(File file) throws IOException {
try (FileInputStream in = new FileInputStream(file)) {
return ImageIO.read(in);
}
}
}

12
src/main/java/jp/sf/fess/solr/IndexUpdater.java
View file

@ -33,7 +33,6 @@ import jp.sf.fess.db.exbhv.pmbean.FavoriteUrlCountPmb;
import jp.sf.fess.db.exentity.customize.FavoriteUrlCount;
import jp.sf.fess.helper.IntervalControlHelper;
import jp.sf.fess.helper.SystemHelper;
import jp.sf.fess.screenshot.ScreenShotManager;
import jp.sf.fess.util.ComponentUtil;
import org.apache.solr.common.SolrInputDocument;
@ -90,10 +89,6 @@ public class IndexUpdater extends Thread {
@Resource
protected SystemHelper systemHelper;
@Binding(bindingType = BindingType.MAY)
@Resource
protected ScreenShotManager screenShotManager;
public int maxDocumentCacheSize = 5;
protected boolean finishCrawling = false;
@ -309,9 +304,6 @@ public class IndexUpdater extends Thread {
forceStop();
} finally {
intervalControlHelper.setCrawlerRunning(true);
if (screenShotManager != null) {
screenShotManager.shutdown();
}
}
if (logger.isInfoEnabled()) {
@ -387,10 +379,6 @@ public class IndexUpdater extends Thread {
map.remove(Constants.INDEXING_TARGET);
}
if (screenShotManager != null) {
screenShotManager.generate(map);
}
final SolrInputDocument doc = createSolrDocument(map);
docList.add(doc);

2
src/main/resources/app.dicon
View file

@ -79,7 +79,7 @@
<property name="responseFields">new String[]{ "id", "docId", "score",
"boost", "contentLength", "host", "site", "lastModified",
"mimetype", "filetype_s", "created", TITLE_FIELD, "digest", "url",
"clickCount_l_x_dv", "favoriteCount_l_x_dv", "screenshot_s_s",
"clickCount_l_x_dv", "favoriteCount_l_x_dv",
"cid_s_s", "lang_s", "hasCache_s_s" }</property>
<property name="responseDocValuesFields">new String[]{
"clickCount_l_x_dv", "favoriteCount_l_x_dv"}</property>

19
src/main/resources/fess.dicon
View file

@ -178,6 +178,25 @@ new String[] {
<arg>htmlScreenShotGenerator</arg>
</initMethod>
</component>
<component name="webDriver" class="org.openqa.selenium.phantomjs.PhantomJSDriver">
<arg>
<component class="org.openqa.selenium.remote.DesiredCapabilities">
<initMethod name="setCapability">
<arg>"phantomjs.binary.path"</arg>
<arg>"/usr/bin/phantomjs"</arg>
</initMethod>
</component>
</arg>
</component>
<component name="htmlScreenShotGenerator" class="jp.sf.fess.screenshot.impl.WebDriverGenerator">
<property name="webDriver">webDriver</property>
<initMethod name="addCondition">
<arg>"mimetype"</arg>
<arg>"text/html"</arg>
</initMethod>
</component>
-->
<!--
<component name="htmlScreenShotGenerator" class="jp.sf.fess.screenshot.impl.CommandGenerator">
<property name="commandList">
{"bash",