#118 and minor fixes

src/main/java/jp/sf/fess/action/admin/IndexAction.java

@@ -24,12 +24,8 @@ import jp.sf.fess.helper.SystemHelper;
 import org.codelibs.sastruts.core.util.ActivityUtil;
 import org.seasar.struts.annotation.Execute;
 import org.seasar.struts.util.RequestUtil;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 public class IndexAction {
-    private static final Logger logger = LoggerFactory
-            .getLogger(IndexAction.class);
 
     @Resource
     protected SystemHelper systemHelper;

src/main/java/jp/sf/fess/db/exentity/CrawlingConfig.java

@@ -21,11 +21,6 @@ import java.util.Map;
 import org.seasar.robot.client.S2RobotClientFactory;
 
 public interface CrawlingConfig {
-    public static final String XPATH_PREFIX = "field.xpath.";
-
-    public static final String SCRIPT_PREFIX = "field.script.";
-
-    public static final String CLIENT_PREFIX = "client.";
 
     Long getId();
 
@@ -69,6 +64,6 @@ public interface CrawlingConfig {
     }
 
     public enum ConfigName {
-        CLIENT, XPATH, SCRIPT;
+        CLIENT, XPATH, SCRIPT, FIELD;
     }
 }

src/main/java/jp/sf/fess/db/exentity/FileCrawlingConfig.java

@@ -270,29 +270,8 @@ public class FileCrawlingConfig extends BsFileCrawlingConfig implements
     @Override
     public Map<String, String> getConfigParameterMap(final ConfigName name) {
         if (configParameterMap == null) {
-            final Map<ConfigName, Map<String, String>> map = new HashMap<>();
-            final Map<String, String> clientConfigMap = new HashMap<>();
-            final Map<String, String> xpathConfigMap = new HashMap<>();
-            final Map<String, String> scriptConfigMap = new HashMap<>();
-            map.put(ConfigName.CLIENT, clientConfigMap);
-            map.put(ConfigName.XPATH, xpathConfigMap);
-            map.put(ConfigName.SCRIPT, scriptConfigMap);
-            for (final Map.Entry<String, String> entry : ParameterUtil.parse(
-                    getConfigParameter()).entrySet()) {
-                final String key = entry.getKey();
-                if (key.startsWith(CLIENT_PREFIX)) {
-                    clientConfigMap.put(key.substring(CLIENT_PREFIX.length()),
-                            entry.getValue());
-                } else if (key.startsWith(XPATH_PREFIX)) {
-                    xpathConfigMap.put(key.substring(XPATH_PREFIX.length()),
-                            entry.getValue());
-                } else if (key.startsWith(SCRIPT_PREFIX)) {
-                    scriptConfigMap.put(key.substring(SCRIPT_PREFIX.length()),
-                            entry.getValue());
-                }
-            }
-
-            configParameterMap = map;
+            configParameterMap = ParameterUtil
+                    .createConfigParameterMap(getConfigParameter());
         }
 
         final Map<String, String> configMap = configParameterMap.get(name);

src/main/java/jp/sf/fess/db/exentity/WebCrawlingConfig.java

@@ -30,8 +30,8 @@ import jp.sf.fess.service.RequestHeaderService;
 import jp.sf.fess.service.WebAuthenticationService;
 import jp.sf.fess.util.ParameterUtil;
 
-import org.seasar.framework.container.SingletonS2Container;
 import org.codelibs.core.util.StringUtil;
+import org.seasar.framework.container.SingletonS2Container;
 import org.seasar.robot.client.S2RobotClientFactory;
 import org.seasar.robot.client.http.Authentication;
 import org.seasar.robot.client.http.HcHttpClient;
@@ -280,29 +280,8 @@ public class WebCrawlingConfig extends BsWebCrawlingConfig implements
     @Override
     public Map<String, String> getConfigParameterMap(final ConfigName name) {
         if (configParameterMap == null) {
-            final Map<ConfigName, Map<String, String>> map = new HashMap<>();
-            final Map<String, String> clientConfigMap = new HashMap<>();
-            final Map<String, String> xpathConfigMap = new HashMap<>();
-            final Map<String, String> scriptConfigMap = new HashMap<>();
-            map.put(ConfigName.CLIENT, clientConfigMap);
-            map.put(ConfigName.XPATH, xpathConfigMap);
-            map.put(ConfigName.SCRIPT, scriptConfigMap);
-            for (final Map.Entry<String, String> entry : ParameterUtil.parse(
-                    getConfigParameter()).entrySet()) {
-                final String key = entry.getKey();
-                if (key.startsWith(CLIENT_PREFIX)) {
-                    clientConfigMap.put(key.substring(CLIENT_PREFIX.length()),
-                            entry.getValue());
-                } else if (key.startsWith(XPATH_PREFIX)) {
-                    xpathConfigMap.put(key.substring(XPATH_PREFIX.length()),
-                            entry.getValue());
-                } else if (key.startsWith(SCRIPT_PREFIX)) {
-                    scriptConfigMap.put(key.substring(SCRIPT_PREFIX.length()),
-                            entry.getValue());
-                }
-            }
-
-            configParameterMap = map;
+            configParameterMap = ParameterUtil
+                    .createConfigParameterMap(getConfigParameter());
         }
 
         final Map<String, String> configMap = configParameterMap.get(name);
@@ -311,4 +290,5 @@ public class WebCrawlingConfig extends BsWebCrawlingConfig implements
         }
         return configMap;
     }
+
 }

src/main/java/jp/sf/fess/helper/SystemHelper.java

@@ -134,6 +134,8 @@ public class SystemHelper implements Serializable {
 
     public String langField = "lang_s";
 
+    public String hasCacheField = "hasCache_s_s";
+
     protected String[] supportedLanguages = new String[] { "ar", "bg", "ca",
             "da", "de", "el", "en", "es", "eu", "fa", "fi", "fr", "ga", "gl",
             "hi", "hu", "hy", "id", "it", "ja", "lv", "ko", "nl", "no", "pt",

src/main/java/jp/sf/fess/helper/impl/QueryHelperImpl.java

@@ -56,7 +56,7 @@ public class QueryHelperImpl implements QueryHelper, Serializable {
 
     private static final String INURL_FIELD = "inurl";
 
-    private static final String TTTLE_FIELD = "title";
+    private static final String TITLE_FIELD = "title";
 
     private static final String CONTENT_FIELD = "content";
 
@@ -91,9 +91,9 @@ public class QueryHelperImpl implements QueryHelper, Serializable {
 
     protected String[] responseFields = new String[] { "id", "docId", "score",
             "boost", "contentLength", "host", "site", "lastModified",
-            "mimetype", "filetype_s", "created", TTTLE_FIELD, "digest", "url",
+            "mimetype", "filetype_s", "created", TITLE_FIELD, "digest", "url",
             "clickCount_l_x_dv", "favoriteCount_l_x_dv", "screenshot_s_s",
-            "cid_s_s", "lang_s" };
+            "cid_s_s", "lang_s", "hasCache_s_s" };
 
     protected String[] responseDocValuesFields = new String[] {
             "clickCount_l_x_dv", "favoriteCount_l_x_dv" };
@@ -101,11 +101,11 @@ public class QueryHelperImpl implements QueryHelper, Serializable {
     protected String[] highlightingFields = new String[] { CONTENT_FIELD };
 
     protected String[] searchFields = new String[] { "url", "docId", "host",
-            TTTLE_FIELD, CONTENT_FIELD, "contentLength", "lastModified",
+            TITLE_FIELD, CONTENT_FIELD, "contentLength", "lastModified",
             "mimetype", "filetype_s", LABEL_FIELD, "segment",
             "clickCount_l_x_dv", "favoriteCount_l_x_dv", INURL_FIELD, "lang_s" };
 
-    protected String[] facetFields = new String[] { "url", "host", TTTLE_FIELD,
+    protected String[] facetFields = new String[] { "url", "host", TITLE_FIELD,
             CONTENT_FIELD, "contentLength", "lastModified", "mimetype",
             "filetype_s", LABEL_FIELD, "segment" };
 
@@ -896,7 +896,7 @@ public class QueryHelperImpl implements QueryHelper, Serializable {
     protected void buildContentQueryWithLang(final StringBuilder buf,
             final String value, final String queryLanguage) {
         buf.append('(');
-        buf.append(TTTLE_FIELD).append(':');
+        buf.append(TITLE_FIELD).append(':');
         appendQueryValue(buf, value);
         buf.append(_OR_);
         buf.append(CONTENT_FIELD).append(':');

src/main/java/jp/sf/fess/transformer/AbstractFessFileTransformer.java

@@ -32,6 +32,7 @@ import jcifs.smb.ACE;
 import jcifs.smb.SID;
 import jp.sf.fess.Constants;
 import jp.sf.fess.db.exentity.CrawlingConfig;
+import jp.sf.fess.db.exentity.CrawlingConfig.ConfigName;
 import jp.sf.fess.helper.CrawlingConfigHelper;
 import jp.sf.fess.helper.CrawlingSessionHelper;
 import jp.sf.fess.helper.FileTypeHelper;
@@ -191,6 +192,9 @@ public abstract class AbstractFessFileTransformer extends
         final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
         final FileTypeHelper fileTypeHelper = ComponentUtil.getFileTypeHelper();
 
+        final Map<String, String> fieldConfigMap = crawlingConfig
+                .getConfigParameterMap(ConfigName.FIELD);
+
         String urlEncoding;
         final UrlQueue urlQueue = CrawlingParameterUtil.getUrlQueue();
         if (urlQueue != null && urlQueue.getEncoding() != null) {
@@ -228,14 +232,22 @@ public abstract class AbstractFessFileTransformer extends
         } else {
             putResultDataBody(dataMap, "content", StringUtil.EMPTY);
         }
-        final String cache = normalizeContent(content);
-        if (enableCache) {
-            // cache 
+        if (Constants.TRUE.equalsIgnoreCase(fieldConfigMap.get("cache"))
+                || enableCache) {
+            String cache;
+            if (content == null) {
+                cache = StringUtil.EMPTY;
+            } else {
+                cache = content.trim().replaceAll("[ \\t\\x0B\\f]+", " ");
+            }
+            // text cache 
             putResultDataBody(dataMap, "cache", cache);
+            putResultDataBody(dataMap, systemHelper.hasCacheField,
+                    Constants.TRUE);
         }
         // digest
         putResultDataBody(dataMap, "digest", Constants.DIGEST_PREFIX
-                + abbreviate(cache, maxDigestLength));
+                + abbreviate(normalizeContent(content), maxDigestLength));
         // title
         if (!dataMap.containsKey("title")) {
             if (url.endsWith("/")) {

src/main/java/jp/sf/fess/transformer/FessXpathTransformer.java

@@ -17,13 +17,9 @@
 package jp.sf.fess.transformer;
 
 import java.io.BufferedInputStream;
-import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.UnsupportedEncodingException;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.util.ArrayList;
@@ -54,6 +50,7 @@ import org.apache.commons.lang.StringUtils;
 import org.apache.xpath.objects.XObject;
 import org.codelibs.core.util.StringUtil;
 import org.cyberneko.html.parsers.DOMParser;
+import org.seasar.framework.util.InputStreamUtil;
 import org.seasar.framework.util.OgnlUtil;
 import org.seasar.framework.util.SerializeUtil;
 import org.seasar.robot.RobotCrawlAccessException;
@@ -231,6 +228,9 @@ public class FessXpathTransformer extends AbstractFessXpathTransformer {
         final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
         final FileTypeHelper fileTypeHelper = ComponentUtil.getFileTypeHelper();
 
+        final Map<String, String> fieldConfigMap = crawlingConfig
+                .getConfigParameterMap(ConfigName.FIELD);
+
         String urlEncoding;
         final UrlQueue urlQueue = CrawlingParameterUtil.getUrlQueue();
         if (urlQueue != null && urlQueue.getEncoding() != null) {
@@ -259,10 +259,25 @@ public class FessXpathTransformer extends AbstractFessXpathTransformer {
         // content
         putResultDataBody(dataMap, "content",
                 getDocumentContent(responseData, document));
-        if (enableCache) {
-            // cache 
-            putResultDataBody(dataMap, "cache",
-                    getDocumentCache(responseData, document));
+        if (Constants.TRUE.equalsIgnoreCase(fieldConfigMap.get("cache"))
+                || enableCache) {
+            String charSet = responseData.getCharSet();
+            if (charSet == null) {
+                charSet = Constants.UTF_8;
+            }
+            try {
+                // cache 
+                putResultDataBody(
+                        dataMap,
+                        "cache",
+                        new String(InputStreamUtil.getBytes(responseData
+                                .getResponseBody()), charSet));
+                putResultDataBody(dataMap, systemHelper.hasCacheField,
+                        Constants.TRUE);
+            } catch (final Exception e) {
+                logger.warn("Failed to write a cache: " + sessionId + ":"
+                        + responseData, e);
+            }
         }
         // digest
         putResultDataBody(dataMap, "digest",
@@ -424,38 +439,6 @@ public class FessXpathTransformer extends AbstractFessXpathTransformer {
         return normalizeContent(getSingleNodeValue(document, contentXpath, true));
     }
 
-    private String getDocumentCache(final ResponseData responseData,
-            final Document document) {
-        final InputStream is = responseData.getResponseBody();
-        final ByteArrayOutputStream baos = new ByteArrayOutputStream();
-
-        byte[] bytes = new byte[1024];
-        try {
-            int length = is.read(bytes);
-            while (length != -1) {
-                if (length != 0) {
-                    baos.write(bytes, 0, length);
-                }
-                if (baos.size() > maxCacheLength) {
-                    break;
-                }
-                length = is.read(bytes);
-            }
-        } catch (final IOException e) {
-            logger.warn("Could not create a cache data.", e);
-        } finally {
-            bytes = null;
-            IOUtils.closeQuietly(is);
-        }
-
-        try {
-            return baos.toString(responseData.getCharSet());
-        } catch (final UnsupportedEncodingException e) {
-            logger.warn("Unsupported encoding: " + responseData.getCharSet(), e);
-            return baos.toString();
-        }
-    }
-
     protected String getSingleNodeValue(final Document document,
             final String xpath, final boolean pruned) {
         Node value = null;

src/main/java/jp/sf/fess/util/ParameterUtil.java

@@ -20,10 +20,19 @@ import java.util.HashMap;
 import java.util.Map;
 
 import jp.sf.fess.FessSystemException;
+import jp.sf.fess.db.exentity.CrawlingConfig.ConfigName;
 
 import org.codelibs.core.util.StringUtil;
 
 public class ParameterUtil {
+    protected static final String XPATH_PREFIX = "field.xpath.";
+
+    protected static final String SCRIPT_PREFIX = "field.script.";
+
+    protected static final String CLIENT_PREFIX = "client.";
+
+    protected static final String FIELD_PREFIX = "field.config.";
+
     protected ParameterUtil() {
         // nothing
     }
@@ -62,4 +71,36 @@ public class ParameterUtil {
             paramMap.putAll(map);
         }
     }
+
+    public static Map<ConfigName, Map<String, String>> createConfigParameterMap(
+            final String configParameters) {
+        final Map<ConfigName, Map<String, String>> map = new HashMap<>();
+        final Map<String, String> clientConfigMap = new HashMap<>();
+        final Map<String, String> xpathConfigMap = new HashMap<>();
+        final Map<String, String> scriptConfigMap = new HashMap<>();
+        final Map<String, String> fieldConfigMap = new HashMap<>();
+        map.put(ConfigName.CLIENT, clientConfigMap);
+        map.put(ConfigName.XPATH, xpathConfigMap);
+        map.put(ConfigName.SCRIPT, scriptConfigMap);
+        map.put(ConfigName.FIELD, fieldConfigMap);
+        for (final Map.Entry<String, String> entry : ParameterUtil.parse(
+                configParameters).entrySet()) {
+            final String key = entry.getKey();
+            if (key.startsWith(CLIENT_PREFIX)) {
+                clientConfigMap.put(key.substring(CLIENT_PREFIX.length()),
+                        entry.getValue());
+            } else if (key.startsWith(XPATH_PREFIX)) {
+                xpathConfigMap.put(key.substring(XPATH_PREFIX.length()),
+                        entry.getValue());
+            } else if (key.startsWith(SCRIPT_PREFIX)) {
+                scriptConfigMap.put(key.substring(SCRIPT_PREFIX.length()),
+                        entry.getValue());
+            } else if (key.startsWith(FIELD_PREFIX)) {
+                fieldConfigMap.put(key.substring(FIELD_PREFIX.length()),
+                        entry.getValue());
+            }
+        }
+
+        return map;
+    }
 }

src/main/resources/app.dicon

@@ -74,9 +74,11 @@
 		</initMethod>
 		<!-- 
 		<property name="additionalGeoQuery">"location_i_i:1"</property>
-		<property name="responseFields">new String[]{"id", "docId", "score", "boost",
-            "contentLength", "host", "site", "lastModified", "mimetype", "filetype_s",
-            "created", "title", "digest", "url", "screenshot_s_s", "cid_s_s", "lang_s"}</property>
+		<property name="responseFields">new String[]{ "id", "docId", "score",
+            "boost", "contentLength", "host", "site", "lastModified",
+            "mimetype", "filetype_s", "created", TITLE_FIELD, "digest", "url",
+            "clickCount_l_x_dv", "favoriteCount_l_x_dv", "screenshot_s_s",
+            "cid_s_s", "lang_s", "hasCache_s_s" }</property>
 		<property name="responseDocValuesFields">new String[]{
             "clickCount_l_x_dv", "favoriteCount_l_x_dv"}</property>
 		<property name="highlightingFields">new String[]{"digest", "cache" }</property>

src/main/resources/application.properties

@@ -359,6 +359,7 @@ labels.search_result_favorite=Vote it
 labels.search_result_favorited=Voted
 labels.search_click_count=Click: {0}
 labels.search_result_more=more..
+labels.search_result_cache=Cache
 
 label.facet_label_title=Label
 label.facet_lastModified_title=Term

src/main/resources/application_ja.properties

@@ -359,6 +359,7 @@ labels.search_result_favorite=1\u7968\u5165\u308c\u308b
 labels.search_result_favorited=\u6295\u7968\u6e08\u307f
 labels.search_click_count=\u30af\u30ea\u30c3\u30af\u6570: {0}
 labels.search_result_more=\u8a73\u7d30..
+labels.search_result_cache=\u30ad\u30e3\u30c3\u30b7\u30e5
 
 label.facet_label_title=\u30e9\u30d9\u30eb
 label.facet_lastModified_title=\u671f\u9593

src/main/webapp/WEB-INF/view/searchResults.jsp

@@ -35,6 +35,10 @@
 						<div class="description">${doc.contentDescription}</div>
 						<div class="site ellipsis">
 							<cite>${f:h(doc.site)}</cite>
+<!--							<c:if test="${doc.hasCache_s_s=='true'}">
+								<a href="#${doc.docId}" class="cache"><bean:message
+										key="labels.search_result_cache" /></a>
+							</c:if> -->
 						</div>
 						<div class="more visible-phone">
 							<a href="#result${s.index}"><bean:message key="labels.search_result_more" /></a>

src/main/webapp/css/style.css

@@ -47,6 +47,10 @@ h3 {
 	color: #093;
 }
 
+#result .body a.cache {
+	color: #093;
+}
+
 #result .favorited {
 	display: none;
 }