fix #778 add important_content

plugin.xml

@@ -71,8 +71,8 @@
 			<param name="plugin.groupId" value="org/codelibs" />
 			<param name="plugin.name.prefix" value="elasticsearch-" />
 			<param name="plugin.name" value="langfield" />
-			<param name="plugin.version" value="2.4.0" />
-			<param name="plugin.zip.version" value="2.4.0" />
+			<param name="plugin.version" value="2.4.1" />
+			<param name="plugin.zip.version" value="2.4.1" />
 		</antcall>
 		<!-- kopf -->
 		<get dest="${target.dir}">

src/main/java/org/codelibs/fess/app/web/admin/upgrade/AdminUpgradeAction.java

@@ -234,6 +234,18 @@ public class AdminUpgradeAction extends FessAdminAction {
                 "{\"properties\":{\"filename\":{\"type\":\"string\",\"index\":\"not_analyzed\"}}}");
         addFieldMapping(indicesClient, searchIndex, "doc", "filename",
                 "{\"properties\":{\"filename\":{\"type\":\"string\",\"index\":\"not_analyzed\"}}}");
+        addFieldMapping(
+                indicesClient,
+                updateIndex,
+                "doc",
+                "important_content",
+                "{\"properties\":{\"important_content\":{\"type\":\"langstring\",\"lang_field\":\"lang\",\"lang_base_name\":\"content\",\"index\":\"no\"}}}");
+        addFieldMapping(
+                indicesClient,
+                searchIndex,
+                "doc",
+                "important_content",
+                "{\"properties\":{\"important_content\":{\"type\":\"langstring\",\"lang_field\":\"lang\",\"lang_base_name\":\"content\",\"index\":\"no\"}}}");
         addFieldMapping(indicesClient, configIndex, "job_log", "lastUpdated", "{\"properties\":{\"lastUpdated\":{\"type\":\"long\"}}}");
 
         // data migration

src/main/java/org/codelibs/fess/es/client/FessEsClient.java

@@ -61,6 +61,7 @@ import org.codelibs.fess.exception.SearchQueryException;
 import org.codelibs.fess.helper.QueryHelper;
 import org.codelibs.fess.mylasta.direction.FessConfig;
 import org.codelibs.fess.util.ComponentUtil;
+import org.codelibs.fess.util.DocMap;
 import org.dbflute.exception.IllegalBehaviorStateException;
 import org.dbflute.optional.OptionalEntity;
 import org.elasticsearch.ElasticsearchException;
@@ -858,7 +859,7 @@ public class FessEsClient implements Client {
         final BulkRequestBuilder bulkRequestBuilder = client.prepareBulk();
         for (final Map<String, Object> doc : docList) {
             final Object id = doc.remove(fessConfig.getIndexFieldId());
-            bulkRequestBuilder.add(client.prepareIndex(index, type, id.toString()).setSource(doc));
+            bulkRequestBuilder.add(client.prepareIndex(index, type, id.toString()).setSource(new DocMap(doc)));
         }
         final BulkResponse response = bulkRequestBuilder.execute().actionGet(ComponentUtil.getFessConfig().getIndexBulkTimeout());
         if (response.hasFailures()) {
@@ -1027,12 +1028,12 @@ public class FessEsClient implements Client {
             if (id == null) {
                 // create
                 response =
-                        client.prepareIndex(index, type).setSource(source).setRefresh(true).setOpType(OpType.CREATE).execute()
+                        client.prepareIndex(index, type).setSource(new DocMap(source)).setRefresh(true).setOpType(OpType.CREATE).execute()
                                 .actionGet(fessConfig.getIndexIndexTimeout());
             } else {
                 // create or update
                 final IndexRequestBuilder builder =
-                        client.prepareIndex(index, type, id).setSource(source).setRefresh(true).setOpType(OpType.INDEX);
+                        client.prepareIndex(index, type, id).setSource(new DocMap(source)).setRefresh(true).setOpType(OpType.INDEX);
                 if (version != null && version.longValue() > 0) {
                     builder.setVersion(version);
                 }

src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java

@@ -319,6 +319,9 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
     /** The key of the configuration. e.g. parent_id */
     String INDEX_FIELD_parent_id = "index.field.parent_id";
 
+    /** The key of the configuration. e.g. important_content */
+    String INDEX_FIELD_important_content = "index.field.important_content";
+
     /** The key of the configuration. e.g. content */
     String INDEX_FIELD_CONTENT = "index.field.content";
 
@@ -1995,6 +1998,13 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
      */
     String getIndexFieldParentId();
 
+    /**
+     * Get the value for the key 'index.field.important_content'. <br>
+     * The value is, e.g. important_content <br>
+     * @return The value of found property. (NotNull: if not found, exception but basically no way)
+     */
+    String getIndexFieldImportantContent();
+
     /**
      * Get the value for the key 'index.field.content'. <br>
      * The value is, e.g. content <br>
@@ -4842,6 +4852,10 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
             return get(FessConfig.INDEX_FIELD_parent_id);
         }
 
+        public String getIndexFieldImportantContent() {
+            return get(FessConfig.INDEX_FIELD_important_content);
+        }
+
         public String getIndexFieldContent() {
             return get(FessConfig.INDEX_FIELD_CONTENT);
         }

src/main/java/org/codelibs/fess/util/DocMap.java

@@ -0,0 +1,110 @@
+/*
+ * Copyright 2012-2016 CodeLibs Project and the Others.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+ * either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+package org.codelibs.fess.util;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+public class DocMap implements Map<String, Object> {
+
+    private static final String LANG_KEY = "lang";
+    private Map<String, Object> parent;
+
+    public DocMap(final Map<String, Object> parent) {
+        this.parent = parent;
+    }
+
+    @Override
+    public int size() {
+        return parent.size();
+    }
+
+    @Override
+    public boolean isEmpty() {
+        return parent.isEmpty();
+    }
+
+    @Override
+    public boolean containsKey(Object key) {
+        return parent.containsKey(key);
+    }
+
+    @Override
+    public boolean containsValue(Object value) {
+        return parent.containsValue(value);
+    }
+
+    @Override
+    public Object get(Object key) {
+        return parent.get(key);
+    }
+
+    @Override
+    public Object put(String key, Object value) {
+        return parent.put(key, value);
+    }
+
+    @Override
+    public Object remove(Object key) {
+        return parent.remove(key);
+    }
+
+    @Override
+    public void putAll(Map<? extends String, ? extends Object> m) {
+        parent.putAll(m);
+    }
+
+    @Override
+    public void clear() {
+        parent.clear();
+    }
+
+    @Override
+    public Set<String> keySet() {
+        return parent.keySet();
+    }
+
+    @Override
+    public Collection<Object> values() {
+        return parent.values();
+    }
+
+    @Override
+    public Set<java.util.Map.Entry<String, Object>> entrySet() {
+        if (parent.containsKey(LANG_KEY)) {
+            final List<java.util.Map.Entry<String, Object>> list = new ArrayList<>(parent.entrySet());
+            Collections.sort(list, (o1, o2) -> {
+                final String k1 = o1.getKey();
+                if (LANG_KEY.equals(k1)) {
+                    return -1;
+                }
+                final String k2 = o2.getKey();
+                if (LANG_KEY.equals(k2)) {
+                    return -1;
+                }
+                return k1.compareTo(k2);
+            });
+            return new LinkedHashSet<>(list);
+        }
+        return parent.entrySet();
+    }
+
+}

src/main/resources/crawler/transformer.xml

@@ -9,7 +9,7 @@
 		<property name="featureMap">defaultFeatureMap</property>
 		<property name="propertyMap">defaultPropertyMap</property>
 		<property name="childUrlRuleMap">htmlUrlRuleMap</property>
-		<!-- 
+		<!--
 		<property name="invalidUrlPattern">@java.util.regex.Pattern@compile("^\\s*javascript:|^\\s*mailto:|^\\s*irc:|^\\s*skype:|^\\s*callto:",@java.util.regex.Pattern@CASE_INSENSITIVE)</property>
 		 -->
 		<property name="convertUrlMap">
@@ -20,6 +20,10 @@
 			<arg>"title"</arg>
 			<arg>"//TITLE"</arg>
 		</postConstruct>
+		<postConstruct name="addFieldRule">
+			<arg>"important_content"</arg>
+			<arg>"//*[self::H1 or self::H2 or self::H3]"</arg>
+		</postConstruct>
 	</component>
 
 	<component name="fessFileTransformer" class="org.codelibs.fess.crawler.transformer.FessFileTransformer" instance="singleton">

src/main/resources/fess_config.properties

@@ -157,6 +157,7 @@ index.field.timestamp=timestamp
 index.field.label=label
 index.field.mimetype=mimetype
 index.field.parent_id=parent_id
+index.field.important_content=important_content
 index.field.content=content
 index.field.cache=cache
 index.field.digest=digest

src/main/resources/fess_indices/fess/doc.json

@@ -446,6 +446,12 @@
         "type": "string",
         "index": "not_analyzed"
       },
+      "important_content": {
+        "type": "langstring",
+        "lang_field": "lang",
+        "lang_base_name": "title",
+        "index": "no"
+      },
       "content": {
         "type": "langstring",
         "lang_field": "lang",