fix #2044 add LanguageHelper

src/main/java/org/codelibs/fess/ds/callback/IndexUpdateCallbackImpl.java

@@ -87,6 +87,8 @@ public class IndexUpdateCallbackImpl implements IndexUpdateCallback {
             dataMap.put(fessConfig.getIndexFieldDocId(), systemHelper.generateDocId(dataMap));
         }
 
+        ComponentUtil.getLanguageHelper().updateDocument(dataMap);
+
         synchronized (docList) {
             docList.add(dataMap);
             if (logger.isDebugEnabled()) {

src/main/java/org/codelibs/fess/helper/LanguageHelper.java

@@ -0,0 +1,63 @@
+/*
+ * Copyright 2012-2019 CodeLibs Project and the Others.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+ * either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+package org.codelibs.fess.helper;
+
+import java.util.Map;
+
+import javax.annotation.PostConstruct;
+
+import org.codelibs.fess.mylasta.direction.FessConfig;
+import org.codelibs.fess.util.ComponentUtil;
+import org.codelibs.fess.util.DocumentUtil;
+
+public class LanguageHelper {
+
+    protected String[] langFields;
+
+    protected String[] supportedLanguages;
+
+    @PostConstruct
+    public void init() {
+        final FessConfig fessConfig = ComponentUtil.getFessConfig();
+        langFields = fessConfig.getIndexerLanguageFieldsAsArray();
+        supportedLanguages = fessConfig.getSupportedLanguagesAsArray();
+    }
+
+    public void updateDocument(final Map<String, Object> doc) {
+        final String language =
+                getSupportedLanguage(DocumentUtil.getValue(doc, ComponentUtil.getFessConfig().getIndexFieldLang(), String.class));
+        if (language == null) {
+            return;
+        }
+
+        for (final String f : langFields) {
+            final String lf = f + "_" + language;
+            if (doc.containsKey(f) && !doc.containsKey(lf)) {
+                doc.put(lf, doc.get(f));
+            }
+        }
+    }
+
+    protected String getSupportedLanguage(final String lang) {
+        for (final String l : supportedLanguages) {
+            if (l.equals(lang)) {
+                return l;
+            }
+        }
+        return null;
+    }
+
+}

src/main/java/org/codelibs/fess/indexer/IndexUpdater.java

@@ -419,6 +419,8 @@ public class IndexUpdater extends Thread {
         if (!map.containsKey(fessConfig.getIndexFieldDocId())) {
             map.put(fessConfig.getIndexFieldDocId(), systemHelper.generateDocId(map));
         }
+
+        ComponentUtil.getLanguageHelper().updateDocument(map);
     }
 
     protected void addBoostValue(final Map<String, Object> map, final float documentBoost) {

src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java

@@ -384,6 +384,9 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
     /** The key of the configuration. e.g. 10485760 */
     String INDEXER_DATA_MAX_DOCUMENT_REQUEST_SIZE = "indexer.data.max.document.request.size";
 
+    /** The key of the configuration. e.g. content,important_content,title */
+    String INDEXER_LANGUAGE_FIELDS = "indexer.language.fields";
+
     /** The key of the configuration. e.g. default */
     String INDEX_CODEC = "index.codec";
 
@@ -2609,6 +2612,13 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
      */
     Integer getIndexerDataMaxDocumentRequestSizeAsInteger();
 
+    /**
+     * Get the value for the key 'indexer.language.fields'. <br>
+     * The value is, e.g. content,important_content,title <br>
+     * @return The value of found property. (NotNull: if not found, exception but basically no way)
+     */
+    String getIndexerLanguageFields();
+
     /**
      * Get the value for the key 'index.codec'. <br>
      * The value is, e.g. default <br>
@@ -6683,6 +6693,10 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
             return getAsInteger(FessConfig.INDEXER_DATA_MAX_DOCUMENT_REQUEST_SIZE);
         }
 
+        public String getIndexerLanguageFields() {
+            return get(FessConfig.INDEXER_LANGUAGE_FIELDS);
+        }
+
         public String getIndexCodec() {
             return get(FessConfig.INDEX_CODEC);
         }
@@ -8668,6 +8682,7 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
             defaultMap.put(FessConfig.INDEXER_WEBFS_MAX_DOCUMENT_REQUEST_SIZE, "10485760");
             defaultMap.put(FessConfig.INDEXER_DATA_MAX_DOCUMENT_CACHE_SIZE, "5");
             defaultMap.put(FessConfig.INDEXER_DATA_MAX_DOCUMENT_REQUEST_SIZE, "10485760");
+            defaultMap.put(FessConfig.INDEXER_LANGUAGE_FIELDS, "content,important_content,title");
             defaultMap.put(FessConfig.INDEX_CODEC, "default");
             defaultMap.put(FessConfig.INDEX_number_of_shards, "5");
             defaultMap.put(FessConfig.INDEX_auto_expand_replicas, "0-1");

src/main/java/org/codelibs/fess/mylasta/direction/FessProp.java

@@ -1984,4 +1984,12 @@ public interface FessProp {
         }
         return list;
     }
+
+    String getIndexerLanguageFields();
+
+    default String[] getIndexerLanguageFieldsAsArray() {
+        return split(getIndexerLanguageFields(), ",").get(stream -> stream.map(String::trim).toArray(n -> new String[n]));
+
+    }
+
 }

src/main/java/org/codelibs/fess/util/ComponentUtil.java

@@ -47,6 +47,7 @@ import org.codelibs.fess.helper.IntervalControlHelper;
 import org.codelibs.fess.helper.JobHelper;
 import org.codelibs.fess.helper.KeyMatchHelper;
 import org.codelibs.fess.helper.LabelTypeHelper;
+import org.codelibs.fess.helper.LanguageHelper;
 import org.codelibs.fess.helper.PathMappingHelper;
 import org.codelibs.fess.helper.PermissionHelper;
 import org.codelibs.fess.helper.PopularWordHelper;
@@ -88,6 +89,8 @@ public final class ComponentUtil {
 
     private static Map<String, Object> componentMap = new HashMap<>();
 
+    private static final String LANGUAGE_HELPER = "languageHelper";
+
     private static final String CURL_HELPER = "curlHelper";
 
     private static final String QUERY_STRING_BUILDER = "queryStringBuilder";
@@ -434,6 +437,11 @@ public final class ComponentUtil {
         return getComponent(CURL_HELPER);
     }
 
+    public static LanguageHelper getLanguageHelper() {
+        return getComponent(LANGUAGE_HELPER);
+
+    }
+
     public static <T> T getComponent(final Class<T> clazz) {
         try {
             return SingletonLaContainer.getComponent(clazz);

src/main/resources/fess.xml

@@ -10,6 +10,8 @@
 
 	<component name="curlHelper" class="org.codelibs.fess.helper.CurlHelper">
 	</component>
+	<component name="languageHelper" class="org.codelibs.fess.helper.LanguageHelper">
+	</component>
 	<component name="searchLogHelper" class="org.codelibs.fess.helper.SearchLogHelper">
 		<!-- 
 		<property name="userCheckInterval">5 * 60 * 1000</property>

src/main/resources/fess_config.properties

@@ -219,6 +219,7 @@ indexer.webfs.max.document.cache.size=20
 indexer.webfs.max.document.request.size=10485760
 indexer.data.max.document.cache.size=5
 indexer.data.max.document.request.size=10485760
+indexer.language.fields=content,important_content,title
 
 # index setting
 index.codec=default

src/main/resources/fess_indices/fess.json

@@ -1168,5 +1168,16 @@
         }
       }
     }
+  },
+  "mappings": {
+    "_doc": {
+      "_source": {
+        "excludes": [
+          "content_*",
+          "important_content_*",
+          "title_*"
+        ]
+      }
+    }
   }
 }