diff --git a/src/main/java/org/codelibs/fess/ds/callback/IndexUpdateCallbackImpl.java b/src/main/java/org/codelibs/fess/ds/callback/IndexUpdateCallbackImpl.java index 0bb0710ab..73cc1ace5 100644 --- a/src/main/java/org/codelibs/fess/ds/callback/IndexUpdateCallbackImpl.java +++ b/src/main/java/org/codelibs/fess/ds/callback/IndexUpdateCallbackImpl.java @@ -87,6 +87,8 @@ public class IndexUpdateCallbackImpl implements IndexUpdateCallback { dataMap.put(fessConfig.getIndexFieldDocId(), systemHelper.generateDocId(dataMap)); } + ComponentUtil.getLanguageHelper().updateDocument(dataMap); + synchronized (docList) { docList.add(dataMap); if (logger.isDebugEnabled()) { diff --git a/src/main/java/org/codelibs/fess/helper/LanguageHelper.java b/src/main/java/org/codelibs/fess/helper/LanguageHelper.java new file mode 100644 index 000000000..323a7c2f2 --- /dev/null +++ b/src/main/java/org/codelibs/fess/helper/LanguageHelper.java @@ -0,0 +1,63 @@ +/* + * Copyright 2012-2019 CodeLibs Project and the Others. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ +package org.codelibs.fess.helper; + +import java.util.Map; + +import javax.annotation.PostConstruct; + +import org.codelibs.fess.mylasta.direction.FessConfig; +import org.codelibs.fess.util.ComponentUtil; +import org.codelibs.fess.util.DocumentUtil; + +public class LanguageHelper { + + protected String[] langFields; + + protected String[] supportedLanguages; + + @PostConstruct + public void init() { + final FessConfig fessConfig = ComponentUtil.getFessConfig(); + langFields = fessConfig.getIndexerLanguageFieldsAsArray(); + supportedLanguages = fessConfig.getSupportedLanguagesAsArray(); + } + + public void updateDocument(final Map doc) { + final String language = + getSupportedLanguage(DocumentUtil.getValue(doc, ComponentUtil.getFessConfig().getIndexFieldLang(), String.class)); + if (language == null) { + return; + } + + for (final String f : langFields) { + final String lf = f + "_" + language; + if (doc.containsKey(f) && !doc.containsKey(lf)) { + doc.put(lf, doc.get(f)); + } + } + } + + protected String getSupportedLanguage(final String lang) { + for (final String l : supportedLanguages) { + if (l.equals(lang)) { + return l; + } + } + return null; + } + +} diff --git a/src/main/java/org/codelibs/fess/indexer/IndexUpdater.java b/src/main/java/org/codelibs/fess/indexer/IndexUpdater.java index 9f19c0192..b52d59918 100644 --- a/src/main/java/org/codelibs/fess/indexer/IndexUpdater.java +++ b/src/main/java/org/codelibs/fess/indexer/IndexUpdater.java @@ -419,6 +419,8 @@ public class IndexUpdater extends Thread { if (!map.containsKey(fessConfig.getIndexFieldDocId())) { map.put(fessConfig.getIndexFieldDocId(), systemHelper.generateDocId(map)); } + + ComponentUtil.getLanguageHelper().updateDocument(map); } protected void addBoostValue(final Map map, final float documentBoost) { diff --git a/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java b/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java index af0fdd9b3..de51ffe76 100644 --- a/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java +++ b/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java @@ -384,6 +384,9 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction /** The key of the configuration. e.g. 10485760 */ String INDEXER_DATA_MAX_DOCUMENT_REQUEST_SIZE = "indexer.data.max.document.request.size"; + /** The key of the configuration. e.g. content,important_content,title */ + String INDEXER_LANGUAGE_FIELDS = "indexer.language.fields"; + /** The key of the configuration. e.g. default */ String INDEX_CODEC = "index.codec"; @@ -2609,6 +2612,13 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction */ Integer getIndexerDataMaxDocumentRequestSizeAsInteger(); + /** + * Get the value for the key 'indexer.language.fields'.
+ * The value is, e.g. content,important_content,title
+ * @return The value of found property. (NotNull: if not found, exception but basically no way) + */ + String getIndexerLanguageFields(); + /** * Get the value for the key 'index.codec'.
* The value is, e.g. default
@@ -6683,6 +6693,10 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction return getAsInteger(FessConfig.INDEXER_DATA_MAX_DOCUMENT_REQUEST_SIZE); } + public String getIndexerLanguageFields() { + return get(FessConfig.INDEXER_LANGUAGE_FIELDS); + } + public String getIndexCodec() { return get(FessConfig.INDEX_CODEC); } @@ -8668,6 +8682,7 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction defaultMap.put(FessConfig.INDEXER_WEBFS_MAX_DOCUMENT_REQUEST_SIZE, "10485760"); defaultMap.put(FessConfig.INDEXER_DATA_MAX_DOCUMENT_CACHE_SIZE, "5"); defaultMap.put(FessConfig.INDEXER_DATA_MAX_DOCUMENT_REQUEST_SIZE, "10485760"); + defaultMap.put(FessConfig.INDEXER_LANGUAGE_FIELDS, "content,important_content,title"); defaultMap.put(FessConfig.INDEX_CODEC, "default"); defaultMap.put(FessConfig.INDEX_number_of_shards, "5"); defaultMap.put(FessConfig.INDEX_auto_expand_replicas, "0-1"); diff --git a/src/main/java/org/codelibs/fess/mylasta/direction/FessProp.java b/src/main/java/org/codelibs/fess/mylasta/direction/FessProp.java index b3329cdfb..e36022297 100644 --- a/src/main/java/org/codelibs/fess/mylasta/direction/FessProp.java +++ b/src/main/java/org/codelibs/fess/mylasta/direction/FessProp.java @@ -1984,4 +1984,12 @@ public interface FessProp { } return list; } + + String getIndexerLanguageFields(); + + default String[] getIndexerLanguageFieldsAsArray() { + return split(getIndexerLanguageFields(), ",").get(stream -> stream.map(String::trim).toArray(n -> new String[n])); + + } + } diff --git a/src/main/java/org/codelibs/fess/util/ComponentUtil.java b/src/main/java/org/codelibs/fess/util/ComponentUtil.java index 17082302a..664ff7aad 100644 --- a/src/main/java/org/codelibs/fess/util/ComponentUtil.java +++ b/src/main/java/org/codelibs/fess/util/ComponentUtil.java @@ -47,6 +47,7 @@ import org.codelibs.fess.helper.IntervalControlHelper; import org.codelibs.fess.helper.JobHelper; import org.codelibs.fess.helper.KeyMatchHelper; import org.codelibs.fess.helper.LabelTypeHelper; +import org.codelibs.fess.helper.LanguageHelper; import org.codelibs.fess.helper.PathMappingHelper; import org.codelibs.fess.helper.PermissionHelper; import org.codelibs.fess.helper.PopularWordHelper; @@ -88,6 +89,8 @@ public final class ComponentUtil { private static Map componentMap = new HashMap<>(); + private static final String LANGUAGE_HELPER = "languageHelper"; + private static final String CURL_HELPER = "curlHelper"; private static final String QUERY_STRING_BUILDER = "queryStringBuilder"; @@ -434,6 +437,11 @@ public final class ComponentUtil { return getComponent(CURL_HELPER); } + public static LanguageHelper getLanguageHelper() { + return getComponent(LANGUAGE_HELPER); + + } + public static T getComponent(final Class clazz) { try { return SingletonLaContainer.getComponent(clazz); diff --git a/src/main/resources/fess.xml b/src/main/resources/fess.xml index 000828ee1..d2c30a641 100644 --- a/src/main/resources/fess.xml +++ b/src/main/resources/fess.xml @@ -10,6 +10,8 @@ + +