fix #1124 add languages

src/main/java/org/codelibs/fess/helper/QueryHelper.java

@@ -569,19 +569,17 @@ public class QueryHelper {
                 builder.apply(fessConfig.getIndexFieldContent(), fessConfig.getQueryBoostContentAsDecimal().floatValue());
         boolQuery.should(contentQuery);
         getQueryLanguages().ifPresent(
-                langs -> {
-                    stream(langs).of(
-                            stream -> stream.forEach(lang -> {
-                                final QueryBuilder titleLangQuery =
-                                        builder.apply(fessConfig.getIndexFieldTitle() + "_" + lang, fessConfig
-                                                .getQueryBoostTitleLangAsDecimal().floatValue());
-                                boolQuery.should(titleLangQuery);
-                                final QueryBuilder contentLangQuery =
-                                        builder.apply(fessConfig.getIndexFieldContent() + "_" + lang, fessConfig
-                                                .getQueryBoostContentLangAsDecimal().floatValue());
-                                boolQuery.should(contentLangQuery);
-                            }));
-                });
+                langs -> stream(langs).of(
+                        stream -> stream.forEach(lang -> {
+                            final QueryBuilder titleLangQuery =
+                                    builder.apply(fessConfig.getIndexFieldTitle() + "_" + lang, fessConfig
+                                            .getQueryBoostTitleLangAsDecimal().floatValue());
+                            boolQuery.should(titleLangQuery);
+                            final QueryBuilder contentLangQuery =
+                                    builder.apply(fessConfig.getIndexFieldContent() + "_" + lang, fessConfig
+                                            .getQueryBoostContentLangAsDecimal().floatValue());
+                            boolQuery.should(contentLangQuery);
+                        })));
         return boolQuery;
     }
 

src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java

@@ -124,7 +124,7 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
     /** The key of the configuration. e.g. license.properties */
     String SUPPORTED_UPLOADED_FILES = "supported.uploaded.files";
 
-    /** The key of the configuration. e.g. ar,bg,ca,da,de,el,en,es,eu,fa,fi,fr,ga,gl,hi,hu,hy,id,it,ja,lv,ko,nl,no,pt,ro,ru,sv,th,tr,zh_CN,zh_TW,zh */
+    /** The key of the configuration. e.g. ar,bg,bn,ca,ckb_IQ,cs,da,de,el,en,en_IE,es,et,eu,fa,fi,fr,gl,gu,he,hi,hr,hu,hy,id,it,ja,ko,lt,lv,mk,ml,nl,no,pa,pl,pt,pt_BR,ro,ru,si,sq,sv,ta,te,th,tl,tr,uk,ur,vi,zh_CN,zh_TW,zh */
     String SUPPORTED_LANGUAGES = "supported.languages";
 
     /** The key of the configuration. e.g. 60 */
@@ -538,12 +538,15 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
     bg=bg
     bn=bn
     ca=ca
+    ckb-iq=ckb-iq
+    ckb_IQ=ckb-iq
     cs=cs
     da=da
     de=de
     el=el
     en=en
     en-ie=en-ie
+    en_IE=en-ie
     es=es
     et=et
     eu=eu
@@ -571,6 +574,7 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
     pl=pl
     pt=pt
     pt-br=pt-br
+    pt_BR=pt-br
     ro=ro
     ru=ru
     si=si
@@ -584,9 +588,11 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
     uk=uk
     ur=ur
     vi=vi
-    zh=zh-cn
     zh-cn=zh-cn
+    zh_CN=zh-cn
     zh-tw=zh-tw
+    zh_TW=zh-tw
+    zh=zh
     */
     String QUERY_LANGUAGE_MAPPING = "query.language.mapping";
 
@@ -1469,7 +1475,7 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
 
     /**
      * Get the value for the key 'supported.languages'. <br>
-     * The value is, e.g. ar,bg,ca,da,de,el,en,es,eu,fa,fi,fr,ga,gl,hi,hu,hy,id,it,ja,lv,ko,nl,no,pt,ro,ru,sv,th,tr,zh_CN,zh_TW,zh <br>
+     * The value is, e.g. ar,bg,bn,ca,ckb_IQ,cs,da,de,el,en,en_IE,es,et,eu,fa,fi,fr,gl,gu,he,hi,hr,hu,hy,id,it,ja,ko,lt,lv,mk,ml,nl,no,pa,pl,pt,pt_BR,ro,ru,si,sq,sv,ta,te,th,tl,tr,uk,ur,vi,zh_CN,zh_TW,zh <br>
      * @return The value of found property. (NotNull: if not found, exception but basically no way)
      */
     String getSupportedLanguages();
@@ -2884,12 +2890,15 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
     bg=bg
     bn=bn
     ca=ca
+    ckb-iq=ckb-iq
+    ckb_IQ=ckb-iq
     cs=cs
     da=da
     de=de
     el=el
     en=en
     en-ie=en-ie
+    en_IE=en-ie
     es=es
     et=et
     eu=eu
@@ -2917,6 +2926,7 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
     pl=pl
     pt=pt
     pt-br=pt-br
+    pt_BR=pt-br
     ro=ro
     ru=ru
     si=si
@@ -2930,9 +2940,11 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
     uk=uk
     ur=ur
     vi=vi
-    zh=zh-cn
     zh-cn=zh-cn
+    zh_CN=zh-cn
     zh-tw=zh-tw
+    zh_TW=zh-tw
+    zh=zh
     <br>
      * @return The value of found property. (NotNull: if not found, exception but basically no way)
      */

src/main/java/org/codelibs/fess/mylasta/direction/FessProp.java

@@ -68,6 +68,8 @@ import org.lastaflute.web.validation.theme.typed.LongTypeValidator;
 
 public interface FessProp {
 
+    public static final String QUERY_DEFAULT_LANGUAGES = "queryDefaultLanguages";
+
     public static final String HTML_PROXY = "httpProxy";
 
     public static final String CRAWLER_FAILURE_URL_STATUS_CODES = "crawlerFailureUrlStatusCodes";
@@ -848,55 +850,67 @@ public interface FessProp {
 
     String getQueryLanguageMapping();
 
+    public default String[] normalizeQueryLanguages(final String[] langs) {
+        @SuppressWarnings("unchecked")
+        Map<String, String> params = (Map<String, String>) propMap.get(QUERY_LANGUAGE_MAPPING);
+        if (params == null) {
+            params = stream(getQueryLanguageMapping().split("\n")).get(stream -> stream.filter(StringUtil::isNotBlank).map(v -> {
+                final String[] values = v.split("=");
+                if (values.length == 2) {
+                    return new Pair<>(values[0], values[1]);
+                }
+                return null;
+            }).collect(Collectors.toMap(Pair::getFirst, d -> d.getSecond())));
+            propMap.put(QUERY_LANGUAGE_MAPPING, params);
+        }
+        final Map<String, String> mapping = params;
+        return stream(langs).get(stream -> stream.map(s -> {
+            if (StringUtil.isBlank(s)) {
+                return null;
+            }
+            final String lang1 = mapping.get(s);
+            if (lang1 != null) {
+                return lang1;
+            }
+            final String lang2 = mapping.get(s.split("[\\-_]")[0]);
+            if (lang2 != null) {
+                return lang2;
+            }
+            return null;
+        }).filter(StringUtil::isNotBlank).distinct().toArray(n -> new String[n]));
+    }
+
     public default String[] getQueryLanguages(final Enumeration<Locale> locales, final String[] requestLangs) {
+        // requestLangs > default > browser
         if (StringUtil.isNotBlank(getQueryDefaultLanguages())) {
-            String[] langs = (String[]) propMap.get("queryDefaultLanguages");
+            String[] langs = (String[]) propMap.get(QUERY_DEFAULT_LANGUAGES);
             if (langs == null) {
                 langs = split(getQueryDefaultLanguages(), ",").get(stream -> stream.map(s -> s.trim()).toArray(n -> new String[n]));
-                propMap.put("queryDefaultLanguages", langs);
+                propMap.put(QUERY_DEFAULT_LANGUAGES, langs);
 
             }
-            return langs;
+            return normalizeQueryLanguages(langs);
         }
 
         if (requestLangs != null && requestLangs.length != 0) {
-            return requestLangs;
+            return normalizeQueryLanguages(requestLangs);
         }
 
         if (locales == null) {
             return StringUtil.EMPTY_STRINGS;
         }
 
-        @SuppressWarnings("unchecked")
-        Map<String, String> params = (Map<String, String>) propMap.get(QUERY_LANGUAGE_MAPPING);
-        if (params == null) {
-            params = stream(getQueryLanguageMapping().split("\n")).get(stream -> stream.filter(StringUtil::isNotBlank).map(v -> {
-                final String[] values = v.split("=");
-                if (values.length == 2) {
-                    return new Pair<>(values[0], values[1]);
-                }
-                return null;
-            }).collect(Collectors.toMap(Pair::getFirst, d -> d.getSecond())));
-            propMap.put(QUERY_LANGUAGE_MAPPING, params);
-        }
-
-        final Map<String, String> mapping = params;
-        return Collections.list(locales).stream().map(locale -> {
+        return normalizeQueryLanguages(Collections.list(locales).stream().map(locale -> {
             final String language = locale.getLanguage();
             final String country = locale.getCountry();
             if (StringUtil.isNotBlank(language)) {
                 if (StringUtil.isNotBlank(country)) {
-                    final String lang = language.toLowerCase(Locale.ROOT) + "-" + country.toLowerCase(Locale.ROOT);
-                    if (mapping.containsKey(lang)) {
-                        return mapping.get(lang);
-                    }
-                }
-                if (mapping.containsKey(language)) {
-                    return mapping.get(language);
+                    return language.toLowerCase(Locale.ROOT) + "-" + country.toLowerCase(Locale.ROOT);
                 }
+                return language.toLowerCase(Locale.ROOT);
             }
             return null;
-        }).filter(l -> l != null).distinct().toArray(n -> new String[n]);
+        }).toArray(n -> new String[n]));
     }
 
     String getSupportedUploadedFiles();

src/main/resources/fess_config.properties

@@ -75,7 +75,7 @@ supported.uploaded.js.extentions=js
 supported.uploaded.css.extentions=css
 supported.uploaded.media.extentions=jpg,jpeg,gif,png,swf
 supported.uploaded.files=license.properties
-supported.languages=ar,bg,ca,da,de,el,en,es,eu,fa,fi,fr,ga,gl,hi,hu,hy,id,it,ja,lv,ko,nl,no,pt,ro,ru,sv,th,tr,zh_CN,zh_TW,zh
+supported.languages=ar,bg,bn,ca,ckb_IQ,cs,da,de,el,en,en_IE,es,et,eu,fa,fi,fr,gl,gu,he,hi,hr,hu,hy,id,it,ja,ko,lt,lv,mk,ml,nl,no,pa,pl,pt,pt_BR,ro,ru,si,sq,sv,ta,te,th,tl,tr,uk,ur,vi,zh_CN,zh_TW,zh
 api.access.token.length=60
 api.access.token.required=false
 api.admin.access.permissions=Radmin-api
@@ -246,12 +246,15 @@ ar=ar\n\
 bg=bg\n\
 bn=bn\n\
 ca=ca\n\
+ckb-iq=ckb-iq\n\
+ckb_IQ=ckb-iq\n\
 cs=cs\n\
 da=da\n\
 de=de\n\
 el=el\n\
 en=en\n\
 en-ie=en-ie\n\
+en_IE=en-ie\n\
 es=es\n\
 et=et\n\
 eu=eu\n\
@@ -279,6 +282,7 @@ pa=pa\n\
 pl=pl\n\
 pt=pt\n\
 pt-br=pt-br\n\
+pt_BR=pt-br\n\
 ro=ro\n\
 ru=ru\n\
 si=si\n\
@@ -292,9 +296,11 @@ tr=tr\n\
 uk=uk\n\
 ur=ur\n\
 vi=vi\n\
-zh=zh-cn\n\
 zh-cn=zh-cn\n\
+zh_CN=zh-cn\n\
 zh-tw=zh-tw\n\
+zh_TW=zh-tw\n\
+zh=zh\n\
 
 # boost
 query.boost.title=0.2

src/test/java/org/codelibs/fess/mylasta/direction/FessPropTest.java

@@ -17,6 +17,7 @@ package org.codelibs.fess.mylasta.direction;
 
 import java.io.File;
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.HashMap;
 
 import org.codelibs.core.io.FileUtil;
@@ -119,4 +120,31 @@ public class FessPropTest extends UnitFessTestCase {
         assertEquals(12288, spaceChars[1]);
     }
 
+    public void test_normalizeQueryLanguages() {
+        FessProp.propMap.clear();
+        FessConfig fessConfig = new FessConfig.SimpleImpl() {
+            @Override
+            public String getQueryLanguageMapping() {
+                return "ja=ja\nzh_cn=zh-cn\nzh_TW=zh-tw\nzh=zh-cn";
+            }
+        };
+
+        assertArrays(new String[] {}, fessConfig.normalizeQueryLanguages(new String[] {}));
+        assertArrays(new String[] {}, fessConfig.normalizeQueryLanguages(new String[] { "unknown" }));
+        assertArrays(new String[] { "ja" }, fessConfig.normalizeQueryLanguages(new String[] { "ja" }));
+        assertArrays(new String[] { "ja" }, fessConfig.normalizeQueryLanguages(new String[] { "ja", "ja" }));
+        assertArrays(new String[] { "ja" }, fessConfig.normalizeQueryLanguages(new String[] { "ja-jp" }));
+        assertArrays(new String[] { "ja" }, fessConfig.normalizeQueryLanguages(new String[] { "ja_JP" }));
+        assertArrays(new String[] { "ja", "zh-cn" }, fessConfig.normalizeQueryLanguages(new String[] { "ja", "zh" }));
+        assertArrays(new String[] { "ja", "zh-cn" }, fessConfig.normalizeQueryLanguages(new String[] { "ja", "zh_CN" }));
+        assertArrays(new String[] { "ja", "zh-cn" }, fessConfig.normalizeQueryLanguages(new String[] { "ja", "zh-cn" }));
+        assertArrays(new String[] { "zh-cn" }, fessConfig.normalizeQueryLanguages(new String[] { "zh", "zh-cn" }));
+        assertArrays(new String[] { "zh-tw" }, fessConfig.normalizeQueryLanguages(new String[] { "zh_TW" }));
+    }
+
+    private void assertArrays(final String[] expected, final String[] actual) {
+        Arrays.sort(expected);
+        Arrays.sort(actual);
+        assertEquals(String.join(",", expected), String.join(",", actual));
+    }
 }