0ct0pu5/fess
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix #1124 add languages

Browse source
This commit is contained in:
Shinsuke Sugaya 2017-06-22 21:40:56 +09:00
parent 1967e387e4
commit 219696b906
5 changed files with 106 additions and 48 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
src/main/java/org/codelibs/fess/helper/QueryHelper.java
View file

@ -569,19 +569,17 @@ public class QueryHelper {
builder.apply(fessConfig.getIndexFieldContent(), fessConfig.getQueryBoostContentAsDecimal().floatValue());
boolQuery.should(contentQuery);
getQueryLanguages().ifPresent(
langs -> {
stream(langs).of(
stream -> stream.forEach(lang -> {
final QueryBuilder titleLangQuery =
builder.apply(fessConfig.getIndexFieldTitle() + "_" + lang, fessConfig
.getQueryBoostTitleLangAsDecimal().floatValue());
boolQuery.should(titleLangQuery);
final QueryBuilder contentLangQuery =
builder.apply(fessConfig.getIndexFieldContent() + "_" + lang, fessConfig
.getQueryBoostContentLangAsDecimal().floatValue());
boolQuery.should(contentLangQuery);
}));
});
langs -> stream(langs).of(
stream -> stream.forEach(lang -> {
final QueryBuilder titleLangQuery =
builder.apply(fessConfig.getIndexFieldTitle() + "_" + lang, fessConfig
.getQueryBoostTitleLangAsDecimal().floatValue());
boolQuery.should(titleLangQuery);
final QueryBuilder contentLangQuery =
builder.apply(fessConfig.getIndexFieldContent() + "_" + lang, fessConfig
.getQueryBoostContentLangAsDecimal().floatValue());
boolQuery.should(contentLangQuery);
})));
return boolQuery;
}

20
src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java
View file

@ -124,7 +124,7 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
/** The key of the configuration. e.g. license.properties */
String SUPPORTED_UPLOADED_FILES = "supported.uploaded.files";
/** The key of the configuration. e.g. ar,bg,ca,da,de,el,en,es,eu,fa,fi,fr,ga,gl,hi,hu,hy,id,it,ja,lv,ko,nl,no,pt,ro,ru,sv,th,tr,zh_CN,zh_TW,zh */
/** The key of the configuration. e.g. ar,bg,bn,ca,ckb_IQ,cs,da,de,el,en,en_IE,es,et,eu,fa,fi,fr,gl,gu,he,hi,hr,hu,hy,id,it,ja,ko,lt,lv,mk,ml,nl,no,pa,pl,pt,pt_BR,ro,ru,si,sq,sv,ta,te,th,tl,tr,uk,ur,vi,zh_CN,zh_TW,zh */
String SUPPORTED_LANGUAGES = "supported.languages";
/** The key of the configuration. e.g. 60 */
@ -538,12 +538,15 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
bg=bg
bn=bn
ca=ca
ckb-iq=ckb-iq
ckb_IQ=ckb-iq
cs=cs
da=da
de=de
el=el
en=en
en-ie=en-ie
en_IE=en-ie
es=es
et=et
eu=eu
@ -571,6 +574,7 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
pl=pl
pt=pt
pt-br=pt-br
pt_BR=pt-br
ro=ro
ru=ru
si=si
@ -584,9 +588,11 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
uk=uk
ur=ur
vi=vi
zh=zh-cn
zh-cn=zh-cn
zh_CN=zh-cn
zh-tw=zh-tw
zh_TW=zh-tw
zh=zh
*/
String QUERY_LANGUAGE_MAPPING = "query.language.mapping";
@ -1469,7 +1475,7 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
/**
* Get the value for the key 'supported.languages'. <br>
* The value is, e.g. ar,bg,ca,da,de,el,en,es,eu,fa,fi,fr,ga,gl,hi,hu,hy,id,it,ja,lv,ko,nl,no,pt,ro,ru,sv,th,tr,zh_CN,zh_TW,zh <br>
* The value is, e.g. ar,bg,bn,ca,ckb_IQ,cs,da,de,el,en,en_IE,es,et,eu,fa,fi,fr,gl,gu,he,hi,hr,hu,hy,id,it,ja,ko,lt,lv,mk,ml,nl,no,pa,pl,pt,pt_BR,ro,ru,si,sq,sv,ta,te,th,tl,tr,uk,ur,vi,zh_CN,zh_TW,zh <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getSupportedLanguages();
@ -2884,12 +2890,15 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
bg=bg
bn=bn
ca=ca
ckb-iq=ckb-iq
ckb_IQ=ckb-iq
cs=cs
da=da
de=de
el=el
en=en
en-ie=en-ie
en_IE=en-ie
es=es
et=et
eu=eu
@ -2917,6 +2926,7 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
pl=pl
pt=pt
pt-br=pt-br
pt_BR=pt-br
ro=ro
ru=ru
si=si
@ -2930,9 +2940,11 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
uk=uk
ur=ur
vi=vi
zh=zh-cn
zh-cn=zh-cn
zh_CN=zh-cn
zh-tw=zh-tw
zh_TW=zh-tw
zh=zh
<br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/

72
src/main/java/org/codelibs/fess/mylasta/direction/FessProp.java
View file

@ -68,6 +68,8 @@ import org.lastaflute.web.validation.theme.typed.LongTypeValidator;
public interface FessProp {
public static final String QUERY_DEFAULT_LANGUAGES = "queryDefaultLanguages";
public static final String HTML_PROXY = "httpProxy";
public static final String CRAWLER_FAILURE_URL_STATUS_CODES = "crawlerFailureUrlStatusCodes";
@ -848,25 +850,7 @@ public interface FessProp {
String getQueryLanguageMapping();
public default String[] getQueryLanguages(final Enumeration<Locale> locales, final String[] requestLangs) {
if (StringUtil.isNotBlank(getQueryDefaultLanguages())) {
String[] langs = (String[]) propMap.get("queryDefaultLanguages");
if (langs == null) {
langs = split(getQueryDefaultLanguages(), ",").get(stream -> stream.map(s -> s.trim()).toArray(n -> new String[n]));
propMap.put("queryDefaultLanguages", langs);
}
return langs;
}
if (requestLangs != null && requestLangs.length != 0) {
return requestLangs;
}
if (locales == null) {
return StringUtil.EMPTY_STRINGS;
}
public default String[] normalizeQueryLanguages(final String[] langs) {
@SuppressWarnings("unchecked")
Map<String, String> params = (Map<String, String>) propMap.get(QUERY_LANGUAGE_MAPPING);
if (params == null) {
@ -879,24 +863,54 @@ public interface FessProp {
}).collect(Collectors.toMap(Pair::getFirst, d -> d.getSecond())));
propMap.put(QUERY_LANGUAGE_MAPPING, params);
}
final Map<String, String> mapping = params;
return Collections.list(locales).stream().map(locale -> {
return stream(langs).get(stream -> stream.map(s -> {
if (StringUtil.isBlank(s)) {
return null;
}
final String lang1 = mapping.get(s);
if (lang1 != null) {
return lang1;
}
final String lang2 = mapping.get(s.split("[\\-_]")[0]);
if (lang2 != null) {
return lang2;
}
return null;
}).filter(StringUtil::isNotBlank).distinct().toArray(n -> new String[n]));
}
public default String[] getQueryLanguages(final Enumeration<Locale> locales, final String[] requestLangs) {
// requestLangs > default > browser
if (StringUtil.isNotBlank(getQueryDefaultLanguages())) {
String[] langs = (String[]) propMap.get(QUERY_DEFAULT_LANGUAGES);
if (langs == null) {
langs = split(getQueryDefaultLanguages(), ",").get(stream -> stream.map(s -> s.trim()).toArray(n -> new String[n]));
propMap.put(QUERY_DEFAULT_LANGUAGES, langs);
}
return normalizeQueryLanguages(langs);
}
if (requestLangs != null && requestLangs.length != 0) {
return normalizeQueryLanguages(requestLangs);
}
if (locales == null) {
return StringUtil.EMPTY_STRINGS;
}
return normalizeQueryLanguages(Collections.list(locales).stream().map(locale -> {
final String language = locale.getLanguage();
final String country = locale.getCountry();
if (StringUtil.isNotBlank(language)) {
if (StringUtil.isNotBlank(country)) {
final String lang = language.toLowerCase(Locale.ROOT) + "-" + country.toLowerCase(Locale.ROOT);
if (mapping.containsKey(lang)) {
return mapping.get(lang);
}
}
if (mapping.containsKey(language)) {
return mapping.get(language);
return language.toLowerCase(Locale.ROOT) + "-" + country.toLowerCase(Locale.ROOT);
}
return language.toLowerCase(Locale.ROOT);
}
return null;
}).filter(l -> l != null).distinct().toArray(n -> new String[n]);
}).toArray(n -> new String[n]));
}
String getSupportedUploadedFiles();

10
src/main/resources/fess_config.properties
View file

@ -75,7 +75,7 @@ supported.uploaded.js.extentions=js
supported.uploaded.css.extentions=css
supported.uploaded.media.extentions=jpg,jpeg,gif,png,swf
supported.uploaded.files=license.properties
supported.languages=ar,bg,ca,da,de,el,en,es,eu,fa,fi,fr,ga,gl,hi,hu,hy,id,it,ja,lv,ko,nl,no,pt,ro,ru,sv,th,tr,zh_CN,zh_TW,zh
supported.languages=ar,bg,bn,ca,ckb_IQ,cs,da,de,el,en,en_IE,es,et,eu,fa,fi,fr,gl,gu,he,hi,hr,hu,hy,id,it,ja,ko,lt,lv,mk,ml,nl,no,pa,pl,pt,pt_BR,ro,ru,si,sq,sv,ta,te,th,tl,tr,uk,ur,vi,zh_CN,zh_TW,zh
api.access.token.length=60
api.access.token.required=false
api.admin.access.permissions=Radmin-api
@ -246,12 +246,15 @@ ar=ar\n\
bg=bg\n\
bn=bn\n\
ca=ca\n\
ckb-iq=ckb-iq\n\
ckb_IQ=ckb-iq\n\
cs=cs\n\
da=da\n\
de=de\n\
el=el\n\
en=en\n\
en-ie=en-ie\n\
en_IE=en-ie\n\
es=es\n\
et=et\n\
eu=eu\n\
@ -279,6 +282,7 @@ pa=pa\n\
pl=pl\n\
pt=pt\n\
pt-br=pt-br\n\
pt_BR=pt-br\n\
ro=ro\n\
ru=ru\n\
si=si\n\
@ -292,9 +296,11 @@ tr=tr\n\
uk=uk\n\
ur=ur\n\
vi=vi\n\
zh=zh-cn\n\
zh-cn=zh-cn\n\
zh_CN=zh-cn\n\
zh-tw=zh-tw\n\
zh_TW=zh-tw\n\
zh=zh\n\
# boost
query.boost.title=0.2

28
src/test/java/org/codelibs/fess/mylasta/direction/FessPropTest.java
View file

@ -17,6 +17,7 @@ package org.codelibs.fess.mylasta.direction;
import java.io.File;
import java.io.IOException;
import java.util.Arrays;
import java.util.HashMap;
import org.codelibs.core.io.FileUtil;
@ -119,4 +120,31 @@ public class FessPropTest extends UnitFessTestCase {
assertEquals(12288, spaceChars[1]);
}
public void test_normalizeQueryLanguages() {
FessProp.propMap.clear();
FessConfig fessConfig = new FessConfig.SimpleImpl() {
@Override
public String getQueryLanguageMapping() {
return "ja=ja\nzh_cn=zh-cn\nzh_TW=zh-tw\nzh=zh-cn";
}
};
assertArrays(new String[] {}, fessConfig.normalizeQueryLanguages(new String[] {}));
assertArrays(new String[] {}, fessConfig.normalizeQueryLanguages(new String[] { "unknown" }));
assertArrays(new String[] { "ja" }, fessConfig.normalizeQueryLanguages(new String[] { "ja" }));
assertArrays(new String[] { "ja" }, fessConfig.normalizeQueryLanguages(new String[] { "ja", "ja" }));
assertArrays(new String[] { "ja" }, fessConfig.normalizeQueryLanguages(new String[] { "ja-jp" }));
assertArrays(new String[] { "ja" }, fessConfig.normalizeQueryLanguages(new String[] { "ja_JP" }));
assertArrays(new String[] { "ja", "zh-cn" }, fessConfig.normalizeQueryLanguages(new String[] { "ja", "zh" }));
assertArrays(new String[] { "ja", "zh-cn" }, fessConfig.normalizeQueryLanguages(new String[] { "ja", "zh_CN" }));
assertArrays(new String[] { "ja", "zh-cn" }, fessConfig.normalizeQueryLanguages(new String[] { "ja", "zh-cn" }));
assertArrays(new String[] { "zh-cn" }, fessConfig.normalizeQueryLanguages(new String[] { "zh", "zh-cn" }));
assertArrays(new String[] { "zh-tw" }, fessConfig.normalizeQueryLanguages(new String[] { "zh_TW" }));
}
private void assertArrays(final String[] expected, final String[] actual) {
Arrays.sort(expected);
Arrays.sort(actual);
assertEquals(String.join(",", expected), String.join(",", actual));
}
}