diff --git a/src/main/java/org/codelibs/fess/api/gsa/GsaApiManager.java b/src/main/java/org/codelibs/fess/api/gsa/GsaApiManager.java index 5f0846026..028e6ac6e 100644 --- a/src/main/java/org/codelibs/fess/api/gsa/GsaApiManager.java +++ b/src/main/java/org/codelibs/fess/api/gsa/GsaApiManager.java @@ -16,6 +16,7 @@ package org.codelibs.fess.api.gsa; import java.io.IOException; +import java.io.UnsupportedEncodingException; import java.net.URLDecoder; import java.net.URLEncoder; import java.text.SimpleDateFormat; @@ -60,6 +61,9 @@ import org.slf4j.LoggerFactory; public class GsaApiManager extends BaseApiManager implements WebApiManager { private static final Logger logger = LoggerFactory.getLogger(GsaApiManager.class); + private static final String OUTPUT_XML = "xml"; // or xml_no_dtd + // http://www.google.com/google.dtd. + private static final String GSA_META_SUFFIX = "_s"; protected String gsaPathPrefix = "/gsa"; @@ -94,12 +98,28 @@ public class GsaApiManager extends BaseApiManager implements WebApiManager { } } + protected void appendParam(final StringBuilder buf, final String name, final String value) throws UnsupportedEncodingException { + appendParam(buf, name, value, URLEncoder.encode(value, Constants.UTF_8)); + } + + protected void appendParam(final StringBuilder buf, final String name, final String value, final String original) + throws UnsupportedEncodingException { + buf.append(""); + } + protected void processSearchRequest(final HttpServletRequest request, final HttpServletResponse response, final FilterChain chain) { final SearchService searchService = ComponentUtil.getComponent(SearchService.class); final FessConfig fessConfig = ComponentUtil.getFessConfig(); + final boolean xmlDtd = OUTPUT_XML.equals(request.getParameter("output")); if (!fessConfig.isAcceptedSearchReferer(request.getHeader("referer"))) { - writeXmlResponse(99, false, StringUtil.EMPTY, "Referer is invalid."); + writeXmlResponse(99, xmlDtd, StringUtil.EMPTY, "Referer is invalid."); return; } @@ -108,7 +128,6 @@ public class GsaApiManager extends BaseApiManager implements WebApiManager { String query = null; final StringBuilder buf = new StringBuilder(1000); request.setAttribute(Constants.SEARCH_LOG_ACCESS_TYPE, Constants.SEARCH_LOG_ACCESS_TYPE_GSA); - boolean xmlDtd = false; try { final SearchRenderData data = new SearchRenderData(); final GsaRequestParams params = new GsaRequestParams(request, fessConfig); @@ -124,10 +143,6 @@ public class GsaApiManager extends BaseApiManager implements WebApiManager { if (StringUtil.isNotBlank(getFieldsParam)) { getFields.addAll(Arrays.asList(getFieldsParam.split("\\."))); } - // DTD - if ("xml".equals(request.getParameter("output"))) { - xmlDtd = true; - } final StringBuilder requestUri = new StringBuilder(request.getRequestURI()); if (request.getQueryString() != null) { requestUri.append("?").append(request.getQueryString()); @@ -138,88 +153,92 @@ public class GsaApiManager extends BaseApiManager implements WebApiManager { final String oe = "UTF-8"; // IP address final String ip = ComponentUtil.getViewHelper().getClientIp(request); - final String start = request.getParameter("start"); - long startNumber = 1; - if (StringUtil.isNotBlank(start)) { - startNumber = Long.parseLong(start) + 1; - } - long endNumber = startNumber + data.getPageSize() - 1; - if (endNumber > allRecordCount) { - endNumber = allRecordCount; - } - buf.append(""); - buf.append(escapeXml(query)); - buf.append(""); buf.append(""); buf.append(execTime); buf.append(""); + buf.append(""); + buf.append(escapeXml(query)); + buf.append(""); for (final Entry entry : request.getParameterMap().entrySet()) { final String[] values = entry.getValue(); if (values == null) { continue; } final String key = entry.getKey(); + if ("sort".equals(key)) { + continue; + } for (final String value : values) { - buf.append(""); + appendParam(buf, key, value); } } - buf.append(""); - buf.append(""); - buf.append(""); + appendParam(buf, "ie", ie); + if (request.getParameter("oe") == null) { + appendParam(buf, "oe", oe); + } + final String[] langs = params.getLanguages(); + if (langs.length > 0) { + appendParam(buf, "inlang", langs[0]); + appendParam(buf, "ulang", langs[0]); + } + appendParam(buf, "ip", ip); + appendParam(buf, "access", "p"); + appendParam(buf, "sort", params.getSortParam(), params.getSortParam()); + appendParam(buf, "entqr", "3"); + appendParam(buf, "entqrm", "0"); + appendParam(buf, "wc", "200"); + appendParam(buf, "wc_mc", "1"); if (!documentItems.isEmpty()) { buf.append(""); buf.append(""); buf.append(allRecordCount); buf.append(""); - if (endNumber < allRecordCount) { + if (data.isExistPrevPage() || data.isExistNextPage()) { buf.append(""); - buf.append(""); - buf.append(escapeXml(uriQueryString.replaceFirst("start=([^&]+)", "start=" + endNumber))); - buf.append(""); + if (data.isExistPrevPage()) { + long s = data.getCurrentStartRecordNumber() - data.getPageSize() - 1; + if (s < 0) { + s = 0; + } + buf.append(""); + buf.append(escapeXml(uriQueryString.replaceFirst("start=([0-9]+)", "start=" + s))); + buf.append(""); + } + if (data.isExistNextPage()) { + buf.append(""); + buf.append(escapeXml(uriQueryString.replaceFirst("start=([0-9]+)", "start=" + data.getCurrentEndRecordNumber()))); + buf.append(""); + } buf.append(""); } - long recordNumber = startNumber; + long recordNumber = data.getCurrentStartRecordNumber(); for (final Map document : documentItems) { buf.append(""); final String url = DocumentUtil.getValue(document, fessConfig.getIndexFieldUrl(), String.class); - document.remove(fessConfig.getIndexFieldUrl()); document.put("UE", url); document.put("U", URLDecoder.decode(url, Constants.UTF_8)); - document.put("T", DocumentUtil.getValue(document, fessConfig.getIndexFieldTitle(), String.class)); - document.remove(fessConfig.getIndexFieldTitle()); - final float score = DocumentUtil.getValue(document, fessConfig.getIndexFieldBoost(), Float.class, Float.valueOf(0)); - document.remove(fessConfig.getIndexFieldBoost()); - document.put("RK", (int) (score * 10)); - document.put("S", - DocumentUtil - .getValue(document, fessConfig.getResponseFieldContentDescription(), String.class, StringUtil.EMPTY) - .replaceAll("<(/*)em>", "<$1b>")); - document.remove(fessConfig.getResponseFieldContentDescription()); + document.put("T", DocumentUtil.getValue(document, fessConfig.getResponseFieldContentTitle(), String.class)); + final float score = DocumentUtil.getValue(document, Constants.SCORE, Float.class, Float.NaN); + int rk = 10; + if (!Float.isNaN(score)) { + if (score < 0.0) { + rk = 0; + } else if (score > 1.0) { + rk = 10; + } else { + rk = (int) (score * 10.0); + } + } + document.put("RK", rk); + document.put("S", DocumentUtil.getValue(document, fessConfig.getResponseFieldContentDescription(), String.class, + StringUtil.EMPTY)); final String lang = DocumentUtil.getValue(document, fessConfig.getIndexFieldLang(), String.class); if (StringUtil.isNotBlank(lang)) { document.put("LANG", lang); @@ -251,6 +270,14 @@ public class GsaApiManager extends BaseApiManager implements WebApiManager { } } } + buf.append("").append(escapeXml("FESS")).append(""); + String lastModified = DocumentUtil.getValue(document, fessConfig.getIndexFieldLastModified(), String.class); + if (StringUtil.isBlank(lastModified)) { + lastModified = StringUtil.EMPTY; + } + lastModified = lastModified.split("T")[0]; + buf.append(""); + buf.append(""); buf.append(""); buf.append(" 0) { + return langs; + } + if (request.getHeader("Accept-Language") != null) { + return Collections.list(request.getLocales()).stream().map(l -> l.getLanguage()).toArray(n -> new String[n]); + } + return new String[] { fessConfig.getQueryGsaDefaultLang() }; } @Override @@ -463,9 +506,50 @@ public class GsaApiManager extends BaseApiManager implements WebApiManager { return createFacetInfo(request); } + public String getSortParam() { + return sortParam; + } + @Override public String getSort() { - return request.getParameter("sort"); + // Sort By Relevance (Default) + // Sort By Date: date::: + // Sort by Metadata: meta:::::: + if (StringUtil.isBlank(sortParam)) { + return null; + } + + final String[] values = sortParam.split(":"); + if (values.length > 0) { + if ("date".equals(values[0])) { + final StringBuilder buf = new StringBuilder(); + buf.append(fessConfig.getIndexFieldTimestamp()); + if (values.length > 1) { + if ("A".equals(values[1])) { + buf.append(".asc"); + } else if ("D".equals(values[1])) { + buf.append(".desc"); + } + } + buf.append(",score.desc"); + return buf.toString(); + } else if ("meta".equals(values[0]) && values.length > 1) { + final StringBuilder buf = new StringBuilder(); + buf.append(values[1]); + if (values.length > 2) { + if ("A".equals(values[2])) { + buf.append(".asc"); + } else if ("D".equals(values[2])) { + buf.append(".desc"); + } + } + buf.append(",score.desc"); + return buf.toString(); + } + } + + sortParam = ""; + return null; } @Override diff --git a/src/main/java/org/codelibs/fess/app/web/admin/backup/AdminBackupAction.java b/src/main/java/org/codelibs/fess/app/web/admin/backup/AdminBackupAction.java index cac38e368..9ecc61170 100644 --- a/src/main/java/org/codelibs/fess/app/web/admin/backup/AdminBackupAction.java +++ b/src/main/java/org/codelibs/fess/app/web/admin/backup/AdminBackupAction.java @@ -44,12 +44,16 @@ import org.codelibs.core.misc.Pair; import org.codelibs.elasticsearch.runner.net.CurlResponse; import org.codelibs.fess.Constants; import org.codelibs.fess.app.web.base.FessAdminAction; +import org.codelibs.fess.es.config.exbhv.FileConfigBhv; +import org.codelibs.fess.es.config.exbhv.LabelTypeBhv; +import org.codelibs.fess.es.config.exbhv.WebConfigBhv; import org.codelibs.fess.es.log.exbhv.ClickLogBhv; import org.codelibs.fess.es.log.exbhv.FavoriteLogBhv; import org.codelibs.fess.es.log.exbhv.SearchLogBhv; import org.codelibs.fess.es.log.exbhv.UserInfoBhv; import org.codelibs.fess.mylasta.direction.FessConfig; import org.codelibs.fess.util.ComponentUtil; +import org.codelibs.fess.util.GsaConfigParser; import org.codelibs.fess.util.RenderDataUtil; import org.lastaflute.core.magic.async.AsyncManager; import org.lastaflute.web.Execute; @@ -59,6 +63,7 @@ import org.lastaflute.web.response.StreamResponse; import org.lastaflute.web.ruts.process.ActionRuntime; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.xml.sax.InputSource; /** * @author shinsuke @@ -74,6 +79,15 @@ public class AdminBackupAction extends FessAdminAction { @Resource private AsyncManager asyncManager; + @Resource + private WebConfigBhv webConfigBhv; + + @Resource + private FileConfigBhv fileConfigBhv; + + @Resource + private LabelTypeBhv labelTypeBhv; + @Override protected void setupHtmlData(final ActionRuntime runtime) { super.setupHtmlData(runtime); @@ -98,6 +112,16 @@ public class AdminBackupAction extends FessAdminAction { } catch (final IOException e) { logger.warn("Failed to process system.properties file: " + form.bulkFile.getFileName(), e); } + } else if (fileName.startsWith("gsa") && fileName.endsWith(".xml")) { + GsaConfigParser configParser = ComponentUtil.getComponent(GsaConfigParser.class); + try (final InputStream in = form.bulkFile.getInputStream()) { + configParser.parse(new InputSource(in)); + } catch (final IOException e) { + logger.warn("Failed to process gsa.xml file: " + form.bulkFile.getFileName(), e); + } + configParser.getWebConfig().ifPresent(c -> webConfigBhv.insert(c)); + configParser.getFileConfig().ifPresent(c -> fileConfigBhv.insert(c)); + labelTypeBhv.batchInsert(Arrays.stream(configParser.getLabelTypes()).collect(Collectors.toList())); } else { try (CurlResponse response = ComponentUtil.getCurlHelper().post("/_bulk").onConnect((req, con) -> { con.setDoOutput(true); diff --git a/src/main/java/org/codelibs/fess/app/web/admin/group/AdminGroupAction.java b/src/main/java/org/codelibs/fess/app/web/admin/group/AdminGroupAction.java index 12c6d2edc..483fee3f3 100644 --- a/src/main/java/org/codelibs/fess/app/web/admin/group/AdminGroupAction.java +++ b/src/main/java/org/codelibs/fess/app/web/admin/group/AdminGroupAction.java @@ -255,6 +255,7 @@ public class AdminGroupAction extends FessAdminAction { public static OptionalEntity getGroup(final CreateForm form) { return getEntity(form).map(entity -> { + copyMapToBean(form.attributes, entity, op -> op.exclude(Constants.COMMON_CONVERSION_RULE)); copyBeanToBean(form, entity, op -> op.exclude(Constants.COMMON_CONVERSION_RULE)); return entity; }); diff --git a/src/main/java/org/codelibs/fess/app/web/admin/role/AdminRoleAction.java b/src/main/java/org/codelibs/fess/app/web/admin/role/AdminRoleAction.java index 38b378f53..07050d3c0 100644 --- a/src/main/java/org/codelibs/fess/app/web/admin/role/AdminRoleAction.java +++ b/src/main/java/org/codelibs/fess/app/web/admin/role/AdminRoleAction.java @@ -209,6 +209,7 @@ public class AdminRoleAction extends FessAdminAction { public static OptionalEntity getRole(final CreateForm form) { return getEntity(form).map(entity -> { + copyMapToBean(form.attributes, entity, op -> op.exclude(Constants.COMMON_CONVERSION_RULE)); copyBeanToBean(form, entity, op -> op.exclude(Constants.COMMON_CONVERSION_RULE)); return entity; }); diff --git a/src/main/java/org/codelibs/fess/app/web/admin/systeminfo/AdminSysteminfoAction.java b/src/main/java/org/codelibs/fess/app/web/admin/systeminfo/AdminSysteminfoAction.java index 17d995be2..0356bc718 100644 --- a/src/main/java/org/codelibs/fess/app/web/admin/systeminfo/AdminSysteminfoAction.java +++ b/src/main/java/org/codelibs/fess/app/web/admin/systeminfo/AdminSysteminfoAction.java @@ -26,8 +26,10 @@ import org.codelibs.core.lang.StringUtil; import org.codelibs.core.misc.DynamicProperties; import org.codelibs.fess.Constants; import org.codelibs.fess.app.web.base.FessAdminAction; +import org.codelibs.fess.mylasta.direction.FessConfig; import org.codelibs.fess.util.ComponentUtil; import org.codelibs.fess.util.RenderDataUtil; +import org.lastaflute.core.direction.ObjectiveConfig; import org.lastaflute.web.Execute; import org.lastaflute.web.response.HtmlResponse; import org.lastaflute.web.response.render.RenderData; @@ -38,6 +40,8 @@ import org.lastaflute.web.ruts.process.ActionRuntime; */ public class AdminSysteminfoAction extends FessAdminAction { + private static final String MASKED_VALUE = "XXXXXXXX"; + // =================================================================================== // Attribute // ========= @@ -83,7 +87,7 @@ public class AdminSysteminfoAction extends FessAdminAction { } protected void registerFessPropItems(final RenderData data) { - RenderDataUtil.register(data, "fessPropItems", getFessPropItems()); + RenderDataUtil.register(data, "fessPropItems", getFessPropItems(fessConfig)); } protected void registerBugReportItems(final RenderData data) { @@ -106,15 +110,42 @@ public class AdminSysteminfoAction extends FessAdminAction { return itemList; } - public static List> getFessPropItems() { + public static List> getFessPropItems(final FessConfig fessConfig) { final List> itemList = new ArrayList<>(); - final DynamicProperties systemProperties = ComponentUtil.getSystemProperties(); - for (final Map.Entry entry : systemProperties.entrySet()) { - itemList.add(createItem(entry.getKey(), entry.getValue())); + ComponentUtil.getSystemProperties().entrySet().stream().forEach(e -> { + final String k = e.getKey().toString(); + final String value; + if (isMaskedValue(k)) { + value = MASKED_VALUE; + } else { + value = e.getValue().toString(); + } + itemList.add(createItem(k, value)); + }); + if (fessConfig instanceof ObjectiveConfig) { + ObjectiveConfig config = (ObjectiveConfig) fessConfig; + config.keySet().stream().forEach(k -> { + final String value; + if (isMaskedValue(k)) { + value = MASKED_VALUE; + } else { + value = config.get(k); + } + itemList.add(createItem(k, value)); + }); } return itemList; } + protected static boolean isMaskedValue(final String key) { + return "http.proxy.password".equals(key) // + || "ldap.admin.security.credentials".equals(key) // + || "spnego.preauth.password".equals(key) // + || "app.cipher.key".equals(key) // + || "oic.client.id".equals(key) // + || "oic.client.secret".equals(key); + } + public static List> getBugReportItems() { final List> itemList = new ArrayList<>(); for (final String label : bugReportLabels) { diff --git a/src/main/java/org/codelibs/fess/app/web/admin/user/AdminUserAction.java b/src/main/java/org/codelibs/fess/app/web/admin/user/AdminUserAction.java index f368e21f5..6cb87ef95 100644 --- a/src/main/java/org/codelibs/fess/app/web/admin/user/AdminUserAction.java +++ b/src/main/java/org/codelibs/fess/app/web/admin/user/AdminUserAction.java @@ -285,6 +285,7 @@ public class AdminUserAction extends FessAdminAction { public static OptionalEntity getUser(final CreateForm form) { return getEntity(form).map(entity -> { + copyMapToBean(form.attributes, entity, op -> op.exclude(Constants.COMMON_CONVERSION_RULE)); copyBeanToBean(form, entity, op -> op.exclude(ArrayUtils.addAll(Constants.COMMON_CONVERSION_RULE, "password"))); if (form.crudMode.intValue() == CrudMode.CREATE || StringUtil.isNotBlank(form.password)) { final String encodedPassword = ComponentUtil.getComponent(FessLoginAssist.class).encryptPassword(form.password); diff --git a/src/main/java/org/codelibs/fess/app/web/api/admin/systeminfo/ApiAdminSysteminfoAction.java b/src/main/java/org/codelibs/fess/app/web/api/admin/systeminfo/ApiAdminSysteminfoAction.java index 5709d6e49..40464d3f5 100644 --- a/src/main/java/org/codelibs/fess/app/web/api/admin/systeminfo/ApiAdminSysteminfoAction.java +++ b/src/main/java/org/codelibs/fess/app/web/api/admin/systeminfo/ApiAdminSysteminfoAction.java @@ -42,7 +42,7 @@ public class ApiAdminSysteminfoAction extends FessApiAdminAction { public JsonResponse get$index() { final List> bugReportItems = getBugReportItems(); final List> envItems = getEnvItems(); - final List> fessPropItems = getFessPropItems(); + final List> fessPropItems = getFessPropItems(fessConfig); final List> propItems = getPropItems(); return asJson(new ApiResult.ApiSystemInfoResponse().bugReportProps(bugReportItems).envProps(envItems).fessProps(fessPropItems) .systemProps(propItems).status(ApiResult.Status.OK).result()); diff --git a/src/main/java/org/codelibs/fess/app/web/base/FessBaseAction.java b/src/main/java/org/codelibs/fess/app/web/base/FessBaseAction.java index 2251fc1d7..4291ea6db 100644 --- a/src/main/java/org/codelibs/fess/app/web/base/FessBaseAction.java +++ b/src/main/java/org/codelibs/fess/app/web/base/FessBaseAction.java @@ -15,6 +15,7 @@ */ package org.codelibs.fess.app.web.base; +import java.util.Map; import java.util.function.Consumer; import javax.annotation.Resource; @@ -194,6 +195,10 @@ public abstract class FessBaseAction extends TypicalAction // has several interf BeanUtil.copyBeanToBean(src, dest, option); } + protected static void copyMapToBean(final Map src, final Object dest, final Consumer option) { + BeanUtil.copyMapToBean(src, dest, option); + } + protected static T copyBeanToNewBean(final Object src, final Class destClass) { return BeanUtil.copyBeanToNewBean(src, destClass); } diff --git a/src/main/java/org/codelibs/fess/exception/GsaConfigException.java b/src/main/java/org/codelibs/fess/exception/GsaConfigException.java new file mode 100644 index 000000000..32c71bb10 --- /dev/null +++ b/src/main/java/org/codelibs/fess/exception/GsaConfigException.java @@ -0,0 +1,30 @@ +/* + * Copyright 2012-2018 CodeLibs Project and the Others. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ +package org.codelibs.fess.exception; + +public class GsaConfigException extends FessSystemException { + + private static final long serialVersionUID = 1L; + + public GsaConfigException(String message, Throwable cause) { + super(message, cause); + } + + public GsaConfigException(String message) { + super(message); + } + +} diff --git a/src/main/java/org/codelibs/fess/helper/QueryHelper.java b/src/main/java/org/codelibs/fess/helper/QueryHelper.java index 551b848be..3f4f134be 100644 --- a/src/main/java/org/codelibs/fess/helper/QueryHelper.java +++ b/src/main/java/org/codelibs/fess/helper/QueryHelper.java @@ -218,7 +218,6 @@ public class QueryHelper { } if (highlightedFields == null) { highlightedFields = fessConfig.getQueryAdditionalHighlightedFields( // - fessConfig.getIndexFieldTitle(), // fessConfig.getIndexFieldContent()); } if (searchFields == null) { @@ -584,28 +583,31 @@ public class QueryHelper { context.addHighlightedQuery(text); return buildDefaultQueryBuilder((f, b) -> buildMatchPhraseQuery(f, text).boost(b * boost)); } else if ("sort".equals(field)) { - final String[] values = text.split("\\."); - if (values.length > 2) { - throw new InvalidQueryException( - messages -> messages.addErrorsInvalidQuerySortValue(UserMessages.GLOBAL_PROPERTY_KEY, text), "Invalid sort field: " - + termQuery); - } - final String sortField = values[0]; - if (!isSortField(sortField)) { - throw new InvalidQueryException(messages -> messages.addErrorsInvalidQueryUnsupportedSortField( - UserMessages.GLOBAL_PROPERTY_KEY, sortField), "Unsupported sort field: " + termQuery); - } - SortOrder sortOrder; - if (values.length == 2) { - sortOrder = SortOrder.DESC.toString().equalsIgnoreCase(values[1]) ? SortOrder.DESC : SortOrder.ASC; - if (sortOrder == null) { - throw new InvalidQueryException(messages -> messages.addErrorsInvalidQueryUnsupportedSortOrder( - UserMessages.GLOBAL_PROPERTY_KEY, values[1]), "Invalid sort order: " + termQuery); - } - } else { - sortOrder = SortOrder.ASC; - } - context.addSorts(createFieldSortBuilder(sortField, sortOrder)); + split(text, ",").of( + stream -> stream.filter(StringUtil::isNotBlank).forEach( + t -> { + final String[] values = t.split("\\."); + if (values.length > 2) { + throw new InvalidQueryException(messages -> messages.addErrorsInvalidQuerySortValue( + UserMessages.GLOBAL_PROPERTY_KEY, text), "Invalid sort field: " + termQuery); + } + final String sortField = values[0]; + if (!isSortField(sortField)) { + throw new InvalidQueryException(messages -> messages.addErrorsInvalidQueryUnsupportedSortField( + UserMessages.GLOBAL_PROPERTY_KEY, sortField), "Unsupported sort field: " + termQuery); + } + SortOrder sortOrder; + if (values.length == 2) { + sortOrder = SortOrder.DESC.toString().equalsIgnoreCase(values[1]) ? SortOrder.DESC : SortOrder.ASC; + if (sortOrder == null) { + throw new InvalidQueryException(messages -> messages.addErrorsInvalidQueryUnsupportedSortOrder( + UserMessages.GLOBAL_PROPERTY_KEY, values[1]), "Invalid sort order: " + termQuery); + } + } else { + sortOrder = SortOrder.ASC; + } + context.addSorts(createFieldSortBuilder(sortField, sortOrder)); + })); return null; } else if (INURL_FIELD.equals(field) || fessConfig.getIndexFieldUrl().equals(context.getDefaultField())) { return QueryBuilders.wildcardQuery(fessConfig.getIndexFieldUrl(), "*" + text + "*").boost(boost); diff --git a/src/main/java/org/codelibs/fess/helper/ViewHelper.java b/src/main/java/org/codelibs/fess/helper/ViewHelper.java index 1911451c7..4f9fffd20 100644 --- a/src/main/java/org/codelibs/fess/helper/ViewHelper.java +++ b/src/main/java/org/codelibs/fess/helper/ViewHelper.java @@ -148,27 +148,31 @@ public class ViewHelper { public String getContentTitle(final Map document) { final FessConfig fessConfig = ComponentUtil.getFessConfig(); - final int size = fessConfig.getResponseMaxTitleLengthAsInteger(); - String title = - DocumentUtil.getValue(document, ComponentUtil.getQueryHelper().getHighlightPrefix() + fessConfig.getIndexFieldTitle(), - String.class); - if (StringUtil.isBlank(title) || title.length() > size - 3) { - title = DocumentUtil.getValue(document, fessConfig.getIndexFieldTitle(), String.class); + String title = DocumentUtil.getValue(document, fessConfig.getIndexFieldTitle(), String.class); + if (StringUtil.isBlank(title)) { + title = DocumentUtil.getValue(document, fessConfig.getIndexFieldFilename(), String.class); if (StringUtil.isBlank(title)) { - title = DocumentUtil.getValue(document, fessConfig.getIndexFieldFilename(), String.class); - if (StringUtil.isBlank(title)) { - title = DocumentUtil.getValue(document, fessConfig.getIndexFieldUrl(), String.class); - } + title = DocumentUtil.getValue(document, fessConfig.getIndexFieldUrl(), String.class); } - title = LaFunctions.h(title); - } else { - title = escapeHighlight(title).replaceAll("\\.\\.\\.$", StringUtil.EMPTY); } + final int size = fessConfig.getResponseMaxTitleLengthAsInteger(); if (size > -1) { - return StringUtils.abbreviate(title, size); - } else { - return title; + title = StringUtils.abbreviate(title, size); } + final String value = LaFunctions.h(title); + return LaRequestUtil.getOptionalRequest().map(req -> { + @SuppressWarnings("unchecked") + final Set querySet = (Set) req.getAttribute(Constants.HIGHLIGHT_QUERIES); + if (querySet != null) { + String t = value; + for (final String query : querySet) { + final String target = LaFunctions.h(query); + t = t.replace(target, highlightTagPre + target + highlightTagPost); + } + return t; + } + return value; + }).orElse(value); } public String getContentDescription(final Map document) { diff --git a/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java b/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java index ff94433c8..798aae3f8 100644 --- a/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java +++ b/src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java @@ -75,6 +75,8 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction -Dlog4j.shutdownHookEnabled=false -Dlog4j2.disable.jmx=true -Dlog4j.skipJansi=true + -Dsun.java2d.cmm=sun.java2d.cmm.kcms.KcmsServiceProvider + -Dorg.apache.pdfbox.rendering.UsePureJavaCMYKConversion=true */ String JVM_CRAWLER_OPTIONS = "jvm.crawler.options"; @@ -129,6 +131,8 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction -Dlog4j.shutdownHookEnabled=false -Dlog4j2.disable.jmx=true -Dlog4j.skipJansi=true + -Dsun.java2d.cmm=sun.java2d.cmm.kcms.KcmsServiceProvider + -Dorg.apache.pdfbox.rendering.UsePureJavaCMYKConversion=true */ String JVM_THUMBNAIL_OPTIONS = "jvm.thumbnail.options"; @@ -623,6 +627,12 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction /** The key of the configuration. e.g. UE,U,T,RK,S,LANG */ String QUERY_GSA_RESPONSE_FIELDS = "query.gsa.response.fields"; + /** The key of the configuration. e.g. en */ + String QUERY_GSA_DEFAULT_LANG = "query.gsa.default.lang"; + + /** The key of the configuration. e.g. */ + String QUERY_GSA_DEFAULT_SORT = "query.gsa.default.sort"; + /** The key of the configuration. e.g. 4 */ String QUERY_COLLAPSE_MAX_CONCURRENT_GROUP_RESULTS = "query.collapse.max.concurrent.group.results"; @@ -1498,6 +1508,8 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction -Dlog4j.shutdownHookEnabled=false -Dlog4j2.disable.jmx=true -Dlog4j.skipJansi=true + -Dsun.java2d.cmm=sun.java2d.cmm.kcms.KcmsServiceProvider + -Dorg.apache.pdfbox.rendering.UsePureJavaCMYKConversion=true
* comment: JVM options * @return The value of found property. (NotNull: if not found, exception but basically no way) @@ -1561,6 +1573,8 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction -Dlog4j.shutdownHookEnabled=false -Dlog4j2.disable.jmx=true -Dlog4j.skipJansi=true + -Dsun.java2d.cmm=sun.java2d.cmm.kcms.KcmsServiceProvider + -Dorg.apache.pdfbox.rendering.UsePureJavaCMYKConversion=true
* @return The value of found property. (NotNull: if not found, exception but basically no way) */ @@ -3238,6 +3252,28 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction */ String getQueryGsaResponseFields(); + /** + * Get the value for the key 'query.gsa.default.lang'.
+ * The value is, e.g. en
+ * @return The value of found property. (NotNull: if not found, exception but basically no way) + */ + String getQueryGsaDefaultLang(); + + /** + * Get the value for the key 'query.gsa.default.sort'.
+ * The value is, e.g.
+ * @return The value of found property. (NotNull: if not found, exception but basically no way) + */ + String getQueryGsaDefaultSort(); + + /** + * Get the value for the key 'query.gsa.default.sort' as {@link Integer}.
+ * The value is, e.g.
+ * @return The value of found property. (NotNull: if not found, exception but basically no way) + * @throws NumberFormatException When the property is not integer. + */ + Integer getQueryGsaDefaultSortAsInteger(); + /** * Get the value for the key 'query.collapse.max.concurrent.group.results'.
* The value is, e.g. 4
@@ -6729,6 +6765,18 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction return get(FessConfig.QUERY_GSA_RESPONSE_FIELDS); } + public String getQueryGsaDefaultLang() { + return get(FessConfig.QUERY_GSA_DEFAULT_LANG); + } + + public String getQueryGsaDefaultSort() { + return get(FessConfig.QUERY_GSA_DEFAULT_SORT); + } + + public Integer getQueryGsaDefaultSortAsInteger() { + return getAsInteger(FessConfig.QUERY_GSA_DEFAULT_SORT); + } + public String getQueryCollapseMaxConcurrentGroupResults() { return get(FessConfig.QUERY_COLLAPSE_MAX_CONCURRENT_GROUP_RESULTS); } @@ -8079,13 +8127,13 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction defaultMap.put(FessConfig.APP_DIGEST_ALGORISM, "sha256"); defaultMap .put(FessConfig.JVM_CRAWLER_OPTIONS, - "-Djava.awt.headless=true\n-Dfile.encoding=UTF-8\n-Djna.nosys=true\n-Djdk.io.permissionsUseCanonicalPath=true\n-server\n-Xmx512m\n-XX:MaxMetaspaceSize=128m\n-XX:CompressedClassSpaceSize=32m\n-XX:-UseGCOverheadLimit\n-XX:+UseConcMarkSweepGC\n-XX:CMSInitiatingOccupancyFraction=75\n-XX:+UseCMSInitiatingOccupancyOnly\n-XX:+UseTLAB\n-XX:+DisableExplicitGC\n-XX:+HeapDumpOnOutOfMemoryError\n-XX:-OmitStackTraceInFastThrow\n-Djcifs.smb.client.connTimeout=60000\n-Djcifs.smb.client.soTimeout=35000\n-Djcifs.smb.client.responseTimeout=30000\n-Dgroovy.use.classvalue=true\n-Dio.netty.noUnsafe=true\n-Dio.netty.noKeySetOptimization=true\n-Dio.netty.recycler.maxCapacityPerThread=0\n-Dlog4j.shutdownHookEnabled=false\n-Dlog4j2.disable.jmx=true\n-Dlog4j.skipJansi=true\n"); + "-Djava.awt.headless=true\n-Dfile.encoding=UTF-8\n-Djna.nosys=true\n-Djdk.io.permissionsUseCanonicalPath=true\n-server\n-Xmx512m\n-XX:MaxMetaspaceSize=128m\n-XX:CompressedClassSpaceSize=32m\n-XX:-UseGCOverheadLimit\n-XX:+UseConcMarkSweepGC\n-XX:CMSInitiatingOccupancyFraction=75\n-XX:+UseCMSInitiatingOccupancyOnly\n-XX:+UseTLAB\n-XX:+DisableExplicitGC\n-XX:+HeapDumpOnOutOfMemoryError\n-XX:-OmitStackTraceInFastThrow\n-Djcifs.smb.client.connTimeout=60000\n-Djcifs.smb.client.soTimeout=35000\n-Djcifs.smb.client.responseTimeout=30000\n-Dgroovy.use.classvalue=true\n-Dio.netty.noUnsafe=true\n-Dio.netty.noKeySetOptimization=true\n-Dio.netty.recycler.maxCapacityPerThread=0\n-Dlog4j.shutdownHookEnabled=false\n-Dlog4j2.disable.jmx=true\n-Dlog4j.skipJansi=true\n-Dsun.java2d.cmm=sun.java2d.cmm.kcms.KcmsServiceProvider\n-Dorg.apache.pdfbox.rendering.UsePureJavaCMYKConversion=true\n"); defaultMap .put(FessConfig.JVM_SUGGEST_OPTIONS, "-Djava.awt.headless=true\n-Dfile.encoding=UTF-8\n-Djna.nosys=true\n-Djdk.io.permissionsUseCanonicalPath=true\n-server\n-Xmx256m\n-XX:MaxMetaspaceSize=128m\n-XX:CompressedClassSpaceSize=32m\n-XX:-UseGCOverheadLimit\n-XX:+UseConcMarkSweepGC\n-XX:CMSInitiatingOccupancyFraction=75\n-XX:+UseCMSInitiatingOccupancyOnly\n-XX:+UseTLAB\n-XX:+DisableExplicitGC\n-XX:+HeapDumpOnOutOfMemoryError\n-Dgroovy.use.classvalue=true\n-Dio.netty.noUnsafe=true\n-Dio.netty.noKeySetOptimization=true\n-Dio.netty.recycler.maxCapacityPerThread=0\n-Dlog4j.shutdownHookEnabled=false\n-Dlog4j2.disable.jmx=true\n-Dlog4j.skipJansi=true\n"); defaultMap .put(FessConfig.JVM_THUMBNAIL_OPTIONS, - "-Djava.awt.headless=true\n-Dfile.encoding=UTF-8\n-Djna.nosys=true\n-Djdk.io.permissionsUseCanonicalPath=true\n-server\n-Xmx128m\n-XX:MaxMetaspaceSize=128m\n-XX:CompressedClassSpaceSize=32m\n-XX:-UseGCOverheadLimit\n-XX:+UseConcMarkSweepGC\n-XX:CMSInitiatingOccupancyFraction=75\n-XX:+UseCMSInitiatingOccupancyOnly\n-XX:+UseTLAB\n-XX:+DisableExplicitGC\n-XX:+HeapDumpOnOutOfMemoryError\n-XX:-OmitStackTraceInFastThrow\n-Djcifs.smb.client.connTimeout=60000\n-Djcifs.smb.client.soTimeout=35000\n-Djcifs.smb.client.responseTimeout=30000\n-Dgroovy.use.classvalue=true\n-Dio.netty.noUnsafe=true\n-Dio.netty.noKeySetOptimization=true\n-Dio.netty.recycler.maxCapacityPerThread=0\n-Dlog4j.shutdownHookEnabled=false\n-Dlog4j2.disable.jmx=true\n-Dlog4j.skipJansi=true\n"); + "-Djava.awt.headless=true\n-Dfile.encoding=UTF-8\n-Djna.nosys=true\n-Djdk.io.permissionsUseCanonicalPath=true\n-server\n-Xmx128m\n-XX:MaxMetaspaceSize=128m\n-XX:CompressedClassSpaceSize=32m\n-XX:-UseGCOverheadLimit\n-XX:+UseConcMarkSweepGC\n-XX:CMSInitiatingOccupancyFraction=75\n-XX:+UseCMSInitiatingOccupancyOnly\n-XX:+UseTLAB\n-XX:+DisableExplicitGC\n-XX:+HeapDumpOnOutOfMemoryError\n-XX:-OmitStackTraceInFastThrow\n-Djcifs.smb.client.connTimeout=60000\n-Djcifs.smb.client.soTimeout=35000\n-Djcifs.smb.client.responseTimeout=30000\n-Dgroovy.use.classvalue=true\n-Dio.netty.noUnsafe=true\n-Dio.netty.noKeySetOptimization=true\n-Dio.netty.recycler.maxCapacityPerThread=0\n-Dlog4j.shutdownHookEnabled=false\n-Dlog4j2.disable.jmx=true\n-Dlog4j.skipJansi=true\n-Dsun.java2d.cmm=sun.java2d.cmm.kcms.KcmsServiceProvider\n-Dorg.apache.pdfbox.rendering.UsePureJavaCMYKConversion=true\n"); defaultMap.put(FessConfig.JOB_SYSTEM_JOB_IDS, "default_crawler"); defaultMap.put(FessConfig.JOB_TEMPLATE_TITLE_WEB, "Web Crawler - {0}"); defaultMap.put(FessConfig.JOB_TEMPLATE_TITLE_FILE, "File Crawler - {0}"); @@ -8255,6 +8303,8 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction defaultMap.put(FessConfig.QUERY_ADDITIONAL_ANALYZED_FIELDS, ""); defaultMap.put(FessConfig.QUERY_ADDITIONAL_NOT_ANALYZED_FIELDS, ""); defaultMap.put(FessConfig.QUERY_GSA_RESPONSE_FIELDS, "UE,U,T,RK,S,LANG"); + defaultMap.put(FessConfig.QUERY_GSA_DEFAULT_LANG, "en"); + defaultMap.put(FessConfig.QUERY_GSA_DEFAULT_SORT, ""); defaultMap.put(FessConfig.QUERY_COLLAPSE_MAX_CONCURRENT_GROUP_RESULTS, "4"); defaultMap.put(FessConfig.QUERY_COLLAPSE_INNER_HITS_NAME, "similar_docs"); defaultMap.put(FessConfig.QUERY_COLLAPSE_INNER_HITS_SIZE, "0"); diff --git a/src/main/java/org/codelibs/fess/util/GsaConfigParser.java b/src/main/java/org/codelibs/fess/util/GsaConfigParser.java new file mode 100644 index 000000000..809a80eeb --- /dev/null +++ b/src/main/java/org/codelibs/fess/util/GsaConfigParser.java @@ -0,0 +1,314 @@ +/* + * Copyright 2012-2018 CodeLibs Project and the Others. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ +package org.codelibs.fess.util; + +import static org.codelibs.core.stream.StreamUtil.split; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import javax.xml.parsers.SAXParser; +import javax.xml.parsers.SAXParserFactory; + +import org.codelibs.core.lang.StringUtil; +import org.codelibs.fess.Constants; +import org.codelibs.fess.es.config.exentity.FileConfig; +import org.codelibs.fess.es.config.exentity.LabelType; +import org.codelibs.fess.es.config.exentity.WebConfig; +import org.codelibs.fess.exception.GsaConfigException; +import org.dbflute.optional.OptionalEntity; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.xml.sax.Attributes; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; +import org.xml.sax.helpers.DefaultHandler; + +public class GsaConfigParser extends DefaultHandler { + + private static final Logger logger = LoggerFactory.getLogger(GsaConfigParser.class); + + protected static final String REGEXP = "regexp:"; + + protected static final String REGEXP_IGNORE_CASE = "regexpIgnoreCase:"; + + protected static final String CONTAINS = "contains:"; + + protected static final String COLLECTIONS = "collections"; + + protected static final String COLLECTION = "collection"; + + protected static final String GLOBALPARAMS = "globalparams"; + + protected static final String START_URLS = "start_urls"; + + protected static final String GOOD_URLS = "good_urls"; + + protected static final String BAD_URLS = "bad_urls"; + + protected String[] webProtocols = new String[] { "http:", "https:" }; + + protected String[] fileProtocols = new String[] { "file:", "smb:" }; + + protected LinkedList tagQueue; + + protected List labelList; + + protected LabelType labelType; + + protected Map globalParams = new HashMap<>(); + + protected WebConfig webConfig = null; + + protected FileConfig fileConfig = null; + + protected StringBuilder textBuf = new StringBuilder(1000); + + protected String userAgent = "gsa-crawler"; + + public void parse(final InputSource is) { + try { + final SAXParserFactory factory = SAXParserFactory.newInstance(); + final SAXParser parser = factory.newSAXParser(); + parser.parse(is, this); + } catch (final Exception e) { + throw new GsaConfigException("Failed to parse XML file.", e); + } + } + + @Override + public void startDocument() throws SAXException { + tagQueue = new LinkedList<>(); + labelList = new ArrayList<>(); + labelType = null; + } + + @Override + public void endDocument() throws SAXException { + globalParams.clear(); + tagQueue.clear(); + } + + @Override + public void startElement(final String uri, final String localName, final String qName, final Attributes attributes) throws SAXException { + if (logger.isDebugEnabled()) { + logger.debug("Start Element: " + qName); + } + if (tagQueue.isEmpty() && !"eef".equalsIgnoreCase(qName)) { + throw new GsaConfigException("Invalid format."); + } else if (COLLECTION.equalsIgnoreCase(qName) && COLLECTIONS.equalsIgnoreCase(tagQueue.peekLast())) { + final long now = System.currentTimeMillis(); + final String name = attributes.getValue("Name"); + labelType = new LabelType(); + labelType.setName(name); + labelType.setValue(name); + labelType.setCreatedBy(Constants.SYSTEM_USER); + labelType.setCreatedTime(now); + labelType.setUpdatedBy(Constants.SYSTEM_USER); + labelType.setUpdatedTime(now); + } + tagQueue.offer(qName); + } + + @Override + public void endElement(final String uri, final String localName, final String qName) throws SAXException { + if (logger.isDebugEnabled()) { + logger.debug("End Element: " + qName); + } + if (GOOD_URLS.equalsIgnoreCase(qName)) { + if (labelType != null) { + labelType.setIncludedPaths(parseFilterPaths(textBuf.toString(), true, true)); + } else if (GLOBALPARAMS.equalsIgnoreCase(tagQueue.get(tagQueue.size() - 2))) { + globalParams.put(GOOD_URLS, textBuf.toString()); + } + } else if (BAD_URLS.equalsIgnoreCase(qName)) { + if (labelType != null) { + labelType.setExcludedPaths(parseFilterPaths(textBuf.toString(), true, true)); + } else if (GLOBALPARAMS.equalsIgnoreCase(tagQueue.get(tagQueue.size() - 2))) { + globalParams.put(BAD_URLS, textBuf.toString()); + } + } else if (START_URLS.equalsIgnoreCase(qName) && GLOBALPARAMS.equalsIgnoreCase(tagQueue.get(tagQueue.size() - 2))) { + globalParams.put(START_URLS, textBuf.toString()); + } else if (labelType != null && COLLECTION.equalsIgnoreCase(qName)) { + labelList.add(labelType); + labelType = null; + } else if (GLOBALPARAMS.equalsIgnoreCase(qName)) { + final Object startUrls = globalParams.get(START_URLS); + if (startUrls != null) { + final long now = System.currentTimeMillis(); + final List urlList = + split(startUrls.toString(), "\n").get( + stream -> stream.map(String::trim).filter(StringUtil::isNotBlank).collect(Collectors.toList())); + + final String webUrls = + urlList.stream().filter(s -> Arrays.stream(webProtocols).anyMatch(p -> s.startsWith(p))) + .collect(Collectors.joining("\n")); + if (StringUtil.isNotBlank(webUrls)) { + webConfig = new WebConfig(); + webConfig.setName("Default"); + webConfig.setAvailable(true); + webConfig.setBoost(1.0f); + webConfig.setConfigParameter(StringUtil.EMPTY); + webConfig.setIntervalTime(1000); + webConfig.setNumOfThread(3); + webConfig.setSortOrder(1); + webConfig.setUrls(webUrls); + webConfig.setIncludedUrls(parseFilterPaths(globalParams.get(GOOD_URLS), true, false)); + webConfig.setIncludedDocUrls(StringUtil.EMPTY); + webConfig.setExcludedUrls(parseFilterPaths(globalParams.get(BAD_URLS), true, false)); + webConfig.setExcludedDocUrls(StringUtil.EMPTY); + webConfig.setUserAgent(userAgent); + webConfig.setCreatedBy(Constants.SYSTEM_USER); + webConfig.setCreatedTime(now); + webConfig.setUpdatedBy(Constants.SYSTEM_USER); + webConfig.setUpdatedTime(now); + } + + final String fileUrls = + urlList.stream().filter(s -> Arrays.stream(fileProtocols).anyMatch(p -> s.startsWith(p))) + .collect(Collectors.joining("\n")); + if (StringUtil.isNotBlank(fileUrls)) { + fileConfig = new FileConfig(); + fileConfig.setName("Default"); + fileConfig.setAvailable(true); + fileConfig.setBoost(1.0f); + fileConfig.setConfigParameter(StringUtil.EMPTY); + fileConfig.setIntervalTime(0); + fileConfig.setNumOfThread(5); + fileConfig.setSortOrder(2); + fileConfig.setPaths(fileUrls); + fileConfig.setIncludedPaths(parseFilterPaths(globalParams.get(GOOD_URLS), false, true)); + fileConfig.setIncludedDocPaths(StringUtil.EMPTY); + fileConfig.setExcludedPaths(parseFilterPaths(globalParams.get(BAD_URLS), false, true)); + fileConfig.setExcludedDocPaths(StringUtil.EMPTY); + fileConfig.setCreatedBy(Constants.SYSTEM_USER); + fileConfig.setCreatedTime(now); + fileConfig.setUpdatedBy(Constants.SYSTEM_USER); + fileConfig.setUpdatedTime(now); + } + } + } else if ("user_agent".equalsIgnoreCase(qName) && GLOBALPARAMS.equalsIgnoreCase(tagQueue.get(tagQueue.size() - 2))) { + userAgent = textBuf.toString().trim(); + } + tagQueue.pollLast(); + textBuf.setLength(0); + } + + @Override + public void characters(final char[] ch, final int start, final int length) throws SAXException { + String text = new String(ch, start, length); + if (logger.isDebugEnabled()) { + logger.debug("Text: " + text); + } + textBuf.append(text); + } + + protected String parseFilterPaths(final String text, final boolean web, final boolean file) { + return split(text, "\n").get(stream -> stream.map(String::trim).filter(StringUtil::isNotBlank).map(s -> { + if (s.startsWith("#")) { + return null; + } else if (s.startsWith(CONTAINS)) { + final String v = s.substring(CONTAINS.length()); + final StringBuilder buf = new StringBuilder(100); + return appendFileterPath(buf, escape(v)); + } else if (s.startsWith(REGEXP_IGNORE_CASE)) { + final String v = s.substring(REGEXP_IGNORE_CASE.length()); + final StringBuilder buf = new StringBuilder(100); + buf.append("(?i)"); + return appendFileterPath(buf, unescape(v)); + } else if (s.startsWith(REGEXP)) { + final String v = s.substring(REGEXP.length()); + final StringBuilder buf = new StringBuilder(100); + return appendFileterPath(buf, unescape(v)); + } else if (Arrays.stream(webProtocols).anyMatch(p -> s.startsWith(p))) { + return escape(s) + ".*"; + } else if (Arrays.stream(fileProtocols).anyMatch(p -> s.startsWith(p))) { + return escape(s) + ".*"; + } else { + final StringBuilder buf = new StringBuilder(100); + return appendFileterPath(buf, escape(s)); + } + }).filter(s -> { + if (StringUtil.isBlank(s)) { + return false; + } + if (Arrays.stream(webProtocols).anyMatch(p -> s.startsWith(p))) { + return web; + } + if (Arrays.stream(fileProtocols).anyMatch(p -> s.startsWith(p))) { + return file; + } + return true; + }).collect(Collectors.joining("\n"))); + } + + protected String escape(final String s) { + return s.replace(".", "\\.")// + .replace("+", "\\+")// + .replace("*", "\\*")// + .replace("[", "\\[")// + .replace("]", "\\]")// + .replace("(", "\\(")// + .replace("(", "\\)")// + .replace("?", "\\?"); + } + + protected String unescape(final String s) { + return s.replace("\\\\", "\\"); + } + + protected String appendFileterPath(final StringBuilder buf, final String v) { + if (!v.startsWith("^")) { + buf.append(".*"); + } + buf.append(v); + if (!v.endsWith("$")) { + buf.append(".*"); + } + return buf.toString(); + } + + public void setWebProtocols(final String[] webProtocols) { + this.webProtocols = webProtocols; + } + + public void setFileProtocols(final String[] fileProtocols) { + this.fileProtocols = fileProtocols; + } + + @Override + public String toString() { + return "GsaConfigParser [labelList=" + labelList + ", webConfig=" + webConfig + ", fileConfig=" + fileConfig + "]"; + } + + public OptionalEntity getWebConfig() { + return OptionalUtil.ofNullable(webConfig); + } + + public OptionalEntity getFileConfig() { + return OptionalUtil.ofNullable(fileConfig); + } + + public LabelType[] getLabelTypes() { + return labelList.toArray(new LabelType[labelList.size()]); + } + +} diff --git a/src/main/java/org/codelibs/fess/util/QueryResponseList.java b/src/main/java/org/codelibs/fess/util/QueryResponseList.java index 8ae9bb1ee..18dff3981 100644 --- a/src/main/java/org/codelibs/fess/util/QueryResponseList.java +++ b/src/main/java/org/codelibs/fess/util/QueryResponseList.java @@ -202,8 +202,8 @@ public class QueryResponseList implements List> { existNextPage = false; allPageCount = currentPageNumber; } - currentStartRecordNumber = allRecordCount != 0 ? (currentPageNumber - 1) * pageSize + 1 : 0; - currentEndRecordNumber = (long) currentPageNumber * pageSize; + currentStartRecordNumber = allRecordCount != 0 ? start + 1 : 0; + currentEndRecordNumber = currentStartRecordNumber + (long) pageSize - 1; currentEndRecordNumber = allRecordCount < currentEndRecordNumber ? allRecordCount : currentEndRecordNumber; final int pageRangeSize = 5; diff --git a/src/main/resources/app.xml b/src/main/resources/app.xml index 605272e2d..8b9497a32 100644 --- a/src/main/resources/app.xml +++ b/src/main/resources/app.xml @@ -383,5 +383,7 @@ + + diff --git a/src/main/resources/fess_config.properties b/src/main/resources/fess_config.properties index c9526ab7b..20691cbe1 100644 --- a/src/main/resources/fess_config.properties +++ b/src/main/resources/fess_config.properties @@ -48,6 +48,8 @@ jvm.crawler.options=\ -Dlog4j.shutdownHookEnabled=false\n\ -Dlog4j2.disable.jmx=true\n\ -Dlog4j.skipJansi=true\n\ +-Dsun.java2d.cmm=sun.java2d.cmm.kcms.KcmsServiceProvider\n\ +-Dorg.apache.pdfbox.rendering.UsePureJavaCMYKConversion=true\n\ jvm.suggest.options=\ @@ -102,6 +104,8 @@ jvm.thumbnail.options=\ -Dlog4j.shutdownHookEnabled=false\n\ -Dlog4j2.disable.jmx=true\n\ -Dlog4j.skipJansi=true\n\ +-Dsun.java2d.cmm=sun.java2d.cmm.kcms.KcmsServiceProvider\n\ +-Dorg.apache.pdfbox.rendering.UsePureJavaCMYKConversion=true\n\ #-Xdebug\n\ @@ -304,6 +308,8 @@ query.additional.sort.fields= query.additional.analyzed.fields= query.additional.not.analyzed.fields= query.gsa.response.fields=UE,U,T,RK,S,LANG +query.gsa.default.lang=en +query.gsa.default.sort= query.collapse.max.concurrent.group.results=4 query.collapse.inner.hits.name=similar_docs query.collapse.inner.hits.size=0 diff --git a/src/test/java/org/codelibs/fess/util/GsaConfigParserTest.java b/src/test/java/org/codelibs/fess/util/GsaConfigParserTest.java new file mode 100644 index 000000000..c40eb0098 --- /dev/null +++ b/src/test/java/org/codelibs/fess/util/GsaConfigParserTest.java @@ -0,0 +1,43 @@ +/* + * Copyright 2012-2018 CodeLibs Project and the Others. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ +package org.codelibs.fess.util; + +import java.io.IOException; +import java.io.InputStream; + +import org.codelibs.core.io.ResourceUtil; +import org.codelibs.fess.es.config.exentity.LabelType; +import org.codelibs.fess.unit.UnitFessTestCase; +import org.xml.sax.InputSource; + +public class GsaConfigParserTest extends UnitFessTestCase { + + public void test_parse() throws IOException { + GsaConfigParser parser = new GsaConfigParser(); + try (InputStream is = ResourceUtil.getResourceAsStream("data/gsaconfig.xml")) { + parser.parse(new InputSource(is)); + } + parser.getWebConfig().ifPresent(c -> { + System.out.println(c.toString()); + }).orElse(() -> fail()); + parser.getFileConfig().ifPresent(c -> { + System.out.println(c.toString()); + }).orElse(() -> fail()); + LabelType[] labelTypes = parser.getLabelTypes(); + assertEquals(3, labelTypes.length); + } + +} diff --git a/src/test/resources/data/gsaconfig.xml b/src/test/resources/data/gsaconfig.xml new file mode 100644 index 000000000..5c888df12 --- /dev/null +++ b/src/test/resources/data/gsaconfig.xml @@ -0,0 +1,86 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +