0ct0pu5/fess
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge remote-tracking branch 'refs/remotes/codelibs/master'

Browse source
This commit is contained in:
igarash1 2018-05-16 10:46:11 +09:00
commit ec82017146
18 changed files with 793 additions and 109 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

204
src/main/java/org/codelibs/fess/api/gsa/GsaApiManager.java
View file

@ -16,6 +16,7 @@
package org.codelibs.fess.api.gsa;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLDecoder;
import java.net.URLEncoder;
import java.text.SimpleDateFormat;
@ -60,6 +61,9 @@ import org.slf4j.LoggerFactory;
public class GsaApiManager extends BaseApiManager implements WebApiManager {
private static final Logger logger = LoggerFactory.getLogger(GsaApiManager.class);
private static final String OUTPUT_XML = "xml"; // or xml_no_dtd
// http://www.google.com/google.dtd.
private static final String GSA_META_SUFFIX = "_s";
protected String gsaPathPrefix = "/gsa";
@ -94,12 +98,28 @@ public class GsaApiManager extends BaseApiManager implements WebApiManager {
}
}
protected void appendParam(final StringBuilder buf, final String name, final String value) throws UnsupportedEncodingException {
appendParam(buf, name, value, URLEncoder.encode(value, Constants.UTF_8));
}
protected void appendParam(final StringBuilder buf, final String name, final String value, final String original)
throws UnsupportedEncodingException {
buf.append("<PARAM name=\"");
buf.append(escapeXml(name));
buf.append("\" value=\"");
buf.append(escapeXml(value));
buf.append("\" original_value=\"");
buf.append(escapeXml(original));
buf.append("\"/>");
}
protected void processSearchRequest(final HttpServletRequest request, final HttpServletResponse response, final FilterChain chain) {
final SearchService searchService = ComponentUtil.getComponent(SearchService.class);
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final boolean xmlDtd = OUTPUT_XML.equals(request.getParameter("output"));
if (!fessConfig.isAcceptedSearchReferer(request.getHeader("referer"))) {
writeXmlResponse(99, false, StringUtil.EMPTY, "Referer is invalid.");
writeXmlResponse(99, xmlDtd, StringUtil.EMPTY, "Referer is invalid.");
return;
}
@ -108,7 +128,6 @@ public class GsaApiManager extends BaseApiManager implements WebApiManager {
String query = null;
final StringBuilder buf = new StringBuilder(1000);
request.setAttribute(Constants.SEARCH_LOG_ACCESS_TYPE, Constants.SEARCH_LOG_ACCESS_TYPE_GSA);
boolean xmlDtd = false;
try {
final SearchRenderData data = new SearchRenderData();
final GsaRequestParams params = new GsaRequestParams(request, fessConfig);
@ -124,10 +143,6 @@ public class GsaApiManager extends BaseApiManager implements WebApiManager {
if (StringUtil.isNotBlank(getFieldsParam)) {
getFields.addAll(Arrays.asList(getFieldsParam.split("\\.")));
}
// DTD
if ("xml".equals(request.getParameter("output"))) {
xmlDtd = true;
}
final StringBuilder requestUri = new StringBuilder(request.getRequestURI());
if (request.getQueryString() != null) {
requestUri.append("?").append(request.getQueryString());
@ -138,88 +153,92 @@ public class GsaApiManager extends BaseApiManager implements WebApiManager {
final String oe = "UTF-8";
// IP address
final String ip = ComponentUtil.getViewHelper().getClientIp(request);
final String start = request.getParameter("start");
long startNumber = 1;
if (StringUtil.isNotBlank(start)) {
startNumber = Long.parseLong(start) + 1;
}
long endNumber = startNumber + data.getPageSize() - 1;
if (endNumber > allRecordCount) {
endNumber = allRecordCount;
}
buf.append("<Q>");
buf.append(escapeXml(query));
buf.append("</Q>");
buf.append("<TM>");
buf.append(execTime);
buf.append("</TM>");
buf.append("<Q>");
buf.append(escapeXml(query));
buf.append("</Q>");
for (final Entry<String, String[]> entry : request.getParameterMap().entrySet()) {
final String[] values = entry.getValue();
if (values == null) {
continue;
}
final String key = entry.getKey();
if ("sort".equals(key)) {
continue;
}
for (final String value : values) {
buf.append("<PARAM name=\"");
buf.append(key);
buf.append("\" value=\"");
buf.append(value);
buf.append("\" original_value=\"");
buf.append(URLEncoder.encode(value, Constants.UTF_8));
buf.append("\"/>");
appendParam(buf, key, value);
}
}
buf.append("<PARAM name=\"ie\" value=\"");
buf.append(ie);
buf.append("\" original_value=\"");
buf.append(URLEncoder.encode(ie, Constants.UTF_8));
buf.append("\"/>");
buf.append("<PARAM name=\"oe\" value=\"");
buf.append(oe);
buf.append("\" original_value=\"");
buf.append(URLEncoder.encode(ie, Constants.UTF_8));
buf.append("\"/>");
buf.append("<PARAM name=\"ip\" value=\"");
buf.append(ip);
buf.append("\" original_value=\"");
buf.append(URLEncoder.encode(ie, Constants.UTF_8));
buf.append("\"/>");
appendParam(buf, "ie", ie);
if (request.getParameter("oe") == null) {
appendParam(buf, "oe", oe);
}
final String[] langs = params.getLanguages();
if (langs.length > 0) {
appendParam(buf, "inlang", langs[0]);
appendParam(buf, "ulang", langs[0]);
}
appendParam(buf, "ip", ip);
appendParam(buf, "access", "p");
appendParam(buf, "sort", params.getSortParam(), params.getSortParam());
appendParam(buf, "entqr", "3");
appendParam(buf, "entqrm", "0");
appendParam(buf, "wc", "200");
appendParam(buf, "wc_mc", "1");
if (!documentItems.isEmpty()) {
buf.append("<RES SN=\"");
buf.append(startNumber);
buf.append(data.getCurrentStartRecordNumber());
buf.append("\" EN=\"");
buf.append(endNumber);
buf.append(data.getCurrentEndRecordNumber());
buf.append("\">");
buf.append("<M>");
buf.append(allRecordCount);
buf.append("</M>");
if (endNumber < allRecordCount) {
if (data.isExistPrevPage() || data.isExistNextPage()) {
buf.append("<NB>");
buf.append("<NU>");
buf.append(escapeXml(uriQueryString.replaceFirst("start=([^&]+)", "start=" + endNumber)));
buf.append("</NU>");
if (data.isExistPrevPage()) {
long s = data.getCurrentStartRecordNumber() - data.getPageSize() - 1;
if (s < 0) {
s = 0;
}
buf.append("<PU>");
buf.append(escapeXml(uriQueryString.replaceFirst("start=([0-9]+)", "start=" + s)));
buf.append("</PU>");
}
if (data.isExistNextPage()) {
buf.append("<NU>");
buf.append(escapeXml(uriQueryString.replaceFirst("start=([0-9]+)", "start=" + data.getCurrentEndRecordNumber())));
buf.append("</NU>");
}
buf.append("</NB>");
}
long recordNumber = startNumber;
long recordNumber = data.getCurrentStartRecordNumber();
for (final Map<String, Object> document : documentItems) {
buf.append("<R N=\"");
buf.append(recordNumber);
buf.append("\">");
final String url = DocumentUtil.getValue(document, fessConfig.getIndexFieldUrl(), String.class);
document.remove(fessConfig.getIndexFieldUrl());
document.put("UE", url);
document.put("U", URLDecoder.decode(url, Constants.UTF_8));
document.put("T", DocumentUtil.getValue(document, fessConfig.getIndexFieldTitle(), String.class));
document.remove(fessConfig.getIndexFieldTitle());
final float score = DocumentUtil.getValue(document, fessConfig.getIndexFieldBoost(), Float.class, Float.valueOf(0));
document.remove(fessConfig.getIndexFieldBoost());
document.put("RK", (int) (score * 10));
document.put("S",
DocumentUtil
.getValue(document, fessConfig.getResponseFieldContentDescription(), String.class, StringUtil.EMPTY)
.replaceAll("<(/*)em>", "<$1b>"));
document.remove(fessConfig.getResponseFieldContentDescription());
document.put("T", DocumentUtil.getValue(document, fessConfig.getResponseFieldContentTitle(), String.class));
final float score = DocumentUtil.getValue(document, Constants.SCORE, Float.class, Float.NaN);
int rk = 10;
if (!Float.isNaN(score)) {
if (score < 0.0) {
rk = 0;
} else if (score > 1.0) {
rk = 10;
} else {
rk = (int) (score * 10.0);
}
}
document.put("RK", rk);
document.put("S", DocumentUtil.getValue(document, fessConfig.getResponseFieldContentDescription(), String.class,
StringUtil.EMPTY));
final String lang = DocumentUtil.getValue(document, fessConfig.getIndexFieldLang(), String.class);
if (StringUtil.isNotBlank(lang)) {
document.put("LANG", lang);
@ -251,6 +270,14 @@ public class GsaApiManager extends BaseApiManager implements WebApiManager {
}
}
}
buf.append("<ENT_SOURCE>").append(escapeXml("FESS")).append("</ENT_SOURCE>");
String lastModified = DocumentUtil.getValue(document, fessConfig.getIndexFieldLastModified(), String.class);
if (StringUtil.isBlank(lastModified)) {
lastModified = StringUtil.EMPTY;
}
lastModified = lastModified.split("T")[0];
buf.append("<FS NAME=\"date\" VALUE=\"").append(escapeXml(lastModified)).append("\"/>");
buf.append("<HAS>");
buf.append("<L/>");
buf.append("<C SZ=\"");
@ -407,9 +434,15 @@ public class GsaApiManager extends BaseApiManager implements WebApiManager {
private int pageSize = -1;
private String sortParam;
protected GsaRequestParams(final HttpServletRequest request, final FessConfig fessConfig) {
this.request = request;
this.fessConfig = fessConfig;
this.sortParam = request.getParameter("sort");
if (this.sortParam == null) {
this.sortParam = fessConfig.getQueryGsaDefaultSort();
}
}
@Override
@ -430,6 +463,9 @@ public class GsaApiManager extends BaseApiManager implements WebApiManager {
if (key.startsWith("fields.")) {
final String[] value = simplifyArray(entry.getValue());
fields.put(key.substring("fields.".length()), value);
} else if ("site".equals(key)) {
final String[] value = simplifyArray(entry.getValue());
fields.put("label", value);
}
}
return fields;
@ -450,7 +486,14 @@ public class GsaApiManager extends BaseApiManager implements WebApiManager {
@Override
public String[] getLanguages() {
return getParamValueArray(request, "lang");
final String[] langs = getParamValueArray(request, "ulang");
if (langs.length > 0) {
return langs;
}
if (request.getHeader("Accept-Language") != null) {
return Collections.list(request.getLocales()).stream().map(l -> l.getLanguage()).toArray(n -> new String[n]);
}
return new String[] { fessConfig.getQueryGsaDefaultLang() };
}
@Override
@ -463,9 +506,50 @@ public class GsaApiManager extends BaseApiManager implements WebApiManager {
return createFacetInfo(request);
}
public String getSortParam() {
return sortParam;
}
@Override
public String getSort() {
return request.getParameter("sort");
// Sort By Relevance (Default)
// Sort By Date: date:<direction>:<mode>:<format>
// Sort by Metadata: meta:<name>:<direction>:<mode>:<language>:<case>:<numeric>
if (StringUtil.isBlank(sortParam)) {
return null;
}
final String[] values = sortParam.split(":");
if (values.length > 0) {
if ("date".equals(values[0])) {
final StringBuilder buf = new StringBuilder();
buf.append(fessConfig.getIndexFieldTimestamp());
if (values.length > 1) {
if ("A".equals(values[1])) {
buf.append(".asc");
} else if ("D".equals(values[1])) {
buf.append(".desc");
}
}
buf.append(",score.desc");
return buf.toString();
} else if ("meta".equals(values[0]) && values.length > 1) {
final StringBuilder buf = new StringBuilder();
buf.append(values[1]);
if (values.length > 2) {
if ("A".equals(values[2])) {
buf.append(".asc");
} else if ("D".equals(values[2])) {
buf.append(".desc");
}
}
buf.append(",score.desc");
return buf.toString();
}
}
sortParam = "";
return null;
}
@Override

24
src/main/java/org/codelibs/fess/app/web/admin/backup/AdminBackupAction.java
View file

@ -44,12 +44,16 @@ import org.codelibs.core.misc.Pair;
import org.codelibs.elasticsearch.runner.net.CurlResponse;
import org.codelibs.fess.Constants;
import org.codelibs.fess.app.web.base.FessAdminAction;
import org.codelibs.fess.es.config.exbhv.FileConfigBhv;
import org.codelibs.fess.es.config.exbhv.LabelTypeBhv;
import org.codelibs.fess.es.config.exbhv.WebConfigBhv;
import org.codelibs.fess.es.log.exbhv.ClickLogBhv;
import org.codelibs.fess.es.log.exbhv.FavoriteLogBhv;
import org.codelibs.fess.es.log.exbhv.SearchLogBhv;
import org.codelibs.fess.es.log.exbhv.UserInfoBhv;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.util.ComponentUtil;
import org.codelibs.fess.util.GsaConfigParser;
import org.codelibs.fess.util.RenderDataUtil;
import org.lastaflute.core.magic.async.AsyncManager;
import org.lastaflute.web.Execute;
@ -59,6 +63,7 @@ import org.lastaflute.web.response.StreamResponse;
import org.lastaflute.web.ruts.process.ActionRuntime;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.xml.sax.InputSource;
/**
* @author shinsuke
@ -74,6 +79,15 @@ public class AdminBackupAction extends FessAdminAction {
@Resource
private AsyncManager asyncManager;
@Resource
private WebConfigBhv webConfigBhv;
@Resource
private FileConfigBhv fileConfigBhv;
@Resource
private LabelTypeBhv labelTypeBhv;
@Override
protected void setupHtmlData(final ActionRuntime runtime) {
super.setupHtmlData(runtime);
@ -98,6 +112,16 @@ public class AdminBackupAction extends FessAdminAction {
} catch (final IOException e) {
logger.warn("Failed to process system.properties file: " + form.bulkFile.getFileName(), e);
}
} else if (fileName.startsWith("gsa") && fileName.endsWith(".xml")) {
GsaConfigParser configParser = ComponentUtil.getComponent(GsaConfigParser.class);
try (final InputStream in = form.bulkFile.getInputStream()) {
configParser.parse(new InputSource(in));
} catch (final IOException e) {
logger.warn("Failed to process gsa.xml file: " + form.bulkFile.getFileName(), e);
}
configParser.getWebConfig().ifPresent(c -> webConfigBhv.insert(c));
configParser.getFileConfig().ifPresent(c -> fileConfigBhv.insert(c));
labelTypeBhv.batchInsert(Arrays.stream(configParser.getLabelTypes()).collect(Collectors.toList()));
} else {
try (CurlResponse response = ComponentUtil.getCurlHelper().post("/_bulk").onConnect((req, con) -> {
con.setDoOutput(true);

1
src/main/java/org/codelibs/fess/app/web/admin/group/AdminGroupAction.java
View file

@ -255,6 +255,7 @@ public class AdminGroupAction extends FessAdminAction {
public static OptionalEntity<Group> getGroup(final CreateForm form) {
return getEntity(form).map(entity -> {
copyMapToBean(form.attributes, entity, op -> op.exclude(Constants.COMMON_CONVERSION_RULE));
copyBeanToBean(form, entity, op -> op.exclude(Constants.COMMON_CONVERSION_RULE));
return entity;
});

1
src/main/java/org/codelibs/fess/app/web/admin/role/AdminRoleAction.java
View file

@ -209,6 +209,7 @@ public class AdminRoleAction extends FessAdminAction {
public static OptionalEntity<Role> getRole(final CreateForm form) {
return getEntity(form).map(entity -> {
copyMapToBean(form.attributes, entity, op -> op.exclude(Constants.COMMON_CONVERSION_RULE));
copyBeanToBean(form, entity, op -> op.exclude(Constants.COMMON_CONVERSION_RULE));
return entity;
});

41
src/main/java/org/codelibs/fess/app/web/admin/systeminfo/AdminSysteminfoAction.java
View file

@ -26,8 +26,10 @@ import org.codelibs.core.lang.StringUtil;
import org.codelibs.core.misc.DynamicProperties;
import org.codelibs.fess.Constants;
import org.codelibs.fess.app.web.base.FessAdminAction;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.util.ComponentUtil;
import org.codelibs.fess.util.RenderDataUtil;
import org.lastaflute.core.direction.ObjectiveConfig;
import org.lastaflute.web.Execute;
import org.lastaflute.web.response.HtmlResponse;
import org.lastaflute.web.response.render.RenderData;
@ -38,6 +40,8 @@ import org.lastaflute.web.ruts.process.ActionRuntime;
*/
public class AdminSysteminfoAction extends FessAdminAction {
private static final String MASKED_VALUE = "XXXXXXXX";
// ===================================================================================
// Attribute
// =========
@ -83,7 +87,7 @@ public class AdminSysteminfoAction extends FessAdminAction {
}
protected void registerFessPropItems(final RenderData data) {
RenderDataUtil.register(data, "fessPropItems", getFessPropItems());
RenderDataUtil.register(data, "fessPropItems", getFessPropItems(fessConfig));
}
protected void registerBugReportItems(final RenderData data) {
@ -106,15 +110,42 @@ public class AdminSysteminfoAction extends FessAdminAction {
return itemList;
}
public static List<Map<String, String>> getFessPropItems() {
public static List<Map<String, String>> getFessPropItems(final FessConfig fessConfig) {
final List<Map<String, String>> itemList = new ArrayList<>();
final DynamicProperties systemProperties = ComponentUtil.getSystemProperties();
for (final Map.Entry<Object, Object> entry : systemProperties.entrySet()) {
itemList.add(createItem(entry.getKey(), entry.getValue()));
ComponentUtil.getSystemProperties().entrySet().stream().forEach(e -> {
final String k = e.getKey().toString();
final String value;
if (isMaskedValue(k)) {
value = MASKED_VALUE;
} else {
value = e.getValue().toString();
}
itemList.add(createItem(k, value));
});
if (fessConfig instanceof ObjectiveConfig) {
ObjectiveConfig config = (ObjectiveConfig) fessConfig;
config.keySet().stream().forEach(k -> {
final String value;
if (isMaskedValue(k)) {
value = MASKED_VALUE;
} else {
value = config.get(k);
}
itemList.add(createItem(k, value));
});
}
return itemList;
}
protected static boolean isMaskedValue(final String key) {
return "http.proxy.password".equals(key) //
|| "ldap.admin.security.credentials".equals(key) //
|| "spnego.preauth.password".equals(key) //
|| "app.cipher.key".equals(key) //
|| "oic.client.id".equals(key) //
|| "oic.client.secret".equals(key);
}
public static List<Map<String, String>> getBugReportItems() {
final List<Map<String, String>> itemList = new ArrayList<>();
for (final String label : bugReportLabels) {

1
src/main/java/org/codelibs/fess/app/web/admin/user/AdminUserAction.java
View file

@ -285,6 +285,7 @@ public class AdminUserAction extends FessAdminAction {
public static OptionalEntity<User> getUser(final CreateForm form) {
return getEntity(form).map(entity -> {
copyMapToBean(form.attributes, entity, op -> op.exclude(Constants.COMMON_CONVERSION_RULE));
copyBeanToBean(form, entity, op -> op.exclude(ArrayUtils.addAll(Constants.COMMON_CONVERSION_RULE, "password")));
if (form.crudMode.intValue() == CrudMode.CREATE || StringUtil.isNotBlank(form.password)) {
final String encodedPassword = ComponentUtil.getComponent(FessLoginAssist.class).encryptPassword(form.password);

2
src/main/java/org/codelibs/fess/app/web/api/admin/systeminfo/ApiAdminSysteminfoAction.java
View file

@ -42,7 +42,7 @@ public class ApiAdminSysteminfoAction extends FessApiAdminAction {
public JsonResponse<ApiResult> get$index() {
final List<Map<String, String>> bugReportItems = getBugReportItems();
final List<Map<String, String>> envItems = getEnvItems();
final List<Map<String, String>> fessPropItems = getFessPropItems();
final List<Map<String, String>> fessPropItems = getFessPropItems(fessConfig);
final List<Map<String, String>> propItems = getPropItems();
return asJson(new ApiResult.ApiSystemInfoResponse().bugReportProps(bugReportItems).envProps(envItems).fessProps(fessPropItems)
.systemProps(propItems).status(ApiResult.Status.OK).result());

5
src/main/java/org/codelibs/fess/app/web/base/FessBaseAction.java
View file

@ -15,6 +15,7 @@
*/
package org.codelibs.fess.app.web.base;
import java.util.Map;
import java.util.function.Consumer;
import javax.annotation.Resource;
@ -194,6 +195,10 @@ public abstract class FessBaseAction extends TypicalAction // has several interf
BeanUtil.copyBeanToBean(src, dest, option);
}
protected static void copyMapToBean(final Map<String, ? extends Object> src, final Object dest, final Consumer<CopyOptions> option) {
BeanUtil.copyMapToBean(src, dest, option);
}
protected static <T> T copyBeanToNewBean(final Object src, final Class<T> destClass) {
return BeanUtil.copyBeanToNewBean(src, destClass);
}

30
src/main/java/org/codelibs/fess/exception/GsaConfigException.java Normal file
View file

@ -0,0 +1,30 @@
/*
* Copyright 2012-2018 CodeLibs Project and the Others.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
* either express or implied. See the License for the specific language
* governing permissions and limitations under the License.
*/
package org.codelibs.fess.exception;
public class GsaConfigException extends FessSystemException {
private static final long serialVersionUID = 1L;
public GsaConfigException(String message, Throwable cause) {
super(message, cause);
}
public GsaConfigException(String message) {
super(message);
}
}

48
src/main/java/org/codelibs/fess/helper/QueryHelper.java
View file

@ -218,7 +218,6 @@ public class QueryHelper {
}
if (highlightedFields == null) {
highlightedFields = fessConfig.getQueryAdditionalHighlightedFields( //
fessConfig.getIndexFieldTitle(), //
fessConfig.getIndexFieldContent());
}
if (searchFields == null) {
@ -584,28 +583,31 @@ public class QueryHelper {
context.addHighlightedQuery(text);
return buildDefaultQueryBuilder((f, b) -> buildMatchPhraseQuery(f, text).boost(b * boost));
} else if ("sort".equals(field)) {
final String[] values = text.split("\\.");
if (values.length > 2) {
throw new InvalidQueryException(
messages -> messages.addErrorsInvalidQuerySortValue(UserMessages.GLOBAL_PROPERTY_KEY, text), "Invalid sort field: "
+ termQuery);
}
final String sortField = values[0];
if (!isSortField(sortField)) {
throw new InvalidQueryException(messages -> messages.addErrorsInvalidQueryUnsupportedSortField(
UserMessages.GLOBAL_PROPERTY_KEY, sortField), "Unsupported sort field: " + termQuery);
}
SortOrder sortOrder;
if (values.length == 2) {
sortOrder = SortOrder.DESC.toString().equalsIgnoreCase(values[1]) ? SortOrder.DESC : SortOrder.ASC;
if (sortOrder == null) {
throw new InvalidQueryException(messages -> messages.addErrorsInvalidQueryUnsupportedSortOrder(
UserMessages.GLOBAL_PROPERTY_KEY, values[1]), "Invalid sort order: " + termQuery);
}
} else {
sortOrder = SortOrder.ASC;
}
context.addSorts(createFieldSortBuilder(sortField, sortOrder));
split(text, ",").of(
stream -> stream.filter(StringUtil::isNotBlank).forEach(
t -> {
final String[] values = t.split("\\.");
if (values.length > 2) {
throw new InvalidQueryException(messages -> messages.addErrorsInvalidQuerySortValue(
UserMessages.GLOBAL_PROPERTY_KEY, text), "Invalid sort field: " + termQuery);
}
final String sortField = values[0];
if (!isSortField(sortField)) {
throw new InvalidQueryException(messages -> messages.addErrorsInvalidQueryUnsupportedSortField(
UserMessages.GLOBAL_PROPERTY_KEY, sortField), "Unsupported sort field: " + termQuery);
}
SortOrder sortOrder;
if (values.length == 2) {
sortOrder = SortOrder.DESC.toString().equalsIgnoreCase(values[1]) ? SortOrder.DESC : SortOrder.ASC;
if (sortOrder == null) {
throw new InvalidQueryException(messages -> messages.addErrorsInvalidQueryUnsupportedSortOrder(
UserMessages.GLOBAL_PROPERTY_KEY, values[1]), "Invalid sort order: " + termQuery);
}
} else {
sortOrder = SortOrder.ASC;
}
context.addSorts(createFieldSortBuilder(sortField, sortOrder));
}));
return null;
} else if (INURL_FIELD.equals(field) || fessConfig.getIndexFieldUrl().equals(context.getDefaultField())) {
return QueryBuilders.wildcardQuery(fessConfig.getIndexFieldUrl(), "*" + text + "*").boost(boost);

36
src/main/java/org/codelibs/fess/helper/ViewHelper.java
View file

@ -148,27 +148,31 @@ public class ViewHelper {
public String getContentTitle(final Map<String, Object> document) {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final int size = fessConfig.getResponseMaxTitleLengthAsInteger();
String title =
DocumentUtil.getValue(document, ComponentUtil.getQueryHelper().getHighlightPrefix() + fessConfig.getIndexFieldTitle(),
String.class);
if (StringUtil.isBlank(title) || title.length() > size - 3) {
title = DocumentUtil.getValue(document, fessConfig.getIndexFieldTitle(), String.class);
String title = DocumentUtil.getValue(document, fessConfig.getIndexFieldTitle(), String.class);
if (StringUtil.isBlank(title)) {
title = DocumentUtil.getValue(document, fessConfig.getIndexFieldFilename(), String.class);
if (StringUtil.isBlank(title)) {
title = DocumentUtil.getValue(document, fessConfig.getIndexFieldFilename(), String.class);
if (StringUtil.isBlank(title)) {
title = DocumentUtil.getValue(document, fessConfig.getIndexFieldUrl(), String.class);
}
title = DocumentUtil.getValue(document, fessConfig.getIndexFieldUrl(), String.class);
}
title = LaFunctions.h(title);
} else {
title = escapeHighlight(title).replaceAll("\\.\\.\\.$", StringUtil.EMPTY);
}
final int size = fessConfig.getResponseMaxTitleLengthAsInteger();
if (size > -1) {
return StringUtils.abbreviate(title, size);
} else {
return title;
title = StringUtils.abbreviate(title, size);
}
final String value = LaFunctions.h(title);
return LaRequestUtil.getOptionalRequest().map(req -> {
@SuppressWarnings("unchecked")
final Set<String> querySet = (Set<String>) req.getAttribute(Constants.HIGHLIGHT_QUERIES);
if (querySet != null) {
String t = value;
for (final String query : querySet) {
final String target = LaFunctions.h(query);
t = t.replace(target, highlightTagPre + target + highlightTagPost);
}
return t;
}
return value;
}).orElse(value);
}
public String getContentDescription(final Map<String, Object> document) {

54
src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java
View file

@ -75,6 +75,8 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
-Dlog4j.shutdownHookEnabled=false
-Dlog4j2.disable.jmx=true
-Dlog4j.skipJansi=true
-Dsun.java2d.cmm=sun.java2d.cmm.kcms.KcmsServiceProvider
-Dorg.apache.pdfbox.rendering.UsePureJavaCMYKConversion=true
*/
String JVM_CRAWLER_OPTIONS = "jvm.crawler.options";
@ -129,6 +131,8 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
-Dlog4j.shutdownHookEnabled=false
-Dlog4j2.disable.jmx=true
-Dlog4j.skipJansi=true
-Dsun.java2d.cmm=sun.java2d.cmm.kcms.KcmsServiceProvider
-Dorg.apache.pdfbox.rendering.UsePureJavaCMYKConversion=true
*/
String JVM_THUMBNAIL_OPTIONS = "jvm.thumbnail.options";
@ -623,6 +627,12 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
/** The key of the configuration. e.g. UE,U,T,RK,S,LANG */
String QUERY_GSA_RESPONSE_FIELDS = "query.gsa.response.fields";
/** The key of the configuration. e.g. en */
String QUERY_GSA_DEFAULT_LANG = "query.gsa.default.lang";
/** The key of the configuration. e.g. */
String QUERY_GSA_DEFAULT_SORT = "query.gsa.default.sort";
/** The key of the configuration. e.g. 4 */
String QUERY_COLLAPSE_MAX_CONCURRENT_GROUP_RESULTS = "query.collapse.max.concurrent.group.results";
@ -1498,6 +1508,8 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
-Dlog4j.shutdownHookEnabled=false
-Dlog4j2.disable.jmx=true
-Dlog4j.skipJansi=true
-Dsun.java2d.cmm=sun.java2d.cmm.kcms.KcmsServiceProvider
-Dorg.apache.pdfbox.rendering.UsePureJavaCMYKConversion=true
<br>
* comment: JVM options
* @return The value of found property. (NotNull: if not found, exception but basically no way)
@ -1561,6 +1573,8 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
-Dlog4j.shutdownHookEnabled=false
-Dlog4j2.disable.jmx=true
-Dlog4j.skipJansi=true
-Dsun.java2d.cmm=sun.java2d.cmm.kcms.KcmsServiceProvider
-Dorg.apache.pdfbox.rendering.UsePureJavaCMYKConversion=true
<br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
@ -3238,6 +3252,28 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
*/
String getQueryGsaResponseFields();
/**
* Get the value for the key 'query.gsa.default.lang'. <br>
* The value is, e.g. en <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getQueryGsaDefaultLang();
/**
* Get the value for the key 'query.gsa.default.sort'. <br>
* The value is, e.g. <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getQueryGsaDefaultSort();
/**
* Get the value for the key 'query.gsa.default.sort' as {@link Integer}. <br>
* The value is, e.g. <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
* @throws NumberFormatException When the property is not integer.
*/
Integer getQueryGsaDefaultSortAsInteger();
/**
* Get the value for the key 'query.collapse.max.concurrent.group.results'. <br>
* The value is, e.g. 4 <br>
@ -6729,6 +6765,18 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
return get(FessConfig.QUERY_GSA_RESPONSE_FIELDS);
}
public String getQueryGsaDefaultLang() {
return get(FessConfig.QUERY_GSA_DEFAULT_LANG);
}
public String getQueryGsaDefaultSort() {
return get(FessConfig.QUERY_GSA_DEFAULT_SORT);
}
public Integer getQueryGsaDefaultSortAsInteger() {
return getAsInteger(FessConfig.QUERY_GSA_DEFAULT_SORT);
}
public String getQueryCollapseMaxConcurrentGroupResults() {
return get(FessConfig.QUERY_COLLAPSE_MAX_CONCURRENT_GROUP_RESULTS);
}
@ -8079,13 +8127,13 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
defaultMap.put(FessConfig.APP_DIGEST_ALGORISM, "sha256");
defaultMap
.put(FessConfig.JVM_CRAWLER_OPTIONS,
"-Djava.awt.headless=true\n-Dfile.encoding=UTF-8\n-Djna.nosys=true\n-Djdk.io.permissionsUseCanonicalPath=true\n-server\n-Xmx512m\n-XX:MaxMetaspaceSize=128m\n-XX:CompressedClassSpaceSize=32m\n-XX:-UseGCOverheadLimit\n-XX:+UseConcMarkSweepGC\n-XX:CMSInitiatingOccupancyFraction=75\n-XX:+UseCMSInitiatingOccupancyOnly\n-XX:+UseTLAB\n-XX:+DisableExplicitGC\n-XX:+HeapDumpOnOutOfMemoryError\n-XX:-OmitStackTraceInFastThrow\n-Djcifs.smb.client.connTimeout=60000\n-Djcifs.smb.client.soTimeout=35000\n-Djcifs.smb.client.responseTimeout=30000\n-Dgroovy.use.classvalue=true\n-Dio.netty.noUnsafe=true\n-Dio.netty.noKeySetOptimization=true\n-Dio.netty.recycler.maxCapacityPerThread=0\n-Dlog4j.shutdownHookEnabled=false\n-Dlog4j2.disable.jmx=true\n-Dlog4j.skipJansi=true\n");
"-Djava.awt.headless=true\n-Dfile.encoding=UTF-8\n-Djna.nosys=true\n-Djdk.io.permissionsUseCanonicalPath=true\n-server\n-Xmx512m\n-XX:MaxMetaspaceSize=128m\n-XX:CompressedClassSpaceSize=32m\n-XX:-UseGCOverheadLimit\n-XX:+UseConcMarkSweepGC\n-XX:CMSInitiatingOccupancyFraction=75\n-XX:+UseCMSInitiatingOccupancyOnly\n-XX:+UseTLAB\n-XX:+DisableExplicitGC\n-XX:+HeapDumpOnOutOfMemoryError\n-XX:-OmitStackTraceInFastThrow\n-Djcifs.smb.client.connTimeout=60000\n-Djcifs.smb.client.soTimeout=35000\n-Djcifs.smb.client.responseTimeout=30000\n-Dgroovy.use.classvalue=true\n-Dio.netty.noUnsafe=true\n-Dio.netty.noKeySetOptimization=true\n-Dio.netty.recycler.maxCapacityPerThread=0\n-Dlog4j.shutdownHookEnabled=false\n-Dlog4j2.disable.jmx=true\n-Dlog4j.skipJansi=true\n-Dsun.java2d.cmm=sun.java2d.cmm.kcms.KcmsServiceProvider\n-Dorg.apache.pdfbox.rendering.UsePureJavaCMYKConversion=true\n");
defaultMap
.put(FessConfig.JVM_SUGGEST_OPTIONS,
"-Djava.awt.headless=true\n-Dfile.encoding=UTF-8\n-Djna.nosys=true\n-Djdk.io.permissionsUseCanonicalPath=true\n-server\n-Xmx256m\n-XX:MaxMetaspaceSize=128m\n-XX:CompressedClassSpaceSize=32m\n-XX:-UseGCOverheadLimit\n-XX:+UseConcMarkSweepGC\n-XX:CMSInitiatingOccupancyFraction=75\n-XX:+UseCMSInitiatingOccupancyOnly\n-XX:+UseTLAB\n-XX:+DisableExplicitGC\n-XX:+HeapDumpOnOutOfMemoryError\n-Dgroovy.use.classvalue=true\n-Dio.netty.noUnsafe=true\n-Dio.netty.noKeySetOptimization=true\n-Dio.netty.recycler.maxCapacityPerThread=0\n-Dlog4j.shutdownHookEnabled=false\n-Dlog4j2.disable.jmx=true\n-Dlog4j.skipJansi=true\n");
defaultMap
.put(FessConfig.JVM_THUMBNAIL_OPTIONS,
"-Djava.awt.headless=true\n-Dfile.encoding=UTF-8\n-Djna.nosys=true\n-Djdk.io.permissionsUseCanonicalPath=true\n-server\n-Xmx128m\n-XX:MaxMetaspaceSize=128m\n-XX:CompressedClassSpaceSize=32m\n-XX:-UseGCOverheadLimit\n-XX:+UseConcMarkSweepGC\n-XX:CMSInitiatingOccupancyFraction=75\n-XX:+UseCMSInitiatingOccupancyOnly\n-XX:+UseTLAB\n-XX:+DisableExplicitGC\n-XX:+HeapDumpOnOutOfMemoryError\n-XX:-OmitStackTraceInFastThrow\n-Djcifs.smb.client.connTimeout=60000\n-Djcifs.smb.client.soTimeout=35000\n-Djcifs.smb.client.responseTimeout=30000\n-Dgroovy.use.classvalue=true\n-Dio.netty.noUnsafe=true\n-Dio.netty.noKeySetOptimization=true\n-Dio.netty.recycler.maxCapacityPerThread=0\n-Dlog4j.shutdownHookEnabled=false\n-Dlog4j2.disable.jmx=true\n-Dlog4j.skipJansi=true\n");
"-Djava.awt.headless=true\n-Dfile.encoding=UTF-8\n-Djna.nosys=true\n-Djdk.io.permissionsUseCanonicalPath=true\n-server\n-Xmx128m\n-XX:MaxMetaspaceSize=128m\n-XX:CompressedClassSpaceSize=32m\n-XX:-UseGCOverheadLimit\n-XX:+UseConcMarkSweepGC\n-XX:CMSInitiatingOccupancyFraction=75\n-XX:+UseCMSInitiatingOccupancyOnly\n-XX:+UseTLAB\n-XX:+DisableExplicitGC\n-XX:+HeapDumpOnOutOfMemoryError\n-XX:-OmitStackTraceInFastThrow\n-Djcifs.smb.client.connTimeout=60000\n-Djcifs.smb.client.soTimeout=35000\n-Djcifs.smb.client.responseTimeout=30000\n-Dgroovy.use.classvalue=true\n-Dio.netty.noUnsafe=true\n-Dio.netty.noKeySetOptimization=true\n-Dio.netty.recycler.maxCapacityPerThread=0\n-Dlog4j.shutdownHookEnabled=false\n-Dlog4j2.disable.jmx=true\n-Dlog4j.skipJansi=true\n-Dsun.java2d.cmm=sun.java2d.cmm.kcms.KcmsServiceProvider\n-Dorg.apache.pdfbox.rendering.UsePureJavaCMYKConversion=true\n");
defaultMap.put(FessConfig.JOB_SYSTEM_JOB_IDS, "default_crawler");
defaultMap.put(FessConfig.JOB_TEMPLATE_TITLE_WEB, "Web Crawler - {0}");
defaultMap.put(FessConfig.JOB_TEMPLATE_TITLE_FILE, "File Crawler - {0}");
@ -8255,6 +8303,8 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
defaultMap.put(FessConfig.QUERY_ADDITIONAL_ANALYZED_FIELDS, "");
defaultMap.put(FessConfig.QUERY_ADDITIONAL_NOT_ANALYZED_FIELDS, "");
defaultMap.put(FessConfig.QUERY_GSA_RESPONSE_FIELDS, "UE,U,T,RK,S,LANG");
defaultMap.put(FessConfig.QUERY_GSA_DEFAULT_LANG, "en");
defaultMap.put(FessConfig.QUERY_GSA_DEFAULT_SORT, "");
defaultMap.put(FessConfig.QUERY_COLLAPSE_MAX_CONCURRENT_GROUP_RESULTS, "4");
defaultMap.put(FessConfig.QUERY_COLLAPSE_INNER_HITS_NAME, "similar_docs");
defaultMap.put(FessConfig.QUERY_COLLAPSE_INNER_HITS_SIZE, "0");

314
src/main/java/org/codelibs/fess/util/GsaConfigParser.java Normal file
View file

@ -0,0 +1,314 @@
/*
* Copyright 2012-2018 CodeLibs Project and the Others.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
* either express or implied. See the License for the specific language
* governing permissions and limitations under the License.
*/
package org.codelibs.fess.util;
import static org.codelibs.core.stream.StreamUtil.split;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import javax.xml.parsers.SAXParser;
import javax.xml.parsers.SAXParserFactory;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.fess.Constants;
import org.codelibs.fess.es.config.exentity.FileConfig;
import org.codelibs.fess.es.config.exentity.LabelType;
import org.codelibs.fess.es.config.exentity.WebConfig;
import org.codelibs.fess.exception.GsaConfigException;
import org.dbflute.optional.OptionalEntity;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.xml.sax.Attributes;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import org.xml.sax.helpers.DefaultHandler;
public class GsaConfigParser extends DefaultHandler {
private static final Logger logger = LoggerFactory.getLogger(GsaConfigParser.class);
protected static final String REGEXP = "regexp:";
protected static final String REGEXP_IGNORE_CASE = "regexpIgnoreCase:";
protected static final String CONTAINS = "contains:";
protected static final String COLLECTIONS = "collections";
protected static final String COLLECTION = "collection";
protected static final String GLOBALPARAMS = "globalparams";
protected static final String START_URLS = "start_urls";
protected static final String GOOD_URLS = "good_urls";
protected static final String BAD_URLS = "bad_urls";
protected String[] webProtocols = new String[] { "http:", "https:" };
protected String[] fileProtocols = new String[] { "file:", "smb:" };
protected LinkedList<String> tagQueue;
protected List<LabelType> labelList;
protected LabelType labelType;
protected Map<String, String> globalParams = new HashMap<>();
protected WebConfig webConfig = null;
protected FileConfig fileConfig = null;
protected StringBuilder textBuf = new StringBuilder(1000);
protected String userAgent = "gsa-crawler";
public void parse(final InputSource is) {
try {
final SAXParserFactory factory = SAXParserFactory.newInstance();
final SAXParser parser = factory.newSAXParser();
parser.parse(is, this);
} catch (final Exception e) {
throw new GsaConfigException("Failed to parse XML file.", e);
}
}
@Override
public void startDocument() throws SAXException {
tagQueue = new LinkedList<>();
labelList = new ArrayList<>();
labelType = null;
}
@Override
public void endDocument() throws SAXException {
globalParams.clear();
tagQueue.clear();
}
@Override
public void startElement(final String uri, final String localName, final String qName, final Attributes attributes) throws SAXException {
if (logger.isDebugEnabled()) {
logger.debug("Start Element: " + qName);
}
if (tagQueue.isEmpty() && !"eef".equalsIgnoreCase(qName)) {
throw new GsaConfigException("Invalid format.");
} else if (COLLECTION.equalsIgnoreCase(qName) && COLLECTIONS.equalsIgnoreCase(tagQueue.peekLast())) {
final long now = System.currentTimeMillis();
final String name = attributes.getValue("Name");
labelType = new LabelType();
labelType.setName(name);
labelType.setValue(name);
labelType.setCreatedBy(Constants.SYSTEM_USER);
labelType.setCreatedTime(now);
labelType.setUpdatedBy(Constants.SYSTEM_USER);
labelType.setUpdatedTime(now);
}
tagQueue.offer(qName);
}
@Override
public void endElement(final String uri, final String localName, final String qName) throws SAXException {
if (logger.isDebugEnabled()) {
logger.debug("End Element: " + qName);
}
if (GOOD_URLS.equalsIgnoreCase(qName)) {
if (labelType != null) {
labelType.setIncludedPaths(parseFilterPaths(textBuf.toString(), true, true));
} else if (GLOBALPARAMS.equalsIgnoreCase(tagQueue.get(tagQueue.size() - 2))) {
globalParams.put(GOOD_URLS, textBuf.toString());
}
} else if (BAD_URLS.equalsIgnoreCase(qName)) {
if (labelType != null) {
labelType.setExcludedPaths(parseFilterPaths(textBuf.toString(), true, true));
} else if (GLOBALPARAMS.equalsIgnoreCase(tagQueue.get(tagQueue.size() - 2))) {
globalParams.put(BAD_URLS, textBuf.toString());
}
} else if (START_URLS.equalsIgnoreCase(qName) && GLOBALPARAMS.equalsIgnoreCase(tagQueue.get(tagQueue.size() - 2))) {
globalParams.put(START_URLS, textBuf.toString());
} else if (labelType != null && COLLECTION.equalsIgnoreCase(qName)) {
labelList.add(labelType);
labelType = null;
} else if (GLOBALPARAMS.equalsIgnoreCase(qName)) {
final Object startUrls = globalParams.get(START_URLS);
if (startUrls != null) {
final long now = System.currentTimeMillis();
final List<String> urlList =
split(startUrls.toString(), "\n").get(
stream -> stream.map(String::trim).filter(StringUtil::isNotBlank).collect(Collectors.toList()));
final String webUrls =
urlList.stream().filter(s -> Arrays.stream(webProtocols).anyMatch(p -> s.startsWith(p)))
.collect(Collectors.joining("\n"));
if (StringUtil.isNotBlank(webUrls)) {
webConfig = new WebConfig();
webConfig.setName("Default");
webConfig.setAvailable(true);
webConfig.setBoost(1.0f);
webConfig.setConfigParameter(StringUtil.EMPTY);
webConfig.setIntervalTime(1000);
webConfig.setNumOfThread(3);
webConfig.setSortOrder(1);
webConfig.setUrls(webUrls);
webConfig.setIncludedUrls(parseFilterPaths(globalParams.get(GOOD_URLS), true, false));
webConfig.setIncludedDocUrls(StringUtil.EMPTY);
webConfig.setExcludedUrls(parseFilterPaths(globalParams.get(BAD_URLS), true, false));
webConfig.setExcludedDocUrls(StringUtil.EMPTY);
webConfig.setUserAgent(userAgent);
webConfig.setCreatedBy(Constants.SYSTEM_USER);
webConfig.setCreatedTime(now);
webConfig.setUpdatedBy(Constants.SYSTEM_USER);
webConfig.setUpdatedTime(now);
}
final String fileUrls =
urlList.stream().filter(s -> Arrays.stream(fileProtocols).anyMatch(p -> s.startsWith(p)))
.collect(Collectors.joining("\n"));
if (StringUtil.isNotBlank(fileUrls)) {
fileConfig = new FileConfig();
fileConfig.setName("Default");
fileConfig.setAvailable(true);
fileConfig.setBoost(1.0f);
fileConfig.setConfigParameter(StringUtil.EMPTY);
fileConfig.setIntervalTime(0);
fileConfig.setNumOfThread(5);
fileConfig.setSortOrder(2);
fileConfig.setPaths(fileUrls);
fileConfig.setIncludedPaths(parseFilterPaths(globalParams.get(GOOD_URLS), false, true));
fileConfig.setIncludedDocPaths(StringUtil.EMPTY);
fileConfig.setExcludedPaths(parseFilterPaths(globalParams.get(BAD_URLS), false, true));
fileConfig.setExcludedDocPaths(StringUtil.EMPTY);
fileConfig.setCreatedBy(Constants.SYSTEM_USER);
fileConfig.setCreatedTime(now);
fileConfig.setUpdatedBy(Constants.SYSTEM_USER);
fileConfig.setUpdatedTime(now);
}
}
} else if ("user_agent".equalsIgnoreCase(qName) && GLOBALPARAMS.equalsIgnoreCase(tagQueue.get(tagQueue.size() - 2))) {
userAgent = textBuf.toString().trim();
}
tagQueue.pollLast();
textBuf.setLength(0);
}
@Override
public void characters(final char[] ch, final int start, final int length) throws SAXException {
String text = new String(ch, start, length);
if (logger.isDebugEnabled()) {
logger.debug("Text: " + text);
}
textBuf.append(text);
}
protected String parseFilterPaths(final String text, final boolean web, final boolean file) {
return split(text, "\n").get(stream -> stream.map(String::trim).filter(StringUtil::isNotBlank).map(s -> {
if (s.startsWith("#")) {
return null;
} else if (s.startsWith(CONTAINS)) {
final String v = s.substring(CONTAINS.length());
final StringBuilder buf = new StringBuilder(100);
return appendFileterPath(buf, escape(v));
} else if (s.startsWith(REGEXP_IGNORE_CASE)) {
final String v = s.substring(REGEXP_IGNORE_CASE.length());
final StringBuilder buf = new StringBuilder(100);
buf.append("(?i)");
return appendFileterPath(buf, unescape(v));
} else if (s.startsWith(REGEXP)) {
final String v = s.substring(REGEXP.length());
final StringBuilder buf = new StringBuilder(100);
return appendFileterPath(buf, unescape(v));
} else if (Arrays.stream(webProtocols).anyMatch(p -> s.startsWith(p))) {
return escape(s) + ".*";
} else if (Arrays.stream(fileProtocols).anyMatch(p -> s.startsWith(p))) {
return escape(s) + ".*";
} else {
final StringBuilder buf = new StringBuilder(100);
return appendFileterPath(buf, escape(s));
}
}).filter(s -> {
if (StringUtil.isBlank(s)) {
return false;
}
if (Arrays.stream(webProtocols).anyMatch(p -> s.startsWith(p))) {
return web;
}
if (Arrays.stream(fileProtocols).anyMatch(p -> s.startsWith(p))) {
return file;
}
return true;
}).collect(Collectors.joining("\n")));
}
protected String escape(final String s) {
return s.replace(".", "\\.")//
.replace("+", "\\+")//
.replace("*", "\\*")//
.replace("[", "\\[")//
.replace("]", "\\]")//
.replace("(", "\\(")//
.replace("(", "\\)")//
.replace("?", "\\?");
}
protected String unescape(final String s) {
return s.replace("\\\\", "\\");
}
protected String appendFileterPath(final StringBuilder buf, final String v) {
if (!v.startsWith("^")) {
buf.append(".*");
}
buf.append(v);
if (!v.endsWith("$")) {
buf.append(".*");
}
return buf.toString();
}
public void setWebProtocols(final String[] webProtocols) {
this.webProtocols = webProtocols;
}
public void setFileProtocols(final String[] fileProtocols) {
this.fileProtocols = fileProtocols;
}
@Override
public String toString() {
return "GsaConfigParser [labelList=" + labelList + ", webConfig=" + webConfig + ", fileConfig=" + fileConfig + "]";
}
public OptionalEntity<WebConfig> getWebConfig() {
return OptionalUtil.ofNullable(webConfig);
}
public OptionalEntity<FileConfig> getFileConfig() {
return OptionalUtil.ofNullable(fileConfig);
}
public LabelType[] getLabelTypes() {
return labelList.toArray(new LabelType[labelList.size()]);
}
}

4
src/main/java/org/codelibs/fess/util/QueryResponseList.java
View file

@ -202,8 +202,8 @@ public class QueryResponseList implements List<Map<String, Object>> {
existNextPage = false;
allPageCount = currentPageNumber;
}
currentStartRecordNumber = allRecordCount != 0 ? (currentPageNumber - 1) * pageSize + 1 : 0;
currentEndRecordNumber = (long) currentPageNumber * pageSize;
currentStartRecordNumber = allRecordCount != 0 ? start + 1 : 0;
currentEndRecordNumber = currentStartRecordNumber + (long) pageSize - 1;
currentEndRecordNumber = allRecordCount < currentEndRecordNumber ? allRecordCount : currentEndRecordNumber;
final int pageRangeSize = 5;

2
src/main/resources/app.xml
View file

@ -383,5 +383,7 @@
</component>
<component name="queryResponseList" class="org.codelibs.fess.util.QueryResponseList" instance="prototype">
</component>
<component name="gsaConfigParser" class="org.codelibs.fess.util.GsaConfigParser" instance="prototype">
</component>
</components>

6
src/main/resources/fess_config.properties
View file

@ -48,6 +48,8 @@ jvm.crawler.options=\
-Dlog4j.shutdownHookEnabled=false\n\
-Dlog4j2.disable.jmx=true\n\
-Dlog4j.skipJansi=true\n\
-Dsun.java2d.cmm=sun.java2d.cmm.kcms.KcmsServiceProvider\n\
-Dorg.apache.pdfbox.rendering.UsePureJavaCMYKConversion=true\n\
jvm.suggest.options=\
@ -102,6 +104,8 @@ jvm.thumbnail.options=\
-Dlog4j.shutdownHookEnabled=false\n\
-Dlog4j2.disable.jmx=true\n\
-Dlog4j.skipJansi=true\n\
-Dsun.java2d.cmm=sun.java2d.cmm.kcms.KcmsServiceProvider\n\
-Dorg.apache.pdfbox.rendering.UsePureJavaCMYKConversion=true\n\
#-Xdebug\n\
@ -304,6 +308,8 @@ query.additional.sort.fields=
query.additional.analyzed.fields=
query.additional.not.analyzed.fields=
query.gsa.response.fields=UE,U,T,RK,S,LANG
query.gsa.default.lang=en
query.gsa.default.sort=
query.collapse.max.concurrent.group.results=4
query.collapse.inner.hits.name=similar_docs
query.collapse.inner.hits.size=0

43
src/test/java/org/codelibs/fess/util/GsaConfigParserTest.java Normal file
View file

@ -0,0 +1,43 @@
/*
* Copyright 2012-2018 CodeLibs Project and the Others.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
* either express or implied. See the License for the specific language
* governing permissions and limitations under the License.
*/
package org.codelibs.fess.util;
import java.io.IOException;
import java.io.InputStream;
import org.codelibs.core.io.ResourceUtil;
import org.codelibs.fess.es.config.exentity.LabelType;
import org.codelibs.fess.unit.UnitFessTestCase;
import org.xml.sax.InputSource;
public class GsaConfigParserTest extends UnitFessTestCase {
public void test_parse() throws IOException {
GsaConfigParser parser = new GsaConfigParser();
try (InputStream is = ResourceUtil.getResourceAsStream("data/gsaconfig.xml")) {
parser.parse(new InputSource(is));
}
parser.getWebConfig().ifPresent(c -> {
System.out.println(c.toString());
}).orElse(() -> fail());
parser.getFileConfig().ifPresent(c -> {
System.out.println(c.toString());
}).orElse(() -> fail());
LabelType[] labelTypes = parser.getLabelTypes();
assertEquals(3, labelTypes.length);
}
}

86
src/test/resources/data/gsaconfig.xml Normal file
View file

@ -0,0 +1,86 @@
<?xml version="1.0" encoding="UTF-8" ?>
<eef>
<config Schema="2.0" EnterpriseVersion="'7.6.50'">
<collections Count="3">
<collection Name="fess">
<bad_urls><![CDATA[
https://fess.codelibs.org/images/
]]></bad_urls>
<good_urls><![CDATA[
https://fess.codelibs.org/
https://www.codelibs.org/
]]></good_urls>
<prerequisite_results><![CDATA[
20
]]></prerequisite_results>
<testwords><![CDATA[
]]></testwords>
</collection>
<collection Name="n2sm">
<bad_urls><![CDATA[
contains:\\.xml$
]]></bad_urls>
<good_urls><![CDATA[
https://www.n2sm.net/
]]></good_urls>
<prerequisite_results><![CDATA[
20
]]></prerequisite_results>
<testwords><![CDATA[
]]></testwords>
</collection>
<collection Name="smb">
<bad_urls><![CDATA[
smb://storage/sample/
]]></bad_urls>
<good_urls><![CDATA[
smb://storage/
]]></good_urls>
<prerequisite_results><![CDATA[
20
]]></prerequisite_results>
<testwords><![CDATA[
]]></testwords>
</collection>
</collections>
<globalparams>
<bad_urls><![CDATA[
contains:/images/
contains:?
contains:\\.xml$
# test
regexp:/([^/]*)/\\1/\\1/
.gif$
.jpg$
.jpeg$
.png$
regexpIgnoreCase:\\.dll$
regexpIgnoreCase:\\.exe$
/?S=A$
/?S=D$
contains:\001
contains:\002
contains:\003
.html/$
]]></bad_urls>
<good_urls><![CDATA[
https://fess.codelibs.org/
https://www.codelibs.org/
https://www.n2sm.net/
smb://storage/
]]></good_urls>
<start_urls><![CDATA[
https://fess.codelibs.org/
https://www.codelibs.org/
https://www.n2sm.net/
smb://storage/
]]></start_urls>
</globalparams>
</config>
</eef>