0ct0pu5/fess
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix #1539 add occt and fix #1537 add sitesearch

Browse source
This commit is contained in:
Shinsuke Sugaya 2018-03-06 22:45:41 +09:00
parent 6284f2e6f7
commit c038c72675
10 changed files with 167 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

36
src/main/java/org/codelibs/fess/entity/QueryContext.java
View file

@ -26,7 +26,9 @@ import java.util.Map;
import java.util.Set;
import java.util.function.Consumer;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.fess.Constants;
import org.codelibs.fess.util.ComponentUtil;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.MatchAllQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
@ -36,11 +38,16 @@ import org.elasticsearch.search.sort.SortBuilder;
import org.lastaflute.web.util.LaRequestUtil;
public class QueryContext {
protected static final String ALLINURL_FIELD_PREFIX = "allinurl:";
protected static final String ALLINTITLE_FIELD_PREFIX = "allintitle:";
private QueryBuilder queryBuilder;
private final List<SortBuilder<?>> sortBuilderList = new ArrayList<>();
private final String queryString;
private String queryString;
private Set<String> highlightedQuerySet = null;
@ -48,9 +55,26 @@ public class QueryContext {
private boolean disableRoleQuery = false;
private String defaultField = null;
@SuppressWarnings("unchecked")
public QueryContext(final String queryString, final boolean isQuery) {
this.queryString = queryString;
if (queryString != null) {
if (queryString.startsWith(ALLINURL_FIELD_PREFIX)) {
this.defaultField = ComponentUtil.getFessConfig().getIndexFieldUrl();
this.queryString = queryString.substring(ALLINURL_FIELD_PREFIX.length());
} else if (queryString.startsWith(ALLINTITLE_FIELD_PREFIX)) {
this.defaultField = ComponentUtil.getFessConfig().getIndexFieldTitle();
this.queryString = queryString.substring(ALLINTITLE_FIELD_PREFIX.length());
} else {
this.queryString = queryString;
}
} else {
this.queryString = queryString;
}
if (StringUtil.isBlank(this.queryString)) {
this.queryString = "*";
}
if (isQuery) {
LaRequestUtil.getOptionalRequest().ifPresent(request -> {
highlightedQuerySet = new HashSet<>();
@ -140,4 +164,12 @@ public class QueryContext {
public void skipRoleQuery() {
disableRoleQuery = true;
}
public String getDefaultField() {
return defaultField;
}
public void setDefaultField(String defaultField) {
this.defaultField = defaultField;
}
}

5
src/main/java/org/codelibs/fess/entity/SearchRequestParams.java
View file

@ -36,6 +36,10 @@ public interface SearchRequestParams {
String AS_FILETYPE = "filetype";
String AS_SITESEARCH = "sitesearch";
String AS_OCCURRENCE = "occt";
String getQuery();
Map<String, String[]> getFields();
@ -70,6 +74,7 @@ public interface SearchRequestParams {
|| !isEmptyArray(conditions.get(AS_EPQ))//
|| !isEmptyArray(conditions.get(AS_OQ))//
|| !isEmptyArray(conditions.get(AS_NQ))//
|| !isEmptyArray(conditions.get(AS_SITESEARCH))//
|| !isEmptyArray(conditions.get(AS_FILETYPE));
}

33
src/main/java/org/codelibs/fess/helper/QueryHelper.java
View file

@ -82,6 +82,8 @@ public class QueryHelper {
protected static final String INURL_FIELD = "inurl";
protected static final String SITE_FIELD = "site";
@Resource
protected FessConfig fessConfig;
@ -193,6 +195,7 @@ public class QueryHelper {
fessConfig.getIndexFieldUrl(), //
fessConfig.getIndexFieldDocId(), //
fessConfig.getIndexFieldHost(), //
fessConfig.getIndexFieldSite(), //
fessConfig.getIndexFieldTitle(), //
fessConfig.getIndexFieldContent(), //
fessConfig.getIndexFieldContentLength(), //
@ -415,7 +418,10 @@ public class QueryHelper {
}
}
}
return boolQuery;
if (boolQuery.hasClauses()) {
return boolQuery;
}
return null;
}
protected String toLowercaseWildcard(final String value) {
@ -425,8 +431,15 @@ public class QueryHelper {
return value;
}
protected String getSearchField(final QueryContext context, final String field) {
if (Constants.DEFAULT_FIELD.equals(field) && context.getDefaultField() != null) {
return context.getDefaultField();
}
return field;
}
protected QueryBuilder convertWildcardQuery(final QueryContext context, final WildcardQuery wildcardQuery, final float boost) {
final String field = wildcardQuery.getField();
final String field = getSearchField(context, wildcardQuery.getField());
if (Constants.DEFAULT_FIELD.equals(field)) {
context.addFieldLog(field, wildcardQuery.getTerm().text());
return buildDefaultQueryBuilder((f, b) -> QueryBuilders.wildcardQuery(f, toLowercaseWildcard(wildcardQuery.getTerm().text()))
@ -444,7 +457,7 @@ public class QueryHelper {
}
protected QueryBuilder convertPrefixQuery(final QueryContext context, final PrefixQuery prefixQuery, final float boost) {
final String field = prefixQuery.getField();
final String field = getSearchField(context, prefixQuery.getField());
if (Constants.DEFAULT_FIELD.equals(field)) {
context.addFieldLog(field, prefixQuery.getPrefix().text());
return buildDefaultQueryBuilder((f, b) -> QueryBuilders.prefixQuery(f, toLowercaseWildcard(prefixQuery.getPrefix().text()))
@ -463,7 +476,7 @@ public class QueryHelper {
protected QueryBuilder convertFuzzyQuery(final QueryContext context, final FuzzyQuery fuzzyQuery, final float boost) {
final Term term = fuzzyQuery.getTerm();
final String field = term.field();
final String field = getSearchField(context, term.field());
// TODO fuzzy value
if (Constants.DEFAULT_FIELD.equals(field)) {
context.addFieldLog(field, term.text());
@ -482,7 +495,7 @@ public class QueryHelper {
}
protected QueryBuilder convertTermRangeQuery(final QueryContext context, final TermRangeQuery termRangeQuery, final float boost) {
final String field = termRangeQuery.getField();
final String field = getSearchField(context, termRangeQuery.getField());
if (isSearchField(field)) {
context.addFieldLog(field, termRangeQuery.toString(field));
final RangeQueryBuilder rangeQuery = QueryBuilders.rangeQuery(field);
@ -530,7 +543,7 @@ public class QueryHelper {
}
protected QueryBuilder convertTermQuery(final QueryContext context, final TermQuery termQuery, final float boost) {
final String field = termQuery.getTerm().field();
final String field = getSearchField(context, termQuery.getTerm().field());
final String text = termQuery.getTerm().text();
if (fessConfig.getQueryReplaceTermWithPrefixQueryAsBoolean() && text.length() > 1 && text.endsWith("*")) {
return convertPrefixQuery(context, new PrefixQuery(new Term(field, text.substring(0, text.length() - 1))), boost);
@ -562,8 +575,10 @@ public class QueryHelper {
}
context.addSorts(createFieldSortBuilder(sortField, sortOrder));
return null;
} else if (INURL_FIELD.equals(field)) {
} else if (INURL_FIELD.equals(field) || fessConfig.getIndexFieldUrl().equals(context.getDefaultField())) {
return QueryBuilders.wildcardQuery(fessConfig.getIndexFieldUrl(), "*" + text + "*").boost(boost);
} else if (SITE_FIELD.equals(field)) {
return convertSiteQuery(context, text, boost);
} else if (isSearchField(field)) {
context.addFieldLog(field, text);
context.addHighlightedQuery(text);
@ -580,6 +595,10 @@ public class QueryHelper {
}
}
protected QueryBuilder convertSiteQuery(final QueryContext context, final String text, final float boost) {
return QueryBuilders.prefixQuery(fessConfig.getIndexFieldSite(), text).boost(boost);
}
private QueryBuilder convertPhraseQuery(final QueryContext context, final PhraseQuery query, final float boost) {
final Term[] terms = query.getTerms();
if (terms.length == 0) {

15
src/main/java/org/codelibs/fess/mylasta/action/FessLabels.java
View file

@ -2718,6 +2718,21 @@ public class FessLabels extends UserMessages {
/** The key of the message: MS PowerPoint */
public static final String LABELS_advance_search_filetype_powerpoint = "{labels.advance_search_filetype_powerpoint}";
/** The key of the message: Terms appearing */
public static final String LABELS_advance_search_occt = "{labels.advance_search_occt}";
/** The key of the message: anywhere in the page */
public static final String LABELS_advance_search_occt_default = "{labels.advance_search_occt_default}";
/** The key of the message: in the title of the page */
public static final String LABELS_advance_search_occt_allintitle = "{labels.advance_search_occt_allintitle}";
/** The key of the message: in the url of the page */
public static final String LABELS_advance_search_occt_allinurl = "{labels.advance_search_occt_allinurl}";
/** The key of the message: Site or domain */
public static final String LABELS_advance_search_sitesearch = "{labels.advance_search_sitesearch}";
/**
* Assert the property is not null.
* @param property The value of the property. (NotNull)

13
src/main/java/org/codelibs/fess/util/QueryStringBuilder.java
View file

@ -104,6 +104,9 @@ public class QueryStringBuilder {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final int maxQueryLength = fessConfig.getQueryMaxLengthAsInteger().intValue();
stream(conditions.get(SearchRequestParams.AS_OCCURRENCE)).of(
stream -> stream.filter(q -> isOccurrence(q)).findFirst().ifPresent(q -> queryBuf.insert(0, q + ":")));
stream(conditions.get(SearchRequestParams.AS_Q)).of(
stream -> stream.filter(q -> StringUtil.isNotBlank(q) && q.length() <= maxQueryLength).forEach(
q -> queryBuf.append(' ').append(q)));
@ -127,10 +130,16 @@ public class QueryStringBuilder {
stream(conditions.get(SearchRequestParams.AS_FILETYPE)).of(
stream -> stream.filter(q -> StringUtil.isNotBlank(q) && q.length() <= maxQueryLength).forEach(
q -> queryBuf.append(" filetype:\"").append(q.trim()).append('"')));
stream(conditions.get(SearchRequestParams.AS_SITESEARCH)).of(
stream -> stream.filter(q -> StringUtil.isNotBlank(q) && q.length() <= maxQueryLength).forEach(
q -> queryBuf.append(" site:").append(q.trim())));
}
private String escape(final String q, final String... values) {
protected boolean isOccurrence(final String value) {
return "allintitle".equals(value) || "allinurl".equals(value);
}
protected String escape(final String q, final String... values) {
String value = q;
for (String s : values) {
value = value.replace(s, "\\" + s);

5
src/main/resources/fess_label.properties
View file

@ -896,3 +896,8 @@ labels.advance_search_filetype_pdf=PDF
labels.advance_search_filetype_word=MS Word
labels.advance_search_filetype_excel=MS Excel
labels.advance_search_filetype_powerpoint=MS PowerPoint
labels.advance_search_occt=Terms appearing
labels.advance_search_occt_default=anywhere in the page
labels.advance_search_occt_allintitle=in the title of the page
labels.advance_search_occt_allinurl=in the url of the page
labels.advance_search_sitesearch=Site or domain

5
src/main/resources/fess_label_en.properties
View file

@ -896,3 +896,8 @@ labels.advance_search_filetype_pdf=PDF
labels.advance_search_filetype_word=MS Word
labels.advance_search_filetype_excel=MS Excel
labels.advance_search_filetype_powerpoint=MS PowerPoint
labels.advance_search_occt=Terms appearing
labels.advance_search_occt_default=anywhere in the page
labels.advance_search_occt_allintitle=in the title of the page
labels.advance_search_occt_allinurl=in the url of the page
labels.advance_search_sitesearch=Site or domain

5
src/main/resources/fess_label_ja.properties
View file

@ -898,3 +898,8 @@ labels.advance_search_filetype_pdf=PDF
labels.advance_search_filetype_word=MS Word
labels.advance_search_filetype_excel=MS Excel
labels.advance_search_filetype_powerpoint=MS PowerPoint
labels.advance_search_occt=\u691c\u7d22\u5bfe\u8c61
labels.advance_search_occt_default=\u30da\u30fc\u30b8\u5168\u4f53
labels.advance_search_occt_allintitle=\u30da\u30fc\u30b8\u5185\u306e\u30bf\u30a4\u30c8\u30eb
labels.advance_search_occt_allinurl=\u30da\u30fc\u30b8\u5185\u306eURL
labels.advance_search_sitesearch=\u30b5\u30a4\u30c8\u307e\u305f\u306f\u30c9\u30e1\u30a4\u30f3

32
src/main/webapp/WEB-INF/view/advance.jsp
View file

@ -239,7 +239,7 @@
/></label>
<div class="col-lg-5 col-md-8 col-sm-7 col-xs-6">
<select id="as_filetype" name="as.filetype" class="form-control">
<option><la:message key="labels.advance_search_filetype_default" /></option>
<option value=""><la:message key="labels.advance_search_filetype_default" /></option>
<option value="html" <c:if test="${as.filetype.contains('html')}">selected</c:if>><la:message
key="labels.advance_search_filetype_html"
/></option>
@ -261,6 +261,36 @@
<!-- TODO -->
</div>
</div>
<div class="form-group row">
<label for="as_filetype" class="col-lg-3 col-md-4 col-sm-5 col-xs-6 col-form-label"><la:message
key="labels.advance_search_occt"
/></label>
<div class="col-lg-5 col-md-8 col-sm-7 col-xs-6">
<select id="as_occt" name="as.occt" class="form-control">
<option value=""><la:message key="labels.advance_search_occt_default" /></option>
<option value="allintitle" <c:if test="${as.occt.contains('allintitle')}">selected</c:if>><la:message
key="labels.advance_search_occt_allintitle"
/></option>
<option value="allinurl" <c:if test="${as.occt.contains('allinurl')}">selected</c:if>><la:message
key="labels.advance_search_occt_allinurl"
/></option>
</select>
</div>
<div class="col-lg-4 hidden-md-down">
<!-- TODO -->
</div>
</div>
<div class="form-group row">
<label for="as_sitesearch" class="col-lg-3 col-md-4 col-sm-5 col-xs-6 col-form-label"><la:message
key="labels.advance_search_sitesearch"
/></label>
<div class="col-lg-5 col-md-8 col-sm-7 col-xs-6">
<input class="form-control" type="text" id="as_sitesearch" name="as.sitesearch" value="${f:h(fe:join(as.sitesearch))}">
</div>
<div class="col-lg-4 hidden-md-down">
<!-- TODO -->
</div>
</div>
<div class="center">
<button type="submit" name="search" id="searchButton" class="btn btn-primary">

30
src/test/java/org/codelibs/fess/helper/QueryHelperTest.java
View file

@ -110,6 +110,36 @@ public class QueryHelperTest extends UnitFessTestCase {
assertQueryBuilder("content", "亜亜", MatchPhraseQueryBuilder.class);
assertQueryBuilder("content", "아", PrefixQueryBuilder.class);
assertQueryBuilder("content", "아아", MatchPhraseQueryBuilder.class);
assertEquals(
"{\"function_score\":{\"query\":{\"prefix\":{\"site\":{\"value\":\"fess.codelibs.org\",\"boost\":1.0}}},\"functions\":[{\"filter\":{\"match_all\":{\"boost\":1.0}},\"field_value_factor\":{\"field\":\"boost\",\"factor\":1.0,\"modifier\":\"none\"}}],\"score_mode\":\"multiply\",\"max_boost\":3.4028235E38,\"boost\":1.0}}",
buildQuery("site:fess.codelibs.org").toString().replaceAll("\\s", ""));
assertEquals(
"{\"function_score\":{\"query\":{\"wildcard\":{\"title\":{\"wildcard\":\"*\",\"boost\":1.0}}},\"functions\":[{\"filter\":{\"match_all\":{\"boost\":1.0}},\"field_value_factor\":{\"field\":\"boost\",\"factor\":1.0,\"modifier\":\"none\"}}],\"score_mode\":\"multiply\",\"max_boost\":3.4028235E38,\"boost\":1.0}}",
buildQuery("allintitle:").toString().replaceAll("\\s", ""));
assertEquals(
"{\"function_score\":{\"query\":{\"match_phrase\":{\"title\":{\"query\":\"test\",\"slop\":0,\"boost\":1.0}}},\"functions\":[{\"filter\":{\"match_all\":{\"boost\":1.0}},\"field_value_factor\":{\"field\":\"boost\",\"factor\":1.0,\"modifier\":\"none\"}}],\"score_mode\":\"multiply\",\"max_boost\":3.4028235E38,\"boost\":1.0}}",
buildQuery("allintitle:test").toString().replaceAll("\\s", ""));
assertEquals(
"{\"function_score\":{\"query\":{\"match_phrase\":{\"title\":{\"query\":\"test\",\"slop\":0,\"boost\":1.0}}},\"functions\":[{\"filter\":{\"match_all\":{\"boost\":1.0}},\"field_value_factor\":{\"field\":\"boost\",\"factor\":1.0,\"modifier\":\"none\"}}],\"score_mode\":\"multiply\",\"max_boost\":3.4028235E38,\"boost\":1.0}}",
buildQuery("allintitle: test").toString().replaceAll("\\s", ""));
assertEquals(
"{\"function_score\":{\"query\":{\"bool\":{\"must\":[{\"match_phrase\":{\"title\":{\"query\":\"aaa\",\"slop\":0,\"boost\":1.0}}},{\"match_phrase\":{\"title\":{\"query\":\"bbb\",\"slop\":0,\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"functions\":[{\"filter\":{\"match_all\":{\"boost\":1.0}},\"field_value_factor\":{\"field\":\"boost\",\"factor\":1.0,\"modifier\":\"none\"}}],\"score_mode\":\"multiply\",\"max_boost\":3.4028235E38,\"boost\":1.0}}",
buildQuery("allintitle: aaa bbb").toString().replaceAll("\\s", ""));
assertEquals(
"{\"function_score\":{\"query\":{\"wildcard\":{\"url\":{\"wildcard\":\"*\",\"boost\":1.0}}},\"functions\":[{\"filter\":{\"match_all\":{\"boost\":1.0}},\"field_value_factor\":{\"field\":\"boost\",\"factor\":1.0,\"modifier\":\"none\"}}],\"score_mode\":\"multiply\",\"max_boost\":3.4028235E38,\"boost\":1.0}}",
buildQuery("allinurl:").toString().replaceAll("\\s", ""));
assertEquals(
"{\"function_score\":{\"query\":{\"wildcard\":{\"url\":{\"wildcard\":\"*test*\",\"boost\":1.0}}},\"functions\":[{\"filter\":{\"match_all\":{\"boost\":1.0}},\"field_value_factor\":{\"field\":\"boost\",\"factor\":1.0,\"modifier\":\"none\"}}],\"score_mode\":\"multiply\",\"max_boost\":3.4028235E38,\"boost\":1.0}}",
buildQuery("allinurl:test").toString().replaceAll("\\s", ""));
assertEquals(
"{\"function_score\":{\"query\":{\"wildcard\":{\"url\":{\"wildcard\":\"*test*\",\"boost\":1.0}}},\"functions\":[{\"filter\":{\"match_all\":{\"boost\":1.0}},\"field_value_factor\":{\"field\":\"boost\",\"factor\":1.0,\"modifier\":\"none\"}}],\"score_mode\":\"multiply\",\"max_boost\":3.4028235E38,\"boost\":1.0}}",
buildQuery("allinurl: test").toString().replaceAll("\\s", ""));
assertEquals(
"{\"function_score\":{\"query\":{\"bool\":{\"must\":[{\"wildcard\":{\"url\":{\"wildcard\":\"*aaa*\",\"boost\":1.0}}},{\"wildcard\":{\"url\":{\"wildcard\":\"*bbb*\",\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"functions\":[{\"filter\":{\"match_all\":{\"boost\":1.0}},\"field_value_factor\":{\"field\":\"boost\",\"factor\":1.0,\"modifier\":\"none\"}}],\"score_mode\":\"multiply\",\"max_boost\":3.4028235E38,\"boost\":1.0}}",
buildQuery("allinurl: aaa bbb").toString().replaceAll("\\s", ""));
}
private void assertQueryBuilder(String field, String value, Class<?> clazz) {