fix #1539 add occt and fix #1537 add sitesearch

src/main/java/org/codelibs/fess/entity/QueryContext.java

@@ -26,7 +26,9 @@ import java.util.Map;
 import java.util.Set;
 import java.util.function.Consumer;
 
+import org.codelibs.core.lang.StringUtil;
 import org.codelibs.fess.Constants;
+import org.codelibs.fess.util.ComponentUtil;
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.MatchAllQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
@@ -36,11 +38,16 @@ import org.elasticsearch.search.sort.SortBuilder;
 import org.lastaflute.web.util.LaRequestUtil;
 
 public class QueryContext {
+
+    protected static final String ALLINURL_FIELD_PREFIX = "allinurl:";
+
+    protected static final String ALLINTITLE_FIELD_PREFIX = "allintitle:";
+
     private QueryBuilder queryBuilder;
 
     private final List<SortBuilder<?>> sortBuilderList = new ArrayList<>();
 
-    private final String queryString;
+    private String queryString;
 
     private Set<String> highlightedQuerySet = null;
 
@@ -48,9 +55,26 @@ public class QueryContext {
 
     private boolean disableRoleQuery = false;
 
+    private String defaultField = null;
+
     @SuppressWarnings("unchecked")
     public QueryContext(final String queryString, final boolean isQuery) {
-        this.queryString = queryString;
+        if (queryString != null) {
+            if (queryString.startsWith(ALLINURL_FIELD_PREFIX)) {
+                this.defaultField = ComponentUtil.getFessConfig().getIndexFieldUrl();
+                this.queryString = queryString.substring(ALLINURL_FIELD_PREFIX.length());
+            } else if (queryString.startsWith(ALLINTITLE_FIELD_PREFIX)) {
+                this.defaultField = ComponentUtil.getFessConfig().getIndexFieldTitle();
+                this.queryString = queryString.substring(ALLINTITLE_FIELD_PREFIX.length());
+            } else {
+                this.queryString = queryString;
+            }
+        } else {
+            this.queryString = queryString;
+        }
+        if (StringUtil.isBlank(this.queryString)) {
+            this.queryString = "*";
+        }
         if (isQuery) {
             LaRequestUtil.getOptionalRequest().ifPresent(request -> {
                 highlightedQuerySet = new HashSet<>();
@@ -140,4 +164,12 @@ public class QueryContext {
     public void skipRoleQuery() {
         disableRoleQuery = true;
     }
+
+    public String getDefaultField() {
+        return defaultField;
+    }
+
+    public void setDefaultField(String defaultField) {
+        this.defaultField = defaultField;
+    }
 }

src/main/java/org/codelibs/fess/entity/SearchRequestParams.java

@@ -36,6 +36,10 @@ public interface SearchRequestParams {
 
     String AS_FILETYPE = "filetype";
 
+    String AS_SITESEARCH = "sitesearch";
+
+    String AS_OCCURRENCE = "occt";
+
     String getQuery();
 
     Map<String, String[]> getFields();
@@ -70,6 +74,7 @@ public interface SearchRequestParams {
                 || !isEmptyArray(conditions.get(AS_EPQ))//
                 || !isEmptyArray(conditions.get(AS_OQ))//
                 || !isEmptyArray(conditions.get(AS_NQ))//
+                || !isEmptyArray(conditions.get(AS_SITESEARCH))//
                 || !isEmptyArray(conditions.get(AS_FILETYPE));
     }
 

src/main/java/org/codelibs/fess/helper/QueryHelper.java

@@ -82,6 +82,8 @@ public class QueryHelper {
 
     protected static final String INURL_FIELD = "inurl";
 
+    protected static final String SITE_FIELD = "site";
+
     @Resource
     protected FessConfig fessConfig;
 
@@ -193,6 +195,7 @@ public class QueryHelper {
                     fessConfig.getIndexFieldUrl(), //
                     fessConfig.getIndexFieldDocId(), //
                     fessConfig.getIndexFieldHost(), //
+                    fessConfig.getIndexFieldSite(), //
                     fessConfig.getIndexFieldTitle(), //
                     fessConfig.getIndexFieldContent(), //
                     fessConfig.getIndexFieldContentLength(), //
@@ -415,7 +418,10 @@ public class QueryHelper {
                 }
             }
         }
-        return boolQuery;
+        if (boolQuery.hasClauses()) {
+            return boolQuery;
+        }
+        return null;
     }
 
     protected String toLowercaseWildcard(final String value) {
@@ -425,8 +431,15 @@ public class QueryHelper {
         return value;
     }
 
+    protected String getSearchField(final QueryContext context, final String field) {
+        if (Constants.DEFAULT_FIELD.equals(field) && context.getDefaultField() != null) {
+            return context.getDefaultField();
+        }
+        return field;
+    }
+
     protected QueryBuilder convertWildcardQuery(final QueryContext context, final WildcardQuery wildcardQuery, final float boost) {
-        final String field = wildcardQuery.getField();
+        final String field = getSearchField(context, wildcardQuery.getField());
         if (Constants.DEFAULT_FIELD.equals(field)) {
             context.addFieldLog(field, wildcardQuery.getTerm().text());
             return buildDefaultQueryBuilder((f, b) -> QueryBuilders.wildcardQuery(f, toLowercaseWildcard(wildcardQuery.getTerm().text()))
@@ -444,7 +457,7 @@ public class QueryHelper {
     }
 
     protected QueryBuilder convertPrefixQuery(final QueryContext context, final PrefixQuery prefixQuery, final float boost) {
-        final String field = prefixQuery.getField();
+        final String field = getSearchField(context, prefixQuery.getField());
         if (Constants.DEFAULT_FIELD.equals(field)) {
             context.addFieldLog(field, prefixQuery.getPrefix().text());
             return buildDefaultQueryBuilder((f, b) -> QueryBuilders.prefixQuery(f, toLowercaseWildcard(prefixQuery.getPrefix().text()))
@@ -463,7 +476,7 @@ public class QueryHelper {
 
     protected QueryBuilder convertFuzzyQuery(final QueryContext context, final FuzzyQuery fuzzyQuery, final float boost) {
         final Term term = fuzzyQuery.getTerm();
-        final String field = term.field();
+        final String field = getSearchField(context, term.field());
         // TODO fuzzy value
         if (Constants.DEFAULT_FIELD.equals(field)) {
             context.addFieldLog(field, term.text());
@@ -482,7 +495,7 @@ public class QueryHelper {
     }
 
     protected QueryBuilder convertTermRangeQuery(final QueryContext context, final TermRangeQuery termRangeQuery, final float boost) {
-        final String field = termRangeQuery.getField();
+        final String field = getSearchField(context, termRangeQuery.getField());
         if (isSearchField(field)) {
             context.addFieldLog(field, termRangeQuery.toString(field));
             final RangeQueryBuilder rangeQuery = QueryBuilders.rangeQuery(field);
@@ -530,7 +543,7 @@ public class QueryHelper {
     }
 
     protected QueryBuilder convertTermQuery(final QueryContext context, final TermQuery termQuery, final float boost) {
-        final String field = termQuery.getTerm().field();
+        final String field = getSearchField(context, termQuery.getTerm().field());
         final String text = termQuery.getTerm().text();
         if (fessConfig.getQueryReplaceTermWithPrefixQueryAsBoolean() && text.length() > 1 && text.endsWith("*")) {
             return convertPrefixQuery(context, new PrefixQuery(new Term(field, text.substring(0, text.length() - 1))), boost);
@@ -562,8 +575,10 @@ public class QueryHelper {
             }
             context.addSorts(createFieldSortBuilder(sortField, sortOrder));
             return null;
-        } else if (INURL_FIELD.equals(field)) {
+        } else if (INURL_FIELD.equals(field) || fessConfig.getIndexFieldUrl().equals(context.getDefaultField())) {
             return QueryBuilders.wildcardQuery(fessConfig.getIndexFieldUrl(), "*" + text + "*").boost(boost);
+        } else if (SITE_FIELD.equals(field)) {
+            return convertSiteQuery(context, text, boost);
         } else if (isSearchField(field)) {
             context.addFieldLog(field, text);
             context.addHighlightedQuery(text);
@@ -580,6 +595,10 @@ public class QueryHelper {
         }
     }
 
+    protected QueryBuilder convertSiteQuery(final QueryContext context, final String text, final float boost) {
+        return QueryBuilders.prefixQuery(fessConfig.getIndexFieldSite(), text).boost(boost);
+    }
+
     private QueryBuilder convertPhraseQuery(final QueryContext context, final PhraseQuery query, final float boost) {
         final Term[] terms = query.getTerms();
         if (terms.length == 0) {

src/main/java/org/codelibs/fess/mylasta/action/FessLabels.java

@@ -2718,6 +2718,21 @@ public class FessLabels extends UserMessages {
     /** The key of the message: MS PowerPoint */
     public static final String LABELS_advance_search_filetype_powerpoint = "{labels.advance_search_filetype_powerpoint}";
 
+    /** The key of the message: Terms appearing */
+    public static final String LABELS_advance_search_occt = "{labels.advance_search_occt}";
+
+    /** The key of the message: anywhere in the page */
+    public static final String LABELS_advance_search_occt_default = "{labels.advance_search_occt_default}";
+
+    /** The key of the message: in the title of the page */
+    public static final String LABELS_advance_search_occt_allintitle = "{labels.advance_search_occt_allintitle}";
+
+    /** The key of the message: in the url of the page */
+    public static final String LABELS_advance_search_occt_allinurl = "{labels.advance_search_occt_allinurl}";
+
+    /** The key of the message: Site or domain */
+    public static final String LABELS_advance_search_sitesearch = "{labels.advance_search_sitesearch}";
+
     /**
      * Assert the property is not null.
      * @param property The value of the property. (NotNull)

src/main/java/org/codelibs/fess/util/QueryStringBuilder.java

@@ -104,6 +104,9 @@ public class QueryStringBuilder {
         final FessConfig fessConfig = ComponentUtil.getFessConfig();
         final int maxQueryLength = fessConfig.getQueryMaxLengthAsInteger().intValue();
 
+        stream(conditions.get(SearchRequestParams.AS_OCCURRENCE)).of(
+                stream -> stream.filter(q -> isOccurrence(q)).findFirst().ifPresent(q -> queryBuf.insert(0, q + ":")));
+
         stream(conditions.get(SearchRequestParams.AS_Q)).of(
                 stream -> stream.filter(q -> StringUtil.isNotBlank(q) && q.length() <= maxQueryLength).forEach(
                         q -> queryBuf.append(' ').append(q)));
@@ -127,10 +130,16 @@ public class QueryStringBuilder {
         stream(conditions.get(SearchRequestParams.AS_FILETYPE)).of(
                 stream -> stream.filter(q -> StringUtil.isNotBlank(q) && q.length() <= maxQueryLength).forEach(
                         q -> queryBuf.append(" filetype:\"").append(q.trim()).append('"')));
+        stream(conditions.get(SearchRequestParams.AS_SITESEARCH)).of(
+                stream -> stream.filter(q -> StringUtil.isNotBlank(q) && q.length() <= maxQueryLength).forEach(
+                        q -> queryBuf.append(" site:").append(q.trim())));
+    }
 
+    protected boolean isOccurrence(final String value) {
+        return "allintitle".equals(value) || "allinurl".equals(value);
     }
 
-    private String escape(final String q, final String... values) {
+    protected String escape(final String q, final String... values) {
         String value = q;
         for (String s : values) {
             value = value.replace(s, "\\" + s);

src/main/resources/fess_label.properties

@@ -896,3 +896,8 @@ labels.advance_search_filetype_pdf=PDF
 labels.advance_search_filetype_word=MS Word
 labels.advance_search_filetype_excel=MS Excel
 labels.advance_search_filetype_powerpoint=MS PowerPoint
+labels.advance_search_occt=Terms appearing
+labels.advance_search_occt_default=anywhere in the page
+labels.advance_search_occt_allintitle=in the title of the page
+labels.advance_search_occt_allinurl=in the url of the page
+labels.advance_search_sitesearch=Site or domain

src/main/resources/fess_label_en.properties

@@ -896,3 +896,8 @@ labels.advance_search_filetype_pdf=PDF
 labels.advance_search_filetype_word=MS Word
 labels.advance_search_filetype_excel=MS Excel
 labels.advance_search_filetype_powerpoint=MS PowerPoint
+labels.advance_search_occt=Terms appearing
+labels.advance_search_occt_default=anywhere in the page
+labels.advance_search_occt_allintitle=in the title of the page
+labels.advance_search_occt_allinurl=in the url of the page
+labels.advance_search_sitesearch=Site or domain

src/main/resources/fess_label_ja.properties

@@ -898,3 +898,8 @@ labels.advance_search_filetype_pdf=PDF
 labels.advance_search_filetype_word=MS Word
 labels.advance_search_filetype_excel=MS Excel
 labels.advance_search_filetype_powerpoint=MS PowerPoint
+labels.advance_search_occt=\u691c\u7d22\u5bfe\u8c61
+labels.advance_search_occt_default=\u30da\u30fc\u30b8\u5168\u4f53
+labels.advance_search_occt_allintitle=\u30da\u30fc\u30b8\u5185\u306e\u30bf\u30a4\u30c8\u30eb
+labels.advance_search_occt_allinurl=\u30da\u30fc\u30b8\u5185\u306eURL
+labels.advance_search_sitesearch=\u30b5\u30a4\u30c8\u307e\u305f\u306f\u30c9\u30e1\u30a4\u30f3

src/main/webapp/WEB-INF/view/advance.jsp

@@ -239,7 +239,7 @@
 							/></label>
 						<div class="col-lg-5 col-md-8 col-sm-7 col-xs-6">
 							<select id="as_filetype" name="as.filetype" class="form-control">
-								<option><la:message key="labels.advance_search_filetype_default" /></option>
+								<option value=""><la:message key="labels.advance_search_filetype_default" /></option>
 								<option value="html" <c:if test="${as.filetype.contains('html')}">selected</c:if>><la:message
 										key="labels.advance_search_filetype_html"
 									/></option>
@@ -261,6 +261,36 @@
 							<!-- TODO -->
 						</div>
 					</div>
+					<div class="form-group row">
+						<label for="as_filetype" class="col-lg-3 col-md-4 col-sm-5 col-xs-6 col-form-label"><la:message
+								key="labels.advance_search_occt"
+							/></label>
+						<div class="col-lg-5 col-md-8 col-sm-7 col-xs-6">
+							<select id="as_occt" name="as.occt" class="form-control">
+								<option value=""><la:message key="labels.advance_search_occt_default" /></option>
+								<option value="allintitle" <c:if test="${as.occt.contains('allintitle')}">selected</c:if>><la:message
+										key="labels.advance_search_occt_allintitle"
+									/></option>
+								<option value="allinurl" <c:if test="${as.occt.contains('allinurl')}">selected</c:if>><la:message
+										key="labels.advance_search_occt_allinurl"
+									/></option>
+							</select>
+						</div>
+						<div class="col-lg-4 hidden-md-down">
+							<!-- TODO -->
+						</div>
+					</div>
+					<div class="form-group row">
+						<label for="as_sitesearch" class="col-lg-3 col-md-4 col-sm-5 col-xs-6 col-form-label"><la:message
+								key="labels.advance_search_sitesearch"
+							/></label>
+						<div class="col-lg-5 col-md-8 col-sm-7 col-xs-6">
+							<input class="form-control" type="text" id="as_sitesearch" name="as.sitesearch" value="${f:h(fe:join(as.sitesearch))}">
+						</div>
+						<div class="col-lg-4 hidden-md-down">
+							<!-- TODO -->
+						</div>
+					</div>
 
 					<div class="center">
 						<button type="submit" name="search" id="searchButton" class="btn btn-primary">

src/test/java/org/codelibs/fess/helper/QueryHelperTest.java

@@ -110,6 +110,36 @@ public class QueryHelperTest extends UnitFessTestCase {
         assertQueryBuilder("content", "亜亜", MatchPhraseQueryBuilder.class);
         assertQueryBuilder("content", "아", PrefixQueryBuilder.class);
         assertQueryBuilder("content", "아아", MatchPhraseQueryBuilder.class);
+
+        assertEquals(
+                "{\"function_score\":{\"query\":{\"prefix\":{\"site\":{\"value\":\"fess.codelibs.org\",\"boost\":1.0}}},\"functions\":[{\"filter\":{\"match_all\":{\"boost\":1.0}},\"field_value_factor\":{\"field\":\"boost\",\"factor\":1.0,\"modifier\":\"none\"}}],\"score_mode\":\"multiply\",\"max_boost\":3.4028235E38,\"boost\":1.0}}",
+                buildQuery("site:fess.codelibs.org").toString().replaceAll("\\s", ""));
+
+        assertEquals(
+                "{\"function_score\":{\"query\":{\"wildcard\":{\"title\":{\"wildcard\":\"*\",\"boost\":1.0}}},\"functions\":[{\"filter\":{\"match_all\":{\"boost\":1.0}},\"field_value_factor\":{\"field\":\"boost\",\"factor\":1.0,\"modifier\":\"none\"}}],\"score_mode\":\"multiply\",\"max_boost\":3.4028235E38,\"boost\":1.0}}",
+                buildQuery("allintitle:").toString().replaceAll("\\s", ""));
+        assertEquals(
+                "{\"function_score\":{\"query\":{\"match_phrase\":{\"title\":{\"query\":\"test\",\"slop\":0,\"boost\":1.0}}},\"functions\":[{\"filter\":{\"match_all\":{\"boost\":1.0}},\"field_value_factor\":{\"field\":\"boost\",\"factor\":1.0,\"modifier\":\"none\"}}],\"score_mode\":\"multiply\",\"max_boost\":3.4028235E38,\"boost\":1.0}}",
+                buildQuery("allintitle:test").toString().replaceAll("\\s", ""));
+        assertEquals(
+                "{\"function_score\":{\"query\":{\"match_phrase\":{\"title\":{\"query\":\"test\",\"slop\":0,\"boost\":1.0}}},\"functions\":[{\"filter\":{\"match_all\":{\"boost\":1.0}},\"field_value_factor\":{\"field\":\"boost\",\"factor\":1.0,\"modifier\":\"none\"}}],\"score_mode\":\"multiply\",\"max_boost\":3.4028235E38,\"boost\":1.0}}",
+                buildQuery("allintitle: test").toString().replaceAll("\\s", ""));
+        assertEquals(
+                "{\"function_score\":{\"query\":{\"bool\":{\"must\":[{\"match_phrase\":{\"title\":{\"query\":\"aaa\",\"slop\":0,\"boost\":1.0}}},{\"match_phrase\":{\"title\":{\"query\":\"bbb\",\"slop\":0,\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"functions\":[{\"filter\":{\"match_all\":{\"boost\":1.0}},\"field_value_factor\":{\"field\":\"boost\",\"factor\":1.0,\"modifier\":\"none\"}}],\"score_mode\":\"multiply\",\"max_boost\":3.4028235E38,\"boost\":1.0}}",
+                buildQuery("allintitle: aaa bbb").toString().replaceAll("\\s", ""));
+
+        assertEquals(
+                "{\"function_score\":{\"query\":{\"wildcard\":{\"url\":{\"wildcard\":\"*\",\"boost\":1.0}}},\"functions\":[{\"filter\":{\"match_all\":{\"boost\":1.0}},\"field_value_factor\":{\"field\":\"boost\",\"factor\":1.0,\"modifier\":\"none\"}}],\"score_mode\":\"multiply\",\"max_boost\":3.4028235E38,\"boost\":1.0}}",
+                buildQuery("allinurl:").toString().replaceAll("\\s", ""));
+        assertEquals(
+                "{\"function_score\":{\"query\":{\"wildcard\":{\"url\":{\"wildcard\":\"*test*\",\"boost\":1.0}}},\"functions\":[{\"filter\":{\"match_all\":{\"boost\":1.0}},\"field_value_factor\":{\"field\":\"boost\",\"factor\":1.0,\"modifier\":\"none\"}}],\"score_mode\":\"multiply\",\"max_boost\":3.4028235E38,\"boost\":1.0}}",
+                buildQuery("allinurl:test").toString().replaceAll("\\s", ""));
+        assertEquals(
+                "{\"function_score\":{\"query\":{\"wildcard\":{\"url\":{\"wildcard\":\"*test*\",\"boost\":1.0}}},\"functions\":[{\"filter\":{\"match_all\":{\"boost\":1.0}},\"field_value_factor\":{\"field\":\"boost\",\"factor\":1.0,\"modifier\":\"none\"}}],\"score_mode\":\"multiply\",\"max_boost\":3.4028235E38,\"boost\":1.0}}",
+                buildQuery("allinurl: test").toString().replaceAll("\\s", ""));
+        assertEquals(
+                "{\"function_score\":{\"query\":{\"bool\":{\"must\":[{\"wildcard\":{\"url\":{\"wildcard\":\"*aaa*\",\"boost\":1.0}}},{\"wildcard\":{\"url\":{\"wildcard\":\"*bbb*\",\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"functions\":[{\"filter\":{\"match_all\":{\"boost\":1.0}},\"field_value_factor\":{\"field\":\"boost\",\"factor\":1.0,\"modifier\":\"none\"}}],\"score_mode\":\"multiply\",\"max_boost\":3.4028235E38,\"boost\":1.0}}",
+                buildQuery("allinurl: aaa bbb").toString().replaceAll("\\s", ""));
     }
 
     private void assertQueryBuilder(String field, String value, Class<?> clazz) {