diff --git a/.gitignore b/.gitignore index c5cfabe0c..2edac019b 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,4 @@ .idea .DS_Store /plugins/ +/tomcat.8080/ diff --git a/pom.xml b/pom.xml index d0f2d4f2c..9a2574eb8 100644 --- a/pom.xml +++ b/pom.xml @@ -446,6 +446,18 @@ 56.1 + + + org.codelibs.fess + fess-suggest + 2.0.0-SNAPSHOT + + + org.codelibs + lucene-analyzers-kuromoji-ipadic-neologd + 4.10.4-20150501 + + diff --git a/src/main/java/org/codelibs/fess/api/json/JsonApiManager.java b/src/main/java/org/codelibs/fess/api/json/JsonApiManager.java index caee64fe8..22cbb62af 100644 --- a/src/main/java/org/codelibs/fess/api/json/JsonApiManager.java +++ b/src/main/java/org/codelibs/fess/api/json/JsonApiManager.java @@ -504,7 +504,7 @@ public class JsonApiManager extends BaseApiManager { } - protected void writeJsonResponse(final int status, final String body, final String errMsg) { + public static void writeJsonResponse(final int status, final String body, final String errMsg) { final String callback = LaRequestUtil.getRequest().getParameter("callback"); final boolean isJsonp = StringUtil.isNotBlank(callback); @@ -538,11 +538,11 @@ public class JsonApiManager extends BaseApiManager { } - protected String escapeCallbackName(final String callbackName) { + protected static String escapeCallbackName(final String callbackName) { return "/**/" + callbackName.replaceAll("[^0-9a-zA-Z_\\$\\.]", StringUtil.EMPTY); } - protected String escapeJson(final Object obj) { + protected static String escapeJson(final Object obj) { if (obj == null) { return "null"; } @@ -583,7 +583,7 @@ public class JsonApiManager extends BaseApiManager { return buf.toString(); } - protected String escapeJsonString(final String str) { + protected static String escapeJsonString(final String str) { final StringWriter out = new StringWriter(str.length() * 2); int sz; @@ -653,7 +653,7 @@ public class JsonApiManager extends BaseApiManager { return out.toString(); } - private String hex(final char ch) { + private static String hex(final char ch) { return Integer.toHexString(ch).toUpperCase(); } diff --git a/src/main/java/org/codelibs/fess/api/suggest/SuggestApiManager.java b/src/main/java/org/codelibs/fess/api/suggest/SuggestApiManager.java index 96c6ca0ce..f09a22916 100644 --- a/src/main/java/org/codelibs/fess/api/suggest/SuggestApiManager.java +++ b/src/main/java/org/codelibs/fess/api/suggest/SuggestApiManager.java @@ -22,7 +22,15 @@ import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import org.apache.commons.lang.StringUtils; +import org.codelibs.core.lang.StringUtil; import org.codelibs.fess.api.BaseApiManager; +import org.codelibs.fess.api.json.JsonApiManager; +import org.codelibs.fess.helper.SuggestHelper; +import org.codelibs.fess.suggest.entity.SuggestItem; +import org.codelibs.fess.suggest.request.suggest.SuggestRequestBuilder; +import org.codelibs.fess.suggest.request.suggest.SuggestResponse; +import org.codelibs.fess.util.ComponentUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,16 +42,143 @@ public class SuggestApiManager extends BaseApiManager { } @Override - public boolean matches(final HttpServletRequest request) { - return false; // TODO remove - // final String servletPath = request.getServletPath(); - // return servletPath.startsWith(pathPrefix); + public boolean matches(HttpServletRequest request) { + final String servletPath = request.getServletPath(); + return servletPath.startsWith(pathPrefix); } @Override - public void process(final HttpServletRequest request, final HttpServletResponse response, final FilterChain chain) throws IOException, - ServletException { - throw new UnsupportedOperationException("TODO"); + public void process(HttpServletRequest request, HttpServletResponse response, FilterChain chain) throws IOException, ServletException { + int status = 0; + String errMsg = StringUtil.EMPTY; + final StringBuilder buf = new StringBuilder(255); + + try { + + final RequestParameter parameter = RequestParameter.parse(request); + + final SuggestHelper suggestHelper = ComponentUtil.getSuggestHelper(); + final SuggestRequestBuilder builder = suggestHelper.suggester().suggest(); + builder.setQuery(parameter.getQuery()); + for (final String field : parameter.getFields()) { + builder.addField(field); + } + builder.setSize(parameter.getNum()); + + final SuggestResponse suggestResponse = builder.execute().getResponse(); + + buf.append("\"result\":{"); + + buf.append("\"index\":\"").append(suggestResponse.getIndex()).append('\"'); + + buf.append(",\"took\":\"").append(suggestResponse.getTookMs()).append('\"'); + + buf.append(",\"total\":\"").append(suggestResponse.getTotal()).append('\"'); + + buf.append(",\"num\":\"").append(suggestResponse.getNum()).append('\"'); + + if (!suggestResponse.getItems().isEmpty()) { + buf.append(",\"hits\":["); + + boolean first = true; + for (final SuggestItem item : suggestResponse.getItems()) { + if (!first) { + buf.append(','); + } + first = false; + + buf.append("{\"text\":\"").append(item.getText()).append('\"'); + buf.append(",\"tags\":["); + for (int i = 0; i < item.getTags().length; i++) { + if (i > 0) { + buf.append(','); + } + buf.append('\"').append(item.getTags()[i]).append('\"'); + } + buf.append(']'); + + buf.append(",\"roles\":["); + for (int i = 0; i < item.getRoles().length; i++) { + if (i > 0) { + buf.append(','); + } + buf.append('\"').append(item.getRoles()[i]).append('\"'); + } + buf.append(']'); + + buf.append(",\"fields\":["); + for (int i = 0; i < item.getFields().length; i++) { + if (i > 0) { + buf.append(','); + } + buf.append('\"').append(item.getFields()[i]).append('\"'); + } + buf.append(']'); + + buf.append('}'); + } + buf.append(']'); + } + + buf.append('}'); + } catch (final Exception e) { + status = 1; + errMsg = e.getMessage(); + if (errMsg == null) { + errMsg = e.getClass().getName(); + } + if (logger.isDebugEnabled()) { + logger.debug("Failed to process a suggest request.", e); + } + } + + JsonApiManager.writeJsonResponse(status, buf.toString(), errMsg); } + protected static class RequestParameter { + private final String query; + + private final String[] fields; + + private final int num; + + protected RequestParameter(final String query, final String[] fields, final int num) { + this.query = query; + this.fields = fields; + this.num = num; + } + + protected static RequestParameter parse(final HttpServletRequest request) { + final String query = request.getParameter("query"); + final String fieldsStr = request.getParameter("fields"); + final String[] fields; + if (StringUtils.isNotBlank(fieldsStr)) { + fields = fieldsStr.split(","); + } else { + fields = new String[0]; + } + + final String numStr = request.getParameter("num"); + final int num; + if (StringUtils.isNotBlank(numStr) && StringUtils.isNumeric(numStr)) { + num = Integer.parseInt(numStr); + } else { + num = 10; + } + + return new RequestParameter(query, fields, num); + } + + protected String getQuery() { + return query; + } + + protected String[] getFields() { + return fields; + } + + protected int getNum() { + return num; + } + } } diff --git a/src/main/java/org/codelibs/fess/entity/QueryContext.java b/src/main/java/org/codelibs/fess/entity/QueryContext.java index 6a83c2178..c8af6b971 100644 --- a/src/main/java/org/codelibs/fess/entity/QueryContext.java +++ b/src/main/java/org/codelibs/fess/entity/QueryContext.java @@ -42,12 +42,19 @@ public class QueryContext { private final Set highlightedQuerySet = new HashSet<>(); - private final Map> fieldLogMap = new HashMap<>(); + private Map> fieldLogMap; public QueryContext(final String queryString) { this.queryString = queryString; LaRequestUtil.getOptionalRequest().ifPresent(request -> { request.setAttribute(Constants.HIGHLIGHT_QUERIES, highlightedQuerySet); + @SuppressWarnings("unchecked") + final Map> existFieldLogMap = (Map>) request.getAttribute(Constants.FIELD_LOGS); + if (existFieldLogMap != null) { + fieldLogMap = existFieldLogMap; + } else { + fieldLogMap = new HashMap<>(); + } request.setAttribute(Constants.FIELD_LOGS, fieldLogMap); }); } diff --git a/src/main/java/org/codelibs/fess/helper/SearchLogHelper.java b/src/main/java/org/codelibs/fess/helper/SearchLogHelper.java index 840ccdbc3..21f6ea5bf 100644 --- a/src/main/java/org/codelibs/fess/helper/SearchLogHelper.java +++ b/src/main/java/org/codelibs/fess/helper/SearchLogHelper.java @@ -190,6 +190,8 @@ public class SearchLogHelper { if (!searchLogList.isEmpty()) { storeSearchLogList(searchLogList); + final SuggestHelper suggestHelper = ComponentUtil.getSuggestHelper(); + suggestHelper.indexFromSearchLog(searchLogList); } } diff --git a/src/main/java/org/codelibs/fess/helper/SuggestHelper.java b/src/main/java/org/codelibs/fess/helper/SuggestHelper.java index 8aac46215..8f94200c4 100644 --- a/src/main/java/org/codelibs/fess/helper/SuggestHelper.java +++ b/src/main/java/org/codelibs/fess/helper/SuggestHelper.java @@ -15,7 +15,6 @@ */ package org.codelibs.fess.helper; -import java.io.File; import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -25,9 +24,13 @@ import javax.annotation.PostConstruct; import javax.annotation.Resource; import org.codelibs.core.lang.StringUtil; +import org.codelibs.fess.es.client.FessEsClient; import org.codelibs.fess.es.config.exbhv.SuggestBadWordBhv; import org.codelibs.fess.es.config.exbhv.SuggestElevateWordBhv; import org.codelibs.fess.es.config.exentity.SuggestBadWord; +import org.codelibs.fess.es.log.exentity.SearchFieldLog; +import org.codelibs.fess.es.log.exentity.SearchLog; +import org.codelibs.fess.suggest.Suggester; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -40,34 +43,68 @@ public class SuggestHelper { @Resource protected SuggestBadWordBhv suggestBadWordBhv; - public String badwordFileDir = "./solr/core1/conf/"; + @Resource + protected FessEsClient fessEsClient; + + @Resource + protected FieldHelper fieldHelper; + + public String[] contentFieldNames = { "_default" }; + + public String[] tagFieldNames = { "label" }; + + public String[] roleFieldNames = { "role" }; + + private static final String TEXT_SEP = " "; + + protected Suggester suggester; @PostConstruct public void init() { final Thread th = new Thread(() -> { - // TODO replace with Elasticsearch - /* - while (true) { - final PingResponse response = searchService.ping(); - final int status = response.getStatus(); - if (status == 0) { - if (logger.isInfoEnabled()) { - logger.info("Solr server was availabled. Refresh suggest words."); - } - refreshWords(); - break; - } - try { - Thread.sleep(1 * 1000); - } catch (final InterruptedException e) { - //ignore - } - } - */ - }); + fessEsClient.admin().cluster().prepareHealth().setWaitForYellowStatus().execute().actionGet(); + suggester = Suggester.builder().build(fessEsClient, fieldHelper.docIndex); + suggester.createIndexIfNothing(); + }); th.start(); } + public Suggester suggester() { + return suggester; + } + + public void indexFromSearchLog(final List searchLogList) { + for (final SearchLog searchLog : searchLogList) { + // TODO if(getHitCount == 0) continue; + + final StringBuilder sb = new StringBuilder(); + final List fields = new ArrayList<>(); + final List tags = new ArrayList<>(); + final List roles = new ArrayList<>(); + + for (final SearchFieldLog searchFieldLog : searchLog.getSearchFieldLogList()) { + final String name = searchFieldLog.getName(); + if (isContentField(name)) { + if (sb.length() > 0) { + sb.append(TEXT_SEP); + } + sb.append(searchFieldLog.getValue()); + fields.add(name); + } else if (isTagField(name)) { + tags.add(searchFieldLog.getValue()); + } else if (isRoleField(name)) { + roles.add(searchFieldLog.getValue()); + } + } + + if (sb.length() > 0) { + suggester.indexer().indexFromSearchWord(sb.toString(), fields.toArray(new String[fields.size()]), + tags.toArray(new String[tags.size()]), roles.toArray(new String[roles.size()]), 1); + } + } + suggester.refresh(); + } + public void refreshWords() { deleteAllBadWord(); storeAllElevateWords(); @@ -143,6 +180,7 @@ public class SuggestHelper { // suggestService.commit(); } + /* public void updateSolrBadwordFile() { suggestBadWordBhv.selectList(cb -> { cb.query().matchAll(); @@ -179,4 +217,32 @@ public class SuggestHelper { // } // } } + */ + + protected boolean isContentField(final String field) { + for (final String contentField : contentFieldNames) { + if (contentField.equals(field)) { + return true; + } + } + return false; + } + + protected boolean isTagField(final String field) { + for (final String tagField : tagFieldNames) { + if (tagField.equals(field)) { + return true; + } + } + return false; + } + + protected boolean isRoleField(final String field) { + for (final String roleField : roleFieldNames) { + if (roleField.equals(field)) { + return true; + } + } + return false; + } } diff --git a/src/main/java/org/codelibs/fess/util/ComponentUtil.java b/src/main/java/org/codelibs/fess/util/ComponentUtil.java index f851e2b2b..7beaa146e 100644 --- a/src/main/java/org/codelibs/fess/util/ComponentUtil.java +++ b/src/main/java/org/codelibs/fess/util/ComponentUtil.java @@ -41,6 +41,7 @@ import org.codelibs.fess.helper.PathMappingHelper; import org.codelibs.fess.helper.QueryHelper; import org.codelibs.fess.helper.SambaHelper; import org.codelibs.fess.helper.SearchLogHelper; +import org.codelibs.fess.helper.SuggestHelper; import org.codelibs.fess.helper.SystemHelper; import org.codelibs.fess.helper.UserAgentHelper; import org.codelibs.fess.helper.UserInfoHelper; @@ -280,4 +281,7 @@ public final class ComponentUtil { return SingletonLaContainerFactory.getContainer().hasComponentDef(QUERY_HELPER); } + public static SuggestHelper getSuggestHelper() { + return getComponent(SuggestHelper.class); + } } diff --git a/src/main/resources/app.xml b/src/main/resources/app.xml index bf23c9532..2ce0822eb 100644 --- a/src/main/resources/app.xml +++ b/src/main/resources/app.xml @@ -240,7 +240,4 @@ "Mozilla/5.0 (compatible; Fess/" + org.codelibs.fess.Constants.FESS_VERSION + "; +http://fess.codelibs.org/bot.html)" - - - diff --git a/src/main/resources/fess.xml b/src/main/resources/fess.xml index acf199866..8aaabd88d 100644 --- a/src/main/resources/fess.xml +++ b/src/main/resources/fess.xml @@ -144,4 +144,6 @@ "," --> + + diff --git a/src/main/webapp/js/index.js b/src/main/webapp/js/index.js index 8fa37aa30..2767f035c 100644 --- a/src/main/webapp/js/index.js +++ b/src/main/webapp/js/index.js @@ -20,8 +20,8 @@ $(function(){ $('#contentQuery').suggestor( { ajaxinfo: { - url: contextPath + '/suggest', // TODO rename - fn: 'keyword', + url: contextPath + '/suggest', + fn: '_default', num: 10 }, boxCssInfo: { diff --git a/src/main/webapp/js/search.js b/src/main/webapp/js/search.js index 555bae941..592fd18c0 100644 --- a/src/main/webapp/js/search.js +++ b/src/main/webapp/js/search.js @@ -173,8 +173,8 @@ $(function(){ $('#query').suggestor( { ajaxinfo: { - url: contextPath + '/json', - fn: 'content', + url: contextPath + '/suggest', + fn: '_default', num: 10 }, boxCssInfo: { diff --git a/src/main/webapp/js/suggestor.js b/src/main/webapp/js/suggestor.js index 455d89cc0..d6548788e 100644 --- a/src/main/webapp/js/suggestor.js +++ b/src/main/webapp/js/suggestor.js @@ -1,7 +1,7 @@ ;(function($){ $.fn.suggestor = function(setting) { - + var $boxElement; var $textArea; var inputText = ""; @@ -11,7 +11,7 @@ $.fn.suggestor = function(setting) { var isMouseHover = false; var started = false; var interval = 5; - + var settingMinTerm = 1; var settingAjaxInfo; var settingAdjustWidthVal; @@ -19,15 +19,15 @@ $.fn.suggestor = function(setting) { var listSelectedCssInfo; var listDeselectedCssInfo; var boxCssInfo; - + var suggestingSts = false; - + var suggestor = { init: function($element, setting) { suggestingSts = false; $boxElement = $("
"); $boxElement.addClass("suggestorBox"); - + //style sheet $boxElement.css("display","none"); $boxElement.css("position","absolute"); @@ -42,14 +42,14 @@ $.fn.suggestor = function(setting) { } else { $boxElement.css(setting.boxCssInfo); } - + $textArea = $element; $textArea.attr("autocomplete","off"); - + isFocusList = false; inputText = $textArea.val(); - - + + //設定 settingAjaxInfo = setting.ajaxinfo; settingMinTerm = setting.minturm; @@ -57,66 +57,71 @@ $.fn.suggestor = function(setting) { listSelectedCssInfo = setting.listSelectedCssInfo; listDeselectedCssInfo = setting.listDeselectedCssInfo; settingAdjustWidthVal = setting.adjustWidthVal; - + boxCssInfo = setting.boxCssInfo; - - + + $boxElement.hover(function() { isMouseHover = true; }, function() { isMouseHover = false; }); - - + + //ポジション設定 this.resize(); var suggestor = this; $(window).resize(function() { suggestor.resize(); }); - + $("body").append($boxElement); }, - + suggest: function() { suggestingSts = true; - + //ポジション設定 this.resize(); - + var suggestor = this; inputText = $textArea.val(); - + listNum = 0; listSelNum = 0; - + if(inputText.length < settingMinTerm) { $boxElement.css("display","none"); suggestingSts = false; return; } - - + + $.ajax({ url: settingAjaxInfo.url, type:"get", dataType: "jsonp", cache : false, - data:{ q: $textArea.val(), + data:{ query: $textArea.val(), fields: settingAjaxInfo.fn, num: settingAjaxInfo.num * 2 } }).done(function(obj) { suggestor.createAutoCompleteList(obj); }).fail(function(a,obj,b) { suggestingSts=false; return; }); - - }, - - + + }, + + createAutoCompleteList: function(obj) { - var hits = obj.hits; + if(obj.response.status != 0) { + $boxElement.css("display","none"); + return; + } + + var hits = obj.response.result.hits; var suggestor = this; var addCount = 0; - - + + listNum = 0; if(typeof hits !== "undefined") { var reslist = new Array(); @@ -127,11 +132,11 @@ $.fn.suggestor = function(setting) { $olEle.css("list-style","none"); $olEle.css("padding","0"); $olEle.css("margin","2px"); - + for(var i=0;i"); $liEle.html(str); @@ -203,14 +208,14 @@ $.fn.suggestor = function(setting) { listSelNum = 0 } }); - + $liEle.css("padding","2px"); - + $olEle.append($liEle); listNum++; } } - + if(listNum>0 && $textArea.val().length >= settingMinTerm) { $boxElement.html(""); $boxElement.append($olEle); @@ -223,15 +228,15 @@ $.fn.suggestor = function(setting) { } //ポジション設定 this.resize(); - + suggestingSts = false; }, - + selectlist: function(direction) { if($boxElement.css("display") == "none") { return; } - + if(direction == "down") { listSelNum++; } else if(direction == "up") { @@ -239,15 +244,15 @@ $.fn.suggestor = function(setting) { } else { return; } - + isFocusList = true; - + if(listSelNum < 0){ listSelNum = listNum; } else if(listSelNum > listNum) { listSelNum = 0; } - + var a = $boxElement.children("ol").children("li"); $boxElement.children("ol").children("li").each(function(i){ if(i == (listSelNum-1)) { @@ -272,20 +277,20 @@ $.fn.suggestor = function(setting) { if(listSelNum == 0) { $textArea.val(inputText); } - + }, - + fixList: function() { if(listSelNum > 0) { $textArea.val($($boxElement.children("ol").children("li").get(listSelNum-1)).html()); } inputText = $textArea.val(); - + isFocusList = false; $boxElement.css("display","none"); listNum = 0; }, - + resize: function() { $boxElement.css("top",$textArea.offset().top + $textArea.height() + 6); $boxElement.css("left",$textArea.offset().left); @@ -296,9 +301,9 @@ $.fn.suggestor = function(setting) { } } } - + suggestor.init($(this), setting); - + $(this).keydown( function(e){ if( ((e.keyCode >= 48) && (e.keyCode <= 90)) || ((e.keyCode >= 96) && (e.keyCode <= 105)) @@ -324,7 +329,7 @@ $.fn.suggestor = function(setting) { if(isFocusList) { suggestor.fixList(); } - } + } }); $(this).keyup( function(e){ if( ((e.keyCode >= 48) && (e.keyCode <= 90)) @@ -350,7 +355,7 @@ $.fn.suggestor = function(setting) { suggestor.fixList(); } }); - + //テキストエリア監視 setInterval( function() { if(interval < 5) { @@ -365,6 +370,6 @@ $.fn.suggestor = function(setting) { } } }, 100); - + } })(jQuery);