Add suggest from searchLog.

This commit is contained in:
y_fujita 2015-11-05 21:17:59 +09:00
parent 13e9b32a60
commit 54c4bd1d6a
13 changed files with 325 additions and 94 deletions

1
.gitignore vendored
View file

@ -20,3 +20,4 @@
.idea
.DS_Store
/plugins/
/tomcat.8080/

12
pom.xml
View file

@ -446,6 +446,18 @@
<version>56.1</version>
</dependency>
<!-- suggest library -->
<dependency>
<groupId>org.codelibs.fess</groupId>
<artifactId>fess-suggest</artifactId>
<version>2.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.codelibs</groupId>
<artifactId>lucene-analyzers-kuromoji-ipadic-neologd</artifactId>
<version>4.10.4-20150501</version>
</dependency>
<!-- fileupload -->
<dependency>
<!-- TODO remove? -->

View file

@ -504,7 +504,7 @@ public class JsonApiManager extends BaseApiManager {
}
protected void writeJsonResponse(final int status, final String body, final String errMsg) {
public static void writeJsonResponse(final int status, final String body, final String errMsg) {
final String callback = LaRequestUtil.getRequest().getParameter("callback");
final boolean isJsonp = StringUtil.isNotBlank(callback);
@ -538,11 +538,11 @@ public class JsonApiManager extends BaseApiManager {
}
protected String escapeCallbackName(final String callbackName) {
protected static String escapeCallbackName(final String callbackName) {
return "/**/" + callbackName.replaceAll("[^0-9a-zA-Z_\\$\\.]", StringUtil.EMPTY);
}
protected String escapeJson(final Object obj) {
protected static String escapeJson(final Object obj) {
if (obj == null) {
return "null";
}
@ -583,7 +583,7 @@ public class JsonApiManager extends BaseApiManager {
return buf.toString();
}
protected String escapeJsonString(final String str) {
protected static String escapeJsonString(final String str) {
final StringWriter out = new StringWriter(str.length() * 2);
int sz;
@ -653,7 +653,7 @@ public class JsonApiManager extends BaseApiManager {
return out.toString();
}
private String hex(final char ch) {
private static String hex(final char ch) {
return Integer.toHexString(ch).toUpperCase();
}

View file

@ -22,7 +22,15 @@ import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.apache.commons.lang.StringUtils;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.fess.api.BaseApiManager;
import org.codelibs.fess.api.json.JsonApiManager;
import org.codelibs.fess.helper.SuggestHelper;
import org.codelibs.fess.suggest.entity.SuggestItem;
import org.codelibs.fess.suggest.request.suggest.SuggestRequestBuilder;
import org.codelibs.fess.suggest.request.suggest.SuggestResponse;
import org.codelibs.fess.util.ComponentUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -34,16 +42,143 @@ public class SuggestApiManager extends BaseApiManager {
}
@Override
public boolean matches(final HttpServletRequest request) {
return false; // TODO remove
// final String servletPath = request.getServletPath();
// return servletPath.startsWith(pathPrefix);
public boolean matches(HttpServletRequest request) {
final String servletPath = request.getServletPath();
return servletPath.startsWith(pathPrefix);
}
@Override
public void process(final HttpServletRequest request, final HttpServletResponse response, final FilterChain chain) throws IOException,
ServletException {
throw new UnsupportedOperationException("TODO");
public void process(HttpServletRequest request, HttpServletResponse response, FilterChain chain) throws IOException, ServletException {
int status = 0;
String errMsg = StringUtil.EMPTY;
final StringBuilder buf = new StringBuilder(255);
try {
final RequestParameter parameter = RequestParameter.parse(request);
final SuggestHelper suggestHelper = ComponentUtil.getSuggestHelper();
final SuggestRequestBuilder builder = suggestHelper.suggester().suggest();
builder.setQuery(parameter.getQuery());
for (final String field : parameter.getFields()) {
builder.addField(field);
}
builder.setSize(parameter.getNum());
final SuggestResponse suggestResponse = builder.execute().getResponse();
buf.append("\"result\":{");
buf.append("\"index\":\"").append(suggestResponse.getIndex()).append('\"');
buf.append(",\"took\":\"").append(suggestResponse.getTookMs()).append('\"');
buf.append(",\"total\":\"").append(suggestResponse.getTotal()).append('\"');
buf.append(",\"num\":\"").append(suggestResponse.getNum()).append('\"');
if (!suggestResponse.getItems().isEmpty()) {
buf.append(",\"hits\":[");
boolean first = true;
for (final SuggestItem item : suggestResponse.getItems()) {
if (!first) {
buf.append(',');
}
first = false;
buf.append("{\"text\":\"").append(item.getText()).append('\"');
buf.append(",\"tags\":[");
for (int i = 0; i < item.getTags().length; i++) {
if (i > 0) {
buf.append(',');
}
buf.append('\"').append(item.getTags()[i]).append('\"');
}
buf.append(']');
buf.append(",\"roles\":[");
for (int i = 0; i < item.getRoles().length; i++) {
if (i > 0) {
buf.append(',');
}
buf.append('\"').append(item.getRoles()[i]).append('\"');
}
buf.append(']');
buf.append(",\"fields\":[");
for (int i = 0; i < item.getFields().length; i++) {
if (i > 0) {
buf.append(',');
}
buf.append('\"').append(item.getFields()[i]).append('\"');
}
buf.append(']');
buf.append('}');
}
buf.append(']');
}
buf.append('}');
} catch (final Exception e) {
status = 1;
errMsg = e.getMessage();
if (errMsg == null) {
errMsg = e.getClass().getName();
}
if (logger.isDebugEnabled()) {
logger.debug("Failed to process a suggest request.", e);
}
}
JsonApiManager.writeJsonResponse(status, buf.toString(), errMsg);
}
protected static class RequestParameter {
private final String query;
private final String[] fields;
private final int num;
protected RequestParameter(final String query, final String[] fields, final int num) {
this.query = query;
this.fields = fields;
this.num = num;
}
protected static RequestParameter parse(final HttpServletRequest request) {
final String query = request.getParameter("query");
final String fieldsStr = request.getParameter("fields");
final String[] fields;
if (StringUtils.isNotBlank(fieldsStr)) {
fields = fieldsStr.split(",");
} else {
fields = new String[0];
}
final String numStr = request.getParameter("num");
final int num;
if (StringUtils.isNotBlank(numStr) && StringUtils.isNumeric(numStr)) {
num = Integer.parseInt(numStr);
} else {
num = 10;
}
return new RequestParameter(query, fields, num);
}
protected String getQuery() {
return query;
}
protected String[] getFields() {
return fields;
}
protected int getNum() {
return num;
}
}
}

View file

@ -42,12 +42,19 @@ public class QueryContext {
private final Set<String> highlightedQuerySet = new HashSet<>();
private final Map<String, List<String>> fieldLogMap = new HashMap<>();
private Map<String, List<String>> fieldLogMap;
public QueryContext(final String queryString) {
this.queryString = queryString;
LaRequestUtil.getOptionalRequest().ifPresent(request -> {
request.setAttribute(Constants.HIGHLIGHT_QUERIES, highlightedQuerySet);
@SuppressWarnings("unchecked")
final Map<String, List<String>> existFieldLogMap = (Map<String, List<String>>) request.getAttribute(Constants.FIELD_LOGS);
if (existFieldLogMap != null) {
fieldLogMap = existFieldLogMap;
} else {
fieldLogMap = new HashMap<>();
}
request.setAttribute(Constants.FIELD_LOGS, fieldLogMap);
});
}

View file

@ -190,6 +190,8 @@ public class SearchLogHelper {
if (!searchLogList.isEmpty()) {
storeSearchLogList(searchLogList);
final SuggestHelper suggestHelper = ComponentUtil.getSuggestHelper();
suggestHelper.indexFromSearchLog(searchLogList);
}
}

View file

@ -15,7 +15,6 @@
*/
package org.codelibs.fess.helper;
import java.io.File;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
@ -25,9 +24,13 @@ import javax.annotation.PostConstruct;
import javax.annotation.Resource;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.fess.es.client.FessEsClient;
import org.codelibs.fess.es.config.exbhv.SuggestBadWordBhv;
import org.codelibs.fess.es.config.exbhv.SuggestElevateWordBhv;
import org.codelibs.fess.es.config.exentity.SuggestBadWord;
import org.codelibs.fess.es.log.exentity.SearchFieldLog;
import org.codelibs.fess.es.log.exentity.SearchLog;
import org.codelibs.fess.suggest.Suggester;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -40,34 +43,68 @@ public class SuggestHelper {
@Resource
protected SuggestBadWordBhv suggestBadWordBhv;
public String badwordFileDir = "./solr/core1/conf/";
@Resource
protected FessEsClient fessEsClient;
@Resource
protected FieldHelper fieldHelper;
public String[] contentFieldNames = { "_default" };
public String[] tagFieldNames = { "label" };
public String[] roleFieldNames = { "role" };
private static final String TEXT_SEP = " ";
protected Suggester suggester;
@PostConstruct
public void init() {
final Thread th = new Thread(() -> {
// TODO replace with Elasticsearch
/*
while (true) {
final PingResponse response = searchService.ping();
final int status = response.getStatus();
if (status == 0) {
if (logger.isInfoEnabled()) {
logger.info("Solr server was availabled. Refresh suggest words.");
}
refreshWords();
break;
}
try {
Thread.sleep(1 * 1000);
} catch (final InterruptedException e) {
//ignore
}
}
*/
});
fessEsClient.admin().cluster().prepareHealth().setWaitForYellowStatus().execute().actionGet();
suggester = Suggester.builder().build(fessEsClient, fieldHelper.docIndex);
suggester.createIndexIfNothing();
});
th.start();
}
public Suggester suggester() {
return suggester;
}
public void indexFromSearchLog(final List<SearchLog> searchLogList) {
for (final SearchLog searchLog : searchLogList) {
// TODO if(getHitCount == 0) continue;
final StringBuilder sb = new StringBuilder();
final List<String> fields = new ArrayList<>();
final List<String> tags = new ArrayList<>();
final List<String> roles = new ArrayList<>();
for (final SearchFieldLog searchFieldLog : searchLog.getSearchFieldLogList()) {
final String name = searchFieldLog.getName();
if (isContentField(name)) {
if (sb.length() > 0) {
sb.append(TEXT_SEP);
}
sb.append(searchFieldLog.getValue());
fields.add(name);
} else if (isTagField(name)) {
tags.add(searchFieldLog.getValue());
} else if (isRoleField(name)) {
roles.add(searchFieldLog.getValue());
}
}
if (sb.length() > 0) {
suggester.indexer().indexFromSearchWord(sb.toString(), fields.toArray(new String[fields.size()]),
tags.toArray(new String[tags.size()]), roles.toArray(new String[roles.size()]), 1);
}
}
suggester.refresh();
}
public void refreshWords() {
deleteAllBadWord();
storeAllElevateWords();
@ -143,6 +180,7 @@ public class SuggestHelper {
// suggestService.commit();
}
/*
public void updateSolrBadwordFile() {
suggestBadWordBhv.selectList(cb -> {
cb.query().matchAll();
@ -179,4 +217,32 @@ public class SuggestHelper {
// }
// }
}
*/
protected boolean isContentField(final String field) {
for (final String contentField : contentFieldNames) {
if (contentField.equals(field)) {
return true;
}
}
return false;
}
protected boolean isTagField(final String field) {
for (final String tagField : tagFieldNames) {
if (tagField.equals(field)) {
return true;
}
}
return false;
}
protected boolean isRoleField(final String field) {
for (final String roleField : roleFieldNames) {
if (roleField.equals(field)) {
return true;
}
}
return false;
}
}

View file

@ -41,6 +41,7 @@ import org.codelibs.fess.helper.PathMappingHelper;
import org.codelibs.fess.helper.QueryHelper;
import org.codelibs.fess.helper.SambaHelper;
import org.codelibs.fess.helper.SearchLogHelper;
import org.codelibs.fess.helper.SuggestHelper;
import org.codelibs.fess.helper.SystemHelper;
import org.codelibs.fess.helper.UserAgentHelper;
import org.codelibs.fess.helper.UserInfoHelper;
@ -280,4 +281,7 @@ public final class ComponentUtil {
return SingletonLaContainerFactory.getContainer().hasComponentDef(QUERY_HELPER);
}
public static SuggestHelper getSuggestHelper() {
return getComponent(SuggestHelper.class);
}
}

View file

@ -240,7 +240,4 @@
<arg>"Mozilla/5.0 (compatible; Fess/" + org.codelibs.fess.Constants.FESS_VERSION + "; +http://fess.codelibs.org/bot.html)"</arg>
</component>
<component name="suggestHelper" class="org.codelibs.fess.helper.SuggestHelper">
</component>
</components>

View file

@ -144,4 +144,6 @@
<property name="roleSeparator">","</property>
-->
</component>
<component name="suggestHelper" class="org.codelibs.fess.helper.SuggestHelper">
</component>
</components>

View file

@ -20,8 +20,8 @@ $(function(){
$('#contentQuery').suggestor( {
ajaxinfo: {
url: contextPath + '/suggest', // TODO rename
fn: 'keyword',
url: contextPath + '/suggest',
fn: '_default',
num: 10
},
boxCssInfo: {

View file

@ -173,8 +173,8 @@ $(function(){
$('#query').suggestor( {
ajaxinfo: {
url: contextPath + '/json',
fn: 'content',
url: contextPath + '/suggest',
fn: '_default',
num: 10
},
boxCssInfo: {

View file

@ -1,7 +1,7 @@
;(function($){
$.fn.suggestor = function(setting) {
var $boxElement;
var $textArea;
var inputText = "";
@ -11,7 +11,7 @@ $.fn.suggestor = function(setting) {
var isMouseHover = false;
var started = false;
var interval = 5;
var settingMinTerm = 1;
var settingAjaxInfo;
var settingAdjustWidthVal;
@ -19,15 +19,15 @@ $.fn.suggestor = function(setting) {
var listSelectedCssInfo;
var listDeselectedCssInfo;
var boxCssInfo;
var suggestingSts = false;
var suggestor = {
init: function($element, setting) {
suggestingSts = false;
$boxElement = $("<div/>");
$boxElement.addClass("suggestorBox");
//style sheet
$boxElement.css("display","none");
$boxElement.css("position","absolute");
@ -42,14 +42,14 @@ $.fn.suggestor = function(setting) {
} else {
$boxElement.css(setting.boxCssInfo);
}
$textArea = $element;
$textArea.attr("autocomplete","off");
isFocusList = false;
inputText = $textArea.val();
//設定
settingAjaxInfo = setting.ajaxinfo;
settingMinTerm = setting.minturm;
@ -57,66 +57,71 @@ $.fn.suggestor = function(setting) {
listSelectedCssInfo = setting.listSelectedCssInfo;
listDeselectedCssInfo = setting.listDeselectedCssInfo;
settingAdjustWidthVal = setting.adjustWidthVal;
boxCssInfo = setting.boxCssInfo;
$boxElement.hover(function() {
isMouseHover = true;
}, function() {
isMouseHover = false;
});
//ポジション設定
this.resize();
var suggestor = this;
$(window).resize(function() {
suggestor.resize();
});
$("body").append($boxElement);
},
suggest: function() {
suggestingSts = true;
//ポジション設定
this.resize();
var suggestor = this;
inputText = $textArea.val();
listNum = 0;
listSelNum = 0;
if(inputText.length < settingMinTerm) {
$boxElement.css("display","none");
suggestingSts = false;
return;
}
$.ajax({
url: settingAjaxInfo.url,
type:"get",
dataType: "jsonp",
cache : false,
data:{ q: $textArea.val(),
data:{ query: $textArea.val(),
fields: settingAjaxInfo.fn,
num: settingAjaxInfo.num * 2
}
}).done(function(obj) { suggestor.createAutoCompleteList(obj); }).fail(function(a,obj,b) { suggestingSts=false; return; });
},
},
createAutoCompleteList: function(obj) {
var hits = obj.hits;
if(obj.response.status != 0) {
$boxElement.css("display","none");
return;
}
var hits = obj.response.result.hits;
var suggestor = this;
var addCount = 0;
listNum = 0;
if(typeof hits !== "undefined") {
var reslist = new Array();
@ -127,11 +132,11 @@ $.fn.suggestor = function(setting) {
$olEle.css("list-style","none");
$olEle.css("padding","0");
$olEle.css("margin","2px");
for(var i=0;i<reslist.length && listNum < settingAjaxInfo.num;i++) {
var str = reslist[i];
var chkCorrectWord = true;
/*
//suggestionの子要素かチェック
var parentEle = $(reslist[i]).closest("arr");
@ -141,7 +146,7 @@ $.fn.suggestor = function(setting) {
continue;
}
*/
//すでに同じ文字が表示されてないかチェック。ゴミ抜き
if(str === $textArea.val()) {
chkCorrectWord = false;
@ -155,7 +160,7 @@ $.fn.suggestor = function(setting) {
}
}
}
if(chkCorrectWord) {
var $liEle = $("<li/>");
$liEle.html(str);
@ -203,14 +208,14 @@ $.fn.suggestor = function(setting) {
listSelNum = 0
}
});
$liEle.css("padding","2px");
$olEle.append($liEle);
listNum++;
}
}
if(listNum>0 && $textArea.val().length >= settingMinTerm) {
$boxElement.html("");
$boxElement.append($olEle);
@ -223,15 +228,15 @@ $.fn.suggestor = function(setting) {
}
//ポジション設定
this.resize();
suggestingSts = false;
},
selectlist: function(direction) {
if($boxElement.css("display") == "none") {
return;
}
if(direction == "down") {
listSelNum++;
} else if(direction == "up") {
@ -239,15 +244,15 @@ $.fn.suggestor = function(setting) {
} else {
return;
}
isFocusList = true;
if(listSelNum < 0){
listSelNum = listNum;
} else if(listSelNum > listNum) {
listSelNum = 0;
}
var a = $boxElement.children("ol").children("li");
$boxElement.children("ol").children("li").each(function(i){
if(i == (listSelNum-1)) {
@ -272,20 +277,20 @@ $.fn.suggestor = function(setting) {
if(listSelNum == 0) {
$textArea.val(inputText);
}
},
fixList: function() {
if(listSelNum > 0) {
$textArea.val($($boxElement.children("ol").children("li").get(listSelNum-1)).html());
}
inputText = $textArea.val();
isFocusList = false;
$boxElement.css("display","none");
listNum = 0;
},
resize: function() {
$boxElement.css("top",$textArea.offset().top + $textArea.height() + 6);
$boxElement.css("left",$textArea.offset().left);
@ -296,9 +301,9 @@ $.fn.suggestor = function(setting) {
}
}
}
suggestor.init($(this), setting);
$(this).keydown( function(e){
if( ((e.keyCode >= 48) && (e.keyCode <= 90))
|| ((e.keyCode >= 96) && (e.keyCode <= 105))
@ -324,7 +329,7 @@ $.fn.suggestor = function(setting) {
if(isFocusList) {
suggestor.fixList();
}
}
}
});
$(this).keyup( function(e){
if( ((e.keyCode >= 48) && (e.keyCode <= 90))
@ -350,7 +355,7 @@ $.fn.suggestor = function(setting) {
suggestor.fixList();
}
});
//テキストエリア監視
setInterval( function() {
if(interval < 5) {
@ -365,6 +370,6 @@ $.fn.suggestor = function(setting) {
}
}
}, 100);
}
})(jQuery);