0ct0pu5/fess
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

working on #68

Browse source
This commit is contained in:
Shinsuke Sugaya 2013-12-08 22:17:23 +09:00
parent 7dff6d385a
commit 2fb512d8aa
13 changed files with 170 additions and 38 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/main/java/jp/sf/fess/Constants.java
View file

@ -320,4 +320,6 @@ public class Constants extends CoreLibConstants {
public static final String DOC_ID = "docId";
public static final String DCF = "dcf";
}

30
src/main/java/jp/sf/fess/action/IndexAction.java
View file

@ -52,12 +52,14 @@ import jp.sf.fess.entity.SuggestResponse;
import jp.sf.fess.form.IndexForm;
import jp.sf.fess.helper.BrowserTypeHelper;
import jp.sf.fess.helper.CrawlingConfigHelper;
import jp.sf.fess.helper.DocumentHelper;
import jp.sf.fess.helper.HotSearchWordHelper;
import jp.sf.fess.helper.HotSearchWordHelper.Range;
import jp.sf.fess.helper.LabelTypeHelper;
import jp.sf.fess.helper.OpenSearchHelper;
import jp.sf.fess.helper.QueryHelper;
import jp.sf.fess.helper.SearchLogHelper;
import jp.sf.fess.helper.SystemHelper;
import jp.sf.fess.helper.UserInfoHelper;
import jp.sf.fess.helper.ViewHelper;
import jp.sf.fess.screenshot.ScreenShotManager;
@ -144,6 +146,9 @@ public class IndexAction {
@Resource
protected OpenSearchHelper openSearchHelper;
@Resource
protected SystemHelper systemHelper;
@Resource
protected SuggesterManager suggesterManager;
@ -589,7 +594,8 @@ public class IndexAction {
try {
final Map<String, Object> doc = indexForm.docId == null ? null
: searchService.getDocument("docId:" + indexForm.docId);
: searchService.getDocument("docId:" + indexForm.docId,
new String[] { systemHelper.favoriteCountField });
final String userCode = userInfoHelper.getUserCode();
final String favoriteUrl = doc == null ? null : (String) doc
.get("url");
@ -621,10 +627,23 @@ public class IndexAction {
return null;
}
if (!favoriteLogService.addUrl(userCode, favoriteUrl, doc)) {
if (!favoriteLogService.addUrl(userCode, favoriteUrl)) {
WebApiUtil.setError(4, "Failed to add url: " + favoriteUrl);
return null;
}
final DocumentHelper documentHelper = SingletonS2Container
.getComponent("documentHelper");
final Object count = doc.get(systemHelper.favoriteCountField);
if (count instanceof Long) {
documentHelper.update(indexForm.docId,
systemHelper.favoriteCountField,
((Long) count).longValue() + 1);
} else {
WebApiUtil
.setError(7, "Failed to update count: " + favoriteUrl);
return null;
}
} catch (final Exception e) {
WebApiUtil.setError(1, e);
}
@ -654,7 +673,9 @@ public class IndexAction {
final String[] docIds = userInfoHelper
.getResultDocIds(indexForm.queryId);
final List<Map<String, Object>> docList = searchService
.getDocumentListByDocIds(docIds, MAX_PAGE_SIZE);
.getDocumentListByDocIds(docIds,
new String[] { systemHelper.favoriteCountField },
MAX_PAGE_SIZE);
List<String> urlList = new ArrayList<String>(docList.size());
for (final Map<String, Object> doc : docList) {
final Object urlObj = doc.get("url");
@ -774,7 +795,8 @@ public class IndexAction {
final int pageNum = Integer.parseInt(indexForm.num);
try {
documentItems = searchService.getDocumentList(query, pageStart,
pageNum, indexForm.facet, indexForm.geo, indexForm.mlt);
pageNum, indexForm.facet, indexForm.geo, indexForm.mlt,
queryHelper.getResponseDocValuesFields());
} catch (final SolrLibQueryException e) {
if (logger.isDebugEnabled()) {
logger.debug(e.getMessage(), e);

7
src/main/java/jp/sf/fess/action/MobileAction.java
View file

@ -32,6 +32,7 @@ import jp.sf.fess.db.allcommon.CDef;
import jp.sf.fess.db.exentity.SearchLog;
import jp.sf.fess.db.exentity.UserInfo;
import jp.sf.fess.form.MobileForm;
import jp.sf.fess.helper.QueryHelper;
import jp.sf.fess.helper.SearchLogHelper;
import jp.sf.fess.helper.UserInfoHelper;
import jp.sf.fess.service.SearchService;
@ -69,6 +70,9 @@ public class MobileAction {
@Resource
protected UserInfoHelper userInfoHelper;
@Resource
protected QueryHelper queryHelper;
@Resource
protected DynamicProperties crawlerProperties;
@ -145,7 +149,8 @@ public class MobileAction {
// TODO add GeoInfo if needed...
try {
documentItems = searchService.getDocumentList(mobileForm.query,
pageStart, pageNum, null, null, null);
pageStart, pageNum, null, null, null,
queryHelper.getResponseDocValuesFields());
} catch (final InvalidQueryException e) {
if (logger.isDebugEnabled()) {
logger.debug(e.getMessage(), e);

4
src/main/java/jp/sf/fess/action/admin/SearchListAction.java
View file

@ -155,7 +155,9 @@ public class SearchListAction implements Serializable {
final int size = Integer.parseInt(searchListForm.num);
try {
documentItems = searchService.getDocumentList(query, offset, size,
null, null, null, false);
null, null, null, new String[] {
systemHelper.clickCountField,
systemHelper.favoriteCountField }, false);
} catch (final InvalidQueryException e) {
if (logger.isDebugEnabled()) {
logger.debug(e.getMessage(), e);

45
src/main/java/jp/sf/fess/helper/DocumentHelper.java Normal file
View file

@ -0,0 +1,45 @@
package jp.sf.fess.helper;
import javax.annotation.Resource;
import jp.sf.fess.FessSystemException;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.common.SolrInputDocument;
import org.codelibs.solr.lib.SolrGroup;
import org.codelibs.solr.lib.SolrGroupManager;
import org.codelibs.solr.lib.policy.QueryType;
public class DocumentHelper {
@Resource
protected SystemHelper systemHelper;
@Resource
protected SolrGroupManager solrGroupManager;
public void update(final String docId, final String fieldName,
final long num) {
final SolrGroup solrGroup = solrGroupManager
.getSolrGroup(QueryType.ADD);
if (!solrGroup.isActive(QueryType.ADD)) {
throw new FessSystemException("SolrGroup "
+ solrGroup.getGroupName() + " is not available.");
}
final SolrInputDocument doc = new SolrInputDocument();
doc.setField(systemHelper.idField, "none");
doc.setField(systemHelper.urlField, "none");
doc.setField(systemHelper.docIdField, docId);
doc.setField(fieldName, num);
final UpdateRequest req = new UpdateRequest();
req.add(doc);
req.setParam("excmd", "update");
req.setParam("term", systemHelper.docIdField);
solrGroup.request(req);
solrGroup.commit(false, false, true);
}
}

2
src/main/java/jp/sf/fess/helper/QueryHelper.java
View file

@ -33,6 +33,8 @@ public interface QueryHelper {
String[] getResponseFields();
String[] getResponseDocValuesFields();
String[] getHighlightingFields();
int getHighlightSnippetSize();

8
src/main/java/jp/sf/fess/helper/SystemHelper.java
View file

@ -105,9 +105,9 @@ public class SystemHelper implements Serializable {
private final AtomicBoolean forceStop = new AtomicBoolean(false);
public String favoriteCountField = "favoriteCount_i";
public String favoriteCountField = "favoriteCount_l_x_dv";
public String clickCountField = "clickCount_i";
public String clickCountField = "clickCount_l_x_dv";
public String screenshotField = "screenshot_s_s";
@ -117,6 +117,10 @@ public class SystemHelper implements Serializable {
public String urlField = "url";
public String docIdField = "docId";
public String idField = "id";
@InitMethod
public void init() {
final File[] files = ResourceUtil.getJarFiles(launcherFileNamePrefix);

23
src/main/java/jp/sf/fess/helper/impl/QueryHelperImpl.java
View file

@ -82,14 +82,19 @@ public class QueryHelperImpl implements QueryHelper, Serializable {
protected String[] responseFields = new String[] { "id", "docId", "score",
"boost", "contentLength", "host", "site", "lastModified",
"mimetype", "created", "title", "digest", "url", "clickCount_i",
"favoriteCount_i", "screenshot_s_s", "cid_s_s" };
"mimetype", "created", "title", "digest", "url",
"clickCount_l_x_dv", "favoriteCount_l_x_dv", "screenshot_s_s",
"cid_s_s" };
protected String[] responseDocValuesFields = new String[] {
"clickCount_l_x_dv", "favoriteCount_l_x_dv" };
protected String[] highlightingFields = new String[] { "content" };
protected String[] searchFields = new String[] { "url", "docId", "host",
"title", "content", "contentLength", "lastModified", "mimetype",
"label", "segment", "clickCount_i", "favoriteCount_i", "inurl" };
"label", "segment", "clickCount_l_x_dv", "favoriteCount_l_x_dv",
"inurl" };
protected String[] facetFields = new String[] { "url", "host", "title",
"content", "contentLength", "lastModified", "mimetype", "label",
@ -98,7 +103,8 @@ public class QueryHelperImpl implements QueryHelper, Serializable {
protected String sortPrefix = "sort:";
protected String[] supportedSortFields = new String[] { "created",
"contentLength", "lastModified", "clickCount_i", "favoriteCount_i" };
"contentLength", "lastModified", "clickCount_l_x_dv",
"favoriteCount_l_x_dv" };
protected String[] supportedMltFields = new String[] { "content",
"content_ja" };
@ -1091,6 +1097,15 @@ public class QueryHelperImpl implements QueryHelper, Serializable {
this.responseFields = responseFields;
}
public String[] getResponseDocValuesFields() {
return responseDocValuesFields;
}
public void setResponseDocValuesFields(
final String[] responseDocValuesFields) {
this.responseDocValuesFields = responseDocValuesFields;
}
/**
* @return the highlightingFields
*/

13
src/main/java/jp/sf/fess/robot/FessS2RobotThread.java
View file

@ -63,10 +63,6 @@ public class FessS2RobotThread extends S2RobotThread {
public int childUrlSize = 10000;
public String clickCountField = "clickCount_i";
public String favoriteCountField = "favoriteCount_i";
@Override
protected boolean isContentUpdated(final S2RobotClient client,
final UrlQueue urlQueue) {
@ -166,7 +162,7 @@ public class FessS2RobotThread extends S2RobotThread {
}
final Integer clickCount = (Integer) solrDocument
.get(clickCountField);
.get(systemHelper.clickCountField);
if (clickCount != null) {
final SearchLogHelper searchLogHelper = SingletonS2Container
.getComponent(SearchLogHelper.class);
@ -179,7 +175,7 @@ public class FessS2RobotThread extends S2RobotThread {
}
final Integer favoriteCount = (Integer) solrDocument
.get(favoriteCountField);
.get(systemHelper.favoriteCountField);
if (favoriteCount != null) {
final SearchLogHelper searchLogHelper = SingletonS2Container
.getComponent(SearchLogHelper.class);
@ -267,6 +263,8 @@ public class FessS2RobotThread extends S2RobotThread {
final boolean wildcard, final String expiresField) {
final SolrGroupManager solrGroupManager = SingletonS2Container
.getComponent(SolrGroupManager.class);
final SystemHelper systemHelper = SingletonS2Container
.getComponent("systemHelper");
final SolrGroup solrGroup = solrGroupManager
.getSolrGroup(QueryType.ADD);
final SolrQuery solrQuery = new SolrQuery();
@ -279,7 +277,8 @@ public class FessS2RobotThread extends S2RobotThread {
queryBuf.append(id);
solrQuery.setQuery(queryBuf.toString());
solrQuery.setFields("id", "lastModified", "anchor", "segment", "role",
expiresField, clickCountField, favoriteCountField);
expiresField, systemHelper.clickCountField,
systemHelper.favoriteCountField);
for (int i = 0; i < maxSolrQueryRetryCount; i++) {
try {
final QueryResponse response = solrGroup.query(solrQuery);

3
src/main/java/jp/sf/fess/service/FavoriteLogService.java
View file

@ -98,8 +98,7 @@ public class FavoriteLogService extends BsFavoriteLogService implements
}
public boolean addUrl(final String userCode, final String url,
final Map<String, Object> doc) {
public boolean addUrl(final String userCode, final String url) {
final UserInfoCB cb = new UserInfoCB();
cb.query().setCode_Equal(userCode);
final UserInfo userInfo = userInfoBhv.selectEntity(cb);

36
src/main/java/jp/sf/fess/service/SearchService.java
View file

@ -61,8 +61,13 @@ public class SearchService implements Serializable {
protected QueryHelper queryHelper;
public Map<String, Object> getDocument(final String query) {
return getDocument(query, null);
}
public Map<String, Object> getDocument(final String query,
final String[] docValuesFields) {
final List<Map<String, Object>> docList = getDocumentList(query, 0, 1,
null, null, null);
null, null, null, docValuesFields);
if (!docList.isEmpty()) {
return docList.get(0);
}
@ -70,7 +75,8 @@ public class SearchService implements Serializable {
}
public List<Map<String, Object>> getDocumentListByDocIds(
final String[] docIds, final int pageSize) {
final String[] docIds, final String[] docValuesFields,
final int pageSize) {
if (docIds == null || docIds.length == 0) {
return Collections.emptyList();
}
@ -82,20 +88,22 @@ public class SearchService implements Serializable {
}
buf.append("docId:").append(docIds[i]);
}
return getDocumentList(buf.toString(), 0, pageSize, null, null, null);
}
public List<Map<String, Object>> getDocumentList(final String query,
final int start, final int rows, final FacetInfo facetInfo,
final GeoInfo geoInfo, final MoreLikeThisInfo mltInfo) {
return getDocumentList(query, start, rows, facetInfo, geoInfo, mltInfo,
true);
return getDocumentList(buf.toString(), 0, pageSize, null, null, null,
docValuesFields);
}
public List<Map<String, Object>> getDocumentList(final String query,
final int start, final int rows, final FacetInfo facetInfo,
final GeoInfo geoInfo, final MoreLikeThisInfo mltInfo,
final boolean forUser) {
final String[] docValuesFields) {
return getDocumentList(query, start, rows, facetInfo, geoInfo, mltInfo,
docValuesFields, true);
}
public List<Map<String, Object>> getDocumentList(final String query,
final int start, final int rows, final FacetInfo facetInfo,
final GeoInfo geoInfo, final MoreLikeThisInfo mltInfo,
final String[] docValuesFields, final boolean forUser) {
if (start > queryHelper.getMaxSearchResultOffset()) {
throw new ResultOffsetExceededException(
"The number of result size is exceeded.");
@ -237,6 +245,12 @@ public class SearchService implements Serializable {
}
}
if (docValuesFields != null) {
for (final String docValuesField : docValuesFields) {
solrQuery.setParam(Constants.DCF, docValuesField);
}
}
queryResponse = solrGroup.query(solrQuery, SolrRequest.METHOD.POST);
}
final long execTime = System.currentTimeMillis() - startTime;

24
src/main/java/jp/sf/fess/util/QueryResponseList.java
View file

@ -39,6 +39,12 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class QueryResponseList implements List<Map<String, Object>> {
private static final String PARTIAL_RESULTS = "partialResults";
private static final String MORE_LIKE_THIS = "moreLikeThis";
private static final String DOC_VALUES = "docValues";
private static final String ID_FIELD = "id";
private static final Logger logger = LoggerFactory
@ -96,7 +102,7 @@ public class QueryResponseList implements List<Map<String, Object>> {
numFound = sdList.getNumFound();
final Object partialResultsValue = queryResponse
.getResponseHeader().get("partialResults");
.getResponseHeader().get(PARTIAL_RESULTS);
if (partialResultsValue != null
&& ((Boolean) partialResultsValue).booleanValue()) {
partialResults = true;
@ -157,7 +163,7 @@ public class QueryResponseList implements List<Map<String, Object>> {
// mlt
final Object moreLikeThisMap = queryResponse.getResponse().get(
"moreLikeThis");
MORE_LIKE_THIS);
if (moreLikeThisMap instanceof SimpleOrderedMap) {
moreLikeThisResponse = new MoreLikeThisResponse();
final int size = ((SimpleOrderedMap<?>) moreLikeThisMap).size();
@ -179,6 +185,20 @@ public class QueryResponseList implements List<Map<String, Object>> {
}
}
}
// docValues
final Object docValuesObj = queryResponse.getResponse().get(
DOC_VALUES);
if (docValuesObj instanceof SimpleOrderedMap) {
final SimpleOrderedMap<List<Long>> docValuesMap = (SimpleOrderedMap<List<Long>>) docValuesObj;
for (int i = 0; i < docValuesMap.size(); i++) {
final String name = docValuesMap.getName(i);
final List<Long> valueList = docValuesMap.getVal(i);
for (int j = 0; j < valueList.size() && j < parent.size(); j++) {
parent.get(j).put(name, valueList.get(j));
}
}
}
}
calculatePageInfo(start, pageSize, numFound);
}

11
src/main/resources/app.dicon
View file

@ -62,14 +62,15 @@
<initMethod name="setApiResponseFields">
<arg>new String[]{"id", "docId", "score", "boost",
"contentLength", "host", "site", "lastModified", "mimetype",
"created", "title", "digest", "url", "clickCount_i", "favoriteCount_i"}</arg>
"created", "title", "digest", "url"}</arg>
</initMethod>
<!--
<property name="additionalGeoQuery">"location_i_i:1"</property>
<property name="responseFields">new String[]{"id", "docId", "score", "boost",
"contentLength", "host", "site", "lastModified", "mimetype",
"created", "title", "digest", "url", "clickCount_i", "favoriteCount_i",
"screenshot_s_s", "cid_s_s"}</property>
"created", "title", "digest", "url", "screenshot_s_s", "cid_s_s"}</property>
<property name="responseDocValuesFields">new String[]{
"clickCount_l_x_dv", "favoriteCount_l_x_dv"}</property>
<property name="highlightingFields">new String[]{"digest", "cache" }</property>
<property name="searchFields">new String[]{"url", "docId", "host",
"title", "content", "contentLength", "lastModified", "mimetype",
@ -79,7 +80,7 @@
"label", "segment" }</property>
<property name="sortPrefix">"sort:"</property>
<property name="supportedSortFields">new String[]{"created",
"contentLength", "lastModified", "clickCount_i" }</property>
"contentLength", "lastModified", "clickCount_l_x_dv" }</property>
<property name="supportedMltFields">new String[]{"content",
"content_ja" }</property>
<property name="supportedSuggestFields">new String[]{"content",
@ -206,6 +207,8 @@
<property name="osddPath">"/WEB-INF/orig/open-search/osdd.xml"</property>
<property name="encoding">"UTF-8"</property>
</component>
<component name="documentHelper" class="jp.sf.fess.helper.DocumentHelper">
</component>
<component name="queryResponseList" class="jp.sf.fess.util.QueryResponseList" instance="prototype">
</component>
<component name="userAgentName" class="java.lang.String">