0ct0pu5/fess
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix #67

Browse source
This commit is contained in:
Shinsuke Sugaya 2014-10-28 22:05:21 +09:00
parent 70bd6fa900
commit f1ae612a23
31 changed files with 459 additions and 298 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
src/main/java/jp/sf/fess/Constants.java
View file

@ -318,8 +318,6 @@ public class Constants extends CoreLibConstants {
public static final int EXIT_FAIL = 1;
public static final String DOC_ID = "docId";
public static final String DCF = "dcf";
public static final String ALL_LANGUAGES = "all";
@ -328,4 +326,6 @@ public class Constants extends CoreLibConstants {
public static final String INVALID_NUMERIC_PARAMETER = "-1";
public static final String NOW = "NOW";
}

51
src/main/java/jp/sf/fess/action/IndexAction.java
View file

@ -57,6 +57,7 @@ import jp.sf.fess.entity.LoginInfo;
import jp.sf.fess.form.IndexForm;
import jp.sf.fess.helper.CrawlingConfigHelper;
import jp.sf.fess.helper.DocumentHelper;
import jp.sf.fess.helper.FieldHelper;
import jp.sf.fess.helper.HotSearchWordHelper;
import jp.sf.fess.helper.HotSearchWordHelper.Range;
import jp.sf.fess.helper.LabelTypeHelper;
@ -163,6 +164,9 @@ public class IndexAction {
@Resource
protected SystemHelper systemHelper;
@Resource
protected FieldHelper fieldHelper;
@Resource
protected Suggester suggester;
@ -351,8 +355,9 @@ public class IndexAction {
public String cache() {
Map<String, Object> doc = null;
try {
doc = searchService.getDocument("docId:" + indexForm.docId,
queryHelper.getCacheResponseFields(), null);
doc = searchService.getDocument(fieldHelper.docIdField + ":"
+ indexForm.docId, queryHelper.getCacheResponseFields(),
null);
} catch (final Exception e) {
logger.warn("Failed to request: " + indexForm.docId, e);
}
@ -379,9 +384,9 @@ public class IndexAction {
public String go() throws IOException {
Map<String, Object> doc = null;
try {
doc = searchService.getDocument("docId:" + indexForm.docId,
queryHelper.getResponseFields(),
new String[] { systemHelper.clickCountField });
doc = searchService.getDocument(fieldHelper.docIdField + ":"
+ indexForm.docId, queryHelper.getResponseFields(),
new String[] { fieldHelper.clickCountField });
} catch (final Exception e) {
logger.warn("Failed to request: " + indexForm.docId, e);
}
@ -391,7 +396,7 @@ public class IndexAction {
indexForm.docId);
return "error.jsp";
}
final Object urlObj = doc.get("url");
final Object urlObj = doc.get(fieldHelper.urlField);
if (urlObj == null) {
errorMessage = MessageResourcesUtil.getMessage(RequestUtil
.getRequest().getLocale(), "errors.document_not_found",
@ -415,7 +420,7 @@ public class IndexAction {
clickLog.setUserSessionId(userSessionId);
clickLog.setDocId(indexForm.docId);
long clickCount = 0;
final Object count = doc.get(systemHelper.clickCountField);
final Object count = doc.get(fieldHelper.clickCountField);
if (count instanceof Long) {
clickCount = ((Long) count).longValue();
}
@ -532,9 +537,10 @@ public class IndexAction {
OutputStream out = null;
BufferedInputStream in = null;
try {
final Map<String, Object> doc = searchService.getDocument("docId:"
+ indexForm.docId);
final String url = doc == null ? null : (String) doc.get("url");
final Map<String, Object> doc = searchService
.getDocument(fieldHelper.docIdField + ":" + indexForm.docId);
final String url = doc == null ? null : (String) doc
.get(fieldHelper.urlField);
if (StringUtil.isBlank(indexForm.queryId)
|| StringUtil.isBlank(url) || screenShotManager == null) {
// 404
@ -798,12 +804,12 @@ public class IndexAction {
try {
final Map<String, Object> doc = indexForm.docId == null ? null
: searchService.getDocument("docId:" + indexForm.docId,
queryHelper.getResponseFields(),
new String[] { systemHelper.favoriteCountField });
: searchService.getDocument(fieldHelper.docIdField + ":"
+ indexForm.docId, queryHelper.getResponseFields(),
new String[] { fieldHelper.favoriteCountField });
final String userCode = userInfoHelper.getUserCode();
final String favoriteUrl = doc == null ? null : (String) doc
.get("url");
.get(fieldHelper.urlField);
if (StringUtil.isBlank(userCode)) {
WebApiUtil.setError(2, "No user session.");
@ -839,10 +845,10 @@ public class IndexAction {
final DocumentHelper documentHelper = ComponentUtil
.getDocumentHelper();
final Object count = doc.get(systemHelper.favoriteCountField);
final Object count = doc.get(fieldHelper.favoriteCountField);
if (count instanceof Long) {
documentHelper.update(indexForm.docId,
systemHelper.favoriteCountField,
fieldHelper.favoriteCountField,
((Long) count).longValue() + 1);
} else {
WebApiUtil
@ -880,11 +886,11 @@ public class IndexAction {
final List<Map<String, Object>> docList = searchService
.getDocumentListByDocIds(docIds,
queryHelper.getResponseFields(),
new String[] { systemHelper.favoriteCountField },
new String[] { fieldHelper.favoriteCountField },
MAX_PAGE_SIZE);
List<String> urlList = new ArrayList<String>(docList.size());
for (final Map<String, Object> doc : docList) {
final Object urlObj = doc.get("url");
final Object urlObj = doc.get(fieldHelper.urlField);
if (urlObj != null) {
urlList.add(urlObj.toString());
}
@ -892,9 +898,9 @@ public class IndexAction {
urlList = favoriteLogService.getUrlList(userCode, urlList);
final List<String> docIdList = new ArrayList<String>(urlList.size());
for (final Map<String, Object> doc : docList) {
final Object urlObj = doc.get("url");
final Object urlObj = doc.get(fieldHelper.urlField);
if (urlObj != null && urlList.contains(urlObj.toString())) {
final Object docIdObj = doc.get(Constants.DOC_ID);
final Object docIdObj = doc.get(fieldHelper.docIdField);
if (docIdObj != null) {
docIdList.add(docIdObj.toString());
}
@ -1173,7 +1179,7 @@ public class IndexAction {
private void appendLangQuery(final StringBuilder queryBuf,
final Set<String> langSet) {
if (langSet.size() == 1) {
queryBuf.append(' ').append(systemHelper.langField).append(':')
queryBuf.append(' ').append(fieldHelper.langField).append(':')
.append(langSet.iterator().next());
} else if (langSet.size() > 1) {
boolean first = true;
@ -1184,8 +1190,7 @@ public class IndexAction {
} else {
queryBuf.append(" OR ");
}
queryBuf.append(systemHelper.langField).append(':')
.append(lang);
queryBuf.append(fieldHelper.langField).append(':').append(lang);
}
queryBuf.append(')');
}

35
src/main/java/jp/sf/fess/action/admin/DocumentAction.java
View file

@ -28,6 +28,7 @@ import javax.annotation.Resource;
import jp.sf.fess.Constants;
import jp.sf.fess.crud.util.SAStrutsUtil;
import jp.sf.fess.form.admin.DocumentForm;
import jp.sf.fess.helper.FieldHelper;
import jp.sf.fess.helper.JobHelper;
import jp.sf.fess.helper.SystemHelper;
import jp.sf.fess.helper.WebManagementHelper;
@ -56,6 +57,12 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class DocumentAction implements Serializable {
private static final String SUGGEST_TYPE_ALL = "all";
private static final String SUGGEST_TYPE_SEARCH_LOG = "searchLog";
private static final String SUGGEST_TYPE_CONTENT = "content";
private static final Logger logger = LoggerFactory
.getLogger(DocumentAction.class);
@ -77,6 +84,9 @@ public class DocumentAction implements Serializable {
@Resource
protected SystemHelper systemHelper;
@Resource
protected FieldHelper fieldHelper;
@Resource
protected JobHelper jobHelper;
@ -266,7 +276,8 @@ public class DocumentAction implements Serializable {
if ("*".equals(documentForm.sessionId)) {
deleteQuery = "*:*";
} else {
deleteQuery = "segment:" + documentForm.sessionId;
deleteQuery = fieldHelper.segmentField + ":"
+ documentForm.sessionId;
}
return deleteByQuery(deleteQuery);
}
@ -274,7 +285,8 @@ public class DocumentAction implements Serializable {
@Token(save = false, validate = true)
@Execute(validator = true, input = "index")
public String confirmByUrl() {
final String confirmQuery = "url:\"" + documentForm.deleteUrl + "\"";
final String confirmQuery = fieldHelper.urlField + ":\""
+ documentForm.deleteUrl + "\"";
return "/admin/searchList/search?query=" + S2Functions.u(confirmQuery)
+ "&redirect=true";
}
@ -283,7 +295,8 @@ public class DocumentAction implements Serializable {
@Execute(validator = true, input = "index")
public String deleteByUrl() {
final String deleteUrl = documentForm.deleteUrl;
final String deleteQuery = "url:\"" + deleteUrl + "\"";
final String deleteQuery = fieldHelper.urlField + ":\"" + deleteUrl
+ "\"";
return deleteByQuery(deleteQuery);
}
@ -367,8 +380,8 @@ public class DocumentAction implements Serializable {
final SolrQuery query = new SolrQuery();
query.setQuery("*:*");
query.setFacet(true);
query.addFacetField("segment");
query.addSort("segment", ORDER.desc);
query.addFacetField(fieldHelper.segmentField);
query.addSort(fieldHelper.segmentField, ORDER.desc);
final QueryResponse queryResponse = serverGroup.query(query);
final List<FacetField> facets = queryResponse.getFacetFields();
@ -400,9 +413,10 @@ public class DocumentAction implements Serializable {
protected Map<String, Long> getSuggestDocumentNum() {
final Map<String, Long> map = new HashMap<String, Long>();
map.put("content", suggestService.getContentDocumentNum());
map.put("searchLog", suggestService.getSearchLogDocumentNum());
map.put("all", suggestService.getDocumentNum());
map.put(SUGGEST_TYPE_CONTENT, suggestService.getContentDocumentNum());
map.put(SUGGEST_TYPE_SEARCH_LOG,
suggestService.getSearchLogDocumentNum());
map.put(SUGGEST_TYPE_ALL, suggestService.getDocumentNum());
return map;
}
@ -412,12 +426,13 @@ public class DocumentAction implements Serializable {
final SuggestSolrServer suggestSolrServer = suggestService
.getSuggestSolrServer();
final String query;
if ("content".equals(documentForm.deleteSuggestType)) {
if (SUGGEST_TYPE_CONTENT.equals(documentForm.deleteSuggestType)) {
query = "*:* NOT " + SuggestConstants.SuggestFieldNames.SEGMENT
+ ":" + SuggestConstants.SEGMENT_ELEVATE + " NOT "
+ SuggestConstants.SuggestFieldNames.SEGMENT + ":"
+ SuggestConstants.SEGMENT_QUERY;
} else if ("searchLog".equals(documentForm.deleteSuggestType)) {
} else if (SUGGEST_TYPE_SEARCH_LOG
.equals(documentForm.deleteSuggestType)) {
query = SuggestConstants.SuggestFieldNames.SEGMENT + ":"
+ SuggestConstants.SEGMENT_QUERY;
} else {

8
src/main/java/jp/sf/fess/action/admin/SearchListAction.java
View file

@ -28,6 +28,7 @@ import jp.sf.fess.InvalidQueryException;
import jp.sf.fess.ResultOffsetExceededException;
import jp.sf.fess.crud.util.SAStrutsUtil;
import jp.sf.fess.form.admin.SearchListForm;
import jp.sf.fess.helper.FieldHelper;
import jp.sf.fess.helper.JobHelper;
import jp.sf.fess.helper.QueryHelper;
import jp.sf.fess.helper.SystemHelper;
@ -78,6 +79,9 @@ public class SearchListAction implements Serializable {
@Resource
protected SystemHelper systemHelper;
@Resource
protected FieldHelper fieldHelper;
@Resource
protected QueryHelper queryHelper;
@ -160,8 +164,8 @@ public class SearchListAction implements Serializable {
try {
documentItems = searchService.getDocumentList(query, offset, size,
null, null, null, queryHelper.getResponseFields(),
new String[] { systemHelper.clickCountField,
systemHelper.favoriteCountField }, false);
new String[] { fieldHelper.clickCountField,
fieldHelper.favoriteCountField }, false);
} catch (final InvalidQueryException e) {
if (logger.isDebugEnabled()) {
logger.debug(e.getMessage(), e);

22
src/main/java/jp/sf/fess/ds/impl/AbstractDataStoreImpl.java
View file

@ -27,7 +27,7 @@ import jp.sf.fess.db.exentity.DataCrawlingConfig;
import jp.sf.fess.ds.DataStore;
import jp.sf.fess.ds.IndexUpdateCallback;
import jp.sf.fess.helper.CrawlingSessionHelper;
import jp.sf.fess.helper.SystemHelper;
import jp.sf.fess.helper.FieldHelper;
import jp.sf.fess.taglib.FessFunctions;
import jp.sf.fess.util.ComponentUtil;
@ -61,7 +61,7 @@ public abstract class AbstractDataStoreImpl implements DataStore {
final CrawlingSessionHelper crawlingSessionHelper = ComponentUtil
.getCrawlingSessionHelper();
final Date documentExpires = crawlingSessionHelper.getDocumentExpires();
final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
initParamMap.putAll(configParamMap);
final Map<String, String> paramMap = initParamMap;
@ -72,33 +72,35 @@ public abstract class AbstractDataStoreImpl implements DataStore {
// cid
final String configId = config.getConfigId();
if (configId != null) {
defaultDataMap.put(systemHelper.configIdField, configId);
defaultDataMap.put(fieldHelper.configIdField, configId);
}
// expires
if (documentExpires != null) {
defaultDataMap.put(systemHelper.expiresField,
defaultDataMap.put(fieldHelper.expiresField,
FessFunctions.formatDate(documentExpires));
}
// segment
defaultDataMap.put("segment", initParamMap.get(Constants.SESSION_ID));
defaultDataMap.put(fieldHelper.segmentField,
initParamMap.get(Constants.SESSION_ID));
// created
defaultDataMap.put("created", "NOW");
defaultDataMap.put(fieldHelper.createdField, "NOW");
// boost
defaultDataMap.put("boost", config.getBoost().toString());
defaultDataMap
.put(fieldHelper.boostField, config.getBoost().toString());
// label: labelType
final List<String> labelTypeList = new ArrayList<String>();
for (final String labelType : config.getLabelTypeValues()) {
labelTypeList.add(labelType);
}
defaultDataMap.put("label", labelTypeList);
defaultDataMap.put(fieldHelper.labelField, labelTypeList);
// role: roleType
final List<String> roleTypeList = new ArrayList<String>();
for (final String roleType : config.getRoleTypeValues()) {
roleTypeList.add(roleType);
}
defaultDataMap.put("role", roleTypeList);
defaultDataMap.put(fieldHelper.roleField, roleTypeList);
// mimetype
defaultDataMap.put("mimetype", mimeType);
defaultDataMap.put(fieldHelper.mimetypeField, mimeType);
// title
// content
// cache

16
src/main/java/jp/sf/fess/ds/impl/FileListDataStoreImpl.java
View file

@ -28,6 +28,8 @@ import jp.sf.fess.ds.DataStoreCrawlingException;
import jp.sf.fess.ds.DataStoreException;
import jp.sf.fess.ds.IndexUpdateCallback;
import jp.sf.fess.helper.CrawlingSessionHelper;
import jp.sf.fess.helper.FieldHelper;
import jp.sf.fess.util.ComponentUtil;
import jp.sf.orangesignal.csv.CsvConfig;
import org.codelibs.robot.RobotSystemException;
@ -67,8 +69,6 @@ public class FileListDataStoreImpl extends CsvDataStoreImpl {
public String eventTypeField = "event_type";
public String urlField = "url";
public int maxDeleteDocumentCacheSize = 100;
protected S2RobotClientFactory robotClientFactory;
@ -179,15 +179,16 @@ public class FileListDataStoreImpl extends CsvDataStoreImpl {
}
protected boolean addDocument(final Map<String, Object> dataMap) {
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
synchronized (indexUpdateCallback) {
// required check
if (!dataMap.containsKey(urlField)
|| dataMap.get(urlField) == null) {
if (!dataMap.containsKey(fieldHelper.urlField)
|| dataMap.get(fieldHelper.urlField) == null) {
logger.warn("Could not add a doc. Invalid data: " + dataMap);
return false;
}
final String url = dataMap.get(urlField).toString();
final String url = dataMap.get(fieldHelper.urlField).toString();
try {
final S2RobotClient client = robotClientFactory
.getClient(url);
@ -259,8 +260,11 @@ public class FileListDataStoreImpl extends CsvDataStoreImpl {
logger.debug("Deleting " + dataMap);
}
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
// required check
if (!dataMap.containsKey(urlField) || dataMap.get(urlField) == null) {
if (!dataMap.containsKey(fieldHelper.urlField)
|| dataMap.get(fieldHelper.urlField) == null) {
logger.warn("Could not delete a doc. Invalid data: " + dataMap);
return false;
}

38
src/main/java/jp/sf/fess/ds/impl/IndexUpdateCallbackImpl.java
View file

@ -21,10 +21,10 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
import jp.sf.fess.Constants;
import jp.sf.fess.FessSystemException;
import jp.sf.fess.ds.IndexUpdateCallback;
import jp.sf.fess.helper.CrawlingSessionHelper;
import jp.sf.fess.helper.FieldHelper;
import jp.sf.fess.helper.IndexingHelper;
import jp.sf.fess.helper.SearchLogHelper;
import jp.sf.fess.helper.SystemHelper;
@ -32,6 +32,7 @@ import jp.sf.fess.util.ComponentUtil;
import org.apache.solr.common.SolrInputDocument;
import org.codelibs.solr.lib.SolrGroup;
import org.seasar.framework.container.annotation.tiger.InitMethod;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -55,6 +56,13 @@ public class IndexUpdateCallbackImpl implements IndexUpdateCallback {
final List<SolrInputDocument> docList = new ArrayList<SolrInputDocument>();
private FieldHelper fieldHelper;
@InitMethod
public void init() {
fieldHelper = ComponentUtil.getFieldHelper();
}
/* (non-Javadoc)
* @see jp.sf.fess.ds.impl.IndexUpdateCallback#store(java.util.Map)
*/
@ -67,7 +75,7 @@ public class IndexUpdateCallbackImpl implements IndexUpdateCallback {
}
// required check
final Object urlObj = dataMap.get("url");
final Object urlObj = dataMap.get(fieldHelper.urlField);
if (urlObj == null) {
throw new FessSystemException("url is null. dataMap=" + dataMap);
}
@ -75,7 +83,8 @@ public class IndexUpdateCallbackImpl implements IndexUpdateCallback {
final IndexingHelper indexingHelper = ComponentUtil.getIndexingHelper();
final CrawlingSessionHelper crawlingSessionHelper = ComponentUtil
.getCrawlingSessionHelper();
dataMap.put("id", crawlingSessionHelper.generateId(dataMap));
dataMap.put(fieldHelper.idField,
crawlingSessionHelper.generateId(dataMap));
final SolrInputDocument doc = createSolrDocument(dataMap);
@ -108,11 +117,11 @@ public class IndexUpdateCallbackImpl implements IndexUpdateCallback {
protected SolrInputDocument createSolrDocument(
final Map<String, Object> dataMap) {
final String url = dataMap.get("url").toString();
final String url = dataMap.get(fieldHelper.urlField).toString();
final SolrInputDocument doc = new SolrInputDocument();
for (final Map.Entry<String, Object> entry : dataMap.entrySet()) {
if ("boost".equals(entry.getKey())) {
if (fieldHelper.boostField.equals(entry.getKey())) {
// boost
final float documentBoost = Float.valueOf(entry.getValue()
.toString());
@ -126,16 +135,17 @@ public class IndexUpdateCallbackImpl implements IndexUpdateCallback {
}
if (clickCountEnabled) {
addClickCountField(doc, url);
addClickCountField(doc, url, fieldHelper.clickCountField);
}
if (favoriteCountEnabled) {
addFavoriteCountField(doc, url);
addFavoriteCountField(doc, url, fieldHelper.favoriteCountField);
}
if (!dataMap.containsKey(Constants.DOC_ID)) {
if (!dataMap.containsKey(fieldHelper.docIdField)) {
final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
doc.addField(Constants.DOC_ID, systemHelper.generateDocId(dataMap));
doc.addField(fieldHelper.docIdField,
systemHelper.generateDocId(dataMap));
}
return doc;
@ -166,24 +176,22 @@ public class IndexUpdateCallbackImpl implements IndexUpdateCallback {
}
protected void addClickCountField(final SolrInputDocument doc,
final String url) {
final String url, final String clickCountField) {
final SearchLogHelper searchLogHelper = ComponentUtil
.getSearchLogHelper();
final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
final int count = searchLogHelper.getClickCount(url);
doc.addField(systemHelper.clickCountField, count);
doc.addField(clickCountField, count);
if (logger.isDebugEnabled()) {
logger.debug("Click Count: " + count + ", url: " + url);
}
}
protected void addFavoriteCountField(final SolrInputDocument doc,
final String url) {
final String url, final String favoriteCountField) {
final SearchLogHelper searchLogHelper = ComponentUtil
.getSearchLogHelper();
final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
final long count = searchLogHelper.getFavoriteCount(url);
doc.addField(systemHelper.favoriteCountField, count);
doc.addField(favoriteCountField, count);
if (logger.isDebugEnabled()) {
logger.debug("Favorite Count: " + count + ", url: " + url);
}

6
src/main/java/jp/sf/fess/exec/Crawler.java
View file

@ -39,10 +39,10 @@ import jp.sf.fess.db.allcommon.CDef;
import jp.sf.fess.helper.CrawlingSessionHelper;
import jp.sf.fess.helper.DataIndexHelper;
import jp.sf.fess.helper.DatabaseHelper;
import jp.sf.fess.helper.FieldHelper;
import jp.sf.fess.helper.MailHelper;
import jp.sf.fess.helper.OverlappingHostHelper;
import jp.sf.fess.helper.PathMappingHelper;
import jp.sf.fess.helper.SystemHelper;
import jp.sf.fess.helper.WebFsIndexHelper;
import jp.sf.fess.screenshot.ScreenShotManager;
import jp.sf.fess.service.CrawlingSessionService;
@ -389,7 +389,7 @@ public class Crawler implements Serializable {
final CrawlingSessionHelper crawlingSessionHelper = ComponentUtil
.getCrawlingSessionHelper();
final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
boolean completed = false;
int exitCode = Constants.EXIT_OK;
@ -471,7 +471,7 @@ public class Crawler implements Serializable {
// clean up
try {
updateSolrGroup.deleteByQuery(systemHelper.expiresField
updateSolrGroup.deleteByQuery(fieldHelper.expiresField
+ ":[* TO " + FessFunctions.formatDate(new Date())
+ "] NOT segment:" + options.sessionId);
} catch (final Exception e) {

6
src/main/java/jp/sf/fess/helper/CrawlingConfigHelper.java
View file

@ -137,8 +137,8 @@ public class CrawlingConfigHelper implements Serializable {
if (logger.isDebugEnabled()) {
logger.debug("writing the content of: " + doc);
}
final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
final Object configIdObj = doc.get(systemHelper.configIdField);
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final Object configIdObj = doc.get(fieldHelper.configIdField);
if (configIdObj == null) {
throw new FessSystemException("configId is null.");
}
@ -171,7 +171,7 @@ public class CrawlingConfigHelper implements Serializable {
if (config == null) {
throw new FessSystemException("No crawlingConfig: " + configIdObj);
}
final String url = (String) doc.get(systemHelper.urlField);
final String url = (String) doc.get(fieldHelper.urlField);
final S2RobotClientFactory robotClientFactory = SingletonS2Container
.getComponent(S2RobotClientFactory.class);
config.initializeClientFactory(robotClientFactory);

16
src/main/java/jp/sf/fess/helper/CrawlingSessionHelper.java
View file

@ -31,6 +31,7 @@ import jp.sf.fess.FessSystemException;
import jp.sf.fess.db.exentity.CrawlingSession;
import jp.sf.fess.db.exentity.CrawlingSessionInfo;
import jp.sf.fess.service.CrawlingSessionService;
import jp.sf.fess.util.ComponentUtil;
import org.apache.commons.lang.time.DateUtils;
import org.apache.solr.client.solrj.SolrQuery;
@ -49,8 +50,6 @@ public class CrawlingSessionHelper implements Serializable {
public static final String FACET_COUNT_KEY = "count";
public static final String FACET_SEGMENT_KEY = "segment";
private static final long serialVersionUID = 1L;
protected Map<String, String> infoMap;
@ -151,21 +150,24 @@ public class CrawlingSessionHelper implements Serializable {
}
public String generateId(final Map<String, Object> dataMap) {
final String url = (String) dataMap.get("url");
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final String url = (String) dataMap.get(fieldHelper.urlField);
@SuppressWarnings("unchecked")
final List<String> roleTypeList = (List<String>) dataMap.get("role");
final List<String> roleTypeList = (List<String>) dataMap
.get(fieldHelper.roleField);
return generateId(url, roleTypeList);
}
public List<Map<String, String>> getSessionIdList(
final SolrGroup serverGroup) {
final List<Map<String, String>> sessionIdList = new ArrayList<Map<String, String>>();
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final SolrQuery query = new SolrQuery();
query.setQuery("*:*");
query.setFacet(true);
query.addFacetField(FACET_SEGMENT_KEY);
query.addSort(FACET_SEGMENT_KEY, ORDER.desc);
query.addFacetField(fieldHelper.segmentField);
query.addSort(fieldHelper.segmentField, ORDER.desc);
final QueryResponse queryResponse = serverGroup.query(query);
final List<FacetField> facets = queryResponse.getFacetFields();
@ -175,7 +177,7 @@ public class CrawlingSessionHelper implements Serializable {
for (final FacetField.Count fcount : facetEntries) {
final Map<String, String> map = new HashMap<String, String>(
2);
map.put(FACET_SEGMENT_KEY, fcount.getName());
map.put(fieldHelper.segmentField, fcount.getName());
map.put(FACET_COUNT_KEY, Long.toString(fcount.getCount()));
sessionIdList.add(map);
}

10
src/main/java/jp/sf/fess/helper/DocumentHelper.java
View file

@ -29,7 +29,7 @@ import org.codelibs.solr.lib.policy.QueryType;
public class DocumentHelper {
@Resource
protected SystemHelper systemHelper;
protected FieldHelper fieldHelper;
@Resource
protected SolrGroupManager solrGroupManager;
@ -45,15 +45,15 @@ public class DocumentHelper {
}
final SolrInputDocument doc = new SolrInputDocument();
doc.setField(systemHelper.idField, "none");
doc.setField(systemHelper.urlField, "none");
doc.setField(systemHelper.docIdField, docId);
doc.setField(fieldHelper.idField, "none");
doc.setField(fieldHelper.urlField, "none");
doc.setField(fieldHelper.docIdField, docId);
doc.setField(fieldName, num);
final UpdateRequest req = new UpdateRequest();
req.add(doc);
req.setParam("excmd", "update");
req.setParam("term", systemHelper.docIdField);
req.setParam("term", fieldHelper.docIdField);
solrGroup.request(req);
solrGroup.commit(false, false, true);
}

56
src/main/java/jp/sf/fess/helper/FieldHelper.java Normal file
View file

@ -0,0 +1,56 @@
package jp.sf.fess.helper;
public class FieldHelper {
public String favoriteCountField = "favoriteCount_l_x_dv";
public String clickCountField = "clickCount_l_x_dv";
public String configIdField = "cid_s";
public String expiresField = "expires_dt";
public String urlField = "url";
public String docIdField = "docId";
public String idField = "id";
public String langField = "lang_s";
public String hasCacheField = "hasCache_s_s";
public String lastModifiedField = "lastModified";
public String anchorField = "anchor";
public String segmentField = "segment";
public String roleField = "role";
public String boostField = "boost";
public String createdField = "created";
public String labelField = "label";
public String mimetypeField = "mimetype";
public String parentIdField = "parentId";
public String contentField = "content";
public String cacheField = "cache";
public String digestField = "digest";
public String titleField = "title";
public String hostField = "host";
public String siteField = "site";
public String contentLengthField = "contentLength";
public String filetypeField = "filetype_s";
}

9
src/main/java/jp/sf/fess/helper/FileTypeHelper.java
View file

@ -22,7 +22,6 @@ import java.util.Map;
import org.codelibs.core.util.StringUtil;
public class FileTypeHelper {
protected String fieldName = "filetype_s";
protected String defaultValue = "others";
@ -40,14 +39,6 @@ public class FileTypeHelper {
return filetype;
}
public String getFieldName() {
return fieldName;
}
public void setFieldName(final String fieldName) {
this.fieldName = fieldName;
}
public String getDefaultValue() {
return defaultValue;
}

37
src/main/java/jp/sf/fess/helper/IndexingHelper.java
View file

@ -62,39 +62,39 @@ public class IndexingHelper {
private void deleteOldDocuments(final SolrGroup solrGroup,
final List<SolrInputDocument> docList) {
final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final List<String> ids = new ArrayList<String>();
final StringBuilder q = new StringBuilder(1000);
final StringBuilder fq = new StringBuilder(100);
for (final SolrInputDocument inputDoc : docList) {
final Object idValue = inputDoc.getFieldValue(systemHelper.idField);
final Object idValue = inputDoc.getFieldValue(fieldHelper.idField);
if (idValue == null) {
continue;
}
final Object configIdValue = inputDoc
.getFieldValue(systemHelper.configIdField);
.getFieldValue(fieldHelper.configIdField);
if (configIdValue == null) {
continue;
}
q.setLength(0);
q.append(systemHelper.urlField).append(":\"");
q.append(fieldHelper.urlField).append(":\"");
q.append(ClientUtils.escapeQueryChars((String) inputDoc
.getFieldValue(systemHelper.urlField)));
.getFieldValue(fieldHelper.urlField)));
q.append('"');
fq.setLength(0);
fq.append(systemHelper.configIdField).append(':');
fq.append(fieldHelper.configIdField).append(':');
fq.append(configIdValue.toString());
final SolrDocumentList docs = getSolrDocumentList(solrGroup,
fq.toString(), q.toString(),
new String[] { systemHelper.idField });
new String[] { fieldHelper.idField });
for (final SolrDocument doc : docs) {
final Object oldIdValue = doc
.getFieldValue(systemHelper.idField);
.getFieldValue(fieldHelper.idField);
if (!idValue.equals(oldIdValue) && oldIdValue != null) {
ids.add(oldIdValue.toString());
}
@ -136,7 +136,8 @@ public class IndexingHelper {
}
public void deleteDocument(final SolrGroup solrGroup, final String id) {
final String query = "{!raw f=id}" + id;
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final String query = "{!raw f=" + fieldHelper.idField + "}" + id;
for (int i = 0; i < maxRetryCount; i++) {
boolean done = true;
try {
@ -168,9 +169,11 @@ public class IndexingHelper {
public SolrDocument getSolrDocument(final SolrGroup solrGroup,
final String id, final String[] fields) {
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final SolrQuery solrQuery = new SolrQuery();
final StringBuilder queryBuf = new StringBuilder(200);
queryBuf.append("{!raw f=id}");
queryBuf.append("{!raw f=").append(fieldHelper.idField).append("}");
queryBuf.append(id);
solrQuery.setQuery(queryBuf.toString());
if (fields != null) {
@ -183,9 +186,8 @@ public class IndexingHelper {
}
if (docList.size() > 1) {
logger.error("Invalid multiple docs for " + id);
final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
for (final SolrDocument doc : docList) {
final Object idValue = doc.getFieldValue(systemHelper.idField);
final Object idValue = doc.getFieldValue(fieldHelper.idField);
if (idValue != null) {
deleteDocument(solrGroup, idValue.toString());
}
@ -197,9 +199,10 @@ public class IndexingHelper {
public SolrDocumentList getSolrDocumentListByPrefixId(
final SolrGroup solrGroup, final String id, final String[] fields) {
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final SolrQuery solrQuery = new SolrQuery();
final StringBuilder queryBuf = new StringBuilder(200);
queryBuf.append("{!prefix f=id}");
queryBuf.append("{!prefix f=").append(fieldHelper.idField).append("}");
queryBuf.append(id);
solrQuery.setQuery(queryBuf.toString());
if (fields != null) {
@ -218,7 +221,9 @@ public class IndexingHelper {
public void deleteChildSolrDocument(final SolrGroup solrGroup,
final String id) {
final String query = "{!raw f=parentId v=\"" + id + "\"}";
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final String query = "{!raw f=" + fieldHelper.parentIdField + " v=\""
+ id + "\"}";
for (final UpdateResponse response : solrGroup.deleteByQuery(query)) {
if (response.getStatus() != 200) {
if (logger.isDebugEnabled()) {
@ -236,8 +241,10 @@ public class IndexingHelper {
protected SolrDocumentList getChildSolrDocumentList(
final SolrGroup solrGroup, final String id, final String[] fields,
final int row) {
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final SolrQuery solrQuery = new SolrQuery();
solrQuery.setQuery("{!raw f=parentId v=\"" + id + "\"}");
solrQuery.setQuery("{!raw f=" + fieldHelper.parentIdField + " v=\""
+ id + "\"}");
if (fields != null) {
solrQuery.setFields(fields);
}

11
src/main/java/jp/sf/fess/helper/KeyMatchHelper.java
View file

@ -22,7 +22,6 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import jp.sf.fess.Constants;
import jp.sf.fess.db.exentity.KeyMatch;
import jp.sf.fess.service.KeyMatchService;
import jp.sf.fess.service.SearchService;
@ -55,6 +54,7 @@ public class KeyMatchHelper {
}
protected void reload(final long interval) {
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final KeyMatchService keyMatchService = SingletonS2Container
.getComponent(KeyMatchService.class);
final List<KeyMatch> list = keyMatchService.getAvailableKeyMatchList();
@ -64,9 +64,9 @@ public class KeyMatchHelper {
final List<Map<String, Object>> documentList = getDocumentList(keyMatch);
final List<String> docIdList = new ArrayList<String>();
for (final Map<String, Object> map : documentList) {
final String docId = (String) map.get(Constants.DOC_ID);
final String docId = (String) map.get(fieldHelper.docIdField);
if (StringUtil.isNotBlank(docId)) {
docIdList.add(Constants.DOC_ID + ":" + docId + "^"
docIdList.add(fieldHelper.docIdField + ":" + docId + "^"
+ keyMatch.getBoost());
}
}
@ -88,10 +88,11 @@ public class KeyMatchHelper {
protected List<Map<String, Object>> getDocumentList(final KeyMatch keyMatch) {
final SearchService searchService = ComponentUtil.getSearchService();
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final List<Map<String, Object>> documentList = searchService
.getDocumentList(keyMatch.getQuery(), 0, keyMatch.getMaxSize(),
null, null, null, new String[] { Constants.DOC_ID },
null, false);
null, null, null,
new String[] { fieldHelper.docIdField }, null, false);
return documentList;
}

129
src/main/java/jp/sf/fess/helper/QueryHelper.java
View file

@ -59,18 +59,15 @@ import org.apache.commons.lang.StringUtils;
import org.codelibs.core.util.StringUtil;
import org.seasar.framework.container.annotation.tiger.Binding;
import org.seasar.framework.container.annotation.tiger.BindingType;
import org.seasar.framework.container.annotation.tiger.InitMethod;
import org.seasar.struts.util.RequestUtil;
public class QueryHelper implements Serializable {
private static final String LABEL_FIELD = "label";
private static final String SCORE_FIELD = "score";
private static final String INURL_FIELD = "inurl";
private static final String TITLE_FIELD = "title";
private static final String CONTENT_FIELD = "content";
private static final String NOT_ = "NOT ";
private static final String AND = "AND";
@ -96,6 +93,9 @@ public class QueryHelper implements Serializable {
@Resource
protected SystemHelper systemHelper;
@Resource
protected FieldHelper fieldHelper;
@Binding(bindingType = BindingType.MAY)
@Resource
protected KeyMatchHelper keyMatchHelper;
@ -104,43 +104,25 @@ public class QueryHelper implements Serializable {
protected Set<String> highlightFieldSet = new HashSet<>();
protected String[] responseFields = new String[] { "id", "docId", "score",
"boost", "contentLength", "host", "site", "lastModified",
"mimetype", "filetype_s", "created", TITLE_FIELD, "digest", "url",
"clickCount_l_x_dv", "favoriteCount_l_x_dv", "cid_s", "lang_s",
"hasCache_s_s" };
protected String[] responseFields;
protected String[] cacheResponseFields = new String[] { "id", "docId",
"score", "boost", "contentLength", "host", "site", "lastModified",
"mimetype", "filetype_s", "created", TITLE_FIELD, "digest", "url",
"clickCount_l_x_dv", "favoriteCount_l_x_dv", "cid_s", "lang_s",
"cache" };
protected String[] cacheResponseFields;
protected String[] responseDocValuesFields = new String[] {
"clickCount_l_x_dv", "favoriteCount_l_x_dv" };
protected String[] responseDocValuesFields;
protected String[] highlightingFields = new String[] { CONTENT_FIELD };
protected String[] highlightingFields;
protected String[] searchFields = new String[] { "url", "docId", "host",
TITLE_FIELD, CONTENT_FIELD, "contentLength", "lastModified",
"mimetype", "filetype_s", LABEL_FIELD, "segment",
"clickCount_l_x_dv", "favoriteCount_l_x_dv", INURL_FIELD, "lang_s" };
protected String[] searchFields;
protected String[] facetFields = new String[] { "url", "host", TITLE_FIELD,
CONTENT_FIELD, "contentLength", "lastModified", "mimetype",
"filetype_s", LABEL_FIELD, "segment" };
protected String[] facetFields;
protected String sortPrefix = "sort:";
protected String[] supportedSortFields = new String[] { "created",
"contentLength", "lastModified", "clickCount_l_x_dv",
"favoriteCount_l_x_dv" };
protected String[] supportedSortFields;
protected String[] supportedMltFields = new String[] { CONTENT_FIELD,
"content_ja" };
protected String[] supportedMltFields;
protected String[] supportedAnalysisFields = new String[] { CONTENT_FIELD,
"content_ja" };
protected String[] supportedAnalysisFields;
protected int highlightSnippetSize = 5;
@ -180,6 +162,73 @@ public class QueryHelper implements Serializable {
protected Map<String, String[]> additionalQueryParamMap = new HashMap<String, String[]>();
@InitMethod
public void init() {
if (responseFields == null) {
responseFields = new String[] { SCORE_FIELD, fieldHelper.idField,
fieldHelper.docIdField, fieldHelper.boostField,
fieldHelper.contentLengthField, fieldHelper.hostField,
fieldHelper.siteField, fieldHelper.lastModifiedField,
fieldHelper.mimetypeField, fieldHelper.filetypeField,
fieldHelper.createdField, fieldHelper.titleField,
fieldHelper.digestField, fieldHelper.urlField,
fieldHelper.clickCountField,
fieldHelper.favoriteCountField, fieldHelper.configIdField,
fieldHelper.langField, fieldHelper.hasCacheField };
}
if (cacheResponseFields == null) {
cacheResponseFields = new String[] { SCORE_FIELD,
fieldHelper.idField, fieldHelper.docIdField,
fieldHelper.boostField, fieldHelper.contentLengthField,
fieldHelper.hostField, fieldHelper.siteField,
fieldHelper.lastModifiedField, fieldHelper.mimetypeField,
fieldHelper.filetypeField, fieldHelper.createdField,
fieldHelper.titleField, fieldHelper.digestField,
fieldHelper.urlField, fieldHelper.clickCountField,
fieldHelper.favoriteCountField, fieldHelper.configIdField,
fieldHelper.langField, fieldHelper.cacheField };
}
if (responseDocValuesFields == null) {
responseDocValuesFields = new String[] {
fieldHelper.clickCountField, fieldHelper.favoriteCountField };
}
if (highlightingFields == null) {
highlightingFields = new String[] { fieldHelper.contentField };
}
if (searchFields == null) {
searchFields = new String[] { INURL_FIELD, fieldHelper.urlField,
fieldHelper.docIdField, fieldHelper.hostField,
fieldHelper.titleField, fieldHelper.contentField,
fieldHelper.contentLengthField,
fieldHelper.lastModifiedField, fieldHelper.mimetypeField,
fieldHelper.filetypeField, fieldHelper.labelField,
fieldHelper.segmentField, fieldHelper.clickCountField,
fieldHelper.favoriteCountField, fieldHelper.langField };
}
if (facetFields == null) {
facetFields = new String[] { fieldHelper.urlField,
fieldHelper.hostField, fieldHelper.titleField,
fieldHelper.contentField, fieldHelper.contentLengthField,
fieldHelper.lastModifiedField, fieldHelper.mimetypeField,
fieldHelper.filetypeField, fieldHelper.labelField,
fieldHelper.segmentField };
}
if (supportedSortFields == null) {
supportedSortFields = new String[] { fieldHelper.createdField,
fieldHelper.contentLengthField,
fieldHelper.lastModifiedField, fieldHelper.clickCountField,
fieldHelper.favoriteCountField };
}
if (supportedMltFields == null) {
supportedMltFields = new String[] { fieldHelper.contentField,
"content_ja" };
}
if (supportedAnalysisFields == null) {
supportedAnalysisFields = new String[] { fieldHelper.contentField,
"content_ja" };
}
}
/*
* (non-Javadoc)
*
@ -261,7 +310,8 @@ public class QueryHelper implements Serializable {
queryBuf.append(_OR_);
}
queryBuf.append("role:");
queryBuf.append(fieldHelper.roleField);
queryBuf.append(':');
queryBuf.append(QueryUtil.escapeValue(role));
}
return queryBuf.toString();
@ -368,7 +418,7 @@ public class QueryHelper implements Serializable {
boolean isInUrl = false;
final String targetWord = value.substring(prefix.length());
if (INURL_FIELD.equals(field)) {
prefix = "url:";
prefix = fieldHelper.urlField + ":";
isInUrl = true;
}
String fieldLogWord;
@ -449,8 +499,8 @@ public class QueryHelper implements Serializable {
highLightQueryList.add(value);
if (fieldLogMap != null) {
addFieldLogValue(fieldLogMap, CONTENT_FIELD, NOT_
+ value);
addFieldLogValue(fieldLogMap, fieldHelper.contentField,
NOT_ + value);
}
} else {
// content
@ -464,7 +514,8 @@ public class QueryHelper implements Serializable {
highLightQueryList.add(value);
if (fieldLogMap != null) {
addFieldLogValue(fieldLogMap, CONTENT_FIELD, value);
addFieldLogValue(fieldLogMap, fieldHelper.contentField,
value);
}
if (keyMatchHelper != null) {
@ -929,10 +980,10 @@ public class QueryHelper implements Serializable {
protected void buildContentQueryWithLang(final StringBuilder buf,
final String value, final String queryLanguage) {
buf.append('(');
buf.append(TITLE_FIELD).append(':');
buf.append(fieldHelper.titleField).append(':');
appendQueryValue(buf, value, useBigram);
buf.append(_OR_);
buf.append(CONTENT_FIELD).append(':');
buf.append(fieldHelper.contentField).append(':');
appendQueryValue(buf, value, useBigram);
if (StringUtil.isNotBlank(queryLanguage)) {
buf.append(_OR_);

26
src/main/java/jp/sf/fess/helper/SystemHelper.java
View file

@ -118,32 +118,6 @@ public class SystemHelper implements Serializable {
private final AtomicBoolean forceStop = new AtomicBoolean(false);
public String favoriteCountField = "favoriteCount_l_x_dv";
public String clickCountField = "clickCount_l_x_dv";
public String configIdField = "cid_s";
public String expiresField = "expires_dt";
public String urlField = "url";
public String docIdField = "docId";
public String idField = "id";
public String langField = "lang_s";
public String hasCacheField = "hasCache_s_s";
public String lastModifiedField = "lastModified";
public String anchorField = "anchor";
public String segmentField = "segment";
public String roleField = "role";
protected String[] supportedLanguages = new String[] { "ar", "bg", "ca",
"da", "de", "el", "en", "es", "eu", "fa", "fi", "fr", "ga", "gl",
"hi", "hu", "hy", "id", "it", "ja", "lv", "ko", "nl", "no", "pt",

21
src/main/java/jp/sf/fess/helper/ViewHelper.java
View file

@ -38,6 +38,7 @@ import jp.sf.fess.Constants;
import jp.sf.fess.FessSystemException;
import jp.sf.fess.entity.FacetQueryView;
import jp.sf.fess.helper.UserAgentHelper.UserAgentType;
import jp.sf.fess.util.ComponentUtil;
import jp.sf.fess.util.ResourceUtil;
import org.apache.commons.lang.StringUtils;
@ -134,12 +135,12 @@ public class ViewHelper implements Serializable {
public String getContentTitle(final Map<String, Object> document) {
final int size = titleLength;
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
String title;
if (StringUtil.isNotBlank(getString(document, "title"))) {
title = getString(document, "title");
if (StringUtil.isNotBlank(getString(document, fieldHelper.titleField))) {
title = getString(document, fieldHelper.titleField);
} else {
title = getString(document, "url");
title = getString(document, fieldHelper.urlField);
}
return StringUtils.abbreviate(title, size);
}
@ -317,7 +318,9 @@ public class ViewHelper implements Serializable {
final String url) {
if (Constants.TRUE.equals(crawlerProperties
.get(Constants.APPEND_QUERY_PARAMETER_PROPERTY))) {
final String mimetype = getString(document, "mimetype");
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final String mimetype = getString(document,
fieldHelper.mimetypeField);
if (StringUtil.isNotBlank(mimetype)) {
if ("application/pdf".equals(mimetype)) {
return appendPDFSearchWord(url);
@ -409,7 +412,7 @@ public class ViewHelper implements Serializable {
public String createCacheContent(final Map<String, Object> doc,
final String[] queries) {
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final FileTemplateLoader loader = new FileTemplateLoader(new File(
ResourceUtil.getViewTemplatePath(StringUtil.EMPTY)));
final Handlebars handlebars = new Handlebars(loader);
@ -423,7 +426,7 @@ public class ViewHelper implements Serializable {
url = MessageResourcesUtil.getMessage(locale,
"labels.search_unknown");
}
Object created = doc.get("created");
Object created = doc.get(fieldHelper.createdField);
if (created instanceof Date) {
final SimpleDateFormat sdf = new SimpleDateFormat(
CoreLibConstants.DATE_FORMAT_ISO_8601_EXTEND);
@ -437,7 +440,7 @@ public class ViewHelper implements Serializable {
doc.put("queries", queries);
String cache = (String) doc.get("cache");
String cache = (String) doc.get(fieldHelper.cacheField);
if (cache != null) {
cache = pathMappingHelper.replaceUrls(cache);
if (queries != null && queries.length > 0) {
@ -446,7 +449,7 @@ public class ViewHelper implements Serializable {
doc.put("hlCache", cache);
}
} else {
doc.put("cache", StringUtil.EMPTY);
doc.put(fieldHelper.cacheField, StringUtil.EMPTY);
doc.put("hlCache", StringUtil.EMPTY);
}

6
src/main/java/jp/sf/fess/helper/impl/CookieUserInfoHelperImpl.java
View file

@ -27,8 +27,10 @@ import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpSession;
import jp.sf.fess.Constants;
import jp.sf.fess.helper.FieldHelper;
import jp.sf.fess.helper.SearchLogHelper;
import jp.sf.fess.helper.UserInfoHelper;
import jp.sf.fess.util.ComponentUtil;
import org.codelibs.core.util.StringUtil;
import org.codelibs.robot.util.LruHashMap;
@ -120,11 +122,13 @@ public class CookieUserInfoHelperImpl implements UserInfoHelper {
final List<Map<String, Object>> documentItems) {
final HttpSession session = RequestUtil.getRequest().getSession(false);
if (session != null) {
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final String queryId = getId();
final List<String> docIdList = new ArrayList<String>();
for (final Map<String, Object> map : documentItems) {
final Object docId = map.get(Constants.DOC_ID);
final Object docId = map.get(fieldHelper.docIdField);
if (docId != null && docId.toString().length() > 0) {
docIdList.add(docId.toString());
}

6
src/main/java/jp/sf/fess/helper/impl/SearchLogHelperImpl.java
View file

@ -34,8 +34,8 @@ import jp.sf.fess.db.exentity.SearchFieldLog;
import jp.sf.fess.db.exentity.SearchLog;
import jp.sf.fess.db.exentity.UserInfo;
import jp.sf.fess.helper.DocumentHelper;
import jp.sf.fess.helper.FieldHelper;
import jp.sf.fess.helper.SearchLogHelper;
import jp.sf.fess.helper.SystemHelper;
import jp.sf.fess.service.SearchLogService;
import jp.sf.fess.service.UserInfoService;
import jp.sf.fess.util.ComponentUtil;
@ -212,11 +212,11 @@ public class SearchLogHelperImpl extends SearchLogHelper {
}
final DocumentHelper documentHelper = ComponentUtil.getDocumentHelper();
final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
for (final Map.Entry<String, Long> entry : clickCountMap.entrySet()) {
try {
documentHelper.update(entry.getKey(),
systemHelper.clickCountField, entry.getValue() + 1);
fieldHelper.clickCountField, entry.getValue() + 1);
} catch (final Exception e) {
logger.warn("Failed to update a clickCount(" + entry.getValue()
+ ") for " + entry.getKey(), e);

46
src/main/java/jp/sf/fess/robot/FessS2RobotThread.java
View file

@ -31,10 +31,10 @@ import jp.sf.fess.Constants;
import jp.sf.fess.db.exentity.CrawlingConfig;
import jp.sf.fess.helper.CrawlingConfigHelper;
import jp.sf.fess.helper.CrawlingSessionHelper;
import jp.sf.fess.helper.FieldHelper;
import jp.sf.fess.helper.IndexingHelper;
import jp.sf.fess.helper.SambaHelper;
import jp.sf.fess.helper.SearchLogHelper;
import jp.sf.fess.helper.SystemHelper;
import jp.sf.fess.util.ComponentUtil;
import org.apache.commons.io.IOUtils;
@ -74,7 +74,7 @@ public class FessS2RobotThread extends S2RobotThread {
.getCrawlingConfigHelper();
final CrawlingSessionHelper crawlingSessionHelper = ComponentUtil
.getCrawlingSessionHelper();
final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final SambaHelper sambaHelper = ComponentUtil.getSambaHelper();
final IndexingHelper indexingHelper = ComponentUtil
.getIndexingHelper();
@ -93,7 +93,7 @@ public class FessS2RobotThread extends S2RobotThread {
final CrawlingConfig crawlingConfig = crawlingConfigHelper
.get(robotContext.getSessionId());
final Map<String, Object> dataMap = new HashMap<String, Object>();
dataMap.put(systemHelper.urlField, url);
dataMap.put(fieldHelper.urlField, url);
final List<String> roleTypeList = new ArrayList<String>();
for (final String roleType : crawlingConfig.getRoleTypeValues()) {
roleTypeList.add(roleType);
@ -119,30 +119,30 @@ public class FessS2RobotThread extends S2RobotThread {
}
}
}
dataMap.put(systemHelper.roleField, roleTypeList);
dataMap.put(fieldHelper.roleField, roleTypeList);
final String id = crawlingSessionHelper.generateId(dataMap);
final SolrDocument solrDocument = indexingHelper
.getSolrDocument(solrGroup, id, new String[] {
systemHelper.idField,
systemHelper.lastModifiedField,
systemHelper.anchorField,
systemHelper.segmentField,
systemHelper.expiresField,
systemHelper.clickCountField,
systemHelper.favoriteCountField });
fieldHelper.idField,
fieldHelper.lastModifiedField,
fieldHelper.anchorField,
fieldHelper.segmentField,
fieldHelper.expiresField,
fieldHelper.clickCountField,
fieldHelper.favoriteCountField });
if (solrDocument == null) {
storeChildUrlsToQueue(urlQueue,
getChildUrlSet(solrGroup, id)); // TODO
getChildUrlSet(solrGroup, id));
return true;
}
final Date expires = (Date) solrDocument
.get(systemHelper.expiresField);
.get(fieldHelper.expiresField);
if (expires != null
&& expires.getTime() < System.currentTimeMillis()) {
final Object idValue = solrDocument
.getFieldValue(systemHelper.idField);
.getFieldValue(fieldHelper.idField);
if (idValue != null) {
indexingHelper.deleteDocument(solrGroup,
idValue.toString());
@ -151,13 +151,13 @@ public class FessS2RobotThread extends S2RobotThread {
}
final Date lastModified = (Date) solrDocument
.get(systemHelper.lastModifiedField);
.get(fieldHelper.lastModifiedField);
if (lastModified == null) {
return true;
}
final Integer clickCount = (Integer) solrDocument
.get(systemHelper.clickCountField);
.get(fieldHelper.clickCountField);
if (clickCount != null) {
final SearchLogHelper searchLogHelper = ComponentUtil
.getSearchLogHelper();
@ -168,7 +168,7 @@ public class FessS2RobotThread extends S2RobotThread {
}
final Integer favoriteCount = (Integer) solrDocument
.get(systemHelper.favoriteCountField);
.get(fieldHelper.favoriteCountField);
if (favoriteCount != null) {
final SearchLogHelper searchLogHelper = ComponentUtil
.getSearchLogHelper();
@ -190,7 +190,8 @@ public class FessS2RobotThread extends S2RobotThread {
final int httpStatusCode = responseData.getHttpStatusCode();
if (httpStatusCode == 404) {
storeChildUrlsToQueue(urlQueue,
getAnchorSet(solrDocument.get("anchor")));
getAnchorSet(solrDocument
.get(fieldHelper.anchorField)));
indexingHelper.deleteDocument(solrGroup, id);
return false;
} else if (responseData.getLastModified() == null) {
@ -209,7 +210,8 @@ public class FessS2RobotThread extends S2RobotThread {
processResponse(urlQueue, responseData);
storeChildUrlsToQueue(urlQueue,
getAnchorSet(solrDocument.get("anchor")));
getAnchorSet(solrDocument
.get(fieldHelper.anchorField)));
return false;
}
@ -260,11 +262,11 @@ public class FessS2RobotThread extends S2RobotThread {
protected Set<RequestData> getChildUrlSet(final SolrGroup solrGroup,
final String id) {
final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final IndexingHelper indexingHelper = ComponentUtil.getIndexingHelper();
final SolrDocumentList docList = indexingHelper
.getChildSolrDocumentList(solrGroup, id,
new String[] { systemHelper.urlField });
new String[] { fieldHelper.urlField });
if (docList.isEmpty()) {
return null;
}
@ -273,7 +275,7 @@ public class FessS2RobotThread extends S2RobotThread {
}
final Set<RequestData> urlSet = new HashSet<>(docList.size());
for (final SolrDocument doc : docList) {
final Object obj = doc.get(systemHelper.urlField);
final Object obj = doc.get(fieldHelper.urlField);
if (obj != null) {
urlSet.add(RequestDataBuilder.newRequestData().get()
.url(obj.toString()).build());

13
src/main/java/jp/sf/fess/screenshot/ScreenShotManager.java
View file

@ -30,7 +30,7 @@ import javax.servlet.http.HttpSession;
import jp.sf.fess.Constants;
import jp.sf.fess.FessSystemException;
import jp.sf.fess.helper.SystemHelper;
import jp.sf.fess.helper.FieldHelper;
import jp.sf.fess.util.ComponentUtil;
import org.codelibs.core.util.StringUtil;
@ -115,9 +115,10 @@ public class ScreenShotManager {
}
public void generate(final Map<String, Object> docMap) {
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
for (final ScreenShotGenerator generator : generatorList) {
if (generator.isTarget(docMap)) {
final String url = (String) docMap.get("url");
final String url = (String) docMap.get(fieldHelper.urlField);
final String path = getImageFilename(docMap);
if (!screenShotTaskQueue.offer(new ScreenShotTask(url,
new File(baseDir, path), generator))) {
@ -131,8 +132,8 @@ public class ScreenShotManager {
protected String getImageFilename(final Map<String, Object> docMap) {
final StringBuilder buf = new StringBuilder(50);
final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
final String docid = (String) docMap.get(systemHelper.docIdField);
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final String docid = (String) docMap.get(fieldHelper.docIdField);
for (int i = 0; i < docid.length(); i++) {
if (i > 0 && i % splitSize == 0) {
buf.append('/');
@ -145,11 +146,11 @@ public class ScreenShotManager {
public void storeRequest(final String queryId,
final List<Map<String, Object>> documentItems) {
final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final Map<String, String> dataMap = new HashMap<String, String>(
documentItems.size());
for (final Map<String, Object> docMap : documentItems) {
final String docid = (String) docMap.get(systemHelper.docIdField);
final String docid = (String) docMap.get(fieldHelper.docIdField);
final String screenShotPath = getImageFilename(docMap);
if (StringUtil.isNotBlank(docid)
&& StringUtil.isNotBlank(screenShotPath)) {

2
src/main/java/jp/sf/fess/service/ClickLogService.java
View file

@ -103,7 +103,7 @@ public class ClickLogService implements Serializable {
try {
final List<String> list = new ArrayList<String>();
list.add("SearchId");
list.add("Url");
list.add("URL");
list.add("RequestedTime");
list.add("QueryRequestedTime");
list.add("UserSessionId");

5
src/main/java/jp/sf/fess/service/SearchService.java
View file

@ -34,6 +34,7 @@ import jp.sf.fess.entity.MoreLikeThisInfo;
import jp.sf.fess.entity.PingResponse;
import jp.sf.fess.entity.SearchQuery;
import jp.sf.fess.entity.SearchQuery.SortField;
import jp.sf.fess.helper.FieldHelper;
import jp.sf.fess.helper.QueryHelper;
import jp.sf.fess.helper.RoleQueryHelper;
import jp.sf.fess.solr.FessSolrQueryException;
@ -89,13 +90,13 @@ public class SearchService implements Serializable {
if (docIds == null || docIds.length == 0) {
return Collections.emptyList();
}
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final StringBuilder buf = new StringBuilder(1000);
for (int i = 0; i < docIds.length; i++) {
if (i != 0) {
buf.append(" OR ");
}
buf.append("docId:").append(docIds[i]);
buf.append(fieldHelper.docIdField + ":").append(docIds[i]);
}
return getDocumentList(buf.toString(), 0, pageSize, null, null, null,
responseFields, docValuesFields);

25
src/main/java/jp/sf/fess/solr/IndexUpdater.java
View file

@ -31,6 +31,7 @@ import jp.sf.fess.db.exbhv.ClickLogBhv;
import jp.sf.fess.db.exbhv.FavoriteLogBhv;
import jp.sf.fess.db.exbhv.pmbean.FavoriteUrlCountPmb;
import jp.sf.fess.db.exentity.customize.FavoriteUrlCount;
import jp.sf.fess.helper.FieldHelper;
import jp.sf.fess.helper.IndexingHelper;
import jp.sf.fess.helper.IntervalControlHelper;
import jp.sf.fess.helper.SystemHelper;
@ -90,6 +91,9 @@ public class IndexUpdater extends Thread {
@Resource
protected SystemHelper systemHelper;
@Resource
protected FieldHelper fieldHelper;
@Resource
protected IndexingHelper indexingHelper;
@ -430,7 +434,7 @@ public class IndexUpdater extends Thread {
float documentBoost = 0.0f;
// add data
for (final Map.Entry<String, Object> entry : map.entrySet()) {
if ("boost".equals(entry.getKey())) {
if (fieldHelper.boostField.equals(entry.getKey())) {
// boost
documentBoost = Float.valueOf(entry.getValue().toString());
} else {
@ -466,8 +470,9 @@ public class IndexUpdater extends Thread {
addBoostValue(map, documentBoost, doc);
}
if (!map.containsKey(Constants.DOC_ID)) {
doc.addField(Constants.DOC_ID, systemHelper.generateDocId(map));
if (!map.containsKey(fieldHelper.docIdField)) {
doc.addField(fieldHelper.docIdField,
systemHelper.generateDocId(map));
}
return doc;
@ -475,7 +480,7 @@ public class IndexUpdater extends Thread {
protected void addBoostValue(final Map<String, Object> map,
final float documentBoost, final SolrInputDocument doc) {
doc.addField("boost", documentBoost);
doc.addField(fieldHelper.boostField, documentBoost);
doc.setDocumentBoost(documentBoost);
if (logger.isDebugEnabled()) {
logger.debug("Set a document boost (" + documentBoost + ").");
@ -484,13 +489,13 @@ public class IndexUpdater extends Thread {
protected void addClickCountField(final Map<String, Object> map,
final SolrInputDocument doc) {
final String url = (String) map.get("url");
final String url = (String) map.get(fieldHelper.urlField);
if (StringUtil.isNotBlank(url)) {
final ClickLogCB cb = new ClickLogCB();
cb.query().setUrl_Equal(url);
final int count = clickLogBhv.selectCount(cb);
doc.addField(systemHelper.clickCountField, count);
map.put(systemHelper.clickCountField, count);
doc.addField(fieldHelper.clickCountField, count);
map.put(fieldHelper.clickCountField, count);
if (logger.isDebugEnabled()) {
logger.debug("Click Count: " + count + ", url: " + url);
}
@ -499,7 +504,7 @@ public class IndexUpdater extends Thread {
protected void addFavoriteCountField(final Map<String, Object> map,
final SolrInputDocument doc) {
final String url = (String) map.get("url");
final String url = (String) map.get(fieldHelper.urlField);
if (StringUtil.isNotBlank(url)) {
final FavoriteUrlCountPmb pmb = new FavoriteUrlCountPmb();
pmb.setUrl(url);
@ -512,8 +517,8 @@ public class IndexUpdater extends Thread {
count = list.get(0).getCnt().longValue();
}
doc.addField(systemHelper.favoriteCountField, count);
map.put(systemHelper.favoriteCountField, count);
doc.addField(fieldHelper.favoriteCountField, count);
map.put(fieldHelper.favoriteCountField, count);
if (logger.isDebugEnabled()) {
logger.debug("Favorite Count: " + count + ", url: " + url);
}

76
src/main/java/jp/sf/fess/transformer/AbstractFessFileTransformer.java
View file

@ -35,11 +35,11 @@ import jp.sf.fess.db.exentity.CrawlingConfig;
import jp.sf.fess.db.exentity.CrawlingConfig.ConfigName;
import jp.sf.fess.helper.CrawlingConfigHelper;
import jp.sf.fess.helper.CrawlingSessionHelper;
import jp.sf.fess.helper.FieldHelper;
import jp.sf.fess.helper.FileTypeHelper;
import jp.sf.fess.helper.LabelTypeHelper;
import jp.sf.fess.helper.PathMappingHelper;
import jp.sf.fess.helper.SambaHelper;
import jp.sf.fess.helper.SystemHelper;
import jp.sf.fess.taglib.FessFunctions;
import jp.sf.fess.util.ComponentUtil;
@ -182,7 +182,7 @@ public abstract class AbstractFessFileTransformer extends
.getCrawlingConfigHelper();
final CrawlingConfig crawlingConfig = crawlingConfigHelper
.get(responseData.getSessionId());
final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final FileTypeHelper fileTypeHelper = ComponentUtil.getFileTypeHelper();
String url = responseData.getUrl();
final String indexingTarget = crawlingConfig.getIndexingTarget(url);
@ -202,15 +202,15 @@ public abstract class AbstractFessFileTransformer extends
// cid
final String configId = crawlingConfig.getConfigId();
if (configId != null) {
putResultDataBody(dataMap, systemHelper.configIdField, configId);
putResultDataBody(dataMap, fieldHelper.configIdField, configId);
}
// expires
if (documentExpires != null) {
putResultDataBody(dataMap, systemHelper.expiresField,
putResultDataBody(dataMap, fieldHelper.expiresField,
FessFunctions.formatDate(documentExpires));
}
// segment
putResultDataBody(dataMap, "segment", sessionId);
putResultDataBody(dataMap, fieldHelper.segmentField, sessionId);
// content
final StringBuilder buf = new StringBuilder(content.length() + 1000);
if (appendBodyContentToContent) {
@ -224,70 +224,78 @@ public abstract class AbstractFessFileTransformer extends
}
final String body = normalizeContent(buf.toString());
if (StringUtil.isNotBlank(body)) {
putResultDataBody(dataMap, "content", body);
putResultDataBody(dataMap, fieldHelper.contentField, body);
} else {
putResultDataBody(dataMap, "content", StringUtil.EMPTY);
putResultDataBody(dataMap, fieldHelper.contentField,
StringUtil.EMPTY);
}
if (Constants.TRUE.equalsIgnoreCase(fieldConfigMap.get("cache"))
|| enableCache) {
if (Constants.TRUE.equalsIgnoreCase(fieldConfigMap
.get(fieldHelper.cacheField)) || enableCache) {
final String cache = content.trim().replaceAll("[ \\t\\x0B\\f]+",
" ");
// text cache
putResultDataBody(dataMap, "cache", cache);
putResultDataBody(dataMap, systemHelper.hasCacheField,
putResultDataBody(dataMap, fieldHelper.cacheField, cache);
putResultDataBody(dataMap, fieldHelper.hasCacheField,
Constants.TRUE);
}
// digest
putResultDataBody(dataMap, "digest", Constants.DIGEST_PREFIX
+ abbreviate(normalizeContent(content), maxDigestLength));
putResultDataBody(
dataMap,
fieldHelper.digestField,
Constants.DIGEST_PREFIX
+ abbreviate(normalizeContent(content), maxDigestLength));
// title
if (!dataMap.containsKey("title")) {
if (!dataMap.containsKey(fieldHelper.titleField)) {
if (url.endsWith("/")) {
if (StringUtil.isNotBlank(content)) {
putResultDataBody(dataMap, "title",
putResultDataBody(dataMap, fieldHelper.titleField,
abbreviate(body, maxTitleLength));
} else {
putResultDataBody(dataMap, "title", noTitleLabel);
putResultDataBody(dataMap, fieldHelper.titleField,
noTitleLabel);
}
} else {
final String u = decodeUrlAsName(url, url.startsWith("file:"));
final int pos = u.lastIndexOf('/');
if (pos == -1) {
putResultDataBody(dataMap, "title", u);
putResultDataBody(dataMap, fieldHelper.titleField, u);
} else {
putResultDataBody(dataMap, "title", u.substring(pos + 1));
putResultDataBody(dataMap, fieldHelper.titleField,
u.substring(pos + 1));
}
}
}
// host
putResultDataBody(dataMap, "host", getHost(url));
putResultDataBody(dataMap, fieldHelper.hostField, getHost(url));
// site
putResultDataBody(dataMap, "site", getSite(url, urlEncoding));
putResultDataBody(dataMap, fieldHelper.siteField,
getSite(url, urlEncoding));
// url
putResultDataBody(dataMap, "url", url);
putResultDataBody(dataMap, fieldHelper.urlField, url);
// created
putResultDataBody(dataMap, "created", "NOW");
putResultDataBody(dataMap, fieldHelper.createdField, Constants.NOW);
// TODO anchor
putResultDataBody(dataMap, "anchor", StringUtil.EMPTY);
putResultDataBody(dataMap, fieldHelper.anchorField, StringUtil.EMPTY);
// mimetype
putResultDataBody(dataMap, "mimetype", mimeType);
putResultDataBody(dataMap, fieldHelper.mimetypeField, mimeType);
if (fileTypeHelper != null) {
// filetype
putResultDataBody(dataMap, fileTypeHelper.getFieldName(),
putResultDataBody(dataMap, fieldHelper.filetypeField,
fileTypeHelper.get(mimeType));
}
// contentLength
putResultDataBody(dataMap, "contentLength",
putResultDataBody(dataMap, fieldHelper.contentLengthField,
Long.toString(responseData.getContentLength()));
// lastModified
if (responseData.getLastModified() != null) {
putResultDataBody(dataMap, "lastModified",
putResultDataBody(dataMap, fieldHelper.lastModifiedField,
FessFunctions.formatDate(responseData.getLastModified()));
}
// indexingTarget
putResultDataBody(dataMap, Constants.INDEXING_TARGET, indexingTarget);
// boost
putResultDataBody(dataMap, "boost", crawlingConfig.getDocumentBoost());
putResultDataBody(dataMap, fieldHelper.boostField,
crawlingConfig.getDocumentBoost());
// label: labelType
final Set<String> labelTypeSet = new HashSet<String>();
for (final String labelType : crawlingConfig.getLabelTypeValues()) {
@ -296,7 +304,7 @@ public abstract class AbstractFessFileTransformer extends
final LabelTypeHelper labelTypeHelper = ComponentUtil
.getLabelTypeHelper();
labelTypeSet.addAll(labelTypeHelper.getMatchedLabelValueSet(url));
putResultDataBody(dataMap, "label", labelTypeSet);
putResultDataBody(dataMap, fieldHelper.labelField, labelTypeSet);
// role: roleType
final List<String> roleTypeList = new ArrayList<String>();
for (final String roleType : crawlingConfig.getRoleTypeValues()) {
@ -316,20 +324,20 @@ public abstract class AbstractFessFileTransformer extends
}
}
}
putResultDataBody(dataMap, "role", roleTypeList);
putResultDataBody(dataMap, fieldHelper.roleField, roleTypeList);
// TODO date
// TODO lang
// id
putResultDataBody(dataMap, "id",
putResultDataBody(dataMap, fieldHelper.idField,
crawlingSessionHelper.generateId(dataMap));
// parentId
String parentUrl = responseData.getParentUrl();
if (StringUtil.isNotBlank(parentUrl)) {
parentUrl = pathMappingHelper.replaceUrl(sessionId, parentUrl);
putResultDataBody(dataMap, "url", parentUrl);
putResultDataBody(dataMap, "parentId",
putResultDataBody(dataMap, fieldHelper.urlField, parentUrl);
putResultDataBody(dataMap, fieldHelper.parentIdField,
crawlingSessionHelper.generateId(dataMap));
putResultDataBody(dataMap, "url", url); // set again
putResultDataBody(dataMap, fieldHelper.urlField, url); // set again
}
// from config

6
src/main/java/jp/sf/fess/transformer/AbstractFessXpathTransformer.java
View file

@ -21,6 +21,9 @@ import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import jp.sf.fess.helper.FieldHelper;
import jp.sf.fess.util.ComponentUtil;
import org.apache.commons.lang.StringUtils;
import org.codelibs.core.util.StringUtil;
import org.codelibs.robot.transformer.impl.XpathTransformer;
@ -118,7 +121,8 @@ public abstract class AbstractFessXpathTransformer extends XpathTransformer {
protected void putResultDataBody(final Map<String, Object> dataMap,
final String key, final Object value) {
if ("url".equals(key)) {
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
if (fieldHelper.urlField.equals(key)) {
dataMap.put(key, value);
} else if (dataMap.containsKey(key)) {
if (appendResultData) {

56
src/main/java/jp/sf/fess/transformer/FessXpathTransformer.java
View file

@ -37,6 +37,7 @@ import jp.sf.fess.db.exentity.CrawlingConfig;
import jp.sf.fess.db.exentity.CrawlingConfig.ConfigName;
import jp.sf.fess.helper.CrawlingConfigHelper;
import jp.sf.fess.helper.CrawlingSessionHelper;
import jp.sf.fess.helper.FieldHelper;
import jp.sf.fess.helper.FileTypeHelper;
import jp.sf.fess.helper.LabelTypeHelper;
import jp.sf.fess.helper.OverlappingHostHelper;
@ -221,6 +222,7 @@ public class FessXpathTransformer extends AbstractFessXpathTransformer {
final CrawlingConfig crawlingConfig = crawlingConfigHelper
.get(responseData.getSessionId());
final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final FileTypeHelper fileTypeHelper = ComponentUtil.getFileTypeHelper();
String url = responseData.getUrl();
final String indexingTarget = crawlingConfig.getIndexingTarget(url);
@ -240,25 +242,25 @@ public class FessXpathTransformer extends AbstractFessXpathTransformer {
// cid
final String configId = crawlingConfig.getConfigId();
if (configId != null) {
putResultDataBody(dataMap, systemHelper.configIdField, configId);
putResultDataBody(dataMap, fieldHelper.configIdField, configId);
}
// expires
if (documentExpires != null) {
putResultDataBody(dataMap, systemHelper.expiresField,
putResultDataBody(dataMap, fieldHelper.expiresField,
FessFunctions.formatDate(documentExpires));
}
// lang
final String lang = systemHelper.normalizeLang(getSingleNodeValue(
document, langXpath, true));
if (lang != null) {
putResultDataBody(dataMap, systemHelper.langField, lang);
putResultDataBody(dataMap, fieldHelper.langField, lang);
}
// title
// content
putResultDataBody(dataMap, "content",
putResultDataBody(dataMap, fieldHelper.contentField,
getDocumentContent(responseData, document));
if (Constants.TRUE.equalsIgnoreCase(fieldConfigMap.get("cache"))
|| enableCache) {
if (Constants.TRUE.equalsIgnoreCase(fieldConfigMap
.get(fieldHelper.cacheField)) || enableCache) {
String charSet = responseData.getCharSet();
if (charSet == null) {
charSet = Constants.UTF_8;
@ -267,10 +269,10 @@ public class FessXpathTransformer extends AbstractFessXpathTransformer {
// cache
putResultDataBody(
dataMap,
"cache",
fieldHelper.cacheField,
new String(InputStreamUtil.getBytes(responseData
.getResponseBody()), charSet));
putResultDataBody(dataMap, systemHelper.hasCacheField,
putResultDataBody(dataMap, fieldHelper.hasCacheField,
Constants.TRUE);
} catch (final Exception e) {
logger.warn("Failed to write a cache: " + sessionId + ":"
@ -278,41 +280,43 @@ public class FessXpathTransformer extends AbstractFessXpathTransformer {
}
}
// digest
putResultDataBody(dataMap, "digest",
putResultDataBody(dataMap, fieldHelper.digestField,
getDocumentDigest(responseData, document));
// segment
putResultDataBody(dataMap, "segment", sessionId);
putResultDataBody(dataMap, fieldHelper.segmentField, sessionId);
// host
putResultDataBody(dataMap, "host", getHost(url));
putResultDataBody(dataMap, fieldHelper.hostField, getHost(url));
// site
putResultDataBody(dataMap, "site", getSite(url, urlEncoding));
putResultDataBody(dataMap, fieldHelper.siteField,
getSite(url, urlEncoding));
// url
putResultDataBody(dataMap, "url", url);
putResultDataBody(dataMap, fieldHelper.urlField, url);
// created
putResultDataBody(dataMap, "created", "NOW");
putResultDataBody(dataMap, fieldHelper.createdField, Constants.NOW);
// anchor
putResultDataBody(dataMap, "anchor",
putResultDataBody(dataMap, fieldHelper.anchorField,
getAnchorList(document, responseData));
// mimetype
final String mimeType = responseData.getMimeType();
putResultDataBody(dataMap, "mimetype", mimeType);
putResultDataBody(dataMap, fieldHelper.mimetypeField, mimeType);
if (fileTypeHelper != null) {
// filetype
putResultDataBody(dataMap, fileTypeHelper.getFieldName(),
putResultDataBody(dataMap, fieldHelper.filetypeField,
fileTypeHelper.get(mimeType));
}
// contentLength
putResultDataBody(dataMap, "contentLength",
putResultDataBody(dataMap, fieldHelper.contentLengthField,
Long.toString(responseData.getContentLength()));
// lastModified
if (responseData.getLastModified() != null) {
putResultDataBody(dataMap, "lastModified",
putResultDataBody(dataMap, fieldHelper.lastModifiedField,
FessFunctions.formatDate(responseData.getLastModified()));
}
// indexingTarget
putResultDataBody(dataMap, Constants.INDEXING_TARGET, indexingTarget);
// boost
putResultDataBody(dataMap, "boost", crawlingConfig.getDocumentBoost());
putResultDataBody(dataMap, fieldHelper.boostField,
crawlingConfig.getDocumentBoost());
// label: labelType
final Set<String> labelTypeSet = new HashSet<String>();
for (final String labelType : crawlingConfig.getLabelTypeValues()) {
@ -321,24 +325,24 @@ public class FessXpathTransformer extends AbstractFessXpathTransformer {
final LabelTypeHelper labelTypeHelper = ComponentUtil
.getLabelTypeHelper();
labelTypeSet.addAll(labelTypeHelper.getMatchedLabelValueSet(url));
putResultDataBody(dataMap, "label", labelTypeSet);
putResultDataBody(dataMap, fieldHelper.labelField, labelTypeSet);
// role: roleType
final List<String> roleTypeList = new ArrayList<String>();
for (final String roleType : crawlingConfig.getRoleTypeValues()) {
roleTypeList.add(roleType);
}
putResultDataBody(dataMap, "role", roleTypeList);
putResultDataBody(dataMap, fieldHelper.roleField, roleTypeList);
// id
putResultDataBody(dataMap, "id",
putResultDataBody(dataMap, fieldHelper.idField,
crawlingSessionHelper.generateId(dataMap));
// parentId
String parentUrl = responseData.getParentUrl();
if (StringUtil.isNotBlank(parentUrl)) {
parentUrl = pathMappingHelper.replaceUrl(sessionId, parentUrl);
putResultDataBody(dataMap, "url", parentUrl);
putResultDataBody(dataMap, "parentId",
putResultDataBody(dataMap, fieldHelper.urlField, parentUrl);
putResultDataBody(dataMap, fieldHelper.parentIdField,
crawlingSessionHelper.generateId(dataMap));
putResultDataBody(dataMap, "url", url); // set again
putResultDataBody(dataMap, fieldHelper.urlField, url); // set again
}
// from config

7
src/main/java/jp/sf/fess/util/ComponentUtil.java
View file

@ -23,6 +23,7 @@ import jp.sf.fess.helper.CrawlingConfigHelper;
import jp.sf.fess.helper.CrawlingSessionHelper;
import jp.sf.fess.helper.DatabaseHelper;
import jp.sf.fess.helper.DocumentHelper;
import jp.sf.fess.helper.FieldHelper;
import jp.sf.fess.helper.FileTypeHelper;
import jp.sf.fess.helper.HotSearchWordHelper;
import jp.sf.fess.helper.IndexingHelper;
@ -114,6 +115,8 @@ public final class ComponentUtil {
private static final String INDEXING_HELPER = "indexingHelper";
private static final String FIELD_HELPER = "fieldHelper";
private ComponentUtil() {
}
@ -249,4 +252,8 @@ public final class ComponentUtil {
public static IndexingHelper getIndexingHelper() {
return SingletonS2Container.getComponent(INDEXING_HELPER);
}
public static FieldHelper getFieldHelper() {
return SingletonS2Container.getComponent(FIELD_HELPER);
}
}

6
src/main/java/jp/sf/fess/util/QueryResponseList.java
View file

@ -24,6 +24,7 @@ import java.util.List;
import java.util.ListIterator;
import java.util.Map;
import jp.sf.fess.helper.FieldHelper;
import jp.sf.fess.helper.QueryHelper;
import jp.sf.fess.helper.ViewHelper;
@ -44,8 +45,6 @@ public class QueryResponseList implements List<Map<String, Object>> {
private static final String DOC_VALUES = "docValues";
private static final String ID_FIELD = "id";
private static final Logger logger = LoggerFactory
.getLogger(QueryResponseList.class);
@ -115,13 +114,14 @@ public class QueryResponseList implements List<Map<String, Object>> {
// build highlighting fields
final QueryHelper queryHelper = ComponentUtil.getQueryHelper();
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final String hlPrefix = queryHelper.getHighlightingPrefix();
for (final SolrDocument solrDocMap : sdList) {
final Map<String, Object> docMap = new HashMap<String, Object>();
docMap.putAll(solrDocMap);
try {
final Object idValue = docMap.get(ID_FIELD);
final Object idValue = docMap.get(fieldHelper.idField);
if (queryResponse.getHighlighting().get(idValue) != null) {
for (final String hf : queryHelper
.getHighlightingFields()) {

2
src/main/resources/fess.dicon
View file

@ -24,6 +24,8 @@
</component>
<component name="jobHelper" class="jp.sf.fess.helper.JobHelper">
</component>
<component name="fieldHelper" class="jp.sf.fess.helper.FieldHelper">
</component>
<component name="systemHelper" class="jp.sf.fess.helper.SystemHelper">
<!--
<property name="javaCommandPath">"java"</property>