0ct0pu5/fess
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

remove FieldHelper

Browse source
This commit is contained in:
Shinsuke Sugaya 2015-11-14 06:56:56 +09:00
parent 0cd9c877fa
commit a773489738
37 changed files with 840 additions and 511 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
src/main/java/org/codelibs/fess/api/json/JsonApiManager.java
View file

@ -45,13 +45,13 @@ import org.codelibs.fess.entity.SearchRenderData;
import org.codelibs.fess.entity.SearchRequestParams;
import org.codelibs.fess.es.client.FessEsClient;
import org.codelibs.fess.exception.WebApiException;
import org.codelibs.fess.helper.FieldHelper;
import org.codelibs.fess.helper.HotSearchWordHelper;
import org.codelibs.fess.helper.HotSearchWordHelper.Range;
import org.codelibs.fess.helper.LabelTypeHelper;
import org.codelibs.fess.helper.QueryHelper;
import org.codelibs.fess.helper.SystemHelper;
import org.codelibs.fess.helper.UserInfoHelper;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.util.ComponentUtil;
import org.codelibs.fess.util.DocumentUtil;
import org.codelibs.fess.util.FacetResponse;
@ -358,8 +358,8 @@ public class JsonApiManager extends BaseApiManager {
return;
}
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final UserInfoHelper userInfoHelper = ComponentUtil.getUserInfoHelper();
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final SearchService searchService = ComponentUtil.getComponent(SearchService.class);
final FavoriteLogService favoriteLogService = ComponentUtil.getComponent(FavoriteLogService.class);
final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
@ -373,8 +373,8 @@ public class JsonApiManager extends BaseApiManager {
throw new WebApiException(6, "No searched urls.");
}
searchService.getDocumentByDocId(docId, new String[] { fieldHelper.urlField }).ifPresent(doc -> {
final String favoriteUrl = DocumentUtil.getValue(doc, fieldHelper.urlField, String.class);
searchService.getDocumentByDocId(docId, new String[] { fessConfig.getIndexFieldUrl() }).ifPresent(doc -> {
final String favoriteUrl = DocumentUtil.getValue(doc, fessConfig.getIndexFieldUrl(), String.class);
final String userCode = userInfoHelper.getUserCode();
if (StringUtil.isBlank(userCode)) {
@ -404,12 +404,12 @@ public class JsonApiManager extends BaseApiManager {
throw new WebApiException(4, "Failed to add url: " + favoriteUrl);
}
final String id = DocumentUtil.getValue(doc, fieldHelper.idField, String.class);
final String id = DocumentUtil.getValue(doc, fessConfig.getIndexFieldId(), String.class);
searchService.update(id, builder -> {
Script script = new Script("ctx._source." + fieldHelper.favoriteCountField + "+=1");
Script script = new Script("ctx._source." + fessConfig.getIndexFieldFavoriteCount() + "+=1");
builder.setScript(script);
Map<String, Object> upsertMap = new HashMap<>();
upsertMap.put(fieldHelper.favoriteCountField, 1);
upsertMap.put(fessConfig.getIndexFieldFavoriteCount(), 1);
builder.setUpsert(upsertMap);
builder.setRefresh(true);
});
@ -442,7 +442,7 @@ public class JsonApiManager extends BaseApiManager {
}
final UserInfoHelper userInfoHelper = ComponentUtil.getUserInfoHelper();
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final SearchService searchService = ComponentUtil.getComponent(SearchService.class);
final FavoriteLogService favoriteLogService = ComponentUtil.getComponent(FavoriteLogService.class);
@ -462,11 +462,13 @@ public class JsonApiManager extends BaseApiManager {
final String[] docIds = userInfoHelper.getResultDocIds(queryId);
final List<Map<String, Object>> docList =
searchService.getDocumentListByDocIds(docIds, new String[] { fieldHelper.urlField, fieldHelper.docIdField,
fieldHelper.favoriteCountField });
searchService.getDocumentListByDocIds(
docIds,
new String[] { fessConfig.getIndexFieldUrl(), fessConfig.getIndexFieldDocId(),
fessConfig.getIndexFieldFavoriteCount() });
List<String> urlList = new ArrayList<>(docList.size());
for (final Map<String, Object> doc : docList) {
final String urlObj = DocumentUtil.getValue(doc, fieldHelper.urlField, String.class);
final String urlObj = DocumentUtil.getValue(doc, fessConfig.getIndexFieldUrl(), String.class);
if (urlObj != null) {
urlList.add(urlObj.toString());
}
@ -474,9 +476,9 @@ public class JsonApiManager extends BaseApiManager {
urlList = favoriteLogService.getUrlList(userCode, urlList);
final List<String> docIdList = new ArrayList<>(urlList.size());
for (final Map<String, Object> doc : docList) {
final String urlObj = DocumentUtil.getValue(doc, fieldHelper.urlField, String.class);
final String urlObj = DocumentUtil.getValue(doc, fessConfig.getIndexFieldUrl(), String.class);
if (urlObj != null && urlList.contains(urlObj)) {
final String docIdObj = DocumentUtil.getValue(doc, fieldHelper.docIdField, String.class);
final String docIdObj = DocumentUtil.getValue(doc, fessConfig.getIndexFieldDocId(), String.class);
if (docIdObj != null) {
docIdList.add(docIdObj);
}

23
src/main/java/org/codelibs/fess/app/service/SearchService.java
View file

@ -42,11 +42,11 @@ import org.codelibs.fess.es.client.FessEsClient;
import org.codelibs.fess.es.client.FessEsClient.SearchConditionBuilder;
import org.codelibs.fess.es.client.FessEsClientException;
import org.codelibs.fess.es.log.exentity.SearchLog;
import org.codelibs.fess.helper.FieldHelper;
import org.codelibs.fess.helper.QueryHelper;
import org.codelibs.fess.helper.SearchLogHelper;
import org.codelibs.fess.helper.SystemHelper;
import org.codelibs.fess.helper.UserInfoHelper;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.util.ComponentUtil;
import org.codelibs.fess.util.QueryResponseList;
import org.codelibs.fess.util.QueryUtil;
@ -80,7 +80,7 @@ public class SearchService {
protected SystemHelper systemHelper;
@Resource
protected FieldHelper fieldHelper;
protected FessConfig fessConfig;
@Resource
protected QueryHelper queryHelper;
@ -123,7 +123,7 @@ public class SearchService {
queryBuf.append(" sort:").append(params.getSort());
}
if (params.getLanguages() != null) {
appendQueries(queryBuf, fieldHelper.langField, params.getLanguages());
appendQueries(queryBuf, fessConfig.getIndexFieldLang(), params.getLanguages());
}
final String query = queryBuf.toString().trim();
@ -132,8 +132,8 @@ public class SearchService {
final int pageSize = params.getPageSize();
final List<Map<String, Object>> documentItems =
fessEsClient.search(
fieldHelper.docIndex,
fieldHelper.docType,
fessConfig.getIndexDocumentIndex(),
fessConfig.getIndexDocumentType(),
searchRequestBuilder -> {
return SearchConditionBuilder.builder(searchRequestBuilder).query(query).offset(pageStart).size(pageSize)
.facetInfo(params.getFacetInfo()).geoInfo(params.getGeoInfo())
@ -327,16 +327,16 @@ public class SearchService {
}
public OptionalEntity<Map<String, Object>> getDocumentByDocId(final String docId, final String[] fields) {
return fessEsClient.getDocument(fieldHelper.docIndex, fieldHelper.docType, builder -> {
builder.setQuery(QueryBuilders.termQuery(fieldHelper.docIdField, docId));
return fessEsClient.getDocument(fessConfig.getIndexDocumentIndex(), fessConfig.getIndexDocumentType(), builder -> {
builder.setQuery(QueryBuilders.termQuery(fessConfig.getIndexFieldDocId(), docId));
builder.addFields(fields);
return true;
});
}
public List<Map<String, Object>> getDocumentListByDocIds(final String[] docIds, final String[] fields) {
return fessEsClient.getDocumentList(fieldHelper.docIndex, fieldHelper.docType, builder -> {
builder.setQuery(QueryBuilders.termsQuery(fieldHelper.docIdField, docIds));
return fessEsClient.getDocumentList(fessConfig.getIndexDocumentIndex(), fessConfig.getIndexDocumentType(), builder -> {
builder.setQuery(QueryBuilders.termsQuery(fessConfig.getIndexFieldDocId(), docIds));
builder.setSize(queryHelper.getMaxPageSize());
builder.addFields(fields);
return true;
@ -344,12 +344,13 @@ public class SearchService {
}
public boolean update(final String id, final String field, final Object value) {
return fessEsClient.update(fieldHelper.docIndex, fieldHelper.docType, id, field, value);
return fessEsClient.update(fessConfig.getIndexDocumentIndex(), fessConfig.getIndexDocumentType(), id, field, value);
}
public boolean update(final String id, Consumer<UpdateRequestBuilder> builderLambda) {
try {
UpdateRequestBuilder builder = fessEsClient.prepareUpdate(fieldHelper.docIndex, fieldHelper.docType, id);
UpdateRequestBuilder builder =
fessEsClient.prepareUpdate(fessConfig.getIndexDocumentIndex(), fessConfig.getIndexDocumentType(), id);
builderLambda.accept(builder);
UpdateResponse response = builder.execute().actionGet();
return response.isCreated();

8
src/main/java/org/codelibs/fess/app/web/admin/searchlist/AdminSearchlistAction.java
View file

@ -28,7 +28,6 @@ import org.codelibs.fess.entity.SearchRenderData;
import org.codelibs.fess.es.client.FessEsClient;
import org.codelibs.fess.exception.InvalidQueryException;
import org.codelibs.fess.exception.ResultOffsetExceededException;
import org.codelibs.fess.helper.FieldHelper;
import org.codelibs.fess.helper.JobHelper;
import org.codelibs.fess.helper.QueryHelper;
import org.codelibs.fess.helper.SystemHelper;
@ -61,9 +60,6 @@ public class AdminSearchlistAction extends FessAdminAction {
@Resource
protected FessEsClient fessEsClient;
@Resource
protected FieldHelper fieldHelper;
@Resource
protected QueryHelper queryHelper;
@ -196,8 +192,8 @@ public class AdminSearchlistAction extends FessAdminAction {
() -> asHtml(path_AdminSearchlist_IndexJsp));
}
try {
final QueryBuilder query = QueryBuilders.termQuery(fieldHelper.docIdField, docId);
fessEsClient.deleteByQuery(fieldHelper.docIndex, fieldHelper.docType, query);
final QueryBuilder query = QueryBuilders.termQuery(fessConfig.getIndexFieldDocId(), docId);
fessEsClient.deleteByQuery(fessConfig.getIndexDocumentIndex(), fessConfig.getIndexDocumentType(), query);
saveInfo(messages -> messages.addSuccessDeleteSolrIndex(GLOBAL));
} catch (final Exception e) {
throwValidationError(messages -> messages.addErrorsFailedToDeleteDocInAdmin(GLOBAL),

4
src/main/java/org/codelibs/fess/app/web/base/FessSearchAction.java
View file

@ -31,7 +31,6 @@ import org.codelibs.core.net.URLUtil;
import org.codelibs.fess.Constants;
import org.codelibs.fess.es.client.FessEsClient;
import org.codelibs.fess.exception.UnsupportedSearchException;
import org.codelibs.fess.helper.FieldHelper;
import org.codelibs.fess.helper.LabelTypeHelper;
import org.codelibs.fess.helper.OpenSearchHelper;
import org.codelibs.fess.helper.QueryHelper;
@ -76,9 +75,6 @@ public abstract class FessSearchAction extends FessBaseAction {
@Resource
protected SystemHelper systemHelper;
@Resource
protected FieldHelper fieldHelper;
@Resource
protected OpenSearchHelper openSearchHelper;

8
src/main/java/org/codelibs/fess/app/web/cache/CacheAction.java vendored
View file

@ -51,8 +51,8 @@ public class CacheAction extends FessSearchAction {
Map<String, Object> doc = null;
try {
doc = fessEsClient.getDocument(fieldHelper.docIndex, fieldHelper.docType, queryRequestBuilder -> {
final TermQueryBuilder termQuery = QueryBuilders.termQuery(fieldHelper.docIdField, form.docId);
doc = fessEsClient.getDocument(fessConfig.getIndexDocumentIndex(), fessConfig.getIndexDocumentType(), queryRequestBuilder -> {
final TermQueryBuilder termQuery = QueryBuilders.termQuery(fessConfig.getIndexFieldDocId(), form.docId);
queryRequestBuilder.setQuery(termQuery);
queryRequestBuilder.addFields(queryHelper.getResponseFields());
return true;
@ -73,8 +73,8 @@ public class CacheAction extends FessSearchAction {
}, () -> asHtml(path_ErrorJsp));
}
return asStream(DocumentUtil.getValue(doc, fieldHelper.docIdField, String.class)).contentType("text/html; charset=UTF-8").data(
content.getBytes(Constants.CHARSET_UTF_8));
return asStream(DocumentUtil.getValue(doc, fessConfig.getIndexFieldDocId(), String.class)).contentType("text/html; charset=UTF-8")
.data(content.getBytes(Constants.CHARSET_UTF_8));
}
}

8
src/main/java/org/codelibs/fess/app/web/go/GoAction.java
View file

@ -68,10 +68,10 @@ public class GoAction extends FessSearchAction {
Map<String, Object> doc = null;
try {
doc = fessEsClient.getDocument(fieldHelper.docIndex, fieldHelper.docType, queryRequestBuilder -> {
final TermQueryBuilder termQuery = QueryBuilders.termQuery(fieldHelper.docIdField, form.docId);
doc = fessEsClient.getDocument(fessConfig.getIndexDocumentIndex(), fessConfig.getIndexDocumentType(), queryRequestBuilder -> {
final TermQueryBuilder termQuery = QueryBuilders.termQuery(fessConfig.getIndexFieldDocId(), form.docId);
queryRequestBuilder.setQuery(termQuery);
queryRequestBuilder.addFields(fieldHelper.urlField);
queryRequestBuilder.addFields(fessConfig.getIndexFieldUrl());
return true;
}).get();
} catch (final Exception e) {
@ -82,7 +82,7 @@ public class GoAction extends FessSearchAction {
messages.addErrorsDocidNotFound(GLOBAL, form.docId);
}, () -> asHtml(path_ErrorJsp));
}
final String url = DocumentUtil.getValue(doc, fieldHelper.urlField, String.class);
final String url = DocumentUtil.getValue(doc, fessConfig.getIndexFieldUrl(), String.class);
if (url == null) {
throwValidationError(messages -> {
messages.addErrorsDocumentNotFound(GLOBAL, form.docId);

16
src/main/java/org/codelibs/fess/app/web/screenshot/ScreenshotAction.java
View file

@ -64,13 +64,15 @@ public class ScreenshotAction extends FessSearchAction {
OutputStream out = null;
BufferedInputStream in = null;
try {
final Map<String, Object> doc = fessEsClient.getDocument(fieldHelper.docIndex, fieldHelper.docType, queryRequestBuilder -> {
final TermQueryBuilder termQuery = QueryBuilders.termQuery(fieldHelper.docIdField, form.docId);
queryRequestBuilder.setQuery(termQuery);
queryRequestBuilder.addFields(queryHelper.getResponseFields());
return true;
}).get();
final String url = DocumentUtil.getValue(doc, fieldHelper.urlField, String.class);
final Map<String, Object> doc =
fessEsClient.getDocument(fessConfig.getIndexDocumentIndex(), fessConfig.getIndexDocumentType(),
queryRequestBuilder -> {
final TermQueryBuilder termQuery = QueryBuilders.termQuery(fessConfig.getIndexFieldDocId(), form.docId);
queryRequestBuilder.setQuery(termQuery);
queryRequestBuilder.addFields(queryHelper.getResponseFields());
return true;
}).get();
final String url = DocumentUtil.getValue(doc, fessConfig.getIndexFieldUrl(), String.class);
if (StringUtil.isBlank(form.queryId) || StringUtil.isBlank(url) || screenShotManager == null) {
// 404
response.sendError(HttpServletResponse.SC_NOT_FOUND);

34
src/main/java/org/codelibs/fess/crawler/FessCrawlerThread.java
View file

@ -38,9 +38,9 @@ import org.codelibs.fess.es.client.FessEsClient;
import org.codelibs.fess.es.config.exentity.CrawlingConfig;
import org.codelibs.fess.helper.CrawlingConfigHelper;
import org.codelibs.fess.helper.CrawlingSessionHelper;
import org.codelibs.fess.helper.FieldHelper;
import org.codelibs.fess.helper.IndexingHelper;
import org.codelibs.fess.helper.SambaHelper;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.util.ComponentUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -59,9 +59,9 @@ public class FessCrawlerThread extends CrawlerThread {
log(logHelper, LogType.CHECK_LAST_MODIFIED, crawlerContext, urlQueue);
final long startTime = System.currentTimeMillis();
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final CrawlingConfigHelper crawlingConfigHelper = ComponentUtil.getCrawlingConfigHelper();
final CrawlingSessionHelper crawlingSessionHelper = ComponentUtil.getCrawlingSessionHelper();
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final SambaHelper sambaHelper = ComponentUtil.getSambaHelper();
final IndexingHelper indexingHelper = ComponentUtil.getIndexingHelper();
final FessEsClient fessEsClient = ComponentUtil.getElasticsearchClient();
@ -72,7 +72,7 @@ public class FessCrawlerThread extends CrawlerThread {
try {
final CrawlingConfig crawlingConfig = crawlingConfigHelper.get(crawlerContext.getSessionId());
final Map<String, Object> dataMap = new HashMap<String, Object>();
dataMap.put(fieldHelper.urlField, url);
dataMap.put(fessConfig.getIndexFieldUrl(), url);
final List<String> roleTypeList = new ArrayList<String>();
for (final String roleType : crawlingConfig.getRoleTypeValues()) {
roleTypeList.add(roleType);
@ -95,28 +95,32 @@ public class FessCrawlerThread extends CrawlerThread {
}
}
}
dataMap.put(fieldHelper.roleField, roleTypeList);
dataMap.put(fessConfig.getIndexFieldRole(), roleTypeList);
final String id = crawlingSessionHelper.generateId(dataMap);
final Map<String, Object> document =
indexingHelper.getDocument(fessEsClient, id, new String[] { fieldHelper.idField, fieldHelper.lastModifiedField,
fieldHelper.anchorField, fieldHelper.segmentField, fieldHelper.expiresField, fieldHelper.clickCountField,
fieldHelper.favoriteCountField });
indexingHelper.getDocument(
fessEsClient,
id,
new String[] { fessConfig.getIndexFieldId(), fessConfig.getIndexFieldLastModified(),
fessConfig.getIndexFieldAnchor(), fessConfig.getIndexFieldSegment(),
fessConfig.getIndexFieldExpires(), fessConfig.getIndexFieldClickCount(),
fessConfig.getIndexFieldFavoriteCount() });
if (document == null) {
storeChildUrlsToQueue(urlQueue, getChildUrlSet(fessEsClient, id));
return true;
}
final Date expires = (Date) document.get(fieldHelper.expiresField);
final Date expires = (Date) document.get(fessConfig.getIndexFieldExpires());
if (expires != null && expires.getTime() < System.currentTimeMillis()) {
final Object idValue = document.get(fieldHelper.idField);
final Object idValue = document.get(fessConfig.getIndexFieldId());
if (idValue != null) {
indexingHelper.deleteDocument(fessEsClient, idValue.toString());
}
return true;
}
final Date lastModified = (Date) document.get(fieldHelper.lastModifiedField);
final Date lastModified = (Date) document.get(fessConfig.getIndexFieldLastModified());
if (lastModified == null) {
return true;
}
@ -131,7 +135,7 @@ public class FessCrawlerThread extends CrawlerThread {
final int httpStatusCode = responseData.getHttpStatusCode();
if (httpStatusCode == 404) {
storeChildUrlsToQueue(urlQueue, getAnchorSet(document.get(fieldHelper.anchorField)));
storeChildUrlsToQueue(urlQueue, getAnchorSet(document.get(fessConfig.getIndexFieldAnchor())));
indexingHelper.deleteDocument(fessEsClient, id);
return false;
} else if (responseData.getLastModified() == null) {
@ -146,7 +150,7 @@ public class FessCrawlerThread extends CrawlerThread {
responseData.setHttpStatusCode(org.codelibs.fess.crawler.Constants.NOT_MODIFIED_STATUS);
processResponse(urlQueue, responseData);
storeChildUrlsToQueue(urlQueue, getAnchorSet(document.get(fieldHelper.anchorField)));
storeChildUrlsToQueue(urlQueue, getAnchorSet(document.get(fessConfig.getIndexFieldAnchor())));
return false;
}
@ -192,10 +196,10 @@ public class FessCrawlerThread extends CrawlerThread {
}
protected Set<RequestData> getChildUrlSet(final FessEsClient fessEsClient, final String id) {
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final IndexingHelper indexingHelper = ComponentUtil.getIndexingHelper();
final List<Map<String, Object>> docList =
indexingHelper.getChildDocumentList(fessEsClient, id, new String[] { fieldHelper.urlField });
indexingHelper.getChildDocumentList(fessEsClient, id, new String[] { fessConfig.getIndexFieldUrl() });
if (docList.isEmpty()) {
return null;
}
@ -204,7 +208,7 @@ public class FessCrawlerThread extends CrawlerThread {
}
final Set<RequestData> urlSet = new HashSet<>(docList.size());
for (final Map<String, Object> doc : docList) {
final Object obj = doc.get(fieldHelper.urlField);
final Object obj = doc.get(fessConfig.getIndexFieldUrl());
if (obj != null) {
urlSet.add(RequestDataBuilder.newRequestData().get().url(obj.toString()).build());
}

69
src/main/java/org/codelibs/fess/crawler/transformer/AbstractFessFileTransformer.java
View file

@ -51,12 +51,12 @@ import org.codelibs.fess.es.config.exentity.CrawlingConfig;
import org.codelibs.fess.es.config.exentity.CrawlingConfig.ConfigName;
import org.codelibs.fess.helper.CrawlingConfigHelper;
import org.codelibs.fess.helper.CrawlingSessionHelper;
import org.codelibs.fess.helper.FieldHelper;
import org.codelibs.fess.helper.FileTypeHelper;
import org.codelibs.fess.helper.LabelTypeHelper;
import org.codelibs.fess.helper.PathMappingHelper;
import org.codelibs.fess.helper.SambaHelper;
import org.codelibs.fess.helper.SystemHelper;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.util.ComponentUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -84,8 +84,6 @@ public abstract class AbstractFessFileTransformer extends AbstractFessXpathTrans
public boolean appendBodyContentToContent = true;
public boolean enableCache = false;
public Map<String, String> parentEncodingMap = Collections.synchronizedMap(new LruHashMap<String, String>(1000));
protected Map<String, String> metaContentMapping;
@ -157,6 +155,7 @@ public abstract class AbstractFessFileTransformer extends AbstractFessXpathTrans
final ResultData resultData = new ResultData();
resultData.setTransformerName(getName());
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final CrawlingSessionHelper crawlingSessionHelper = ComponentUtil.getCrawlingSessionHelper();
final String sessionId = crawlingSessionHelper.getCanonicalSessionId(responseData.getSessionId());
final Long documentExpires = crawlingSessionHelper.getDocumentExpires();
@ -166,7 +165,6 @@ public abstract class AbstractFessFileTransformer extends AbstractFessXpathTrans
final boolean useAclAsRole = crawlerProperties.getProperty(Constants.USE_ACL_AS_ROLE, Constants.FALSE).equals(Constants.TRUE);
final CrawlingConfigHelper crawlingConfigHelper = ComponentUtil.getCrawlingConfigHelper();
final CrawlingConfig crawlingConfig = crawlingConfigHelper.get(responseData.getSessionId());
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
final FileTypeHelper fileTypeHelper = ComponentUtil.getFileTypeHelper();
String url = responseData.getUrl();
@ -186,14 +184,14 @@ public abstract class AbstractFessFileTransformer extends AbstractFessXpathTrans
// cid
final String configId = crawlingConfig.getConfigId();
if (configId != null) {
putResultDataBody(dataMap, fieldHelper.configIdField, configId);
putResultDataBody(dataMap, fessConfig.getIndexFieldConfigId(), configId);
}
// expires
if (documentExpires != null) {
putResultDataBody(dataMap, fieldHelper.expiresField, new Date(documentExpires));
putResultDataBody(dataMap, fessConfig.getIndexFieldExpires(), new Date(documentExpires));
}
// segment
putResultDataBody(dataMap, fieldHelper.segmentField, sessionId);
putResultDataBody(dataMap, fessConfig.getIndexFieldSegment(), sessionId);
// content
final StringBuilder buf = new StringBuilder(content.length() + 1000);
if (appendBodyContentToContent) {
@ -207,63 +205,64 @@ public abstract class AbstractFessFileTransformer extends AbstractFessXpathTrans
}
final String body = normalizeContent(buf.toString());
if (StringUtil.isNotBlank(body)) {
putResultDataBody(dataMap, fieldHelper.contentField, body);
putResultDataBody(dataMap, fessConfig.getIndexFieldContent(), body);
} else {
putResultDataBody(dataMap, fieldHelper.contentField, StringUtil.EMPTY);
putResultDataBody(dataMap, fessConfig.getIndexFieldContent(), StringUtil.EMPTY);
}
if (Constants.TRUE.equalsIgnoreCase(fieldConfigMap.get(fieldHelper.cacheField)) || enableCache) {
if (Constants.TRUE.equalsIgnoreCase(fieldConfigMap.get(fessConfig.getIndexFieldCache()))
|| fessConfig.isCrawlerDocumentCacheEnable()) {
final String cache = content.trim().replaceAll("[ \\t\\x0B\\f]+", " ");
// text cache
putResultDataBody(dataMap, fieldHelper.cacheField, cache);
putResultDataBody(dataMap, fieldHelper.hasCacheField, Constants.TRUE);
putResultDataBody(dataMap, fessConfig.getIndexFieldCache(), cache);
putResultDataBody(dataMap, fessConfig.getIndexFieldHasCache(), Constants.TRUE);
}
// digest
putResultDataBody(dataMap, fieldHelper.digestField, Constants.DIGEST_PREFIX
+ abbreviate(normalizeContent(content), maxDigestLength));
putResultDataBody(dataMap, fessConfig.getIndexFieldDigest(),
Constants.DIGEST_PREFIX + abbreviate(normalizeContent(content), maxDigestLength));
// title
if (!dataMap.containsKey(fieldHelper.titleField)) {
if (!dataMap.containsKey(fessConfig.getIndexFieldTitle())) {
if (url.endsWith("/")) {
if (StringUtil.isNotBlank(content)) {
putResultDataBody(dataMap, fieldHelper.titleField, abbreviate(body, maxTitleLength));
putResultDataBody(dataMap, fessConfig.getIndexFieldTitle(), abbreviate(body, maxTitleLength));
} else {
putResultDataBody(dataMap, fieldHelper.titleField, noTitleLabel);
putResultDataBody(dataMap, fessConfig.getIndexFieldTitle(), noTitleLabel);
}
} else {
final String u = decodeUrlAsName(url, url.startsWith("file:"));
final int pos = u.lastIndexOf('/');
if (pos == -1) {
putResultDataBody(dataMap, fieldHelper.titleField, u);
putResultDataBody(dataMap, fessConfig.getIndexFieldTitle(), u);
} else {
putResultDataBody(dataMap, fieldHelper.titleField, u.substring(pos + 1));
putResultDataBody(dataMap, fessConfig.getIndexFieldTitle(), u.substring(pos + 1));
}
}
}
// host
putResultDataBody(dataMap, fieldHelper.hostField, getHost(url));
putResultDataBody(dataMap, fessConfig.getIndexFieldHost(), getHost(url));
// site
putResultDataBody(dataMap, fieldHelper.siteField, getSite(url, urlEncoding));
putResultDataBody(dataMap, fessConfig.getIndexFieldSite(), getSite(url, urlEncoding));
// url
putResultDataBody(dataMap, fieldHelper.urlField, url);
putResultDataBody(dataMap, fessConfig.getIndexFieldUrl(), url);
// created
putResultDataBody(dataMap, fieldHelper.createdField, systemHelper.getCurrentTime());
putResultDataBody(dataMap, fessConfig.getIndexFieldCreated(), systemHelper.getCurrentTime());
// TODO anchor
putResultDataBody(dataMap, fieldHelper.anchorField, StringUtil.EMPTY);
putResultDataBody(dataMap, fessConfig.getIndexFieldAnchor(), StringUtil.EMPTY);
// mimetype
putResultDataBody(dataMap, fieldHelper.mimetypeField, mimeType);
putResultDataBody(dataMap, fessConfig.getIndexFieldMimetype(), mimeType);
if (fileTypeHelper != null) {
// filetype
putResultDataBody(dataMap, fieldHelper.filetypeField, fileTypeHelper.get(mimeType));
putResultDataBody(dataMap, fessConfig.getIndexFieldFiletype(), fileTypeHelper.get(mimeType));
}
// contentLength
putResultDataBody(dataMap, fieldHelper.contentLengthField, Long.toString(responseData.getContentLength()));
putResultDataBody(dataMap, fessConfig.getIndexFieldContentLength(), Long.toString(responseData.getContentLength()));
// lastModified
if (responseData.getLastModified() != null) {
putResultDataBody(dataMap, fieldHelper.lastModifiedField, responseData.getLastModified());
putResultDataBody(dataMap, fessConfig.getIndexFieldLastModified(), responseData.getLastModified());
}
// indexingTarget
putResultDataBody(dataMap, Constants.INDEXING_TARGET, indexingTarget);
// boost
putResultDataBody(dataMap, fieldHelper.boostField, crawlingConfig.getDocumentBoost());
putResultDataBody(dataMap, fessConfig.getIndexFieldBoost(), crawlingConfig.getDocumentBoost());
// label: labelType
final Set<String> labelTypeSet = new HashSet<String>();
for (final String labelType : crawlingConfig.getLabelTypeValues()) {
@ -271,7 +270,7 @@ public abstract class AbstractFessFileTransformer extends AbstractFessXpathTrans
}
final LabelTypeHelper labelTypeHelper = ComponentUtil.getLabelTypeHelper();
labelTypeSet.addAll(labelTypeHelper.getMatchedLabelValueSet(url));
putResultDataBody(dataMap, fieldHelper.labelField, labelTypeSet);
putResultDataBody(dataMap, fessConfig.getIndexFieldLabel(), labelTypeSet);
// role: roleType
final List<String> roleTypeList = new ArrayList<String>();
for (final String roleType : crawlingConfig.getRoleTypeValues()) {
@ -289,18 +288,18 @@ public abstract class AbstractFessFileTransformer extends AbstractFessXpathTrans
}
}
}
putResultDataBody(dataMap, fieldHelper.roleField, roleTypeList);
putResultDataBody(dataMap, fessConfig.getIndexFieldRole(), roleTypeList);
// TODO date
// TODO lang
// id
putResultDataBody(dataMap, fieldHelper.idField, crawlingSessionHelper.generateId(dataMap));
putResultDataBody(dataMap, fessConfig.getIndexFieldId(), crawlingSessionHelper.generateId(dataMap));
// parentId
String parentUrl = responseData.getParentUrl();
if (StringUtil.isNotBlank(parentUrl)) {
parentUrl = pathMappingHelper.replaceUrl(sessionId, parentUrl);
putResultDataBody(dataMap, fieldHelper.urlField, parentUrl);
putResultDataBody(dataMap, fieldHelper.parentIdField, crawlingSessionHelper.generateId(dataMap));
putResultDataBody(dataMap, fieldHelper.urlField, url); // set again
putResultDataBody(dataMap, fessConfig.getIndexFieldUrl(), parentUrl);
putResultDataBody(dataMap, fessConfig.getIndexFieldParentId(), crawlingSessionHelper.generateId(dataMap));
putResultDataBody(dataMap, fessConfig.getIndexFieldUrl(), url); // set again
}
// from config

6
src/main/java/org/codelibs/fess/crawler/transformer/AbstractFessXpathTransformer.java
View file

@ -23,7 +23,7 @@ import java.util.Map;
import org.apache.commons.lang3.StringUtils;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.fess.crawler.transformer.impl.XpathTransformer;
import org.codelibs.fess.helper.FieldHelper;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.util.ComponentUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -117,8 +117,8 @@ public abstract class AbstractFessXpathTransformer extends XpathTransformer {
}
protected void putResultDataBody(final Map<String, Object> dataMap, final String key, final Object value) {
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
if (fieldHelper.urlField.equals(key)) {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
if (fessConfig.getIndexFieldUrl().equals(key)) {
dataMap.put(key, value);
} else if (dataMap.containsKey(key)) {
if (appendResultData) {

59
src/main/java/org/codelibs/fess/crawler/transformer/FessXpathTransformer.java
View file

@ -53,12 +53,12 @@ import org.codelibs.fess.es.config.exentity.CrawlingConfig;
import org.codelibs.fess.es.config.exentity.CrawlingConfig.ConfigName;
import org.codelibs.fess.helper.CrawlingConfigHelper;
import org.codelibs.fess.helper.CrawlingSessionHelper;
import org.codelibs.fess.helper.FieldHelper;
import org.codelibs.fess.helper.FileTypeHelper;
import org.codelibs.fess.helper.LabelTypeHelper;
import org.codelibs.fess.helper.OverlappingHostHelper;
import org.codelibs.fess.helper.PathMappingHelper;
import org.codelibs.fess.helper.SystemHelper;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.util.ComponentUtil;
import org.cyberneko.html.parsers.DOMParser;
import org.slf4j.Logger;
@ -91,8 +91,6 @@ public class FessXpathTransformer extends AbstractFessXpathTransformer {
public int maxCacheLength = 2621440; // 2.5Mbytes
public boolean enableCache = false;
public Map<String, String> convertUrlMap = new HashMap<String, String>();
@Override
@ -192,6 +190,7 @@ public class FessXpathTransformer extends AbstractFessXpathTransformer {
}
}
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final CrawlingSessionHelper crawlingSessionHelper = ComponentUtil.getCrawlingSessionHelper();
final String sessionId = crawlingSessionHelper.getCanonicalSessionId(responseData.getSessionId());
final Long documentExpires = crawlingSessionHelper.getDocumentExpires();
@ -199,7 +198,6 @@ public class FessXpathTransformer extends AbstractFessXpathTransformer {
final CrawlingConfigHelper crawlingConfigHelper = ComponentUtil.getCrawlingConfigHelper();
final CrawlingConfig crawlingConfig = crawlingConfigHelper.get(responseData.getSessionId());
final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final FileTypeHelper fileTypeHelper = ComponentUtil.getFileTypeHelper();
String url = responseData.getUrl();
final String indexingTarget = crawlingConfig.getIndexingTarget(url);
@ -218,65 +216,66 @@ public class FessXpathTransformer extends AbstractFessXpathTransformer {
// cid
final String configId = crawlingConfig.getConfigId();
if (configId != null) {
putResultDataBody(dataMap, fieldHelper.configIdField, configId);
putResultDataBody(dataMap, fessConfig.getIndexFieldConfigId(), configId);
}
// expires
if (documentExpires != null) {
putResultDataBody(dataMap, fieldHelper.expiresField, new Date(documentExpires));
putResultDataBody(dataMap, fessConfig.getIndexFieldExpires(), new Date(documentExpires));
}
// lang
final String lang = systemHelper.normalizeLang(getSingleNodeValue(document, langXpath, true));
if (lang != null) {
putResultDataBody(dataMap, fieldHelper.langField, lang);
putResultDataBody(dataMap, fessConfig.getIndexFieldLang(), lang);
}
// title
// content
putResultDataBody(dataMap, fieldHelper.contentField, getDocumentContent(responseData, document));
if (Constants.TRUE.equalsIgnoreCase(fieldConfigMap.get(fieldHelper.cacheField)) || enableCache) {
putResultDataBody(dataMap, fessConfig.getIndexFieldContent(), getDocumentContent(responseData, document));
if (Constants.TRUE.equalsIgnoreCase(fieldConfigMap.get(fessConfig.getIndexFieldCache()))
|| fessConfig.isCrawlerDocumentCacheEnable()) {
String charSet = responseData.getCharSet();
if (charSet == null) {
charSet = Constants.UTF_8;
}
try {
// cache
putResultDataBody(dataMap, fieldHelper.cacheField, new String(InputStreamUtil.getBytes(responseData.getResponseBody()),
charSet));
putResultDataBody(dataMap, fieldHelper.hasCacheField, Constants.TRUE);
putResultDataBody(dataMap, fessConfig.getIndexFieldCache(),
new String(InputStreamUtil.getBytes(responseData.getResponseBody()), charSet));
putResultDataBody(dataMap, fessConfig.getIndexFieldHasCache(), Constants.TRUE);
} catch (final Exception e) {
logger.warn("Failed to write a cache: " + sessionId + ":" + responseData, e);
}
}
// digest
putResultDataBody(dataMap, fieldHelper.digestField, getDocumentDigest(responseData, document));
putResultDataBody(dataMap, fessConfig.getIndexFieldDigest(), getDocumentDigest(responseData, document));
// segment
putResultDataBody(dataMap, fieldHelper.segmentField, sessionId);
putResultDataBody(dataMap, fessConfig.getIndexFieldSegment(), sessionId);
// host
putResultDataBody(dataMap, fieldHelper.hostField, getHost(url));
putResultDataBody(dataMap, fessConfig.getIndexFieldHost(), getHost(url));
// site
putResultDataBody(dataMap, fieldHelper.siteField, getSite(url, urlEncoding));
putResultDataBody(dataMap, fessConfig.getIndexFieldSite(), getSite(url, urlEncoding));
// url
putResultDataBody(dataMap, fieldHelper.urlField, url);
putResultDataBody(dataMap, fessConfig.getIndexFieldUrl(), url);
// created
putResultDataBody(dataMap, fieldHelper.createdField, systemHelper.getCurrentTime());
putResultDataBody(dataMap, fessConfig.getIndexFieldCreated(), systemHelper.getCurrentTime());
// anchor
putResultDataBody(dataMap, fieldHelper.anchorField, getAnchorList(document, responseData));
putResultDataBody(dataMap, fessConfig.getIndexFieldAnchor(), getAnchorList(document, responseData));
// mimetype
final String mimeType = responseData.getMimeType();
putResultDataBody(dataMap, fieldHelper.mimetypeField, mimeType);
putResultDataBody(dataMap, fessConfig.getIndexFieldMimetype(), mimeType);
if (fileTypeHelper != null) {
// filetype
putResultDataBody(dataMap, fieldHelper.filetypeField, fileTypeHelper.get(mimeType));
putResultDataBody(dataMap, fessConfig.getIndexFieldFiletype(), fileTypeHelper.get(mimeType));
}
// contentLength
putResultDataBody(dataMap, fieldHelper.contentLengthField, Long.toString(responseData.getContentLength()));
putResultDataBody(dataMap, fessConfig.getIndexFieldContentLength(), Long.toString(responseData.getContentLength()));
// lastModified
if (responseData.getLastModified() != null) {
putResultDataBody(dataMap, fieldHelper.lastModifiedField, responseData.getLastModified());
putResultDataBody(dataMap, fessConfig.getIndexFieldLastModified(), responseData.getLastModified());
}
// indexingTarget
putResultDataBody(dataMap, Constants.INDEXING_TARGET, indexingTarget);
// boost
putResultDataBody(dataMap, fieldHelper.boostField, crawlingConfig.getDocumentBoost());
putResultDataBody(dataMap, fessConfig.getIndexFieldBoost(), crawlingConfig.getDocumentBoost());
// label: labelType
final Set<String> labelTypeSet = new HashSet<String>();
for (final String labelType : crawlingConfig.getLabelTypeValues()) {
@ -284,22 +283,22 @@ public class FessXpathTransformer extends AbstractFessXpathTransformer {
}
final LabelTypeHelper labelTypeHelper = ComponentUtil.getLabelTypeHelper();
labelTypeSet.addAll(labelTypeHelper.getMatchedLabelValueSet(url));
putResultDataBody(dataMap, fieldHelper.labelField, labelTypeSet);
putResultDataBody(dataMap, fessConfig.getIndexFieldLabel(), labelTypeSet);
// role: roleType
final List<String> roleTypeList = new ArrayList<String>();
for (final String roleType : crawlingConfig.getRoleTypeValues()) {
roleTypeList.add(roleType);
}
putResultDataBody(dataMap, fieldHelper.roleField, roleTypeList);
putResultDataBody(dataMap, fessConfig.getIndexFieldRole(), roleTypeList);
// id
putResultDataBody(dataMap, fieldHelper.idField, crawlingSessionHelper.generateId(dataMap));
putResultDataBody(dataMap, fessConfig.getIndexFieldId(), crawlingSessionHelper.generateId(dataMap));
// parentId
String parentUrl = responseData.getParentUrl();
if (StringUtil.isNotBlank(parentUrl)) {
parentUrl = pathMappingHelper.replaceUrl(sessionId, parentUrl);
putResultDataBody(dataMap, fieldHelper.urlField, parentUrl);
putResultDataBody(dataMap, fieldHelper.parentIdField, crawlingSessionHelper.generateId(dataMap));
putResultDataBody(dataMap, fieldHelper.urlField, url); // set again
putResultDataBody(dataMap, fessConfig.getIndexFieldUrl(), parentUrl);
putResultDataBody(dataMap, fessConfig.getIndexFieldParentId(), crawlingSessionHelper.generateId(dataMap));
putResultDataBody(dataMap, fessConfig.getIndexFieldUrl(), url); // set again
}
// from config

20
src/main/java/org/codelibs/fess/ds/impl/AbstractDataStoreImpl.java
View file

@ -27,8 +27,8 @@ import org.codelibs.fess.ds.DataStore;
import org.codelibs.fess.ds.IndexUpdateCallback;
import org.codelibs.fess.es.config.exentity.DataConfig;
import org.codelibs.fess.helper.CrawlingSessionHelper;
import org.codelibs.fess.helper.FieldHelper;
import org.codelibs.fess.helper.SystemHelper;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.util.ComponentUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -56,7 +56,7 @@ public abstract class AbstractDataStoreImpl implements DataStore {
final CrawlingSessionHelper crawlingSessionHelper = ComponentUtil.getCrawlingSessionHelper();
final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
final Long documentExpires = crawlingSessionHelper.getDocumentExpires();
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final FessConfig fessConfig = ComponentUtil.getFessConfig();
initParamMap.putAll(configParamMap);
final Map<String, String> paramMap = initParamMap;
@ -67,32 +67,32 @@ public abstract class AbstractDataStoreImpl implements DataStore {
// cid
final String configId = config.getConfigId();
if (configId != null) {
defaultDataMap.put(fieldHelper.configIdField, configId);
defaultDataMap.put(fessConfig.getIndexFieldConfigId(), configId);
}
// expires
if (documentExpires != null) {
defaultDataMap.put(fieldHelper.expiresField, new Date(documentExpires));
defaultDataMap.put(fessConfig.getIndexFieldExpires(), new Date(documentExpires));
}
// segment
defaultDataMap.put(fieldHelper.segmentField, initParamMap.get(Constants.SESSION_ID));
defaultDataMap.put(fessConfig.getIndexFieldSegment(), initParamMap.get(Constants.SESSION_ID));
// created
defaultDataMap.put(fieldHelper.createdField, systemHelper.getCurrentTime());
defaultDataMap.put(fessConfig.getIndexFieldCreated(), systemHelper.getCurrentTime());
// boost
defaultDataMap.put(fieldHelper.boostField, config.getBoost().toString());
defaultDataMap.put(fessConfig.getIndexFieldBoost(), config.getBoost().toString());
// label: labelType
final List<String> labelTypeList = new ArrayList<String>();
for (final String labelType : config.getLabelTypeValues()) {
labelTypeList.add(labelType);
}
defaultDataMap.put(fieldHelper.labelField, labelTypeList);
defaultDataMap.put(fessConfig.getIndexFieldLabel(), labelTypeList);
// role: roleType
final List<String> roleTypeList = new ArrayList<String>();
for (final String roleType : config.getRoleTypeValues()) {
roleTypeList.add(roleType);
}
defaultDataMap.put(fieldHelper.roleField, roleTypeList);
defaultDataMap.put(fessConfig.getIndexFieldRole(), roleTypeList);
// mimetype
defaultDataMap.put(fieldHelper.mimetypeField, mimeType);
defaultDataMap.put(fessConfig.getIndexFieldMimetype(), mimeType);
// title
// content
// cache

12
src/main/java/org/codelibs/fess/ds/impl/FileListDataStoreImpl.java
View file

@ -41,8 +41,8 @@ import org.codelibs.fess.ds.IndexUpdateCallback;
import org.codelibs.fess.es.client.FessEsClient;
import org.codelibs.fess.es.config.exentity.DataConfig;
import org.codelibs.fess.helper.CrawlingSessionHelper;
import org.codelibs.fess.helper.FieldHelper;
import org.codelibs.fess.helper.IndexingHelper;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.util.ComponentUtil;
import org.lastaflute.di.core.SingletonLaContainer;
import org.slf4j.Logger;
@ -159,15 +159,15 @@ public class FileListDataStoreImpl extends CsvDataStoreImpl {
}
protected boolean addDocument(final Map<String, Object> dataMap) {
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final FessConfig fessConfig = ComponentUtil.getFessConfig();
synchronized (indexUpdateCallback) {
// required check
if (!dataMap.containsKey(fieldHelper.urlField) || dataMap.get(fieldHelper.urlField) == null) {
if (!dataMap.containsKey(fessConfig.getIndexFieldUrl()) || dataMap.get(fessConfig.getIndexFieldUrl()) == null) {
logger.warn("Could not add a doc. Invalid data: " + dataMap);
return false;
}
final String url = dataMap.get(fieldHelper.urlField).toString();
final String url = dataMap.get(fessConfig.getIndexFieldUrl()).toString();
try {
final CrawlerClient client = crawlerClientFactory.getClient(url);
if (client == null) {
@ -227,10 +227,10 @@ public class FileListDataStoreImpl extends CsvDataStoreImpl {
logger.debug("Deleting " + dataMap);
}
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final FessConfig fessConfig = ComponentUtil.getFessConfig();
// required check
if (!dataMap.containsKey(fieldHelper.urlField) || dataMap.get(fieldHelper.urlField) == null) {
if (!dataMap.containsKey(fessConfig.getIndexFieldUrl()) || dataMap.get(fessConfig.getIndexFieldUrl()) == null) {
logger.warn("Could not delete a doc. Invalid data: " + dataMap);
return false;
}

49
src/main/java/org/codelibs/fess/ds/impl/IndexUpdateCallbackImpl.java
View file

@ -20,16 +20,14 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
import javax.annotation.PostConstruct;
import org.codelibs.fess.ds.IndexUpdateCallback;
import org.codelibs.fess.es.client.FessEsClient;
import org.codelibs.fess.exception.FessSystemException;
import org.codelibs.fess.helper.CrawlingSessionHelper;
import org.codelibs.fess.helper.FieldHelper;
import org.codelibs.fess.helper.IndexingHelper;
import org.codelibs.fess.helper.SearchLogHelper;
import org.codelibs.fess.helper.SystemHelper;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.util.ComponentUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -51,35 +49,42 @@ public class IndexUpdateCallbackImpl implements IndexUpdateCallback {
final List<Map<String, Object>> docList = new ArrayList<>();
private FieldHelper fieldHelper;
@PostConstruct
public void init() {
fieldHelper = ComponentUtil.getFieldHelper();
}
/* (non-Javadoc)
* @see org.codelibs.fess.ds.impl.IndexUpdateCallback#store(java.util.Map)
*/
@Override
public synchronized boolean store(final Map<String, Object> dataMap) {
final long startTime = System.currentTimeMillis();
final FessConfig fessConfig = ComponentUtil.getFessConfig();
if (logger.isDebugEnabled()) {
logger.debug("Adding " + dataMap);
}
// required check
final Object urlObj = dataMap.get(fieldHelper.urlField);
final Object urlObj = dataMap.get(fessConfig.getIndexFieldUrl());
if (urlObj == null) {
throw new FessSystemException("url is null. dataMap=" + dataMap);
}
final IndexingHelper indexingHelper = ComponentUtil.getIndexingHelper();
final CrawlingSessionHelper crawlingSessionHelper = ComponentUtil.getCrawlingSessionHelper();
dataMap.put(fieldHelper.idField, crawlingSessionHelper.generateId(dataMap));
dataMap.put(fessConfig.getIndexFieldId(), crawlingSessionHelper.generateId(dataMap));
updateDocument(dataMap);
final String url = dataMap.get(fessConfig.getIndexFieldUrl()).toString();
if (clickCountEnabled) {
addClickCountField(dataMap, url, fessConfig.getIndexFieldClickCount());
}
if (favoriteCountEnabled) {
addFavoriteCountField(dataMap, url, fessConfig.getIndexFieldFavoriteCount());
}
if (!dataMap.containsKey(fessConfig.getIndexFieldDocId())) {
final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
dataMap.put(fessConfig.getIndexFieldDocId(), systemHelper.generateDocId(dataMap));
}
docList.add(dataMap);
if (logger.isDebugEnabled()) {
@ -102,24 +107,6 @@ public class IndexUpdateCallbackImpl implements IndexUpdateCallback {
return true;
}
protected void updateDocument(final Map<String, Object> dataMap) {
final String url = dataMap.get(fieldHelper.urlField).toString();
if (clickCountEnabled) {
addClickCountField(dataMap, url, fieldHelper.clickCountField);
}
if (favoriteCountEnabled) {
addFavoriteCountField(dataMap, url, fieldHelper.favoriteCountField);
}
if (!dataMap.containsKey(fieldHelper.docIdField)) {
final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
dataMap.put(fieldHelper.docIdField, systemHelper.generateDocId(dataMap));
}
}
@Override
public void commit() {
if (!docList.isEmpty()) {

28
src/main/java/org/codelibs/fess/es/client/FessEsClient.java
View file

@ -50,9 +50,9 @@ import org.codelibs.fess.entity.PingResponse;
import org.codelibs.fess.entity.QueryContext;
import org.codelibs.fess.exception.FessSystemException;
import org.codelibs.fess.exception.ResultOffsetExceededException;
import org.codelibs.fess.helper.FieldHelper;
import org.codelibs.fess.helper.QueryHelper;
import org.codelibs.fess.indexer.FessSearchQueryException;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.util.ComponentUtil;
import org.codelibs.fess.util.StreamUtil;
import org.dbflute.exception.IllegalBehaviorStateException;
@ -563,11 +563,11 @@ public class FessEsClient implements Client {
type,
condition,
(response, hit) -> {
FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final Map<String, Object> source = hit.getSource();
if (source != null) {
Map<String, Object> docMap = new HashMap<>(source);
docMap.put(fieldHelper.idField, hit.getId());
docMap.put(fessConfig.getIndexFieldId(), hit.getId());
return docMap;
}
final Map<String, SearchHitField> fields = hit.getFields();
@ -575,7 +575,7 @@ public class FessEsClient implements Client {
Map<String, Object> docMap =
fields.entrySet().stream()
.collect(Collectors.toMap(e -> e.getKey(), e -> (Object) e.getValue().getValues()));
docMap.put(fieldHelper.idField, hit.getId());
docMap.put(fessConfig.getIndexFieldId(), hit.getId());
return docMap;
}
return null;
@ -603,11 +603,11 @@ public class FessEsClient implements Client {
id,
condition,
(response, result) -> {
FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final Map<String, Object> source = response.getSource();
if (source != null) {
Map<String, Object> docMap = new HashMap<>(source);
docMap.put(fieldHelper.idField, response.getId());
docMap.put(fessConfig.getIndexFieldId(), response.getId());
return docMap;
}
final Map<String, GetField> fields = response.getFields();
@ -615,7 +615,7 @@ public class FessEsClient implements Client {
Map<String, Object> docMap =
fields.entrySet().stream()
.collect(Collectors.toMap(e -> e.getKey(), e -> (Object) e.getValue().getValues()));
docMap.put(fieldHelper.idField, response.getId());
docMap.put(fessConfig.getIndexFieldId(), response.getId());
return docMap;
}
return null;
@ -639,11 +639,11 @@ public class FessEsClient implements Client {
type,
condition,
(response, hit) -> {
FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final Map<String, Object> source = hit.getSource();
if (source != null) {
Map<String, Object> docMap = new HashMap<>(source);
docMap.put(fieldHelper.idField, hit.getId());
docMap.put(fessConfig.getIndexFieldId(), hit.getId());
return docMap;
}
final Map<String, SearchHitField> fields = hit.getFields();
@ -651,7 +651,7 @@ public class FessEsClient implements Client {
Map<String, Object> docMap =
fields.entrySet().stream()
.collect(Collectors.toMap(e -> e.getKey(), e -> (Object) e.getValue().getValues()));
docMap.put(fieldHelper.idField, hit.getId());
docMap.put(fessConfig.getIndexFieldId(), hit.getId());
return docMap;
}
return null;
@ -741,10 +741,10 @@ public class FessEsClient implements Client {
}
public void addAll(final String index, final String type, final List<Map<String, Object>> docList) {
FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final BulkRequestBuilder bulkRequestBuilder = client.prepareBulk();
for (final Map<String, Object> doc : docList) {
Object id = doc.remove(fieldHelper.idField);
Object id = doc.remove(fessConfig.getIndexFieldId());
bulkRequestBuilder.add(client.prepareIndex(index, type, id.toString()).setSource(doc));
}
final BulkResponse response = bulkRequestBuilder.execute().actionGet();
@ -877,9 +877,9 @@ public class FessEsClient implements Client {
}
public boolean store(final String index, final String type, final Object obj) {
FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final Map<String, Object> source = BeanUtil.copyBeanToNewMap(obj);
final String id = (String) source.remove(fieldHelper.idField);
final String id = (String) source.remove(fessConfig.getIndexFieldId());
final Long version = (Long) source.remove("version");
IndexResponse response;
try {

9
src/main/java/org/codelibs/fess/exec/Crawler.java
View file

@ -40,7 +40,6 @@ import org.codelibs.fess.crawler.client.EsClient;
import org.codelibs.fess.es.client.FessEsClient;
import org.codelibs.fess.helper.CrawlingSessionHelper;
import org.codelibs.fess.helper.DataIndexHelper;
import org.codelibs.fess.helper.FieldHelper;
import org.codelibs.fess.helper.OverlappingHostHelper;
import org.codelibs.fess.helper.PathMappingHelper;
import org.codelibs.fess.helper.WebFsIndexHelper;
@ -321,7 +320,7 @@ public class Crawler implements Serializable {
final long totalTime = System.currentTimeMillis();
final CrawlingSessionHelper crawlingSessionHelper = ComponentUtil.getCrawlingSessionHelper();
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final FessConfig fessConfig = ComponentUtil.getFessConfig();
boolean completed = false;
int exitCode = Constants.EXIT_OK;
@ -379,10 +378,10 @@ public class Crawler implements Serializable {
// clean up
final QueryBuilder queryBuilder =
QueryBuilders.boolQuery().must(QueryBuilders.rangeQuery(fieldHelper.expiresField).to(new Date()))
.mustNot(QueryBuilders.termQuery(fieldHelper.segmentField, options.sessionId));
QueryBuilders.boolQuery().must(QueryBuilders.rangeQuery(fessConfig.getIndexFieldExpires()).to(new Date()))
.mustNot(QueryBuilders.termQuery(fessConfig.getIndexFieldSegment(), options.sessionId));
try {
fessEsClient.deleteByQuery(fieldHelper.docIndex, fieldHelper.docType, queryBuilder);
fessEsClient.deleteByQuery(fessConfig.getIndexDocumentIndex(), fessConfig.getIndexDocumentType(), queryBuilder);
} catch (final Exception e) {
if (logger.isWarnEnabled()) {
logger.warn("Could not delete expired sessions: " + queryBuilder.toString(), e);

23
src/main/java/org/codelibs/fess/helper/CrawlingSessionHelper.java
View file

@ -30,6 +30,7 @@ import org.codelibs.fess.es.client.FessEsClient;
import org.codelibs.fess.es.config.exentity.CrawlingSession;
import org.codelibs.fess.es.config.exentity.CrawlingSessionInfo;
import org.codelibs.fess.exception.FessSystemException;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.util.ComponentUtil;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.aggregations.AggregationBuilders;
@ -52,7 +53,7 @@ public class CrawlingSessionHelper implements Serializable {
protected Long documentExpires;
private final int maxSessionIdsInList = 100;
public int maxSessionIdsInList;
protected CrawlingSessionService getCrawlingSessionService() {
return SingletonLaContainer.getComponent(CrawlingSessionService.class);
@ -142,30 +143,32 @@ public class CrawlingSessionHelper implements Serializable {
}
public String generateId(final Map<String, Object> dataMap) {
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final String url = (String) dataMap.get(fieldHelper.urlField);
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final String url = (String) dataMap.get(fessConfig.getIndexFieldUrl());
@SuppressWarnings("unchecked")
final List<String> roleTypeList = (List<String>) dataMap.get(fieldHelper.roleField);
final List<String> roleTypeList = (List<String>) dataMap.get(fessConfig.getIndexFieldRole());
return generateId(url, roleTypeList);
}
public List<Map<String, String>> getSessionIdList(final FessEsClient fessEsClient) {
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
return fessEsClient.search(fieldHelper.docIndex, fieldHelper.docType,
final FessConfig fessConfig = ComponentUtil.getFessConfig();
return fessEsClient.search(
fessConfig.getIndexDocumentIndex(),
fessConfig.getIndexDocumentType(),
queryRequestBuilder -> {
queryRequestBuilder.setQuery(QueryBuilders.matchAllQuery());
final TermsBuilder termsBuilder =
AggregationBuilders.terms(fieldHelper.segmentField).field(fieldHelper.segmentField).size(maxSessionIdsInList)
.order(Order.term(false));
AggregationBuilders.terms(fessConfig.getIndexFieldSegment()).field(fessConfig.getIndexFieldSegment())
.size(maxSessionIdsInList).order(Order.term(false));
queryRequestBuilder.addAggregation(termsBuilder);
return true;
}, (queryRequestBuilder, execTime, searchResponse) -> {
final List<Map<String, String>> sessionIdList = new ArrayList<Map<String, String>>();
searchResponse.ifPresent(response -> {
final Terms terms = response.getAggregations().get(fieldHelper.segmentField);
final Terms terms = response.getAggregations().get(fessConfig.getIndexFieldSegment());
for (final Bucket bucket : terms.getBuckets()) {
final Map<String, String> map = new HashMap<String, String>(2);
map.put(fieldHelper.segmentField, bucket.getKey().toString());
map.put(fessConfig.getIndexFieldSegment(), bucket.getKey().toString());
map.put(FACET_COUNT_KEY, Long.toString(bucket.getDocCount()));
sessionIdList.add(map);
}

10
src/main/java/org/codelibs/fess/helper/DataIndexHelper.java
View file

@ -32,6 +32,7 @@ import org.codelibs.fess.ds.DataStore;
import org.codelibs.fess.ds.DataStoreFactory;
import org.codelibs.fess.ds.IndexUpdateCallback;
import org.codelibs.fess.es.config.exentity.DataConfig;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.util.ComponentUtil;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
@ -252,12 +253,13 @@ public class DataIndexHelper implements Serializable {
logger.warn("Invalid sessionId at " + dataConfig);
return;
}
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final QueryBuilder queryBuilder =
QueryBuilders.boolQuery().must(QueryBuilders.termQuery(fieldHelper.configIdField, dataConfig.getConfigId()))
.mustNot(QueryBuilders.termQuery(fieldHelper.segmentField, sessionId));
QueryBuilders.boolQuery().must(QueryBuilders.termQuery(fessConfig.getIndexFieldConfigId(), dataConfig.getConfigId()))
.mustNot(QueryBuilders.termQuery(fessConfig.getIndexFieldSegment(), sessionId));
try {
ComponentUtil.getElasticsearchClient().deleteByQuery(fieldHelper.docIndex, fieldHelper.docType, queryBuilder);
ComponentUtil.getElasticsearchClient().deleteByQuery(fessConfig.getIndexDocumentIndex(), fessConfig.getIndexDocumentType(),
queryBuilder);
} catch (final Exception e) {
logger.error("Could not delete old docs at " + dataConfig, e);
}

80
src/main/java/org/codelibs/fess/helper/FieldHelper.java
View file

@ -1,80 +0,0 @@
/*
* Copyright 2012-2015 CodeLibs Project and the Others.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
* either express or implied. See the License for the specific language
* governing permissions and limitations under the License.
*/
package org.codelibs.fess.helper;
// TODO move to properties file
public class FieldHelper {
public String favoriteCountField = "favorite_count";
public String clickCountField = "click_count";
public String configIdField = "config_id";
public String expiresField = "expires";
public String urlField = "url";
public String docIdField = "doc_id";
public String idField = "_id";
public String langField = "lang";
public String hasCacheField = "has_cache";
public String lastModifiedField = "last_modified";
public String anchorField = "anchor";
public String segmentField = "segment";
public String roleField = "role";
public String boostField = "boost";
public String createdField = "created";
public String labelField = "label";
public String mimetypeField = "mimetype";
public String parentIdField = "parent_id";
public String contentField = "content";
public String cacheField = "cache";
public String digestField = "digest";
public String titleField = "title";
public String hostField = "host";
public String siteField = "site";
public String contentLengthField = "content_length";
public String filetypeField = "filetype";
public String docIndex = "fess";
public String docType = "doc";
public String configIndex = ".fess_config";
public String boostDocumentRuleType = "boost_document_rule";
}

61
src/main/java/org/codelibs/fess/helper/IndexingHelper.java
View file

@ -20,6 +20,7 @@ import java.util.List;
import java.util.Map;
import org.codelibs.fess.es.client.FessEsClient;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.util.ComponentUtil;
import org.elasticsearch.action.count.CountResponse;
import org.elasticsearch.index.query.QueryBuilder;
@ -43,8 +44,8 @@ public class IndexingHelper {
}
synchronized (fessEsClient) {
deleteOldDocuments(fessEsClient, docList);
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
fessEsClient.addAll(fieldHelper.docIndex, fieldHelper.docType, docList);
final FessConfig fessConfig = ComponentUtil.getFessConfig();
fessEsClient.addAll(fessConfig.getIndexDocumentIndex(), fessConfig.getIndexDocumentType(), docList);
}
if (logger.isInfoEnabled()) {
logger.info("Sent " + docList.size() + " docs (" + (System.currentTimeMillis() - execTime) + "ms)");
@ -53,30 +54,32 @@ public class IndexingHelper {
}
private void deleteOldDocuments(final FessEsClient fessEsClient, final List<Map<String, Object>> docList) {
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final List<String> docIdList = new ArrayList<>();
for (final Map<String, Object> inputDoc : docList) {
final Object idValue = inputDoc.get(fieldHelper.idField);
final Object idValue = inputDoc.get(fessConfig.getIndexFieldId());
if (idValue == null) {
continue;
}
final Object configIdValue = inputDoc.get(fieldHelper.configIdField);
final Object configIdValue = inputDoc.get(fessConfig.getIndexFieldConfigId());
if (configIdValue == null) {
continue;
}
final QueryBuilder queryBuilder =
QueryBuilders.boolQuery().must(QueryBuilders.termQuery(fieldHelper.urlField, inputDoc.get(fieldHelper.urlField)))
.filter(QueryBuilders.termQuery(fieldHelper.configIdField, configIdValue));
QueryBuilders.boolQuery()
.must(QueryBuilders.termQuery(fessConfig.getIndexFieldUrl(), inputDoc.get(fessConfig.getIndexFieldUrl())))
.filter(QueryBuilders.termQuery(fessConfig.getIndexFieldConfigId(), configIdValue));
final List<Map<String, Object>> docs =
getDocumentListByQuery(fessEsClient, queryBuilder, new String[] { fieldHelper.idField, fieldHelper.docIdField });
getDocumentListByQuery(fessEsClient, queryBuilder,
new String[] { fessConfig.getIndexFieldId(), fessConfig.getIndexFieldDocId() });
for (final Map<String, Object> doc : docs) {
final Object oldIdValue = doc.get(fieldHelper.idField);
final Object oldIdValue = doc.get(fessConfig.getIndexFieldId());
if (!idValue.equals(oldIdValue) && oldIdValue != null) {
final Object oldDocIdValue = doc.get(fieldHelper.docIdField);
final Object oldDocIdValue = doc.get(fessConfig.getIndexFieldDocId());
if (oldDocIdValue != null) {
docIdList.add(oldDocIdValue.toString());
}
@ -87,57 +90,59 @@ public class IndexingHelper {
}
}
if (!docIdList.isEmpty()) {
fessEsClient.deleteByQuery(fieldHelper.docIndex, fieldHelper.docType,
QueryBuilders.idsQuery(fieldHelper.docType).ids(docIdList.stream().toArray(n -> new String[n])));
fessEsClient.deleteByQuery(fessConfig.getIndexDocumentIndex(), fessConfig.getIndexDocumentType(),
QueryBuilders.idsQuery(fessConfig.getIndexDocumentType()).ids(docIdList.stream().toArray(n -> new String[n])));
}
}
public void deleteDocument(final FessEsClient fessEsClient, final String id) {
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
fessEsClient.delete(fieldHelper.docIndex, fieldHelper.docType, id, 0);
final FessConfig fessConfig = ComponentUtil.getFessConfig();
fessEsClient.delete(fessConfig.getIndexDocumentIndex(), fessConfig.getIndexDocumentType(), id, 0);
}
public void deleteDocumentsByDocId(final FessEsClient fessEsClient, final List<String> docIdList) {
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
fessEsClient.deleteByQuery(fieldHelper.docIndex, fieldHelper.docType,
QueryBuilders.idsQuery(fieldHelper.docType).ids(docIdList.stream().toArray(n -> new String[n])));
final FessConfig fessConfig = ComponentUtil.getFessConfig();
fessEsClient.deleteByQuery(fessConfig.getIndexDocumentIndex(), fessConfig.getIndexDocumentType(),
QueryBuilders.idsQuery(fessConfig.getIndexDocumentType()).ids(docIdList.stream().toArray(n -> new String[n])));
}
public Map<String, Object> getDocument(final FessEsClient fessEsClient, final String id, final String[] fields) {
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
return fessEsClient.getDocument(fieldHelper.docIndex, fieldHelper.docType, id, requestBuilder -> {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
return fessEsClient.getDocument(fessConfig.getIndexDocumentIndex(), fessConfig.getIndexDocumentType(), id, requestBuilder -> {
return true;
}).orElseGet(() -> null);
}
public List<Map<String, Object>> getDocumentListByPrefixId(final FessEsClient fessEsClient, final String id, final String[] fields) {
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final QueryBuilder queryBuilder = QueryBuilders.prefixQuery(fieldHelper.idField, id);
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final QueryBuilder queryBuilder = QueryBuilders.prefixQuery(fessConfig.getIndexFieldId(), id);
return getDocumentListByQuery(fessEsClient, queryBuilder, fields);
}
public void deleteChildDocument(final FessEsClient fessEsClient, final String id) {
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
fessEsClient.deleteByQuery(fieldHelper.docIndex, fieldHelper.docType, QueryBuilders.termQuery(fieldHelper.parentIdField, id));
final FessConfig fessConfig = ComponentUtil.getFessConfig();
fessEsClient.deleteByQuery(fessConfig.getIndexDocumentIndex(), fessConfig.getIndexDocumentType(),
QueryBuilders.termQuery(fessConfig.getIndexFieldParentId(), id));
}
public List<Map<String, Object>> getChildDocumentList(final FessEsClient fessEsClient, final String id, final String[] fields) {
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final QueryBuilder queryBuilder = QueryBuilders.termQuery(fieldHelper.parentIdField, id);
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final QueryBuilder queryBuilder = QueryBuilders.termQuery(fessConfig.getIndexFieldParentId(), id);
return getDocumentListByQuery(fessEsClient, queryBuilder, fields);
}
protected List<Map<String, Object>> getDocumentListByQuery(final FessEsClient fessEsClient, final QueryBuilder queryBuilder,
final String[] fields) {
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final CountResponse countResponse =
fessEsClient.prepareCount(fieldHelper.docIndex).setTypes(fieldHelper.docType).setQuery(queryBuilder).execute().actionGet();
fessEsClient.prepareCount(fessConfig.getIndexDocumentIndex()).setTypes(fessConfig.getIndexDocumentType())
.setQuery(queryBuilder).execute().actionGet();
final long numFound = countResponse.getCount();
// TODO max threshold
return fessEsClient.getDocumentList(fieldHelper.docIndex, fieldHelper.docType, requestBuilder -> {
return fessEsClient.getDocumentList(fessConfig.getIndexDocumentIndex(), fessConfig.getIndexDocumentType(), requestBuilder -> {
requestBuilder.setQuery(queryBuilder).setSize((int) numFound);
if (fields != null) {
requestBuilder.addFields(fields);

13
src/main/java/org/codelibs/fess/helper/KeyMatchHelper.java
View file

@ -28,6 +28,7 @@ import org.codelibs.fess.app.service.KeyMatchService;
import org.codelibs.fess.es.client.FessEsClient;
import org.codelibs.fess.es.client.FessEsClient.SearchConditionBuilder;
import org.codelibs.fess.es.config.exentity.KeyMatch;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.util.ComponentUtil;
import org.lastaflute.di.core.SingletonLaContainer;
@ -48,7 +49,7 @@ public class KeyMatchHelper {
}
protected void reload(final long interval) {
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final KeyMatchService keyMatchService = SingletonLaContainer.getComponent(KeyMatchService.class);
final List<KeyMatch> list = keyMatchService.getAvailableKeyMatchList();
final Map<String, String[]> keyMatchQueryMap = new HashMap<String, String[]>(list.size());
@ -56,9 +57,9 @@ public class KeyMatchHelper {
final List<Map<String, Object>> documentList = getDocumentList(keyMatch);
final List<String> docIdList = new ArrayList<String>();
for (final Map<String, Object> map : documentList) {
final String docId = (String) map.get(fieldHelper.docIdField);
final String docId = (String) map.get(fessConfig.getIndexFieldDocId());
if (StringUtil.isNotBlank(docId)) {
docIdList.add(fieldHelper.docIdField + ":" + docId + "^" + keyMatch.getBoost());
docIdList.add(fessConfig.getIndexFieldDocId() + ":" + docId + "^" + keyMatch.getBoost());
}
}
if (!docIdList.isEmpty()) {
@ -78,12 +79,12 @@ public class KeyMatchHelper {
protected List<Map<String, Object>> getDocumentList(final KeyMatch keyMatch) {
final FessEsClient fessEsClient = ComponentUtil.getElasticsearchClient();
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final List<Map<String, Object>> documentList =
fessEsClient.getDocumentList(fieldHelper.docIndex, fieldHelper.docType,
fessEsClient.getDocumentList(fessConfig.getIndexDocumentIndex(), fessConfig.getIndexDocumentType(),
searchRequestBuilder -> {
return SearchConditionBuilder.builder(searchRequestBuilder).administrativeAccess().size(keyMatch.getMaxSize())
.query(keyMatch.getQuery()).responseFields(new String[] { fieldHelper.docIdField }).build();
.query(keyMatch.getQuery()).responseFields(new String[] { fessConfig.getIndexFieldDocId() }).build();
});
return documentList;
}

68
src/main/java/org/codelibs/fess/helper/QueryHelper.java
View file

@ -53,6 +53,8 @@ import org.codelibs.fess.entity.FacetInfo;
import org.codelibs.fess.entity.GeoInfo;
import org.codelibs.fess.entity.QueryContext;
import org.codelibs.fess.exception.InvalidQueryException;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.util.ComponentUtil;
import org.codelibs.fess.util.StreamUtil;
import org.dbflute.optional.OptionalEntity;
import org.dbflute.optional.OptionalThing;
@ -104,9 +106,6 @@ public class QueryHelper implements Serializable {
@Resource
protected SystemHelper systemHelper;
@Resource
protected FieldHelper fieldHelper;
@Resource
protected KeyMatchHelper keyMatchHelper;
@ -162,45 +161,54 @@ public class QueryHelper implements Serializable {
@PostConstruct
public void init() {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
if (responseFields == null) {
responseFields =
new String[] { SCORE_FIELD, fieldHelper.idField, fieldHelper.docIdField, fieldHelper.boostField,
fieldHelper.contentLengthField, fieldHelper.hostField, fieldHelper.siteField, fieldHelper.lastModifiedField,
fieldHelper.mimetypeField, fieldHelper.filetypeField, fieldHelper.createdField, fieldHelper.titleField,
fieldHelper.digestField, fieldHelper.urlField, fieldHelper.clickCountField, fieldHelper.favoriteCountField,
fieldHelper.configIdField, fieldHelper.langField, fieldHelper.hasCacheField };
new String[] { SCORE_FIELD, fessConfig.getIndexFieldId(), fessConfig.getIndexFieldDocId(),
fessConfig.getIndexFieldBoost(), fessConfig.getIndexFieldContentLength(), fessConfig.getIndexFieldHost(),
fessConfig.getIndexFieldSite(), fessConfig.getIndexFieldLastModified(), fessConfig.getIndexFieldMimetype(),
fessConfig.getIndexFieldFiletype(), fessConfig.getIndexFieldCreated(), fessConfig.getIndexFieldTitle(),
fessConfig.getIndexFieldDigest(), fessConfig.getIndexFieldUrl(), fessConfig.getIndexFieldClickCount(),
fessConfig.getIndexFieldFavoriteCount(), fessConfig.getIndexFieldConfigId(), fessConfig.getIndexFieldLang(),
fessConfig.getIndexFieldHasCache() };
}
if (cacheResponseFields == null) {
cacheResponseFields =
new String[] { SCORE_FIELD, fieldHelper.idField, fieldHelper.docIdField, fieldHelper.boostField,
fieldHelper.contentLengthField, fieldHelper.hostField, fieldHelper.siteField, fieldHelper.lastModifiedField,
fieldHelper.mimetypeField, fieldHelper.filetypeField, fieldHelper.createdField, fieldHelper.titleField,
fieldHelper.digestField, fieldHelper.urlField, fieldHelper.clickCountField, fieldHelper.favoriteCountField,
fieldHelper.configIdField, fieldHelper.langField, fieldHelper.cacheField };
new String[] { SCORE_FIELD, fessConfig.getIndexFieldId(), fessConfig.getIndexFieldDocId(),
fessConfig.getIndexFieldBoost(), fessConfig.getIndexFieldContentLength(), fessConfig.getIndexFieldHost(),
fessConfig.getIndexFieldSite(), fessConfig.getIndexFieldLastModified(), fessConfig.getIndexFieldMimetype(),
fessConfig.getIndexFieldFiletype(), fessConfig.getIndexFieldCreated(), fessConfig.getIndexFieldTitle(),
fessConfig.getIndexFieldDigest(), fessConfig.getIndexFieldUrl(), fessConfig.getIndexFieldClickCount(),
fessConfig.getIndexFieldFavoriteCount(), fessConfig.getIndexFieldConfigId(), fessConfig.getIndexFieldLang(),
fessConfig.getIndexFieldCache() };
}
if (responseDocValuesFields == null) {
responseDocValuesFields = new String[] { fieldHelper.clickCountField, fieldHelper.favoriteCountField };
responseDocValuesFields = new String[] { fessConfig.getIndexFieldClickCount(), fessConfig.getIndexFieldFavoriteCount() };
}
if (highlightedFields == null) {
highlightedFields = new String[] { fieldHelper.contentField };
highlightedFields = new String[] { fessConfig.getIndexFieldContent() };
}
if (searchFields == null) {
searchFields =
new String[] { INURL_FIELD, fieldHelper.urlField, fieldHelper.docIdField, fieldHelper.hostField,
fieldHelper.titleField, fieldHelper.contentField, fieldHelper.contentLengthField,
fieldHelper.lastModifiedField, fieldHelper.mimetypeField, fieldHelper.filetypeField, fieldHelper.labelField,
fieldHelper.segmentField, fieldHelper.clickCountField, fieldHelper.favoriteCountField, fieldHelper.langField };
new String[] { INURL_FIELD, fessConfig.getIndexFieldUrl(), fessConfig.getIndexFieldDocId(),
fessConfig.getIndexFieldHost(), fessConfig.getIndexFieldTitle(), fessConfig.getIndexFieldContent(),
fessConfig.getIndexFieldContentLength(), fessConfig.getIndexFieldLastModified(),
fessConfig.getIndexFieldMimetype(), fessConfig.getIndexFieldFiletype(), fessConfig.getIndexFieldLabel(),
fessConfig.getIndexFieldSegment(), fessConfig.getIndexFieldClickCount(),
fessConfig.getIndexFieldFavoriteCount(), fessConfig.getIndexFieldLang() };
}
if (facetFields == null) {
facetFields =
new String[] { fieldHelper.urlField, fieldHelper.hostField, fieldHelper.titleField, fieldHelper.contentField,
fieldHelper.contentLengthField, fieldHelper.lastModifiedField, fieldHelper.mimetypeField,
fieldHelper.filetypeField, fieldHelper.labelField, fieldHelper.segmentField };
new String[] { fessConfig.getIndexFieldUrl(), fessConfig.getIndexFieldHost(), fessConfig.getIndexFieldTitle(),
fessConfig.getIndexFieldContent(), fessConfig.getIndexFieldContentLength(),
fessConfig.getIndexFieldLastModified(), fessConfig.getIndexFieldMimetype(), fessConfig.getIndexFieldFiletype(),
fessConfig.getIndexFieldLabel(), fessConfig.getIndexFieldSegment() };
}
if (supportedSortFields == null) {
supportedSortFields =
new String[] { fieldHelper.createdField, fieldHelper.contentLengthField, fieldHelper.lastModifiedField,
fieldHelper.clickCountField, fieldHelper.favoriteCountField };
new String[] { fessConfig.getIndexFieldCreated(), fessConfig.getIndexFieldContentLength(),
fessConfig.getIndexFieldLastModified(), fessConfig.getIndexFieldClickCount(),
fessConfig.getIndexFieldFavoriteCount() };
}
}
@ -230,10 +238,11 @@ public class QueryHelper implements Serializable {
if (roleQueryHelper != null) {
final Set<String> roleSet = roleQueryHelper.build();
if (!roleSet.isEmpty()) {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
queryContext.addQuery(boolQuery -> {
BoolQueryBuilder roleQuery = QueryBuilders.boolQuery();
roleSet.stream().forEach(name -> {
roleQuery.filter(QueryBuilders.termQuery(fieldHelper.roleField, name));
roleQuery.filter(QueryBuilders.termQuery(fessConfig.getIndexFieldRole(), name));
});
boolQuery.filter(roleQuery);
});
@ -447,16 +456,17 @@ public class QueryHelper implements Serializable {
}
private QueryBuilder buildDefaultQueryBuilder(final Function<String, QueryBuilder> builder) {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
// TODO boost
final BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
final QueryBuilder titleQuery = builder.apply(fieldHelper.titleField);
final QueryBuilder titleQuery = builder.apply(fessConfig.getIndexFieldTitle());
boolQuery.should(titleQuery);
final QueryBuilder contentQuery = builder.apply(fieldHelper.contentField);
final QueryBuilder contentQuery = builder.apply(fessConfig.getIndexFieldContent());
boolQuery.should(contentQuery);
getQueryLanguage().ifPresent(lang -> {
final QueryBuilder titleLangQuery = builder.apply(fieldHelper.titleField + "_" + lang);
final QueryBuilder titleLangQuery = builder.apply(fessConfig.getIndexFieldTitle() + "_" + lang);
boolQuery.should(titleLangQuery);
final QueryBuilder contentLangQuery = builder.apply(fieldHelper.contentField + "_" + lang);
final QueryBuilder contentLangQuery = builder.apply(fessConfig.getIndexFieldContent() + "_" + lang);
boolQuery.should(contentLangQuery);
});
return boolQuery;

32
src/main/java/org/codelibs/fess/helper/SearchLogHelper.java
View file

@ -40,6 +40,7 @@ import org.codelibs.fess.es.log.exentity.ClickLog;
import org.codelibs.fess.es.log.exentity.SearchFieldLog;
import org.codelibs.fess.es.log.exentity.SearchLog;
import org.codelibs.fess.es.log.exentity.UserInfo;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.util.ComponentUtil;
import org.codelibs.fess.util.DocumentUtil;
import org.codelibs.fess.util.StreamUtil;
@ -55,9 +56,6 @@ public class SearchLogHelper {
@Resource
protected DynamicProperties crawlerProperties;
@Resource
protected FieldHelper fieldHelper;
public long userCheckInterval = 5 * 60 * 1000;// 5 min
public int userInfoCacheSize = 1000;
@ -244,18 +242,24 @@ public class SearchLogHelper {
final SearchService searchService = ComponentUtil.getComponent(SearchService.class);
try {
searchService.bulkUpdate(builder -> {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
searchService.getDocumentListByDocIds(clickCountMap.keySet().toArray(new String[clickCountMap.size()]),
new String[] { fieldHelper.docIdField }).forEach(doc -> {
String id = DocumentUtil.getValue(doc, fieldHelper.docIdField, String.class);
String docId = DocumentUtil.getValue(doc, fieldHelper.docIdField, String.class);
if (id != null && docId != null && clickCountMap.containsKey(docId)) {
Integer count = clickCountMap.get(docId);
Script script = new Script("ctx._source." + fieldHelper.clickCountField + "+=" + count.toString());
Map<String, Object> upsertMap = new HashMap<>();
upsertMap.put(fieldHelper.clickCountField, 1);
builder.add(new UpdateRequest(fieldHelper.docIndex, fieldHelper.docType, id).script(script).upsert(upsertMap));
}
});
new String[] { fessConfig.getIndexFieldDocId() })
.forEach(
doc -> {
String id = DocumentUtil.getValue(doc, fessConfig.getIndexFieldDocId(), String.class);
String docId = DocumentUtil.getValue(doc, fessConfig.getIndexFieldDocId(), String.class);
if (id != null && docId != null && clickCountMap.containsKey(docId)) {
Integer count = clickCountMap.get(docId);
Script script =
new Script("ctx._source." + fessConfig.getIndexFieldClickCount() + "+="
+ count.toString());
Map<String, Object> upsertMap = new HashMap<>();
upsertMap.put(fessConfig.getIndexFieldClickCount(), 1);
builder.add(new UpdateRequest(fessConfig.getIndexDocumentIndex(), fessConfig
.getIndexDocumentType(), id).script(script).upsert(upsertMap));
}
});
});
} catch (final Exception e) {
logger.warn("Failed to update clickCounts", e);

13
src/main/java/org/codelibs/fess/helper/SuggestHelper.java
View file

@ -34,6 +34,7 @@ import org.codelibs.fess.es.config.exbhv.SuggestElevateWordBhv;
import org.codelibs.fess.es.config.exentity.SuggestBadWord;
import org.codelibs.fess.es.log.exentity.SearchFieldLog;
import org.codelibs.fess.es.log.exentity.SearchLog;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.suggest.Suggester;
import org.codelibs.fess.suggest.constants.FieldNames;
import org.codelibs.fess.suggest.entity.SuggestItem;
@ -41,6 +42,7 @@ import org.codelibs.fess.suggest.index.contents.document.DocumentReader;
import org.codelibs.fess.suggest.index.contents.document.ESSourceReader;
import org.codelibs.fess.suggest.settings.SuggestSettings;
import org.codelibs.fess.suggest.util.SuggestUtil;
import org.codelibs.fess.util.ComponentUtil;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.slf4j.Logger;
@ -58,9 +60,6 @@ public class SuggestHelper {
@Resource
protected FessEsClient fessEsClient;
@Resource
protected FieldHelper fieldHelper;
public String[] contentFieldNames = { "_default" };
public String[] tagFieldNames = { "label" };
@ -78,8 +77,9 @@ public class SuggestHelper {
@PostConstruct
public void init() {
final Thread th = new Thread(() -> {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
fessEsClient.admin().cluster().prepareHealth().setWaitForYellowStatus().execute().actionGet();
suggester = Suggester.builder().build(fessEsClient, fieldHelper.docIndex);
suggester = Suggester.builder().build(fessEsClient, fessConfig.getIndexDocumentIndex());
suggester.settings().array().delete(SuggestSettings.DefaultKeys.SUPPORTED_FIELDS);
for (final String field : contentsIndex) {
suggester.settings().array().add(SuggestSettings.DefaultKeys.SUPPORTED_FIELDS, field);
@ -136,7 +136,10 @@ public class SuggestHelper {
}
}
DocumentReader reader = new ESSourceReader(fessEsClient, suggester.settings(), fieldHelper.docIndex, fieldHelper.docType);
final FessConfig fessConfig = ComponentUtil.getFessConfig();
DocumentReader reader =
new ESSourceReader(fessEsClient, suggester.settings(), fessConfig.getIndexDocumentIndex(),
fessConfig.getIndexDocumentType());
suggester.indexer().indexFromDocument(reader, 2, 100).done(response -> {
suggester.refresh();

29
src/main/java/org/codelibs/fess/helper/ViewHelper.java
View file

@ -57,6 +57,7 @@ import org.codelibs.fess.es.config.exentity.CrawlingConfig;
import org.codelibs.fess.es.config.exentity.CrawlingConfig.ConfigType;
import org.codelibs.fess.exception.FessSystemException;
import org.codelibs.fess.helper.UserAgentHelper.UserAgentType;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.util.ComponentUtil;
import org.codelibs.fess.util.DocumentUtil;
import org.codelibs.fess.util.ResourceUtil;
@ -146,12 +147,12 @@ public class ViewHelper implements Serializable {
public String getContentTitle(final Map<String, Object> document) {
final int size = titleLength;
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final FessConfig fessConfig = ComponentUtil.getFessConfig();
String title;
if (StringUtil.isNotBlank(getString(document, fieldHelper.titleField))) {
title = getString(document, fieldHelper.titleField);
if (StringUtil.isNotBlank(getString(document, fessConfig.getIndexFieldTitle()))) {
title = getString(document, fessConfig.getIndexFieldTitle());
} else {
title = getString(document, fieldHelper.urlField);
title = getString(document, fessConfig.getIndexFieldUrl());
}
return StringUtils.abbreviate(title, size);
}
@ -159,6 +160,7 @@ public class ViewHelper implements Serializable {
public String getContentDescription(final Map<String, Object> document) {
final Set<String> queries = new HashSet<>();
LaRequestUtil.getOptionalRequest().ifPresent(request -> {
@SuppressWarnings("unchecked")
Set<String> set = (Set<String>) request.getAttribute(Constants.HIGHLIGHT_QUERIES);
if (set != null) {
queries.addAll(set);
@ -303,8 +305,8 @@ public class ViewHelper implements Serializable {
protected String appendQueryParameter(final Map<String, Object> document, final String url) {
if (Constants.TRUE.equals(crawlerProperties.get(Constants.APPEND_QUERY_PARAMETER_PROPERTY))) {
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final String mimetype = getString(document, fieldHelper.mimetypeField);
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final String mimetype = getString(document, fessConfig.getIndexFieldMimetype());
if (StringUtil.isNotBlank(mimetype)) {
if ("application/pdf".equals(mimetype)) {
return appendPDFSearchWord(url);
@ -318,6 +320,7 @@ public class ViewHelper implements Serializable {
}
protected String appendPDFSearchWord(final String url) {
@SuppressWarnings("unchecked")
final Set<String> queries = (Set<String>) LaRequestUtil.getRequest().getAttribute(Constants.HIGHLIGHT_QUERIES);
if (queries != null) {
final StringBuilder buf = new StringBuilder(url.length() + 100);
@ -387,7 +390,7 @@ public class ViewHelper implements Serializable {
}
public String createCacheContent(final Map<String, Object> doc, final String[] queries) {
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final FileTemplateLoader loader = new FileTemplateLoader(ResourceUtil.getViewTemplatePath().toFile());
final Handlebars handlebars = new Handlebars(loader);
@ -400,7 +403,7 @@ public class ViewHelper implements Serializable {
url = ComponentUtil.getMessageManager().getMessage(locale, "labels.search_unknown");
}
String createdStr;
final Long created = DocumentUtil.getValue(doc, fieldHelper.createdField, Long.class);
final Long created = DocumentUtil.getValue(doc, fessConfig.getIndexFieldCreated(), Long.class);
if (created != null) {
final SimpleDateFormat sdf = new SimpleDateFormat(CoreLibConstants.DATE_FORMAT_ISO_8601_EXTEND);
createdStr = sdf.format(new Date(created.longValue()));
@ -412,7 +415,7 @@ public class ViewHelper implements Serializable {
doc.put("queries", queries);
String cache = (String) doc.get(fieldHelper.cacheField);
String cache = (String) doc.get(fessConfig.getIndexFieldCache());
if (cache != null) {
cache = pathMappingHelper.replaceUrls(cache);
if (queries != null && queries.length > 0) {
@ -421,7 +424,7 @@ public class ViewHelper implements Serializable {
doc.put("hlCache", cache);
}
} else {
doc.put(fieldHelper.cacheField, StringUtil.EMPTY);
doc.put(fessConfig.getIndexFieldCache(), StringUtil.EMPTY);
doc.put("hlCache", StringUtil.EMPTY);
}
@ -479,9 +482,9 @@ public class ViewHelper implements Serializable {
if (logger.isDebugEnabled()) {
logger.debug("writing the content of: " + doc);
}
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final CrawlingConfigHelper crawlingConfigHelper = ComponentUtil.getCrawlingConfigHelper();
final Object configIdObj = doc.get(fieldHelper.configIdField);
final Object configIdObj = doc.get(fessConfig.getIndexFieldConfigId());
if (configIdObj == null) {
throw new FessSystemException("configId is null.");
}
@ -507,7 +510,7 @@ public class ViewHelper implements Serializable {
if (config == null) {
throw new FessSystemException("No crawlingConfig: " + configIdObj);
}
final String url = (String) doc.get(fieldHelper.urlField);
final String url = (String) doc.get(fessConfig.getIndexFieldUrl());
final CrawlerClientFactory crawlerClientFactory = SingletonLaContainer.getComponent(CrawlerClientFactory.class);
config.initializeClientFactory(crawlerClientFactory);
final CrawlerClient client = crawlerClientFactory.getClient(url);

6
src/main/java/org/codelibs/fess/helper/impl/CookieUserInfoHelperImpl.java
View file

@ -28,9 +28,9 @@ import javax.servlet.http.HttpSession;
import org.codelibs.core.collection.LruHashMap;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.fess.Constants;
import org.codelibs.fess.helper.FieldHelper;
import org.codelibs.fess.helper.SearchLogHelper;
import org.codelibs.fess.helper.UserInfoHelper;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.util.ComponentUtil;
import org.lastaflute.web.util.LaRequestUtil;
import org.lastaflute.web.util.LaResponseUtil;
@ -119,11 +119,11 @@ public class CookieUserInfoHelperImpl implements UserInfoHelper {
public void storeQueryId(final String queryId, final List<Map<String, Object>> documentItems) {
final HttpSession session = LaRequestUtil.getRequest().getSession(false);
if (session != null) {
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final List<String> docIdList = new ArrayList<String>();
for (final Map<String, Object> map : documentItems) {
final Object docId = map.get(fieldHelper.docIdField);
final Object docId = map.get(fessConfig.getIndexFieldDocId());
if (docId != null && docId.toString().length() > 0) {
docIdList.add(docId.toString());
}

23
src/main/java/org/codelibs/fess/indexer/IndexUpdater.java
View file

@ -40,11 +40,11 @@ import org.codelibs.fess.es.client.FessEsClient;
import org.codelibs.fess.es.log.exbhv.ClickLogBhv;
import org.codelibs.fess.es.log.exbhv.FavoriteLogBhv;
import org.codelibs.fess.exception.FessSystemException;
import org.codelibs.fess.helper.FieldHelper;
import org.codelibs.fess.helper.IndexingHelper;
import org.codelibs.fess.helper.IntervalControlHelper;
import org.codelibs.fess.helper.SearchLogHelper;
import org.codelibs.fess.helper.SystemHelper;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.util.ComponentUtil;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.index.query.QueryBuilder;
@ -80,9 +80,6 @@ public class IndexUpdater extends Thread {
@Resource
protected SystemHelper systemHelper;
@Resource
protected FieldHelper fieldHelper;
@Resource
protected IndexingHelper indexingHelper;
@ -405,24 +402,27 @@ public class IndexUpdater extends Thread {
addBoostValue(map, documentBoost);
}
if (!map.containsKey(fieldHelper.docIdField)) {
map.put(fieldHelper.docIdField, systemHelper.generateDocId(map));
final FessConfig fessConfig = ComponentUtil.getFessConfig();
if (!map.containsKey(fessConfig.getIndexFieldDocId())) {
map.put(fessConfig.getIndexFieldDocId(), systemHelper.generateDocId(map));
}
}
protected void addBoostValue(final Map<String, Object> map, final float documentBoost) {
map.put(fieldHelper.boostField, documentBoost);
final FessConfig fessConfig = ComponentUtil.getFessConfig();
map.put(fessConfig.getIndexFieldBoost(), documentBoost);
if (logger.isDebugEnabled()) {
logger.debug("Set a document boost (" + documentBoost + ").");
}
}
protected void addClickCountField(final Map<String, Object> doc) {
final String url = (String) doc.get(fieldHelper.urlField);
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final String url = (String) doc.get(fessConfig.getIndexFieldUrl());
if (StringUtil.isNotBlank(url)) {
final SearchLogHelper searchLogHelper = ComponentUtil.getSearchLogHelper();
final int count = searchLogHelper.getClickCount(url);
doc.put(fieldHelper.clickCountField, count);
doc.put(fessConfig.getIndexFieldClickCount(), count);
if (logger.isDebugEnabled()) {
logger.debug("Click Count: " + count + ", url: " + url);
}
@ -430,11 +430,12 @@ public class IndexUpdater extends Thread {
}
protected void addFavoriteCountField(final Map<String, Object> map) {
final String url = (String) map.get(fieldHelper.urlField);
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final String url = (String) map.get(fessConfig.getIndexFieldUrl());
if (StringUtil.isNotBlank(url)) {
final SearchLogHelper searchLogHelper = ComponentUtil.getSearchLogHelper();
final long count = searchLogHelper.getFavoriteCount(url);
map.put(fieldHelper.favoriteCountField, count);
map.put(fessConfig.getIndexFieldFavoriteCount(), count);
if (logger.isDebugEnabled()) {
logger.debug("Favorite Count: " + count + ", url: " + url);
}

473
src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java
View file

@ -25,11 +25,92 @@ public interface FessConfig extends FessEnv {
/** The key of the configuration. e.g. Fess */
String DOMAIN_TITLE = "domain.title";
/** The key of the configuration. e.g. 100 */
String FORM_ROLE_LIST_SIZE = "form.role.list.size";
/** The key of the configuration. e.g. false */
String CRAWLER_DOCUMENT_CACHE_ENABLE = "crawler.document.cache.enable";
/** The key of the configuration. e.g. 100 */
String FORM_GROUP_LIST_SIZE = "form.group.list.size";
/** The key of the configuration. e.g. favorite_count */
String INDEX_FIELD_favorite_count = "index.field.favorite_count";
/** The key of the configuration. e.g. click_count */
String INDEX_FIELD_click_count = "index.field.click_count";
/** The key of the configuration. e.g. config_id */
String INDEX_FIELD_config_id = "index.field.config_id";
/** The key of the configuration. e.g. expires */
String INDEX_FIELD_EXPIRES = "index.field.expires";
/** The key of the configuration. e.g. url */
String INDEX_FIELD_URL = "index.field.url";
/** The key of the configuration. e.g. doc_id */
String INDEX_FIELD_doc_id = "index.field.doc_id";
/** The key of the configuration. e.g. _id */
String INDEX_FIELD_ID = "index.field.id";
/** The key of the configuration. e.g. lang */
String INDEX_FIELD_LANG = "index.field.lang";
/** The key of the configuration. e.g. has_cache */
String INDEX_FIELD_has_cache = "index.field.has_cache";
/** The key of the configuration. e.g. last_modified */
String INDEX_FIELD_last_modified = "index.field.last_modified";
/** The key of the configuration. e.g. anchor */
String INDEX_FIELD_ANCHOR = "index.field.anchor";
/** The key of the configuration. e.g. segment */
String INDEX_FIELD_SEGMENT = "index.field.segment";
/** The key of the configuration. e.g. role */
String INDEX_FIELD_ROLE = "index.field.role";
/** The key of the configuration. e.g. boost */
String INDEX_FIELD_BOOST = "index.field.boost";
/** The key of the configuration. e.g. created */
String INDEX_FIELD_CREATED = "index.field.created";
/** The key of the configuration. e.g. label */
String INDEX_FIELD_LABEL = "index.field.label";
/** The key of the configuration. e.g. mimetype */
String INDEX_FIELD_MIMETYPE = "index.field.mimetype";
/** The key of the configuration. e.g. parent_id */
String INDEX_FIELD_parent_id = "index.field.parent_id";
/** The key of the configuration. e.g. content */
String INDEX_FIELD_CONTENT = "index.field.content";
/** The key of the configuration. e.g. cache */
String INDEX_FIELD_CACHE = "index.field.cache";
/** The key of the configuration. e.g. digest */
String INDEX_FIELD_DIGEST = "index.field.digest";
/** The key of the configuration. e.g. title */
String INDEX_FIELD_TITLE = "index.field.title";
/** The key of the configuration. e.g. host */
String INDEX_FIELD_HOST = "index.field.host";
/** The key of the configuration. e.g. site */
String INDEX_FIELD_SITE = "index.field.site";
/** The key of the configuration. e.g. content_length */
String INDEX_FIELD_content_length = "index.field.content_length";
/** The key of the configuration. e.g. filetype */
String INDEX_FIELD_FILETYPE = "index.field.filetype";
/** The key of the configuration. e.g. fess */
String INDEX_DOCUMENT_INDEX = "index.document.index";
/** The key of the configuration. e.g. doc */
String INDEX_DOCUMENT_TYPE = "index.document.type";
/** The key of the configuration. e.g. admin */
String AUTHENTICATION_ADMIN_ROLES = "authentication.admin.roles";
@ -55,6 +136,12 @@ public interface FessConfig extends FessEnv {
/** The key of the configuration. e.g. true */
String PAGING_PAGE_RANGE_FILL_LIMIT = "paging.page.range.fill.limit";
/** The key of the configuration. e.g. 100 */
String FORM_ROLE_LIST_SIZE = "form.role.list.size";
/** The key of the configuration. e.g. 100 */
String FORM_GROUP_LIST_SIZE = "form.group.list.size";
/** The key of the configuration. e.g. Administrator */
String MAIL_FROM_NAME = "mail.from.name";
@ -86,34 +173,216 @@ public interface FessConfig extends FessEnv {
String getDomainTitle();
/**
* Get the value for the key 'form.role.list.size'. <br>
* The value is, e.g. 100 <br>
* Get the value for the key 'crawler.document.cache.enable'. <br>
* The value is, e.g. false <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getFormRoleListSize();
String getCrawlerDocumentCacheEnable();
/**
* Get the value for the key 'form.role.list.size' as {@link Integer}. <br>
* The value is, e.g. 100 <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
* @throws NumberFormatException When the property is not integer.
* Is the property for the key 'crawler.document.cache.enable' true? <br>
* The value is, e.g. false <br>
* @return The determination, true or false. (if not found, exception but basically no way)
*/
Integer getFormRoleListSizeAsInteger();
boolean isCrawlerDocumentCacheEnable();
/**
* Get the value for the key 'form.group.list.size'. <br>
* The value is, e.g. 100 <br>
* Get the value for the key 'index.field.favorite_count'. <br>
* The value is, e.g. favorite_count <br>
* comment: field names
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getFormGroupListSize();
String getIndexFieldFavoriteCount();
/**
* Get the value for the key 'form.group.list.size' as {@link Integer}. <br>
* The value is, e.g. 100 <br>
* Get the value for the key 'index.field.click_count'. <br>
* The value is, e.g. click_count <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
* @throws NumberFormatException When the property is not integer.
*/
Integer getFormGroupListSizeAsInteger();
String getIndexFieldClickCount();
/**
* Get the value for the key 'index.field.config_id'. <br>
* The value is, e.g. config_id <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getIndexFieldConfigId();
/**
* Get the value for the key 'index.field.expires'. <br>
* The value is, e.g. expires <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getIndexFieldExpires();
/**
* Get the value for the key 'index.field.url'. <br>
* The value is, e.g. url <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getIndexFieldUrl();
/**
* Get the value for the key 'index.field.doc_id'. <br>
* The value is, e.g. doc_id <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getIndexFieldDocId();
/**
* Get the value for the key 'index.field.id'. <br>
* The value is, e.g. _id <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getIndexFieldId();
/**
* Get the value for the key 'index.field.lang'. <br>
* The value is, e.g. lang <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getIndexFieldLang();
/**
* Get the value for the key 'index.field.has_cache'. <br>
* The value is, e.g. has_cache <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getIndexFieldHasCache();
/**
* Get the value for the key 'index.field.last_modified'. <br>
* The value is, e.g. last_modified <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getIndexFieldLastModified();
/**
* Get the value for the key 'index.field.anchor'. <br>
* The value is, e.g. anchor <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getIndexFieldAnchor();
/**
* Get the value for the key 'index.field.segment'. <br>
* The value is, e.g. segment <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getIndexFieldSegment();
/**
* Get the value for the key 'index.field.role'. <br>
* The value is, e.g. role <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getIndexFieldRole();
/**
* Get the value for the key 'index.field.boost'. <br>
* The value is, e.g. boost <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getIndexFieldBoost();
/**
* Get the value for the key 'index.field.created'. <br>
* The value is, e.g. created <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getIndexFieldCreated();
/**
* Get the value for the key 'index.field.label'. <br>
* The value is, e.g. label <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getIndexFieldLabel();
/**
* Get the value for the key 'index.field.mimetype'. <br>
* The value is, e.g. mimetype <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getIndexFieldMimetype();
/**
* Get the value for the key 'index.field.parent_id'. <br>
* The value is, e.g. parent_id <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getIndexFieldParentId();
/**
* Get the value for the key 'index.field.content'. <br>
* The value is, e.g. content <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getIndexFieldContent();
/**
* Get the value for the key 'index.field.cache'. <br>
* The value is, e.g. cache <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getIndexFieldCache();
/**
* Get the value for the key 'index.field.digest'. <br>
* The value is, e.g. digest <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getIndexFieldDigest();
/**
* Get the value for the key 'index.field.title'. <br>
* The value is, e.g. title <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getIndexFieldTitle();
/**
* Get the value for the key 'index.field.host'. <br>
* The value is, e.g. host <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getIndexFieldHost();
/**
* Get the value for the key 'index.field.site'. <br>
* The value is, e.g. site <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getIndexFieldSite();
/**
* Get the value for the key 'index.field.content_length'. <br>
* The value is, e.g. content_length <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getIndexFieldContentLength();
/**
* Get the value for the key 'index.field.filetype'. <br>
* The value is, e.g. filetype <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getIndexFieldFiletype();
/**
* Get the value for the key 'index.document.index'. <br>
* The value is, e.g. fess <br>
* comment: document index
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getIndexDocumentIndex();
/**
* Get the value for the key 'index.document.type'. <br>
* The value is, e.g. doc <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getIndexDocumentType();
/**
* Get the value for the key 'authentication.admin.roles'. <br>
@ -223,6 +492,36 @@ public interface FessConfig extends FessEnv {
*/
boolean isPagingPageRangeFillLimit();
/**
* Get the value for the key 'form.role.list.size'. <br>
* The value is, e.g. 100 <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getFormRoleListSize();
/**
* Get the value for the key 'form.role.list.size' as {@link Integer}. <br>
* The value is, e.g. 100 <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
* @throws NumberFormatException When the property is not integer.
*/
Integer getFormRoleListSizeAsInteger();
/**
* Get the value for the key 'form.group.list.size'. <br>
* The value is, e.g. 100 <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getFormGroupListSize();
/**
* Get the value for the key 'form.group.list.size' as {@link Integer}. <br>
* The value is, e.g. 100 <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
* @throws NumberFormatException When the property is not integer.
*/
Integer getFormGroupListSizeAsInteger();
/**
* Get the value for the key 'mail.from.name'. <br>
* The value is, e.g. Administrator <br>
@ -251,20 +550,124 @@ public interface FessConfig extends FessEnv {
return get(FessConfig.DOMAIN_TITLE);
}
public String getFormRoleListSize() {
return get(FessConfig.FORM_ROLE_LIST_SIZE);
public String getCrawlerDocumentCacheEnable() {
return get(FessConfig.CRAWLER_DOCUMENT_CACHE_ENABLE);
}
public Integer getFormRoleListSizeAsInteger() {
return getAsInteger(FessConfig.FORM_ROLE_LIST_SIZE);
public boolean isCrawlerDocumentCacheEnable() {
return is(FessConfig.CRAWLER_DOCUMENT_CACHE_ENABLE);
}
public String getFormGroupListSize() {
return get(FessConfig.FORM_GROUP_LIST_SIZE);
public String getIndexFieldFavoriteCount() {
return get(FessConfig.INDEX_FIELD_favorite_count);
}
public Integer getFormGroupListSizeAsInteger() {
return getAsInteger(FessConfig.FORM_GROUP_LIST_SIZE);
public String getIndexFieldClickCount() {
return get(FessConfig.INDEX_FIELD_click_count);
}
public String getIndexFieldConfigId() {
return get(FessConfig.INDEX_FIELD_config_id);
}
public String getIndexFieldExpires() {
return get(FessConfig.INDEX_FIELD_EXPIRES);
}
public String getIndexFieldUrl() {
return get(FessConfig.INDEX_FIELD_URL);
}
public String getIndexFieldDocId() {
return get(FessConfig.INDEX_FIELD_doc_id);
}
public String getIndexFieldId() {
return get(FessConfig.INDEX_FIELD_ID);
}
public String getIndexFieldLang() {
return get(FessConfig.INDEX_FIELD_LANG);
}
public String getIndexFieldHasCache() {
return get(FessConfig.INDEX_FIELD_has_cache);
}
public String getIndexFieldLastModified() {
return get(FessConfig.INDEX_FIELD_last_modified);
}
public String getIndexFieldAnchor() {
return get(FessConfig.INDEX_FIELD_ANCHOR);
}
public String getIndexFieldSegment() {
return get(FessConfig.INDEX_FIELD_SEGMENT);
}
public String getIndexFieldRole() {
return get(FessConfig.INDEX_FIELD_ROLE);
}
public String getIndexFieldBoost() {
return get(FessConfig.INDEX_FIELD_BOOST);
}
public String getIndexFieldCreated() {
return get(FessConfig.INDEX_FIELD_CREATED);
}
public String getIndexFieldLabel() {
return get(FessConfig.INDEX_FIELD_LABEL);
}
public String getIndexFieldMimetype() {
return get(FessConfig.INDEX_FIELD_MIMETYPE);
}
public String getIndexFieldParentId() {
return get(FessConfig.INDEX_FIELD_parent_id);
}
public String getIndexFieldContent() {
return get(FessConfig.INDEX_FIELD_CONTENT);
}
public String getIndexFieldCache() {
return get(FessConfig.INDEX_FIELD_CACHE);
}
public String getIndexFieldDigest() {
return get(FessConfig.INDEX_FIELD_DIGEST);
}
public String getIndexFieldTitle() {
return get(FessConfig.INDEX_FIELD_TITLE);
}
public String getIndexFieldHost() {
return get(FessConfig.INDEX_FIELD_HOST);
}
public String getIndexFieldSite() {
return get(FessConfig.INDEX_FIELD_SITE);
}
public String getIndexFieldContentLength() {
return get(FessConfig.INDEX_FIELD_content_length);
}
public String getIndexFieldFiletype() {
return get(FessConfig.INDEX_FIELD_FILETYPE);
}
public String getIndexDocumentIndex() {
return get(FessConfig.INDEX_DOCUMENT_INDEX);
}
public String getIndexDocumentType() {
return get(FessConfig.INDEX_DOCUMENT_TYPE);
}
public String getAuthenticationAdminRoles() {
@ -319,6 +722,22 @@ public interface FessConfig extends FessEnv {
return is(FessConfig.PAGING_PAGE_RANGE_FILL_LIMIT);
}
public String getFormRoleListSize() {
return get(FessConfig.FORM_ROLE_LIST_SIZE);
}
public Integer getFormRoleListSizeAsInteger() {
return getAsInteger(FessConfig.FORM_ROLE_LIST_SIZE);
}
public String getFormGroupListSize() {
return get(FessConfig.FORM_GROUP_LIST_SIZE);
}
public Integer getFormGroupListSizeAsInteger() {
return getAsInteger(FessConfig.FORM_GROUP_LIST_SIZE);
}
public String getMailFromName() {
return get(FessConfig.MAIL_FROM_NAME);
}

14
src/main/java/org/codelibs/fess/screenshot/ScreenShotManager.java
View file

@ -33,7 +33,7 @@ import org.codelibs.core.collection.LruHashMap;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.fess.Constants;
import org.codelibs.fess.exception.FessSystemException;
import org.codelibs.fess.helper.FieldHelper;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.util.ComponentUtil;
import org.codelibs.fess.util.DocumentUtil;
import org.lastaflute.web.util.LaRequestUtil;
@ -109,10 +109,10 @@ public class ScreenShotManager {
}
public void generate(final Map<String, Object> docMap) {
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final FessConfig fessConfig = ComponentUtil.getFessConfig();
for (final ScreenShotGenerator generator : generatorList) {
if (generator.isTarget(docMap)) {
final String url = DocumentUtil.getValue(docMap, fieldHelper.urlField, String.class);
final String url = DocumentUtil.getValue(docMap, fessConfig.getIndexFieldUrl(), String.class);
final String path = getImageFilename(docMap);
if (!screenShotTaskQueue.offer(new ScreenShotTask(url, new File(baseDir, path), generator))) {
logger.warn("Failed to offer a screenshot task: " + url + " -> " + path);
@ -124,8 +124,8 @@ public class ScreenShotManager {
protected String getImageFilename(final Map<String, Object> docMap) {
final StringBuilder buf = new StringBuilder(50);
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final String docid = DocumentUtil.getValue(docMap, fieldHelper.docIdField, String.class);
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final String docid = DocumentUtil.getValue(docMap, fessConfig.getIndexFieldDocId(), String.class);
for (int i = 0; i < docid.length(); i++) {
if (i > 0 && i % splitSize == 0) {
buf.append('/');
@ -137,10 +137,10 @@ public class ScreenShotManager {
}
public void storeRequest(final String queryId, final List<Map<String, Object>> documentItems) {
final FieldHelper fieldHelper = ComponentUtil.getFieldHelper();
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final Map<String, String> dataMap = new HashMap<String, String>(documentItems.size());
for (final Map<String, Object> docMap : documentItems) {
final String docid = (String) docMap.get(fieldHelper.docIdField);
final String docid = (String) docMap.get(fessConfig.getIndexFieldDocId());
final String screenShotPath = getImageFilename(docMap);
if (StringUtil.isNotBlank(docid) && StringUtil.isNotBlank(screenShotPath)) {
dataMap.put(docid, screenShotPath);

12
src/main/java/org/codelibs/fess/util/ComponentUtil.java
View file

@ -27,7 +27,6 @@ import org.codelibs.fess.es.client.FessEsClient;
import org.codelibs.fess.helper.AdRoleHelper;
import org.codelibs.fess.helper.CrawlingConfigHelper;
import org.codelibs.fess.helper.CrawlingSessionHelper;
import org.codelibs.fess.helper.FieldHelper;
import org.codelibs.fess.helper.FileTypeHelper;
import org.codelibs.fess.helper.HotSearchWordHelper;
import org.codelibs.fess.helper.IndexingHelper;
@ -47,6 +46,7 @@ import org.codelibs.fess.helper.UserInfoHelper;
import org.codelibs.fess.helper.ViewHelper;
import org.codelibs.fess.indexer.IndexUpdater;
import org.codelibs.fess.job.JobExecutor;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.lastaflute.core.message.MessageManager;
import org.lastaflute.di.core.SingletonLaContainer;
import org.lastaflute.di.core.factory.SingletonLaContainerFactory;
@ -116,8 +116,6 @@ public final class ComponentUtil {
private static final String INDEXING_HELPER = "indexingHelper";
private static final String FIELD_HELPER = "fieldHelper";
private static final String ELASTICSEARCH_CLIENT = FESS_ES_CLIENT;
private ComponentUtil() {
@ -235,10 +233,6 @@ public final class ComponentUtil {
return SingletonLaContainer.getComponent(INDEXING_HELPER);
}
public static FieldHelper getFieldHelper() {
return SingletonLaContainer.getComponent(FIELD_HELPER);
}
public static UserInfoHelper getUserInfoHelper() {
return SingletonLaContainer.getComponent(USER_INFO_HELPER);
}
@ -263,6 +257,10 @@ public final class ComponentUtil {
return SingletonLaContainer.getComponent(FESS_ES_CLIENT);
}
public static FessConfig getFessConfig() {
return SingletonLaContainer.getComponent(FessConfig.class);
}
public static <T> T getComponent(final Class<T> clazz) {
return SingletonLaContainer.getComponent(clazz);
}

1
src/main/java/org/codelibs/fess/util/QueryResponseList.java
View file

@ -93,7 +93,6 @@ public class QueryResponseList implements List<Map<String, Object>> {
// build highlighting fields
final QueryHelper queryHelper = ComponentUtil.getQueryHelper();
ComponentUtil.getFieldHelper();
final String hlPrefix = queryHelper.getHighlightPrefix();
for (final SearchHit searchHit : searchHits.getHits()) {
final Map<String, Object> docMap = new HashMap<String, Object>();

61
src/main/resources/app.xml
View file

@ -78,45 +78,6 @@
<arg>"label"</arg>
<arg>"0.0"</arg>
</postConstruct>
<!--
<property name="additionalGeoQuery">"location_i_i:1"</property>
<property name="responseFields">new String[]{ "id", "docId", "score",
"boost", "contentLength", "host", "site", "lastModified",
"mimetype", "filetype_s", "created", "title", "digest", "url",
"clickCount_l_x_dv", "favoriteCount_l_x_dv",
"cid_s", "lang_s", "hasCache_s_s" }</property>
<property name="responseDocValuesFields">new String[]{
"clickCount_l_x_dv", "favoriteCount_l_x_dv"}</property>
<property name="highlightingFields">new String[]{"digest", "cache" }</property>
<property name="searchFields">new String[]{"url", "docId", "host",
"title", "content", "contentLength", "lastModified", "mimetype", "filetype_s",
"label", "segment", "lang_s" }</property>
<property name="facetFields">new String[]{"url", "host",
"title", "content", "contentLength", "lastModified", "mimetype", "filetype_s",
"label", "segment" }</property>
<property name="sortPrefix">"sort:"</property>
<property name="supportedSortFields">new String[]{"created",
"contentLength", "lastModified", "clickCount_l_x_dv" }</property>
<property name="supportedMltFields">new String[]{"content",
"content_ja" }</property>
<property name="shards">"localhost:8983/solr,localhost:7574/solr"</property>
<property name="useBigram">true</property>
<property name="additionalQuery">""</property>
<property name="maxFilterQueriesForRole">3</property>
<property name="timeAllowed">1000</property>
<postConstruct name="addRequestParameter">
<arg>"debug.explain.structured"</arg>
<arg>"true"</arg>
</postConstruct>
<postConstruct name="addDefaultSortField">
<arg>"score"</arg>
<arg>"desc"</arg>
</postConstruct>
<postConstruct name="addQueryParam">
<arg>"word"</arg>
<arg>new String[]{"$query"}</arg>
</postConstruct>
-->
</component>
<component name="viewHelper" class="org.codelibs.fess.helper.ViewHelper">
<postConstruct name="addFacetQueryView">
@ -200,35 +161,13 @@
</component>
</arg>
</postConstruct>
<!--
<property name="encodeUrlLink">false</property>
<property name="useHighlight">false</property>
<property name="descriptionLength">200</property>
<property name="titleLength">50</property>
<property name="urlLinkEncoding">"UTF-8"</property>
<property name="highlightingFields">new String[] { "hl_content", "digest" }</property>
<postConstruct name="addInitFacetParam">
<arg>"facet.field"</arg>
<arg>"label"</arg>
</postConstruct>
-->
</component>
<component name="userAgentHelper" class="org.codelibs.fess.helper.UserAgentHelper">
</component>
<component name="hotSearchWordHelper" class="org.codelibs.fess.helper.HotSearchWordHelper">
<property name="excludedWordPattern">java.util.regex.Pattern.compile(".*[a-zA-Z0-9_]+:.*")</property>
<!--
<property name="size">10</property>
-->
</component>
<component name="userInfoHelper" class="org.codelibs.fess.helper.impl.CookieUserInfoHelperImpl">
<!--
<property name="cookieName">"fu"</property>
<property name="cookieDomain">""</property>
<property name="cookieMaxAge">30 * 24 * 60 * 60</property>
<property name="cookiePath">""</property>
<property name="cookieSecure">false</property>
-->
</component>
<component name="openSearchHelper" class="org.codelibs.fess.helper.OpenSearchHelper">
<property name="osddPath">"/WEB-INF/orig/open-search/osdd.xml"</property>

2
src/main/resources/fess.xml
View file

@ -20,8 +20,6 @@
</component>
<component name="jobHelper" class="org.codelibs.fess.helper.JobHelper">
</component>
<component name="fieldHelper" class="org.codelibs.fess.helper.FieldHelper">
</component>
<component name="systemHelper" class="org.codelibs.fess.helper.SystemHelper">
<!--
<property name="javaCommandPath">"java"</property>

41
src/main/resources/fess_config.properties
View file

@ -10,11 +10,42 @@ domain.title = Fess
# ========================================================================================
# ES
# Crawler
# ====
form.role.list.size = 100
form.group.list.size = 100
crawler.document.cache.enable=false
# field names
index.field.favorite_count=favorite_count
index.field.click_count=click_count
index.field.config_id=config_id
index.field.expires=expires
index.field.url=url
index.field.doc_id=doc_id
index.field.id=_id
index.field.lang=lang
index.field.has_cache=has_cache
index.field.last_modified=last_modified
index.field.anchor=anchor
index.field.segment=segment
index.field.role=role
index.field.boost=boost
index.field.created=created
index.field.label=label
index.field.mimetype=mimetype
index.field.parent_id=parent_id
index.field.content=content
index.field.cache=cache
index.field.digest=digest
index.field.title=title
index.field.host=host
index.field.site=site
index.field.content_length=content_length
index.field.filetype=filetype
# document index
index.document.index=fess
index.document.type=doc
# ========================================================================================
# Web
@ -24,6 +55,7 @@ form.group.list.size = 100
# ------
authentication.admin.roles=admin
# ----------------------------------------------------------
# Cookie
# ------
@ -51,6 +83,9 @@ paging.page.range.size = 3
# The option 'fillLimit' of page range for paging
paging.page.range.fill.limit = true
form.role.list.size = 100
form.group.list.size = 100
# ----------------------------------------------------------
# Mail
# ------

1
src/main/webapp/WEB-INF/crawler/resources/app.xml
View file

@ -3,6 +3,7 @@
"http://dbflute.org/meta/lastadi10.dtd">
<components>
<include path="convention.xml" />
<include path="lastaflute_director.xml"/>
<include path="fess.xml" />
<include path="crawler_es.xml" />

1
src/main/webapp/WEB-INF/suggest/resources/app.xml
View file

@ -3,6 +3,7 @@
"http://dbflute.org/meta/lastadi10.dtd">
<components>
<include path="convention.xml" />
<include path="lastaflute_director.xml"/>
<include path="fess.xml" />
<component name="fessSuggest" class="org.codelibs.fess.exec.SuggestCreater"

6
src/test/java/org/codelibs/fess/helper/QueryHelperTest.java
View file

@ -16,7 +16,9 @@
package org.codelibs.fess.helper;
import org.apache.lucene.queryparser.classic.ParseException;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.unit.UnitFessTestCase;
import org.codelibs.fess.util.ComponentUtil;
public class QueryHelperTest extends UnitFessTestCase {
@ -26,8 +28,8 @@ public class QueryHelperTest extends UnitFessTestCase {
public void setUp() throws Exception {
super.setUp();
queryHelper = new QueryHelper();
FieldHelper fieldHelper = new FieldHelper();
registerMockInstance(fieldHelper);
final FessConfig fessConfig = ComponentUtil.getFessConfig();
registerMockInstance(fessConfig);
registerMockInstance(new SystemHelper());
inject(queryHelper);
queryHelper.init();