0ct0pu5/fess
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

modify docboost and minor fixes

Browse source
This commit is contained in:
Shinsuke Sugaya 2015-11-19 16:13:40 +09:00
parent 048a8b59b7
commit f90a6b8dd3
15 changed files with 114 additions and 70 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
src/main/java/org/codelibs/fess/app/service/BoostDocumentRuleService.java
View file

@ -27,6 +27,7 @@ import org.codelibs.fess.app.pager.BoostDocumentRulePager;
import org.codelibs.fess.es.config.cbean.BoostDocumentRuleCB;
import org.codelibs.fess.es.config.exbhv.BoostDocumentRuleBhv;
import org.codelibs.fess.es.config.exentity.BoostDocumentRule;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.dbflute.cbean.result.PagingResultBean;
import org.dbflute.optional.OptionalEntity;
@ -37,9 +38,8 @@ public class BoostDocumentRuleService implements Serializable {
@Resource
protected BoostDocumentRuleBhv boostDocumentRuleBhv;
public BoostDocumentRuleService() {
super();
}
@Resource
protected FessConfig fessConfig;
public List<BoostDocumentRule> getBoostDocumentRuleList(final BoostDocumentRulePager boostDocumentRulePager) {
@ -112,7 +112,9 @@ public class BoostDocumentRuleService implements Serializable {
public List<BoostDocumentRule> getAvailableBoostDocumentRuleList() {
return boostDocumentRuleBhv.selectList(cb -> {
cb.query().matchAll();
cb.query().addOrderBy_SortOrder_Asc();
cb.fetchFirst(fessConfig.getPageDocboostMaxFetchSizeAsInteger());
});
}

7
src/main/java/org/codelibs/fess/app/web/admin/log/AdminLogAction.java
View file

@ -17,6 +17,7 @@ package org.codelibs.fess.app.web.admin.log;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
@ -40,8 +41,6 @@ import org.lastaflute.web.callback.ActionRuntime;
import org.lastaflute.web.response.ActionResponse;
import org.lastaflute.web.response.HtmlResponse;
import com.google.common.base.Charsets;
/**
* @author codelibs
* @author Keiichi Watanabe
@ -64,7 +63,7 @@ public class AdminLogAction extends FessAdminAction {
@Execute
public ActionResponse download(final String id) {
String filename = new String(Base64.getDecoder().decode(id), Charsets.UTF_8).replace("..", "").replaceAll("\\s", "");
String filename = new String(Base64.getDecoder().decode(id), StandardCharsets.UTF_8).replace("..", "").replaceAll("\\s", "");
final String logFilePath = systemHelper.getLogFilePath();
if (StringUtil.isNotBlank(logFilePath)) {
Path path = Paths.get(logFilePath, filename);
@ -89,7 +88,7 @@ public class AdminLogAction extends FessAdminAction {
stream.filter(entry -> entry.getFileName().toString().endsWith(".log")).forEach(filePath -> {
Map<String, Object> map = new HashMap<>();
String name = filePath.getFileName().toString();
map.put("id", Base64.getEncoder().encodeToString(name.getBytes(Charsets.UTF_8)));
map.put("id", Base64.getEncoder().encodeToString(name.getBytes(StandardCharsets.UTF_8)));
map.put("name", name);
try {
map.put("lastModified", new Date(Files.getLastModifiedTime(filePath).toMillis()));

4
src/main/java/org/codelibs/fess/app/web/admin/user/AdminUserAction.java
View file

@ -121,8 +121,8 @@ public class AdminUserAction extends FessAdminAction {
}
private void registerForms(final RenderData data) {
data.register("roleItems", roleService.getAvailableRoleList(fessConfig.getFormRoleListSizeAsInteger()));
data.register("groupItems", groupService.getAvailableGroupList(fessConfig.getFormGroupListSizeAsInteger()));
data.register("roleItems", roleService.getAvailableRoleList(fessConfig.getPageRoleMaxFetchSizeAsInteger()));
data.register("groupItems", groupService.getAvailableGroupList(fessConfig.getPageGroupMaxFetchSizeAsInteger()));
}
// ===================================================================================

7
src/main/java/org/codelibs/fess/entity/QueryContext.java
View file

@ -29,6 +29,7 @@ import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.MatchAllQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder;
import org.elasticsearch.search.sort.SortBuilder;
import org.lastaflute.web.util.LaRequestUtil;
@ -59,6 +60,12 @@ public class QueryContext {
}
}
public void addFunctionScore(final Consumer<FunctionScoreQueryBuilder> functionScoreQuery) {
FunctionScoreQueryBuilder builder = QueryBuilders.functionScoreQuery(queryBuilder);
functionScoreQuery.accept(builder);
queryBuilder = builder;
}
public void addQuery(final Consumer<BoolQueryBuilder> boolQuery) {
BoolQueryBuilder builder;
if (queryBuilder instanceof BoolQueryBuilder) {

7
src/main/java/org/codelibs/fess/es/client/FessEsClient.java
View file

@ -34,7 +34,6 @@ import java.util.stream.Collectors;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import org.apache.commons.codec.Charsets;
import org.codelibs.core.beans.util.BeanUtil;
import org.codelibs.core.io.FileUtil;
import org.codelibs.core.io.ResourceUtil;
@ -50,8 +49,8 @@ import org.codelibs.fess.entity.PingResponse;
import org.codelibs.fess.entity.QueryContext;
import org.codelibs.fess.exception.FessSystemException;
import org.codelibs.fess.exception.ResultOffsetExceededException;
import org.codelibs.fess.exception.SearchQueryException;
import org.codelibs.fess.helper.QueryHelper;
import org.codelibs.fess.indexer.FessSearchQueryException;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.util.ComponentUtil;
import org.codelibs.fess.util.StreamUtil;
@ -841,7 +840,7 @@ public class FessEsClient implements Client {
if (facetInfo != null) {
StreamUtil.of(facetInfo.field).forEach(f -> {
if (queryHelper.isFacetField(f)) {
final String encodedField = BaseEncoding.base64().encode(f.getBytes(Charsets.UTF_8));
final String encodedField = BaseEncoding.base64().encode(f.getBytes(StandardCharsets.UTF_8));
final TermsBuilder termsBuilder = AggregationBuilders.terms(Constants.FACET_FIELD_PREFIX + encodedField).field(f);
// TODO order
if (facetInfo.limit != null) {
@ -850,7 +849,7 @@ public class FessEsClient implements Client {
}
searchRequestBuilder.addAggregation(termsBuilder);
} else {
throw new FessSearchQueryException("Invalid facet field: " + f);
throw new SearchQueryException("Invalid facet field: " + f);
}
});
StreamUtil.of(facetInfo.query).forEach(

2
src/main/java/org/codelibs/fess/es/config/allcommon/EsAbstractBehavior.java
View file

@ -94,7 +94,7 @@ public abstract class EsAbstractBehavior<ENTITY extends Entity, CB extends Condi
final int from;
final int size;
if (cb.isFetchScopeEffective()) {
from = cb.getFetchStartIndex();
from = cb.getPageStartIndex();
size = cb.getFetchSize();
} else {
from = 0;

2
src/main/java/org/codelibs/fess/es/log/allcommon/EsAbstractBehavior.java
View file

@ -94,7 +94,7 @@ public abstract class EsAbstractBehavior<ENTITY extends Entity, CB extends Condi
final int from;
final int size;
if (cb.isFetchScopeEffective()) {
from = cb.getFetchStartIndex();
from = cb.getPageStartIndex();
size = cb.getFetchSize();
} else {
from = 0;

2
src/main/java/org/codelibs/fess/es/user/allcommon/EsAbstractBehavior.java
View file

@ -94,7 +94,7 @@ public abstract class EsAbstractBehavior<ENTITY extends Entity, CB extends Condi
final int from;
final int size;
if (cb.isFetchScopeEffective()) {
from = cb.getFetchStartIndex();
from = cb.getPageStartIndex();
size = cb.getFetchSize();
} else {
from = 0;

12
src/main/java/org/codelibs/fess/indexer/FessSearchQueryException.java → src/main/java/org/codelibs/fess/exception/SearchQueryException.java
View file

@ -13,23 +13,21 @@
* either express or implied. See the License for the specific language
* governing permissions and limitations under the License.
*/
package org.codelibs.fess.indexer;
package org.codelibs.fess.exception;
import org.codelibs.fess.exception.FessSystemException;
public class FessSearchQueryException extends FessSystemException {
public class SearchQueryException extends FessSystemException {
private static final long serialVersionUID = 1L;
public FessSearchQueryException(final String message, final Throwable cause) {
public SearchQueryException(final String message, final Throwable cause) {
super(message, cause);
}
public FessSearchQueryException(final String message) {
public SearchQueryException(final String message) {
super(message);
}
public FessSearchQueryException(final Throwable cause) {
public SearchQueryException(final Throwable cause) {
super(cause);
}

16
src/main/java/org/codelibs/fess/helper/QueryHelper.java
View file

@ -54,7 +54,6 @@ import org.codelibs.fess.entity.GeoInfo;
import org.codelibs.fess.entity.QueryContext;
import org.codelibs.fess.exception.InvalidQueryException;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.util.ComponentUtil;
import org.codelibs.fess.util.StreamUtil;
import org.dbflute.optional.OptionalEntity;
import org.dbflute.optional.OptionalThing;
@ -63,6 +62,7 @@ import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.RangeQueryBuilder;
import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders;
import org.elasticsearch.search.sort.SortBuilder;
import org.elasticsearch.search.sort.SortBuilders;
import org.elasticsearch.search.sort.SortOrder;
@ -100,6 +100,9 @@ public class QueryHelper implements Serializable {
@Resource
protected DynamicProperties crawlerProperties;
@Resource
protected FessConfig fessConfig;
@Resource
protected RoleQueryHelper roleQueryHelper;
@ -161,7 +164,6 @@ public class QueryHelper implements Serializable {
@PostConstruct
public void init() {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
if (responseFields == null) {
responseFields =
new String[] { SCORE_FIELD, fessConfig.getIndexFieldId(), fessConfig.getIndexFieldDocId(),
@ -223,6 +225,8 @@ public class QueryHelper implements Serializable {
final QueryContext queryContext = new QueryContext(q, true);
buildBaseQuery(queryContext, context);
buildBoostQuery(queryContext);
if (keyMatchHelper != null) {
final List<String> docIdQueryList = keyMatchHelper.getDocIdQueryList();
if (docIdQueryList != null && !docIdQueryList.isEmpty()) {
@ -238,7 +242,6 @@ public class QueryHelper implements Serializable {
if (roleQueryHelper != null) {
final Set<String> roleSet = roleQueryHelper.build();
if (!roleSet.isEmpty()) {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
queryContext.addQuery(boolQuery -> {
BoolQueryBuilder roleQuery = QueryBuilders.boolQuery();
roleSet.stream().forEach(name -> {
@ -255,6 +258,12 @@ public class QueryHelper implements Serializable {
return queryContext;
}
private void buildBoostQuery(final QueryContext queryContext) {
queryContext.addFunctionScore(functionScoreQuery -> {
functionScoreQuery.add(ScoreFunctionBuilders.fieldValueFactorFunction(fessConfig.getIndexFieldBoost()));
});
}
public void buildBaseQuery(final QueryContext queryContext, final Consumer<QueryContext> context) {
final QueryParser queryParser = getQueryParser();
try {
@ -456,7 +465,6 @@ public class QueryHelper implements Serializable {
}
private QueryBuilder buildDefaultQueryBuilder(final Function<String, QueryBuilder> builder) {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
// TODO boost
final BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
final QueryBuilder titleQuery = builder.apply(fessConfig.getIndexFieldTitle());

5
src/main/java/org/codelibs/fess/helper/WebFsIndexHelper.java
View file

@ -36,7 +36,6 @@ import org.codelibs.fess.crawler.interval.FessIntervalController;
import org.codelibs.fess.crawler.service.DataService;
import org.codelibs.fess.crawler.service.UrlFilterService;
import org.codelibs.fess.crawler.service.UrlQueueService;
import org.codelibs.fess.es.config.exentity.BoostDocumentRule;
import org.codelibs.fess.es.config.exentity.FileConfig;
import org.codelibs.fess.es.config.exentity.WebConfig;
import org.codelibs.fess.indexer.IndexUpdater;
@ -386,9 +385,9 @@ public class WebFsIndexHelper implements Serializable {
indexUpdater.setSessionIdList(sessionIdList);
indexUpdater.setDaemon(true);
indexUpdater.setCrawlerList(crawlerList);
for (final BoostDocumentRule rule : boostDocumentRuleService.getAvailableBoostDocumentRuleList()) {
boostDocumentRuleService.getAvailableBoostDocumentRuleList().forEach(rule -> {
indexUpdater.addDocBoostMatcher(new org.codelibs.fess.indexer.DocBoostMatcher(rule));
}
});
indexUpdater.start();
int startedCrawlerNum = 0;

76
src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java
View file

@ -136,11 +136,14 @@ public interface FessConfig extends FessEnv {
/** The key of the configuration. e.g. true */
String PAGING_PAGE_RANGE_FILL_LIMIT = "paging.page.range.fill.limit";
/** The key of the configuration. e.g. 100 */
String FORM_ROLE_LIST_SIZE = "form.role.list.size";
/** The key of the configuration. e.g. 1000 */
String PAGE_DOCBOOST_MAX_FETCH_SIZE = "page.docboost.max.fetch.size";
/** The key of the configuration. e.g. 100 */
String FORM_GROUP_LIST_SIZE = "form.group.list.size";
/** The key of the configuration. e.g. 1000 */
String PAGE_ROLE_MAX_FETCH_SIZE = "page.role.max.fetch.size";
/** The key of the configuration. e.g. 1000 */
String PAGE_GROUP_MAX_FETCH_SIZE = "page.group.max.fetch.size";
/** The key of the configuration. e.g. Administrator */
String MAIL_FROM_NAME = "mail.from.name";
@ -589,34 +592,51 @@ public interface FessConfig extends FessEnv {
boolean isPagingPageRangeFillLimit();
/**
* Get the value for the key 'form.role.list.size'. <br>
* The value is, e.g. 100 <br>
* Get the value for the key 'page.docboost.max.fetch.size'. <br>
* The value is, e.g. 1000 <br>
* comment: max page size
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getFormRoleListSize();
String getPageDocboostMaxFetchSize();
/**
* Get the value for the key 'form.role.list.size' as {@link Integer}. <br>
* The value is, e.g. 100 <br>
* Get the value for the key 'page.docboost.max.fetch.size' as {@link Integer}. <br>
* The value is, e.g. 1000 <br>
* comment: max page size
* @return The value of found property. (NotNull: if not found, exception but basically no way)
* @throws NumberFormatException When the property is not integer.
*/
Integer getFormRoleListSizeAsInteger();
Integer getPageDocboostMaxFetchSizeAsInteger();
/**
* Get the value for the key 'form.group.list.size'. <br>
* The value is, e.g. 100 <br>
* Get the value for the key 'page.role.max.fetch.size'. <br>
* The value is, e.g. 1000 <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getFormGroupListSize();
String getPageRoleMaxFetchSize();
/**
* Get the value for the key 'form.group.list.size' as {@link Integer}. <br>
* The value is, e.g. 100 <br>
* Get the value for the key 'page.role.max.fetch.size' as {@link Integer}. <br>
* The value is, e.g. 1000 <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
* @throws NumberFormatException When the property is not integer.
*/
Integer getFormGroupListSizeAsInteger();
Integer getPageRoleMaxFetchSizeAsInteger();
/**
* Get the value for the key 'page.group.max.fetch.size'. <br>
* The value is, e.g. 1000 <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getPageGroupMaxFetchSize();
/**
* Get the value for the key 'page.group.max.fetch.size' as {@link Integer}. <br>
* The value is, e.g. 1000 <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
* @throws NumberFormatException When the property is not integer.
*/
Integer getPageGroupMaxFetchSizeAsInteger();
/**
* Get the value for the key 'mail.from.name'. <br>
@ -1043,20 +1063,28 @@ public interface FessConfig extends FessEnv {
return is(FessConfig.PAGING_PAGE_RANGE_FILL_LIMIT);
}
public String getFormRoleListSize() {
return get(FessConfig.FORM_ROLE_LIST_SIZE);
public String getPageDocboostMaxFetchSize() {
return get(FessConfig.PAGE_DOCBOOST_MAX_FETCH_SIZE);
}
public Integer getFormRoleListSizeAsInteger() {
return getAsInteger(FessConfig.FORM_ROLE_LIST_SIZE);
public Integer getPageDocboostMaxFetchSizeAsInteger() {
return getAsInteger(FessConfig.PAGE_DOCBOOST_MAX_FETCH_SIZE);
}
public String getFormGroupListSize() {
return get(FessConfig.FORM_GROUP_LIST_SIZE);
public String getPageRoleMaxFetchSize() {
return get(FessConfig.PAGE_ROLE_MAX_FETCH_SIZE);
}
public Integer getFormGroupListSizeAsInteger() {
return getAsInteger(FessConfig.FORM_GROUP_LIST_SIZE);
public Integer getPageRoleMaxFetchSizeAsInteger() {
return getAsInteger(FessConfig.PAGE_ROLE_MAX_FETCH_SIZE);
}
public String getPageGroupMaxFetchSize() {
return get(FessConfig.PAGE_GROUP_MAX_FETCH_SIZE);
}
public Integer getPageGroupMaxFetchSizeAsInteger() {
return getAsInteger(FessConfig.PAGE_GROUP_MAX_FETCH_SIZE);
}
public String getMailFromName() {

26
src/main/java/org/codelibs/fess/util/FacetResponse.java
View file

@ -15,6 +15,7 @@
*/
package org.codelibs.fess.util;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
@ -25,7 +26,6 @@ import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.aggregations.bucket.filter.Filter;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import com.google.common.base.Charsets;
import com.google.common.io.BaseEncoding;
public class FacetResponse {
@ -34,17 +34,19 @@ public class FacetResponse {
protected List<Field> fieldList = new ArrayList<FacetResponse.Field>();
public FacetResponse(final Aggregations aggregations) {
aggregations.forEach(aggregation -> {
if (aggregation.getName().startsWith(Constants.FACET_FIELD_PREFIX)) {
final Terms termFacet = (Terms) aggregation;
fieldList.add(new Field(termFacet));
} else if (aggregation.getName().startsWith(Constants.FACET_QUERY_PREFIX)) {
final Filter queryFacet = (Filter) aggregation;
final String encodedQuery = queryFacet.getName().substring(Constants.FACET_QUERY_PREFIX.length());
queryCountMap.put(new String(BaseEncoding.base64().decode(encodedQuery), Charsets.UTF_8), queryFacet.getDocCount());
}
aggregations
.forEach(aggregation -> {
if (aggregation.getName().startsWith(Constants.FACET_FIELD_PREFIX)) {
final Terms termFacet = (Terms) aggregation;
fieldList.add(new Field(termFacet));
} else if (aggregation.getName().startsWith(Constants.FACET_QUERY_PREFIX)) {
final Filter queryFacet = (Filter) aggregation;
final String encodedQuery = queryFacet.getName().substring(Constants.FACET_QUERY_PREFIX.length());
queryCountMap.put(new String(BaseEncoding.base64().decode(encodedQuery), StandardCharsets.UTF_8),
queryFacet.getDocCount());
}
});
});
}
public boolean hasFacetResponse() {
@ -58,7 +60,7 @@ public class FacetResponse {
public Field(final Terms termFacet) {
final String encodedField = termFacet.getName().substring(Constants.FACET_FIELD_PREFIX.length());
name = new String(BaseEncoding.base64().decode(encodedField), Charsets.UTF_8);
name = new String(BaseEncoding.base64().decode(encodedField), StandardCharsets.UTF_8);
for (final Terms.Bucket tfEntry : termFacet.getBuckets()) {
valueCountMap.put(tfEntry.getKeyAsString(), tfEntry.getDocCount());
}

6
src/main/resources/fess_config.properties
View file

@ -83,8 +83,10 @@ paging.page.range.size = 3
# The option 'fillLimit' of page range for paging
paging.page.range.fill.limit = true
form.role.list.size = 100
form.group.list.size = 100
# max page size
page.docboost.max.fetch.size=1000
page.role.max.fetch.size=1000
page.group.max.fetch.size=1000
# ----------------------------------------------------------
# Mail

4
src/main/webapp/WEB-INF/view/admin/boostdoc/admin_boostdoc_edit.jsp
View file

@ -51,14 +51,14 @@
<label for="urlExpr" class="col-sm-3 control-label"><la:message
key="labels.boost_document_rule_url_expr" /></label>
<div class="col-sm-9">
<la:text property="urlExpr" styleClass="form-control" />
<la:textarea property="urlExpr" styleClass="form-control" rows="5"/>
</div>
</div>
<div class="form-group">
<label for="boostExpr" class="col-sm-3 control-label"><la:message
key="labels.boost_document_rule_boost_expr" /></label>
<div class="col-sm-9">
<la:text property="boostExpr" styleClass="form-control" />
<la:textarea property="boostExpr" styleClass="form-control" rows="5"/>
</div>
</div>
<div class="form-group">