0ct0pu5/fess
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Rebuild suggest documents on new index when execute suggest-job

Browse source
This commit is contained in:
yfujita 2017-07-10 19:08:32 +09:00
parent 9978bda21d
commit 74e2928ff2
6 changed files with 68 additions and 40 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
src/main/java/org/codelibs/fess/app/web/admin/badword/AdminBadwordAction.java
View file

@ -223,7 +223,7 @@ public class AdminBadwordAction extends FessAdminAction {
entity -> {
try {
badWordService.store(entity);
suggestHelper.addBadWord(entity.getSuggestWord());
suggestHelper.addBadWord(entity.getSuggestWord(), false);
saveInfo(messages -> messages.addSuccessCrudCreateCrudTable(GLOBAL));
} catch (final Exception e) {
throwValidationError(messages -> messages.addErrorsCrudFailedToCreateCrudTable(GLOBAL, buildThrowableMessage(e)),
@ -244,7 +244,7 @@ public class AdminBadwordAction extends FessAdminAction {
entity -> {
try {
badWordService.store(entity);
suggestHelper.storeAllBadWords();
suggestHelper.storeAllBadWords(false);
saveInfo(messages -> messages.addSuccessCrudUpdateCrudTable(GLOBAL));
} catch (final Exception e) {
throwValidationError(messages -> messages.addErrorsCrudFailedToUpdateCrudTable(GLOBAL, buildThrowableMessage(e)),
@ -288,7 +288,7 @@ public class AdminBadwordAction extends FessAdminAction {
new Thread(() -> {
try (Reader reader = new BufferedReader(new InputStreamReader(form.badWordFile.getInputStream(), getCsvEncoding()))) {
badWordService.importCsv(reader);
suggestHelper.storeAllBadWords();
suggestHelper.storeAllBadWords(false);
} catch (final Exception e) {
throw new FessSystemException("Failed to import data.", e);
}

12
src/main/java/org/codelibs/fess/app/web/admin/elevateword/AdminElevatewordAction.java
View file

@ -256,7 +256,7 @@ public class AdminElevatewordAction extends FessAdminAction {
try {
elevateWordService.store(entity);
suggestHelper.addElevateWord(entity.getSuggestWord(), entity.getReading(), entity.getLabelTypeValues(),
entity.getPermissions(), entity.getBoost());
entity.getPermissions(), entity.getBoost(), false);
saveInfo(messages -> messages.addSuccessCrudCreateCrudTable(GLOBAL));
} catch (final Exception e) {
throwValidationError(messages -> messages.addErrorsCrudFailedToCreateCrudTable(GLOBAL, buildThrowableMessage(e)),
@ -277,8 +277,8 @@ public class AdminElevatewordAction extends FessAdminAction {
entity -> {
try {
elevateWordService.store(entity);
suggestHelper.deleteAllElevateWord();
suggestHelper.storeAllElevateWords();
suggestHelper.deleteAllElevateWord(false);
suggestHelper.storeAllElevateWords(false);
saveInfo(messages -> messages.addSuccessCrudUpdateCrudTable(GLOBAL));
} catch (final Exception e) {
throwValidationError(messages -> messages.addErrorsCrudFailedToUpdateCrudTable(GLOBAL, buildThrowableMessage(e)),
@ -302,7 +302,7 @@ public class AdminElevatewordAction extends FessAdminAction {
entity -> {
try {
elevateWordService.delete(entity);
suggestHelper.deleteElevateWord(entity.getSuggestWord());
suggestHelper.deleteElevateWord(entity.getSuggestWord(), false);
saveInfo(messages -> messages.addSuccessCrudDeleteCrudTable(GLOBAL));
} catch (final Exception e) {
throwValidationError(
@ -322,8 +322,8 @@ public class AdminElevatewordAction extends FessAdminAction {
new Thread(() -> {
try (Reader reader = new BufferedReader(new InputStreamReader(form.elevateWordFile.getInputStream(), getCsvEncoding()))) {
elevateWordService.importCsv(reader);
suggestHelper.deleteAllElevateWord();
suggestHelper.storeAllElevateWords();
suggestHelper.deleteAllElevateWord(false);
suggestHelper.storeAllElevateWords(false);
} catch (final Exception e) {
throw new FessSystemException("Failed to import data.", e);
}

6
src/main/java/org/codelibs/fess/app/web/api/admin/badword/ApiAdminBadwordAction.java
View file

@ -92,7 +92,7 @@ public class ApiAdminBadwordAction extends FessApiAdminAction {
});
try {
badWordService.store(entity);
suggestHelper.addBadWord(entity.getSuggestWord());
suggestHelper.addBadWord(entity.getSuggestWord(), false);
} catch (final Exception e) {
throwValidationErrorApi(messages -> messages.addErrorsCrudFailedToCreateCrudTable(GLOBAL, buildThrowableMessage(e)));
}
@ -107,7 +107,7 @@ public class ApiAdminBadwordAction extends FessApiAdminAction {
final BadWord badWord = getBadWord(body).map(entity -> {
try {
badWordService.store(entity);
suggestHelper.storeAllBadWords();
suggestHelper.storeAllBadWords(false);
} catch (final Exception e) {
throwValidationErrorApi(messages -> messages.addErrorsCrudFailedToUpdateCrudTable(GLOBAL, buildThrowableMessage(e)));
}
@ -148,7 +148,7 @@ public class ApiAdminBadwordAction extends FessApiAdminAction {
new Thread(() -> {
try (Reader reader = new BufferedReader(new InputStreamReader(body.badWordFile.getInputStream(), getCsvEncoding()))) {
badWordService.importCsv(reader);
suggestHelper.storeAllBadWords();
suggestHelper.storeAllBadWords(false);
} catch (final Exception e) {
throw new FessSystemException("Failed to import data.", e);
}

10
src/main/java/org/codelibs/fess/app/web/api/admin/elevateword/ApiAdminElevatewordAction.java
View file

@ -102,7 +102,7 @@ public class ApiAdminElevatewordAction extends FessApiAdminAction {
try {
elevateWordService.store(entity);
suggestHelper.addElevateWord(entity.getSuggestWord(), entity.getReading(), entity.getLabelTypeValues(),
entity.getPermissions(), entity.getBoost());
entity.getPermissions(), entity.getBoost(), false);
} catch (final Exception e) {
throwValidationErrorApi(messages -> messages.addErrorsCrudFailedToCreateCrudTable(GLOBAL, buildThrowableMessage(e)));
}
@ -117,8 +117,8 @@ public class ApiAdminElevatewordAction extends FessApiAdminAction {
final ElevateWord elevateWord = getElevateWord(body).map(entity -> {
try {
elevateWordService.store(entity);
suggestHelper.deleteAllElevateWord();
suggestHelper.storeAllElevateWords();
suggestHelper.deleteAllElevateWord(false);
suggestHelper.storeAllElevateWords(false);
} catch (final Exception e) {
throwValidationErrorApi(messages -> messages.addErrorsCrudFailedToUpdateCrudTable(GLOBAL, buildThrowableMessage(e)));
}
@ -138,7 +138,7 @@ public class ApiAdminElevatewordAction extends FessApiAdminAction {
elevateWordService.getElevateWord(id).ifPresent(entity -> {
try {
elevateWordService.delete(entity);
suggestHelper.deleteElevateWord(entity.getSuggestWord());
suggestHelper.deleteElevateWord(entity.getSuggestWord(), false);
saveInfo(messages -> messages.addSuccessCrudDeleteCrudTable(GLOBAL));
} catch (final Exception e) {
throwValidationErrorApi(messages -> messages.addErrorsCrudFailedToDeleteCrudTable(GLOBAL, buildThrowableMessage(e)));
@ -159,7 +159,7 @@ public class ApiAdminElevatewordAction extends FessApiAdminAction {
new Thread(() -> {
try (Reader reader = new BufferedReader(new InputStreamReader(body.elevateWordFile.getInputStream(), getCsvEncoding()))) {
elevateWordService.importCsv(reader);
suggestHelper.storeAllElevateWords();
suggestHelper.storeAllElevateWords(false);
} catch (final Exception e) {
throw new FessSystemException("Failed to import data.", e);
}

19
src/main/java/org/codelibs/fess/exec/SuggestCreator.java
View file

@ -183,6 +183,16 @@ public class SuggestCreator {
final SuggestHelper suggestHelper = ComponentUtil.getSuggestHelper();
logger.info("Create update index.");
suggestHelper.suggester().createNextIndex();
logger.info("storeAllBadWords");
suggestHelper.storeAllBadWords(true);
logger.info("storeAllElevateWords");
suggestHelper.storeAllElevateWords(true);
logger.info("indexFromDocuments");
suggestHelper.indexFromDocuments(ret -> {
logger.info("Success index from documents.");
result.set(0);
@ -200,6 +210,15 @@ public class SuggestCreator {
}
}
logger.info("storeSearchLog");
suggestHelper.storeSearchLog();
logger.info("switchIndex");
suggestHelper.suggester().switchIndex();
logger.info("removeDisableIndices");
suggestHelper.suggester().removeDisableIndices();
return result.get();
}

55
src/main/java/org/codelibs/fess/helper/SuggestHelper.java
View file

@ -20,6 +20,7 @@ import static org.codelibs.core.stream.StreamUtil.stream;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
@ -36,6 +37,7 @@ import org.codelibs.fess.es.config.exbhv.BadWordBhv;
import org.codelibs.fess.es.config.exbhv.ElevateWordBhv;
import org.codelibs.fess.es.config.exentity.BadWord;
import org.codelibs.fess.es.config.exentity.ElevateWord;
import org.codelibs.fess.es.log.exbhv.SearchLogBhv;
import org.codelibs.fess.es.log.exentity.SearchFieldLog;
import org.codelibs.fess.es.log.exentity.SearchLog;
import org.codelibs.fess.mylasta.direction.FessConfig;
@ -116,8 +118,18 @@ public class SuggestHelper {
return suggester;
}
public void storeSearchLog() {
final SearchLogBhv searchLogBhv = ComponentUtil.getComponent(SearchLogBhv.class);
searchLogBhv.selectBulk((cb) -> {
final String from = LocalDateTime.now().minusDays(fessConfig.getPurgeSuggestSearchLogDay()).format(DateTimeFormatter.ISO_DATE);
cb.query().addQuery(QueryBuilders.rangeQuery("requestedAt").gte(from));
cb.query().addOrderBy_RequestedAt_Asc();
}, (searchLogsList) -> indexFromSearchLog(searchLogsList));
}
public void indexFromSearchLog(final List<SearchLog> searchLogList) {
final Set<String> sessionIdSet = new HashSet<>();
final Set<String> ipSet = new HashSet<>();
searchLogList.stream().forEach(
searchLog -> {
if (searchLog.getHitCount() == null
@ -125,8 +137,8 @@ public class SuggestHelper {
return;
}
final String sessionId = searchLog.getUserSessionId();
if (sessionId == null || sessionIdSet.contains(sessionId)) {
final String clientIp = searchLog.getClientIp();
if (clientIp == null || ipSet.contains(clientIp)) {
return;
}
@ -156,7 +168,7 @@ public class SuggestHelper {
if (fessConfig.isValidSearchLogPermissions(roles.toArray(new String[roles.size()]))) {
suggester.indexer().indexFromSearchWord(sb.toString(), fields.toArray(new String[fields.size()]),
tags.toArray(new String[tags.size()]), roles.toArray(new String[roles.size()]), 1, langs);
sessionIdSet.add(sessionId);
ipSet.add(clientIp);
}
}
});
@ -257,8 +269,8 @@ public class SuggestHelper {
return true;
}
public void storeAllElevateWords() {
deleteAllBadWords();
public void storeAllElevateWords(final boolean apply) {
deleteAllElevateWord(apply);
final List<ElevateWord> list = elevateWordBhv.selectList(cb -> {
cb.query().matchAll();
@ -267,35 +279,30 @@ public class SuggestHelper {
for (final ElevateWord elevateWord : list) {
addElevateWord(elevateWord.getSuggestWord(), elevateWord.getReading(), elevateWord.getLabelTypeValues(),
elevateWord.getPermissions(), elevateWord.getBoost(), false);
elevateWord.getPermissions(), elevateWord.getBoost(), apply);
}
refresh();
}
public void deleteAllElevateWord() {
public void deleteAllElevateWord(final boolean apply) {
final List<ElevateWord> list = elevateWordBhv.selectList(cb -> {
cb.query().matchAll();
cb.fetchFirst(ComponentUtil.getFessConfig().getPageElevateWordMaxFetchSizeAsInteger());
});
for (final ElevateWord elevateWord : list) {
suggester.indexer().deleteElevateWord(elevateWord.getSuggestWord());
suggester.indexer().deleteElevateWord(elevateWord.getSuggestWord(), apply);
}
refresh();
}
public void deleteElevateWord(final String word) {
suggester.indexer().deleteElevateWord(word);
public void deleteElevateWord(final String word, final boolean apply) {
suggester.indexer().deleteElevateWord(word, apply);
refresh();
}
public void addElevateWord(final String word, final String reading, final String[] tags, final String[] permissions, final Float boost) {
addElevateWord(word, reading, tags, permissions, boost, true);
refresh();
}
protected void addElevateWord(final String word, final String reading, final String[] tags, final String[] permissions,
final float boost, final boolean commit) {
public void addElevateWord(final String word, final String reading, final String[] tags, final String[] permissions, final Float boost,
final boolean apply) {
final String[] readings;
if (StringUtil.isBlank(reading)) {
readings = word.replace("　", TEXT_SEP).replaceAll(TEXT_SEP + "+", TEXT_SEP).split(TEXT_SEP);
@ -314,14 +321,16 @@ public class SuggestHelper {
suggester.indexer().addElevateWord(
new org.codelibs.fess.suggest.entity.ElevateWord(word, boost, Arrays.asList(readings), contentFieldList, labelList,
roleList));
roleList), apply);
refresh();
}
protected void deleteAllBadWords() {
suggester.settings().badword().deleteAll();
}
public void storeAllBadWords() {
public void storeAllBadWords(final boolean apply) {
deleteAllBadWords();
final List<BadWord> list = badWordBhv.selectList(cb -> {
cb.query().matchAll();
@ -329,13 +338,13 @@ public class SuggestHelper {
});
for (final BadWord badWord : list) {
final String word = badWord.getSuggestWord();
suggester.indexer().addBadWord(word);
suggester.indexer().addBadWord(word, apply);
}
refresh();
}
public void addBadWord(final String badWord) {
suggester.indexer().addBadWord(badWord);
public void addBadWord(final String badWord, final boolean apply) {
suggester.indexer().addBadWord(badWord, apply);
refresh();
}