diff --git a/src/main/java/org/codelibs/fess/app/web/admin/badword/AdminBadwordAction.java b/src/main/java/org/codelibs/fess/app/web/admin/badword/AdminBadwordAction.java index 65ed20ce1..3d97e276c 100644 --- a/src/main/java/org/codelibs/fess/app/web/admin/badword/AdminBadwordAction.java +++ b/src/main/java/org/codelibs/fess/app/web/admin/badword/AdminBadwordAction.java @@ -223,7 +223,7 @@ public class AdminBadwordAction extends FessAdminAction { entity -> { try { badWordService.store(entity); - suggestHelper.addBadWord(entity.getSuggestWord()); + suggestHelper.addBadWord(entity.getSuggestWord(), false); saveInfo(messages -> messages.addSuccessCrudCreateCrudTable(GLOBAL)); } catch (final Exception e) { throwValidationError(messages -> messages.addErrorsCrudFailedToCreateCrudTable(GLOBAL, buildThrowableMessage(e)), @@ -244,7 +244,7 @@ public class AdminBadwordAction extends FessAdminAction { entity -> { try { badWordService.store(entity); - suggestHelper.storeAllBadWords(); + suggestHelper.storeAllBadWords(false); saveInfo(messages -> messages.addSuccessCrudUpdateCrudTable(GLOBAL)); } catch (final Exception e) { throwValidationError(messages -> messages.addErrorsCrudFailedToUpdateCrudTable(GLOBAL, buildThrowableMessage(e)), @@ -288,7 +288,7 @@ public class AdminBadwordAction extends FessAdminAction { new Thread(() -> { try (Reader reader = new BufferedReader(new InputStreamReader(form.badWordFile.getInputStream(), getCsvEncoding()))) { badWordService.importCsv(reader); - suggestHelper.storeAllBadWords(); + suggestHelper.storeAllBadWords(false); } catch (final Exception e) { throw new FessSystemException("Failed to import data.", e); } diff --git a/src/main/java/org/codelibs/fess/app/web/admin/elevateword/AdminElevatewordAction.java b/src/main/java/org/codelibs/fess/app/web/admin/elevateword/AdminElevatewordAction.java index 603f3c03d..b6cbbcf2b 100644 --- a/src/main/java/org/codelibs/fess/app/web/admin/elevateword/AdminElevatewordAction.java +++ b/src/main/java/org/codelibs/fess/app/web/admin/elevateword/AdminElevatewordAction.java @@ -256,7 +256,7 @@ public class AdminElevatewordAction extends FessAdminAction { try { elevateWordService.store(entity); suggestHelper.addElevateWord(entity.getSuggestWord(), entity.getReading(), entity.getLabelTypeValues(), - entity.getPermissions(), entity.getBoost()); + entity.getPermissions(), entity.getBoost(), false); saveInfo(messages -> messages.addSuccessCrudCreateCrudTable(GLOBAL)); } catch (final Exception e) { throwValidationError(messages -> messages.addErrorsCrudFailedToCreateCrudTable(GLOBAL, buildThrowableMessage(e)), @@ -277,8 +277,8 @@ public class AdminElevatewordAction extends FessAdminAction { entity -> { try { elevateWordService.store(entity); - suggestHelper.deleteAllElevateWord(); - suggestHelper.storeAllElevateWords(); + suggestHelper.deleteAllElevateWord(false); + suggestHelper.storeAllElevateWords(false); saveInfo(messages -> messages.addSuccessCrudUpdateCrudTable(GLOBAL)); } catch (final Exception e) { throwValidationError(messages -> messages.addErrorsCrudFailedToUpdateCrudTable(GLOBAL, buildThrowableMessage(e)), @@ -302,7 +302,7 @@ public class AdminElevatewordAction extends FessAdminAction { entity -> { try { elevateWordService.delete(entity); - suggestHelper.deleteElevateWord(entity.getSuggestWord()); + suggestHelper.deleteElevateWord(entity.getSuggestWord(), false); saveInfo(messages -> messages.addSuccessCrudDeleteCrudTable(GLOBAL)); } catch (final Exception e) { throwValidationError( @@ -322,8 +322,8 @@ public class AdminElevatewordAction extends FessAdminAction { new Thread(() -> { try (Reader reader = new BufferedReader(new InputStreamReader(form.elevateWordFile.getInputStream(), getCsvEncoding()))) { elevateWordService.importCsv(reader); - suggestHelper.deleteAllElevateWord(); - suggestHelper.storeAllElevateWords(); + suggestHelper.deleteAllElevateWord(false); + suggestHelper.storeAllElevateWords(false); } catch (final Exception e) { throw new FessSystemException("Failed to import data.", e); } diff --git a/src/main/java/org/codelibs/fess/app/web/api/admin/badword/ApiAdminBadwordAction.java b/src/main/java/org/codelibs/fess/app/web/api/admin/badword/ApiAdminBadwordAction.java index a915736a3..8a26b1abd 100644 --- a/src/main/java/org/codelibs/fess/app/web/api/admin/badword/ApiAdminBadwordAction.java +++ b/src/main/java/org/codelibs/fess/app/web/api/admin/badword/ApiAdminBadwordAction.java @@ -92,7 +92,7 @@ public class ApiAdminBadwordAction extends FessApiAdminAction { }); try { badWordService.store(entity); - suggestHelper.addBadWord(entity.getSuggestWord()); + suggestHelper.addBadWord(entity.getSuggestWord(), false); } catch (final Exception e) { throwValidationErrorApi(messages -> messages.addErrorsCrudFailedToCreateCrudTable(GLOBAL, buildThrowableMessage(e))); } @@ -107,7 +107,7 @@ public class ApiAdminBadwordAction extends FessApiAdminAction { final BadWord badWord = getBadWord(body).map(entity -> { try { badWordService.store(entity); - suggestHelper.storeAllBadWords(); + suggestHelper.storeAllBadWords(false); } catch (final Exception e) { throwValidationErrorApi(messages -> messages.addErrorsCrudFailedToUpdateCrudTable(GLOBAL, buildThrowableMessage(e))); } @@ -148,7 +148,7 @@ public class ApiAdminBadwordAction extends FessApiAdminAction { new Thread(() -> { try (Reader reader = new BufferedReader(new InputStreamReader(body.badWordFile.getInputStream(), getCsvEncoding()))) { badWordService.importCsv(reader); - suggestHelper.storeAllBadWords(); + suggestHelper.storeAllBadWords(false); } catch (final Exception e) { throw new FessSystemException("Failed to import data.", e); } diff --git a/src/main/java/org/codelibs/fess/app/web/api/admin/elevateword/ApiAdminElevatewordAction.java b/src/main/java/org/codelibs/fess/app/web/api/admin/elevateword/ApiAdminElevatewordAction.java index 87bfe69ea..f085b8255 100644 --- a/src/main/java/org/codelibs/fess/app/web/api/admin/elevateword/ApiAdminElevatewordAction.java +++ b/src/main/java/org/codelibs/fess/app/web/api/admin/elevateword/ApiAdminElevatewordAction.java @@ -102,7 +102,7 @@ public class ApiAdminElevatewordAction extends FessApiAdminAction { try { elevateWordService.store(entity); suggestHelper.addElevateWord(entity.getSuggestWord(), entity.getReading(), entity.getLabelTypeValues(), - entity.getPermissions(), entity.getBoost()); + entity.getPermissions(), entity.getBoost(), false); } catch (final Exception e) { throwValidationErrorApi(messages -> messages.addErrorsCrudFailedToCreateCrudTable(GLOBAL, buildThrowableMessage(e))); } @@ -117,8 +117,8 @@ public class ApiAdminElevatewordAction extends FessApiAdminAction { final ElevateWord elevateWord = getElevateWord(body).map(entity -> { try { elevateWordService.store(entity); - suggestHelper.deleteAllElevateWord(); - suggestHelper.storeAllElevateWords(); + suggestHelper.deleteAllElevateWord(false); + suggestHelper.storeAllElevateWords(false); } catch (final Exception e) { throwValidationErrorApi(messages -> messages.addErrorsCrudFailedToUpdateCrudTable(GLOBAL, buildThrowableMessage(e))); } @@ -138,7 +138,7 @@ public class ApiAdminElevatewordAction extends FessApiAdminAction { elevateWordService.getElevateWord(id).ifPresent(entity -> { try { elevateWordService.delete(entity); - suggestHelper.deleteElevateWord(entity.getSuggestWord()); + suggestHelper.deleteElevateWord(entity.getSuggestWord(), false); saveInfo(messages -> messages.addSuccessCrudDeleteCrudTable(GLOBAL)); } catch (final Exception e) { throwValidationErrorApi(messages -> messages.addErrorsCrudFailedToDeleteCrudTable(GLOBAL, buildThrowableMessage(e))); @@ -159,7 +159,7 @@ public class ApiAdminElevatewordAction extends FessApiAdminAction { new Thread(() -> { try (Reader reader = new BufferedReader(new InputStreamReader(body.elevateWordFile.getInputStream(), getCsvEncoding()))) { elevateWordService.importCsv(reader); - suggestHelper.storeAllElevateWords(); + suggestHelper.storeAllElevateWords(false); } catch (final Exception e) { throw new FessSystemException("Failed to import data.", e); } diff --git a/src/main/java/org/codelibs/fess/exec/SuggestCreator.java b/src/main/java/org/codelibs/fess/exec/SuggestCreator.java index 83c87181f..fc05fbe5e 100644 --- a/src/main/java/org/codelibs/fess/exec/SuggestCreator.java +++ b/src/main/java/org/codelibs/fess/exec/SuggestCreator.java @@ -183,6 +183,16 @@ public class SuggestCreator { final SuggestHelper suggestHelper = ComponentUtil.getSuggestHelper(); + logger.info("Create update index."); + suggestHelper.suggester().createNextIndex(); + + logger.info("storeAllBadWords"); + suggestHelper.storeAllBadWords(true); + + logger.info("storeAllElevateWords"); + suggestHelper.storeAllElevateWords(true); + + logger.info("indexFromDocuments"); suggestHelper.indexFromDocuments(ret -> { logger.info("Success index from documents."); result.set(0); @@ -200,6 +210,15 @@ public class SuggestCreator { } } + logger.info("storeSearchLog"); + suggestHelper.storeSearchLog(); + + logger.info("switchIndex"); + suggestHelper.suggester().switchIndex(); + + logger.info("removeDisableIndices"); + suggestHelper.suggester().removeDisableIndices(); + return result.get(); } diff --git a/src/main/java/org/codelibs/fess/helper/SuggestHelper.java b/src/main/java/org/codelibs/fess/helper/SuggestHelper.java index a340a1070..1d1599089 100644 --- a/src/main/java/org/codelibs/fess/helper/SuggestHelper.java +++ b/src/main/java/org/codelibs/fess/helper/SuggestHelper.java @@ -20,6 +20,7 @@ import static org.codelibs.core.stream.StreamUtil.stream; import java.time.LocalDateTime; import java.time.ZoneId; +import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; @@ -36,6 +37,7 @@ import org.codelibs.fess.es.config.exbhv.BadWordBhv; import org.codelibs.fess.es.config.exbhv.ElevateWordBhv; import org.codelibs.fess.es.config.exentity.BadWord; import org.codelibs.fess.es.config.exentity.ElevateWord; +import org.codelibs.fess.es.log.exbhv.SearchLogBhv; import org.codelibs.fess.es.log.exentity.SearchFieldLog; import org.codelibs.fess.es.log.exentity.SearchLog; import org.codelibs.fess.mylasta.direction.FessConfig; @@ -116,8 +118,18 @@ public class SuggestHelper { return suggester; } + public void storeSearchLog() { + final SearchLogBhv searchLogBhv = ComponentUtil.getComponent(SearchLogBhv.class); + + searchLogBhv.selectBulk((cb) -> { + final String from = LocalDateTime.now().minusDays(fessConfig.getPurgeSuggestSearchLogDay()).format(DateTimeFormatter.ISO_DATE); + cb.query().addQuery(QueryBuilders.rangeQuery("requestedAt").gte(from)); + cb.query().addOrderBy_RequestedAt_Asc(); + }, (searchLogsList) -> indexFromSearchLog(searchLogsList)); + } + public void indexFromSearchLog(final List searchLogList) { - final Set sessionIdSet = new HashSet<>(); + final Set ipSet = new HashSet<>(); searchLogList.stream().forEach( searchLog -> { if (searchLog.getHitCount() == null @@ -125,8 +137,8 @@ public class SuggestHelper { return; } - final String sessionId = searchLog.getUserSessionId(); - if (sessionId == null || sessionIdSet.contains(sessionId)) { + final String clientIp = searchLog.getClientIp(); + if (clientIp == null || ipSet.contains(clientIp)) { return; } @@ -156,7 +168,7 @@ public class SuggestHelper { if (fessConfig.isValidSearchLogPermissions(roles.toArray(new String[roles.size()]))) { suggester.indexer().indexFromSearchWord(sb.toString(), fields.toArray(new String[fields.size()]), tags.toArray(new String[tags.size()]), roles.toArray(new String[roles.size()]), 1, langs); - sessionIdSet.add(sessionId); + ipSet.add(clientIp); } } }); @@ -257,8 +269,8 @@ public class SuggestHelper { return true; } - public void storeAllElevateWords() { - deleteAllBadWords(); + public void storeAllElevateWords(final boolean apply) { + deleteAllElevateWord(apply); final List list = elevateWordBhv.selectList(cb -> { cb.query().matchAll(); @@ -267,35 +279,30 @@ public class SuggestHelper { for (final ElevateWord elevateWord : list) { addElevateWord(elevateWord.getSuggestWord(), elevateWord.getReading(), elevateWord.getLabelTypeValues(), - elevateWord.getPermissions(), elevateWord.getBoost(), false); + elevateWord.getPermissions(), elevateWord.getBoost(), apply); } refresh(); } - public void deleteAllElevateWord() { + public void deleteAllElevateWord(final boolean apply) { final List list = elevateWordBhv.selectList(cb -> { cb.query().matchAll(); cb.fetchFirst(ComponentUtil.getFessConfig().getPageElevateWordMaxFetchSizeAsInteger()); }); for (final ElevateWord elevateWord : list) { - suggester.indexer().deleteElevateWord(elevateWord.getSuggestWord()); + suggester.indexer().deleteElevateWord(elevateWord.getSuggestWord(), apply); } refresh(); } - public void deleteElevateWord(final String word) { - suggester.indexer().deleteElevateWord(word); + public void deleteElevateWord(final String word, final boolean apply) { + suggester.indexer().deleteElevateWord(word, apply); refresh(); } - public void addElevateWord(final String word, final String reading, final String[] tags, final String[] permissions, final Float boost) { - addElevateWord(word, reading, tags, permissions, boost, true); - refresh(); - } - - protected void addElevateWord(final String word, final String reading, final String[] tags, final String[] permissions, - final float boost, final boolean commit) { + public void addElevateWord(final String word, final String reading, final String[] tags, final String[] permissions, final Float boost, + final boolean apply) { final String[] readings; if (StringUtil.isBlank(reading)) { readings = word.replace("　", TEXT_SEP).replaceAll(TEXT_SEP + "+", TEXT_SEP).split(TEXT_SEP); @@ -314,14 +321,16 @@ public class SuggestHelper { suggester.indexer().addElevateWord( new org.codelibs.fess.suggest.entity.ElevateWord(word, boost, Arrays.asList(readings), contentFieldList, labelList, - roleList)); + roleList), apply); + + refresh(); } protected void deleteAllBadWords() { suggester.settings().badword().deleteAll(); } - public void storeAllBadWords() { + public void storeAllBadWords(final boolean apply) { deleteAllBadWords(); final List list = badWordBhv.selectList(cb -> { cb.query().matchAll(); @@ -329,13 +338,13 @@ public class SuggestHelper { }); for (final BadWord badWord : list) { final String word = badWord.getSuggestWord(); - suggester.indexer().addBadWord(word); + suggester.indexer().addBadWord(word, apply); } refresh(); } - public void addBadWord(final String badWord) { - suggester.indexer().addBadWord(badWord); + public void addBadWord(final String badWord, final boolean apply) { + suggester.indexer().addBadWord(badWord, apply); refresh(); }