0ct0pu5/fess
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix #2834 Added language detection and default value settings to bulk document indexing in Admin API.

Browse source
This commit is contained in:
Shinsuke Sugaya 2024-07-25 10:47:59 +09:00
parent 11beb70eb5
commit 9999aad850
3 changed files with 71 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

56
src/main/java/org/codelibs/fess/app/web/api/admin/documents/ApiAdminDocumentsAction.java
View file

@ -16,6 +16,7 @@
package org.codelibs.fess.app.web.api.admin.documents;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -33,6 +34,7 @@ import org.codelibs.fess.app.web.api.admin.searchlist.ApiAdminSearchlistAction;
import org.codelibs.fess.es.client.SearchEngineClient;
import org.codelibs.fess.helper.CrawlingConfigHelper;
import org.codelibs.fess.helper.CrawlingInfoHelper;
import org.codelibs.fess.helper.LanguageHelper;
import org.codelibs.fess.thumbnail.ThumbnailManager;
import org.codelibs.fess.util.ComponentUtil;
import org.lastaflute.web.Execute;
@ -66,12 +68,60 @@ public class ApiAdminDocumentsAction extends FessApiAdminAction {
throwValidationErrorApi(messages -> messages.addErrorsCrudFailedToCreateCrudTable(GLOBAL, "documents is empty."));
}
final String indexFieldId = fessConfig.getIndexFieldId();
final String indexFieldDocId = fessConfig.getIndexFieldDocId();
final String indexFieldContentLength = fessConfig.getIndexFieldContentLength();
final String indexFieldTitle = fessConfig.getIndexFieldTitle();
final String indexFieldContent = fessConfig.getIndexFieldContent();
final String indexFieldFavoriteCount = fessConfig.getIndexFieldFavoriteCount();
final String indexFieldClickCount = fessConfig.getIndexFieldClickCount();
final String indexFieldBoost = fessConfig.getIndexFieldBoost();
final String indexFieldRole = fessConfig.getIndexFieldRole();
final String indexFieldLastModified = fessConfig.getIndexFieldLastModified();
final String indexFieldTimestamp = fessConfig.getIndexFieldTimestamp();
final String indexFieldLang = fessConfig.getIndexFieldLang();
final List<String> guestRoleList = fessConfig.getSearchGuestRoleList();
final Date now = systemHelper.getCurrentTime();
final CrawlingInfoHelper crawlingInfoHelper = ComponentUtil.getCrawlingInfoHelper();
final LanguageHelper languageHelper = ComponentUtil.getLanguageHelper();
final List<Map<String, Object>> docList = body.documents.stream().map(doc -> {
if (!doc.containsKey(indexFieldContentLength)) {
long contentLength = 0;
if (doc.get(indexFieldTitle) instanceof final String title) {
contentLength += title.length();
}
if (doc.get(indexFieldContent) instanceof final String content) {
contentLength += content.length();
}
doc.put(indexFieldContentLength, contentLength);
}
if (!doc.containsKey(indexFieldFavoriteCount)) {
doc.put(indexFieldFavoriteCount, 0L);
}
if (!doc.containsKey(indexFieldClickCount)) {
doc.put(indexFieldClickCount, 0L);
}
if (!doc.containsKey(indexFieldBoost)) {
doc.put(indexFieldBoost, 1.0f);
}
if (!doc.containsKey(indexFieldRole)) {
doc.put(indexFieldRole, guestRoleList);
}
if (!doc.containsKey(indexFieldLastModified)) {
doc.put(indexFieldLastModified, now);
}
if (!doc.containsKey(indexFieldTimestamp)) {
doc.put(indexFieldTimestamp, now);
}
AdminSearchlistAction.validateFields(doc, this::throwValidationErrorApi);
final Map<String, Object> newDoc = fessConfig.convertToStorableDoc(doc);
final String newId = crawlingInfoHelper.generateId(newDoc);
newDoc.put(indexFieldId, newId);
newDoc.put(indexFieldId, crawlingInfoHelper.generateId(newDoc));
newDoc.put(indexFieldDocId, systemHelper.generateDocId(newDoc));
if (newDoc.get(indexFieldLang) instanceof final List<?> langList) {
if (langList.contains("auto")) {
newDoc.remove(indexFieldLang);
}
languageHelper.updateDocument(newDoc);
}
return newDoc;
}).toList();
if (fessConfig.isThumbnailCrawlerEnabled()) {
@ -96,7 +146,7 @@ public class ApiAdminDocumentsAction extends FessApiAdminAction {
});
return asJson(new ApiBulkResponse().items(Arrays.stream(response.getItems()).map(item -> {
final Map<String, Object> itemMap = new HashMap<>();
itemMap.put("status", item.status().name());
itemMap.put("result", item.status().name());
if (item.isFailed()) {
itemMap.put("message", item.getFailureMessage());
} else {

2
src/main/java/org/codelibs/fess/helper/RoleQueryHelper.java
View file

@ -135,7 +135,7 @@ public class RoleQueryHelper {
throw new InvalidAccessTokenException("invalid_token", "Access token is requried.");
}
if (!hasAccessToken || roleSet.isEmpty()) {
roleSet.addAll(fessConfig.getSearchGuestPermissionList());
roleSet.addAll(fessConfig.getSearchGuestRoleList());
}
});
} catch (final RuntimeException e) {

25
src/main/java/org/codelibs/fess/mylasta/direction/FessProp.java
View file

@ -132,7 +132,7 @@ public interface FessProp {
String AUTHENTICATION_ADMIN_ROLES = "authenticationAdminRoles";
String SEARCH_GUEST_PERMISSION_LIST = "searchGuestPermissionList";
String SEARCH_GUEST_ROLE_LIST = "searchGuestPermissionList";
String SUGGEST_SEARCH_LOG_PERMISSIONS = "suggestSearchLogPermissions";
@ -1276,15 +1276,15 @@ public interface FessProp {
String getRoleSearchGuestPermissions();
default List<String> getSearchGuestPermissionList() {
default List<String> getSearchGuestRoleList() {
@SuppressWarnings("unchecked")
List<String> list = (List<String>) propMap.get(SEARCH_GUEST_PERMISSION_LIST);
List<String> list = (List<String>) propMap.get(SEARCH_GUEST_ROLE_LIST);
if (list == null) {
final PermissionHelper permissionHelper = ComponentUtil.getPermissionHelper();
list = split(getRoleSearchGuestPermissions(), ",")
.get(stream -> stream.map(s -> permissionHelper.encode(s)).filter(StringUtil::isNotBlank).collect(Collectors.toList()));
list.add(getRoleSearchUserPrefix() + Constants.GUEST_USER);
propMap.put(SEARCH_GUEST_PERMISSION_LIST, list);
propMap.put(SEARCH_GUEST_ROLE_LIST, list);
}
return list;
}
@ -1331,9 +1331,14 @@ public interface FessProp {
}
default List<String> invalidIndexDateFields(final Map<String, Object> source) {
return split(getIndexAdminDateFields(), ",")
.get(stream -> stream.filter(StringUtil::isNotBlank).map(String::trim).filter(s -> isNonEmptyValue(source.get(s)))
.filter(s -> !validateDateTimeString(source.get(s))).collect(Collectors.toList()));
return split(getIndexAdminDateFields(), ",").get(
stream -> stream.filter(StringUtil::isNotBlank).map(String::trim).filter(s -> isNonEmptyValue(source.get(s))).filter(s -> {
final Object obj = source.get(s);
if (obj instanceof Date) {
return false;
}
return !validateDateTimeString(source.get(s));
}).collect(Collectors.toList()));
}
default boolean validateDateTimeString(final Object obj) {
@ -1516,7 +1521,11 @@ public interface FessProp {
}
} else if (dateFieldSet.contains(key)) {
// TODO time zone
value = FessFunctions.parseDate(value.toString());
if (value instanceof Date) {
// nothing
} else {
value = FessFunctions.parseDate(value.toString());
}
} else if (integerFieldSet.contains(key)) {
if (value instanceof Number num) {
value = num.intValue();