0ct0pu5/fess
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

#2488 use local data map

Browse source
This commit is contained in:
Shinsuke Sugaya 2020-10-10 15:25:54 +09:00
parent 46a9aee502
commit b7d56472a1
1 changed files with 5 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
src/main/java/org/codelibs/fess/ds/callback/FileListIndexUpdateCallbackImpl.java
View file

@ -130,21 +130,23 @@ public class FileListIndexUpdateCallbackImpl implements IndexUpdateCallback {
final Deque<String> urlQueue = new LinkedList<>();
urlQueue.offer(url);
while (!urlQueue.isEmpty() && (maxAccessCount < 0 || counter < maxAccessCount)) {
final Map<String, Object> localDataMap =
dataMap.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
String processingUrl = urlQueue.poll();
if (deleteUrlList.contains(processingUrl)) {
deleteDocuments(); // delete before indexing
}
try {
for (int i = 0; i < maxRedirectCount; i++) {
processingUrl = processRequest(paramMap, dataMap, processingUrl, client);
processingUrl = processRequest(paramMap, localDataMap, processingUrl, client);
if (processingUrl == null) {
break;
}
counter++;
dataMap.put(fessConfig.getIndexFieldUrl(), processingUrl);
localDataMap.put(fessConfig.getIndexFieldUrl(), processingUrl);
}
} catch (ChildUrlsException e) {
e.getChildUrlList().stream().map(req -> req.getUrl()).forEach(urlQueue::offer);
e.getChildUrlList().stream().map(RequestData::getUrl).forEach(urlQueue::offer);
} catch (DataStoreCrawlingException e) {
Throwable cause = e.getCause();
if (cause instanceof ChildUrlsException) {