0ct0pu5/fess
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix #1080 update expires

Browse source
This commit is contained in:
Shinsuke Sugaya 2017-05-31 06:50:50 +09:00
parent 562479344e
commit 02973940d1
2 changed files with 18 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
src/main/java/org/codelibs/fess/crawler/FessCrawlerThread.java
View file

@ -125,8 +125,8 @@ public class FessCrawlerThread extends CrawlerThread {
final Date expires = DocumentUtil.getValue(document, fessConfig.getIndexFieldExpires(), Date.class);
if (expires != null && expires.getTime() < System.currentTimeMillis()) {
final Object idValue = document.get(fessConfig.getIndexFieldId());
if (idValue != null) {
indexingHelper.deleteDocument(fessEsClient, idValue.toString());
if (idValue != null && !indexingHelper.deleteDocument(fessEsClient, idValue.toString())) {
logger.debug("Failed to delete expired document: " + url);
}
return true;
}
@ -152,7 +152,9 @@ public class FessCrawlerThread extends CrawlerThread {
}
if (httpStatusCode == 404) {
storeChildUrlsToQueue(urlQueue, getAnchorSet(document.get(fessConfig.getIndexFieldAnchor())));
indexingHelper.deleteDocument(fessEsClient, id);
if (!indexingHelper.deleteDocument(fessEsClient, id)) {
logger.debug("Failed to delete 404 document: " + url);
}
return false;
} else if (responseData.getLastModified() == null) {
return true;
@ -168,6 +170,12 @@ public class FessCrawlerThread extends CrawlerThread {
storeChildUrlsToQueue(urlQueue, getAnchorSet(document.get(fessConfig.getIndexFieldAnchor())));
final Date documentExpires = crawlingInfoHelper.getDocumentExpires(crawlingConfig);
if (documentExpires != null
&& !indexingHelper.updateDocument(fessEsClient, id, fessConfig.getIndexFieldExpires(), documentExpires)) {
logger.debug("Failed to update " + fessConfig.getIndexFieldExpires() + " at " + url);
}
return false;
}
} finally {

9
src/main/java/org/codelibs/fess/helper/IndexingHelper.java
View file

@ -116,9 +116,14 @@ public class IndexingHelper {
}
}
public void deleteDocument(final FessEsClient fessEsClient, final String id) {
public boolean updateDocument(final FessEsClient fessEsClient, final String id, final String field, final Object value) {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
fessEsClient.delete(fessConfig.getIndexDocumentUpdateIndex(), fessConfig.getIndexDocumentType(), id, 0);
return fessEsClient.update(fessConfig.getIndexDocumentUpdateIndex(), fessConfig.getIndexDocumentType(), id, field, value);
}
public boolean deleteDocument(final FessEsClient fessEsClient, final String id) {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
return fessEsClient.delete(fessConfig.getIndexDocumentUpdateIndex(), fessConfig.getIndexDocumentType(), id, 0);
}
public int deleteDocumentByUrl(final FessEsClient fessEsClient, final String url) {