fix #616 ignore error name for failure urls

This commit is contained in:
Shinsuke Sugaya 2016-08-18 22:28:01 +09:00
parent bdb03bef36
commit 236cf95e13
3 changed files with 16 additions and 22 deletions

View file

@ -383,8 +383,6 @@ public class Constants extends CoreLibConstants {
public static final String SEARCH_PREFERENCE_PRIMARY = "_primary";
public static final String CONFIG_IGNORE_FAILURE_URLS = "ignore.failureUrls";
public static final String CONFIG_CLEANUP_FILTERS = "cleanup.urlFilters";
public static final String CONFIG_CLEANUP_ALL = "cleanup.all";

View file

@ -146,7 +146,7 @@ public class FailureUrlService {
final List<String> urlList = new ArrayList<>();
for (final FailureUrl failureUrl : list) {
if (pattern != null) {
if (!pattern.matcher(failureUrl.getUrl()).matches()) {
if (!pattern.matcher(failureUrl.getErrorName()).matches()) {
urlList.add(failureUrl.getUrl());
}
} else {

View file

@ -227,15 +227,13 @@ public class WebFsIndexHelper {
}
// failure url
if (!Constants.TRUE.equalsIgnoreCase(configParamMap.get(Constants.CONFIG_IGNORE_FAILURE_URLS))) {
final List<String> excludedUrlList = failureUrlService.getExcludedUrlList(webConfig.getConfigId());
for (final String u : excludedUrlList) {
if (StringUtil.isNotBlank(u)) {
final String urlValue = u.trim();
crawler.addExcludeFilter(urlValue);
if (logger.isInfoEnabled()) {
logger.info("Excluded URL from failures: " + urlValue);
}
final List<String> excludedUrlList = failureUrlService.getExcludedUrlList(webConfig.getConfigId());
for (final String u : excludedUrlList) {
if (StringUtil.isNotBlank(u)) {
final String urlValue = u.trim();
crawler.addExcludeFilter(urlValue);
if (logger.isInfoEnabled()) {
logger.info("Excluded URL from failures: " + urlValue);
}
}
}
@ -371,16 +369,14 @@ public class WebFsIndexHelper {
}
// failure url
if (!Constants.TRUE.equalsIgnoreCase(configParamMap.get(Constants.CONFIG_IGNORE_FAILURE_URLS))) {
final List<String> excludedUrlList = failureUrlService.getExcludedUrlList(fileConfig.getConfigId());
if (excludedUrlList != null) {
for (final String u : excludedUrlList) {
if (StringUtil.isNotBlank(u)) {
final String urlValue = u.trim();
crawler.addExcludeFilter(urlValue);
if (logger.isInfoEnabled()) {
logger.info("Excluded Path from failures: " + urlValue);
}
final List<String> excludedUrlList = failureUrlService.getExcludedUrlList(fileConfig.getConfigId());
if (excludedUrlList != null) {
for (final String u : excludedUrlList) {
if (StringUtil.isNotBlank(u)) {
final String urlValue = u.trim();
crawler.addExcludeFilter(urlValue);
if (logger.isInfoEnabled()) {
logger.info("Excluded Path from failures: " + urlValue);
}
}
}