fix #616 ignore error name for failure urls
This commit is contained in:
parent
bdb03bef36
commit
236cf95e13
3 changed files with 16 additions and 22 deletions
|
@ -383,8 +383,6 @@ public class Constants extends CoreLibConstants {
|
|||
|
||||
public static final String SEARCH_PREFERENCE_PRIMARY = "_primary";
|
||||
|
||||
public static final String CONFIG_IGNORE_FAILURE_URLS = "ignore.failureUrls";
|
||||
|
||||
public static final String CONFIG_CLEANUP_FILTERS = "cleanup.urlFilters";
|
||||
|
||||
public static final String CONFIG_CLEANUP_ALL = "cleanup.all";
|
||||
|
|
|
@ -146,7 +146,7 @@ public class FailureUrlService {
|
|||
final List<String> urlList = new ArrayList<>();
|
||||
for (final FailureUrl failureUrl : list) {
|
||||
if (pattern != null) {
|
||||
if (!pattern.matcher(failureUrl.getUrl()).matches()) {
|
||||
if (!pattern.matcher(failureUrl.getErrorName()).matches()) {
|
||||
urlList.add(failureUrl.getUrl());
|
||||
}
|
||||
} else {
|
||||
|
|
|
@ -227,15 +227,13 @@ public class WebFsIndexHelper {
|
|||
}
|
||||
|
||||
// failure url
|
||||
if (!Constants.TRUE.equalsIgnoreCase(configParamMap.get(Constants.CONFIG_IGNORE_FAILURE_URLS))) {
|
||||
final List<String> excludedUrlList = failureUrlService.getExcludedUrlList(webConfig.getConfigId());
|
||||
for (final String u : excludedUrlList) {
|
||||
if (StringUtil.isNotBlank(u)) {
|
||||
final String urlValue = u.trim();
|
||||
crawler.addExcludeFilter(urlValue);
|
||||
if (logger.isInfoEnabled()) {
|
||||
logger.info("Excluded URL from failures: " + urlValue);
|
||||
}
|
||||
final List<String> excludedUrlList = failureUrlService.getExcludedUrlList(webConfig.getConfigId());
|
||||
for (final String u : excludedUrlList) {
|
||||
if (StringUtil.isNotBlank(u)) {
|
||||
final String urlValue = u.trim();
|
||||
crawler.addExcludeFilter(urlValue);
|
||||
if (logger.isInfoEnabled()) {
|
||||
logger.info("Excluded URL from failures: " + urlValue);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -371,16 +369,14 @@ public class WebFsIndexHelper {
|
|||
}
|
||||
|
||||
// failure url
|
||||
if (!Constants.TRUE.equalsIgnoreCase(configParamMap.get(Constants.CONFIG_IGNORE_FAILURE_URLS))) {
|
||||
final List<String> excludedUrlList = failureUrlService.getExcludedUrlList(fileConfig.getConfigId());
|
||||
if (excludedUrlList != null) {
|
||||
for (final String u : excludedUrlList) {
|
||||
if (StringUtil.isNotBlank(u)) {
|
||||
final String urlValue = u.trim();
|
||||
crawler.addExcludeFilter(urlValue);
|
||||
if (logger.isInfoEnabled()) {
|
||||
logger.info("Excluded Path from failures: " + urlValue);
|
||||
}
|
||||
final List<String> excludedUrlList = failureUrlService.getExcludedUrlList(fileConfig.getConfigId());
|
||||
if (excludedUrlList != null) {
|
||||
for (final String u : excludedUrlList) {
|
||||
if (StringUtil.isNotBlank(u)) {
|
||||
final String urlValue = u.trim();
|
||||
crawler.addExcludeFilter(urlValue);
|
||||
if (logger.isInfoEnabled()) {
|
||||
logger.info("Excluded Path from failures: " + urlValue);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Add table
Reference in a new issue