Browse Source

fix #2171 use duplicate host conversion

Shinsuke Sugaya 6 years ago
parent
commit
5fdfefb925
1 changed files with 5 additions and 2 deletions
  1. 5 2
      src/main/java/org/codelibs/fess/helper/WebFsIndexHelper.java

+ 5 - 2
src/main/java/org/codelibs/fess/helper/WebFsIndexHelper.java

@@ -146,12 +146,15 @@ public class WebFsIndexHelper {
                 }
                 }
             }
             }
 
 
+            final DuplicateHostHelper duplicateHostHelper = ComponentUtil.getDuplicateHostHelper();
+
             // set urls
             // set urls
             split(urlsStr, "[\r\n]").of(stream -> stream.filter(StringUtil::isNotBlank).map(String::trim).distinct().forEach(urlValue -> {
             split(urlsStr, "[\r\n]").of(stream -> stream.filter(StringUtil::isNotBlank).map(String::trim).distinct().forEach(urlValue -> {
                 if (!urlValue.startsWith("#") && fessConfig.isValidCrawlerWebProtocol(urlValue)) {
                 if (!urlValue.startsWith("#") && fessConfig.isValidCrawlerWebProtocol(urlValue)) {
-                    crawler.addUrl(urlValue);
+                    final String u = duplicateHostHelper.convert(urlValue);
+                    crawler.addUrl(u);
                     if (logger.isInfoEnabled()) {
                     if (logger.isInfoEnabled()) {
-                        logger.info("Target URL: " + urlValue);
+                        logger.info("Target URL: " + u);
                     }
                     }
                 }
                 }
             }));
             }));