0ct0pu5/fess
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

WebCrawl support URLEncode and Disabled (#2678)

Browse source
This commit is contained in:
jasongwq 2022-09-03 08:52:18 +08:00 • committed by GitHub
parent 9fa9e17056
commit 63da35d257
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 24 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
src/main/java/org/codelibs/fess/helper/WebFsIndexHelper.java
View file

@ -162,22 +162,42 @@ public class WebFsIndexHelper {
}));
// set included urls
split(includedUrlsStr, "[\r\n]").of(stream -> stream.filter(StringUtil::isNotBlank).map(String::trim).forEach(urlValue -> {
if (!urlValue.startsWith("#")) {
final AtomicBoolean urlEncodeDisabled = new AtomicBoolean(false);
split(includedUrlsStr, "[\r\n]").of(stream -> stream.filter(StringUtil::isNotBlank).map(String::trim).forEach(line -> {
if (!line.startsWith("#")) {
final String urlValue;
if (urlEncodeDisabled.get()) {
urlValue = line;
urlEncodeDisabled.set(false);
} else {
urlValue = systemHelper.encodeUrlFilter(line);
}
crawler.addIncludeFilter(urlValue);
if (logger.isInfoEnabled()) {
logger.info("Included URL: {}", urlValue);
}
} else if (line.startsWith("#DISABLE_URL_ENCODE")) {
urlEncodeDisabled.set(true);
}
}));
// set excluded urls
split(excludedUrlsStr, "[\r\n]").of(stream -> stream.filter(StringUtil::isNotBlank).map(String::trim).forEach(urlValue -> {
if (!urlValue.startsWith("#")) {
urlEncodeDisabled.set(false);
split(excludedUrlsStr, "[\r\n]").of(stream -> stream.filter(StringUtil::isNotBlank).map(String::trim).forEach(line -> {
if (!line.startsWith("#")) {
final String urlValue;
if (urlEncodeDisabled.get()) {
urlValue = line;
urlEncodeDisabled.set(false);
} else {
urlValue = systemHelper.encodeUrlFilter(line);
}
crawler.addExcludeFilter(urlValue);
if (logger.isInfoEnabled()) {
logger.info("Excluded URL: {}", urlValue);
}
} else if (line.startsWith("#DISABLE_URL_ENCODE")) {
urlEncodeDisabled.set(true);
}
}));