0ct0pu5/fess
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix #2271 reuse CrawlerClientFactory

Browse source
This commit is contained in:
Shinsuke Sugaya 2019-10-13 11:04:06 +09:00
parent 416476b195
commit 37dc223d4c
9 changed files with 53 additions and 23 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
src/main/java/org/codelibs/fess/es/config/exentity/CrawlingConfig.java
View file

@ -16,6 +16,7 @@
package org.codelibs.fess.es.config.exentity;
import java.util.Map;
import java.util.function.Supplier;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.codelibs.core.lang.StringUtil;
@ -44,7 +45,7 @@ public interface CrawlingConfig {
Integer getTimeToLive();
Map<String, Object> initializeClientFactory(CrawlerClientFactory crawlerClientFactory);
CrawlerClientFactory initializeClientFactory(Supplier<CrawlerClientFactory> creator);
Map<String, String> getConfigParameterMap(ConfigName name);

5
src/main/java/org/codelibs/fess/es/config/exentity/CrawlingConfigWrapper.java
View file

@ -16,6 +16,7 @@
package org.codelibs.fess.es.config.exentity;
import java.util.Map;
import java.util.function.Supplier;
import org.codelibs.fess.crawler.client.CrawlerClientFactory;
@ -68,8 +69,8 @@ public class CrawlingConfigWrapper implements CrawlingConfig {
}
@Override
public Map<String, Object> initializeClientFactory(final CrawlerClientFactory crawlerClientFactory) {
return crawlingConfig.initializeClientFactory(crawlerClientFactory);
public CrawlerClientFactory initializeClientFactory(final Supplier<CrawlerClientFactory> creator) {
return crawlingConfig.initializeClientFactory(creator);
}
@Override

19
src/main/java/org/codelibs/fess/es/config/exentity/DataConfig.java
View file

@ -22,6 +22,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.function.Supplier;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
@ -76,9 +77,11 @@ public class DataConfig extends BsDataConfig implements CrawlingConfig {
protected Pattern[] excludedDocPathPatterns;
private Map<String, String> handlerParameterMap;
protected Map<String, String> handlerParameterMap;
private Map<String, String> handlerScriptMap;
protected Map<String, String> handlerScriptMap;
protected CrawlerClientFactory crawlerClientFactory = null;
public DataConfig() {
super();
@ -131,11 +134,16 @@ public class DataConfig extends BsDataConfig implements CrawlingConfig {
}
@Override
public Map<String, Object> initializeClientFactory(final CrawlerClientFactory crawlerClientFactory) {
public CrawlerClientFactory initializeClientFactory(final Supplier<CrawlerClientFactory> creator) {
if (crawlerClientFactory != null) {
return crawlerClientFactory;
}
final CrawlerClientFactory factory = creator.get();
final Map<String, String> paramMap = getHandlerParameterMap();
final Map<String, Object> factoryParamMap = new HashMap<>();
crawlerClientFactory.setInitParameterMap(factoryParamMap);
factory.setInitParameterMap(factoryParamMap);
// parameters
for (final Map.Entry<String, String> entry : paramMap.entrySet()) {
@ -284,7 +292,8 @@ public class DataConfig extends BsDataConfig implements CrawlingConfig {
}
}
return factoryParamMap;
crawlerClientFactory = factory;
return factory;
}
private AuthScheme getAuthScheme(final Map<String, String> paramMap, final String webAuthName, final String scheme) {

15
src/main/java/org/codelibs/fess/es/config/exentity/FileConfig.java
View file

@ -21,6 +21,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Supplier;
import java.util.regex.Pattern;
import org.codelibs.core.lang.StringUtil;
@ -47,6 +48,8 @@ public class FileConfig extends BsFileConfig implements CrawlingConfig {
protected transient volatile Map<ConfigName, Map<String, String>> configParameterMap;
protected CrawlerClientFactory crawlerClientFactory = null;
public FileConfig() {
super();
setBoost(1.0f);
@ -142,12 +145,17 @@ public class FileConfig extends BsFileConfig implements CrawlingConfig {
}
@Override
public Map<String, Object> initializeClientFactory(final CrawlerClientFactory clientFactory) {
public CrawlerClientFactory initializeClientFactory(final Supplier<CrawlerClientFactory> creator) {
if (crawlerClientFactory != null) {
return crawlerClientFactory;
}
final CrawlerClientFactory factory = creator.get();
final FileAuthenticationService fileAuthenticationService = ComponentUtil.getComponent(FileAuthenticationService.class);
// Parameters
final Map<String, Object> paramMap = new HashMap<>();
clientFactory.setInitParameterMap(paramMap);
factory.setInitParameterMap(paramMap);
final Map<String, String> clientConfigMap = getConfigParameterMap(ConfigName.CLIENT);
if (clientConfigMap != null) {
@ -193,7 +201,8 @@ public class FileConfig extends BsFileConfig implements CrawlingConfig {
smb1AuthList.toArray(new org.codelibs.fess.crawler.client.smb1.SmbAuthentication[smb1AuthList.size()]));
paramMap.put(Param.Client.FTP_AUTHENTICATIONS, ftpAuthList.toArray(new FtpAuthentication[ftpAuthList.size()]));
return paramMap;
crawlerClientFactory = factory;
return factory;
}
@Override

15
src/main/java/org/codelibs/fess/es/config/exentity/WebConfig.java
View file

@ -21,6 +21,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Supplier;
import java.util.regex.Pattern;
import org.apache.http.auth.UsernamePasswordCredentials;
@ -51,6 +52,8 @@ public class WebConfig extends BsWebConfig implements CrawlingConfig {
protected transient volatile Map<ConfigName, Map<String, String>> configParameterMap;
protected CrawlerClientFactory crawlerClientFactory = null;
public WebConfig() {
super();
setBoost(1.0f);
@ -145,14 +148,19 @@ public class WebConfig extends BsWebConfig implements CrawlingConfig {
}
@Override
public Map<String, Object> initializeClientFactory(final CrawlerClientFactory clientFactory) {
public CrawlerClientFactory initializeClientFactory(final Supplier<CrawlerClientFactory> creator) {
if (crawlerClientFactory != null) {
return crawlerClientFactory;
}
final CrawlerClientFactory factory = creator.get();
final WebAuthenticationService webAuthenticationService = ComponentUtil.getComponent(WebAuthenticationService.class);
final RequestHeaderService requestHeaderService = ComponentUtil.getComponent(RequestHeaderService.class);
final FessConfig fessConfig = ComponentUtil.getFessConfig();
// HttpClient Parameters
final Map<String, Object> paramMap = new HashMap<>();
clientFactory.setInitParameterMap(paramMap);
factory.setInitParameterMap(paramMap);
final Map<String, String> clientConfigMap = getConfigParameterMap(ConfigName.CLIENT);
if (clientConfigMap != null) {
@ -199,7 +207,8 @@ public class WebConfig extends BsWebConfig implements CrawlingConfig {
initializeDefaultHttpProxy(paramMap);
}
return paramMap;
crawlerClientFactory = factory;
return factory;
}
@Override

4
src/main/java/org/codelibs/fess/helper/DocumentHelper.java
View file

@ -178,8 +178,8 @@ public class DocumentHelper {
throw new CrawlingAccessException("sessionId is null.");
}
final CrawlerClientFactory crawlerClientFactory = ComponentUtil.getCrawlerClientFactory();
crawlingConfig.initializeClientFactory(crawlerClientFactory);
final CrawlerClientFactory crawlerClientFactory =
crawlingConfig.initializeClientFactory(() -> ComponentUtil.getCrawlerClientFactory());
final CrawlerClient client = crawlerClientFactory.getClient(url);
if (client == null) {
throw new CrawlingAccessException("CrawlerClient is null for " + url);

4
src/main/java/org/codelibs/fess/helper/ViewHelper.java
View file

@ -606,8 +606,8 @@ public class ViewHelper {
throw new FessSystemException("No crawlingConfig: " + configId);
}
final String url = DocumentUtil.getValue(doc, fessConfig.getIndexFieldUrl(), String.class);
final CrawlerClientFactory crawlerClientFactory = ComponentUtil.getComponent(CrawlerClientFactory.class);
config.initializeClientFactory(crawlerClientFactory);
final CrawlerClientFactory crawlerClientFactory =
config.initializeClientFactory(() -> ComponentUtil.getComponent(CrawlerClientFactory.class));
final CrawlerClient client = crawlerClientFactory.getClient(url);
if (client == null) {
throw new FessSystemException("No CrawlerClient: " + configId + ", url: " + url);

4
src/main/java/org/codelibs/fess/helper/WebFsIndexHelper.java
View file

@ -133,7 +133,7 @@ public class WebFsIndexHelper {
final long maxCount = webConfig.getMaxAccessCount() != null ? webConfig.getMaxAccessCount() : maxAccessCount;
crawlerContext.setMaxAccessCount(maxCount);
webConfig.initializeClientFactory(crawler.getClientFactory());
webConfig.initializeClientFactory(() -> crawler.getClientFactory());
final Map<String, String> configParamMap = webConfig.getConfigParameterMap(ConfigName.CONFIG);
if (Constants.TRUE.equalsIgnoreCase(configParamMap.get(Config.CLEANUP_ALL))) {
@ -241,7 +241,7 @@ public class WebFsIndexHelper {
final long maxCount = fileConfig.getMaxAccessCount() != null ? fileConfig.getMaxAccessCount() : maxAccessCount;
crawlerContext.setMaxAccessCount(maxCount);
fileConfig.initializeClientFactory(crawler.getClientFactory());
fileConfig.initializeClientFactory(() -> crawler.getClientFactory());
final Map<String, String> configParamMap = fileConfig.getConfigParameterMap(ConfigName.CONFIG);
if (Constants.TRUE.equalsIgnoreCase(configParamMap.get(Config.CLEANUP_ALL))) {

7
src/main/java/org/codelibs/fess/thumbnail/impl/BaseThumbnailGenerator.java
View file

@ -191,7 +191,8 @@ public abstract class BaseThumbnailGenerator implements ThumbnailGenerator {
}
protected boolean process(final String id, final Predicate<ResponseData> consumer) {
return process(id,
return process(
id,
(configId, url) -> {
final CrawlingConfigHelper crawlingConfigHelper = ComponentUtil.getCrawlingConfigHelper();
final CrawlingConfig config = crawlingConfigHelper.getCrawlingConfig(configId);
@ -203,8 +204,8 @@ public abstract class BaseThumbnailGenerator implements ThumbnailGenerator {
logger.info("Generating Thumbnail: " + url);
}
final CrawlerClientFactory crawlerClientFactory = ComponentUtil.getComponent(CrawlerClientFactory.class);
config.initializeClientFactory(crawlerClientFactory);
final CrawlerClientFactory crawlerClientFactory =
config.initializeClientFactory(() -> ComponentUtil.getComponent(CrawlerClientFactory.class));
final CrawlerClient client = crawlerClientFactory.getClient(url);
if (client == null) {
throw new ThumbnailGenerationException("No CrawlerClient: " + configId + ", url: " + url);