0ct0pu5/fess
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

#739 add token to header for client

Browse source
This commit is contained in:
Shinsuke Sugaya 2016-10-09 18:38:27 +09:00
parent 033cccea10
commit 7d06ec97a2
6 changed files with 94 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

47
src/main/java/org/codelibs/fess/ds/impl/GitBucketDataStoreImpl.java
View file

@ -16,6 +16,7 @@
package org.codelibs.fess.ds.impl;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
@ -25,7 +26,12 @@ import java.util.function.Consumer;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.elasticsearch.runner.net.Curl;
import org.codelibs.elasticsearch.runner.net.CurlResponse;
import org.codelibs.fess.crawler.client.CrawlerClientFactory;
import org.codelibs.fess.crawler.client.http.HcHttpClient;
import org.codelibs.fess.crawler.client.http.RequestHeader;
import org.codelibs.fess.ds.IndexUpdateCallback;
import org.codelibs.fess.es.config.exentity.CrawlingConfig;
import org.codelibs.fess.es.config.exentity.CrawlingConfigWrapper;
import org.codelibs.fess.es.config.exentity.DataConfig;
import org.codelibs.fess.util.ComponentUtil;
import org.elasticsearch.common.xcontent.json.JsonXContent;
@ -62,14 +68,30 @@ public class GitBucketDataStoreImpl extends AbstractDataStoreImpl {
return;
}
final CrawlingConfig crawlingConfig = new CrawlingConfigWrapper(dataConfig) {
@Override
public Map<String, Object> initializeClientFactory(CrawlerClientFactory crawlerClientFactory) {
final Map<String, Object> paramMap = super.initializeClientFactory(crawlerClientFactory);
List<RequestHeader> headerList = new ArrayList<>();
RequestHeader[] headers = (RequestHeader[]) paramMap.get(HcHttpClient.REQUERT_HEADERS_PROPERTY);
if (headers != null) {
for (RequestHeader header : headers) {
headerList.add(header);
}
}
headerList.add(new RequestHeader("Authorization", "token " + authToken));
paramMap.put(HcHttpClient.REQUERT_HEADERS_PROPERTY, headerList.toArray(new RequestHeader[headerList.size()]));
return paramMap;
}
};
for (final Map<String, Object> repository : repositoryList) {
try {
final String name = (String) repository.get("name");
final String owner = (String) repository.get("owner");
repository.get("is_private");
collectFileNames(rootURL, authToken, owner, name, "", 0, readInterval, path -> {
storeFileContent(rootURL, authToken, owner, name, path, dataConfig, callback, paramMap, scriptMap, defaultDataMap);
collectFileNames(rootURL, authToken, owner, name, StringUtil.EMPTY, 0, readInterval, path -> {
storeFileContent(rootURL, authToken, owner, name, path, crawlingConfig, callback, paramMap, scriptMap, defaultDataMap);
if (readInterval > 0) {
sleep(readInterval);
}
@ -125,23 +147,20 @@ public class GitBucketDataStoreImpl extends AbstractDataStoreImpl {
}
private void storeFileContent(final String rootURL, final String authToken, final String owner, final String name, final String path,
final DataConfig dataConfig, final IndexUpdateCallback callback, final Map<String, String> paramMap,
final CrawlingConfig crawlingConfig, final IndexUpdateCallback callback, final Map<String, String> paramMap,
final Map<String, String> scriptMap, final Map<String, Object> defaultDataMap) {
final String url = rootURL + owner + "/" + name + "/blob/master/" + path;
try (CurlResponse curlResponse = Curl.get(url).param("raw", "true").header("Authorization", "token " + authToken).execute()) {
if (logger.isInfoEnabled()) {
logger.info("Get a content from " + url);
final Map<String, Object> dataMap = new HashMap<>();
dataMap.putAll(defaultDataMap);
dataMap.putAll(ComponentUtil.getDocumentHelper().processRequest(dataConfig, paramMap.get("crawlingInfoId"), url));
// TODO scriptMap
callback.store(paramMap, dataMap);
} catch (final Exception e) {
// TODO CrawlingAccessException?
logger.warn("Failed to parse " + url, e);
}
final Map<String, Object> dataMap = new HashMap<>();
dataMap.putAll(defaultDataMap);
dataMap.putAll(ComponentUtil.getDocumentHelper().processRequest(crawlingConfig, paramMap.get("crawlingInfoId"), url + "?raw=true"));
// TODO scriptMap
callback.store(paramMap, dataMap);
return;
}

2
src/main/java/org/codelibs/fess/es/config/exentity/CrawlingConfig.java
View file

@ -37,7 +37,7 @@ public interface CrawlingConfig {
Integer getTimeToLive();
void initializeClientFactory(CrawlerClientFactory crawlerClientFactory);
Map<String, Object> initializeClientFactory(CrawlerClientFactory crawlerClientFactory);
Map<String, String> getConfigParameterMap(ConfigName name);

54
src/main/java/org/codelibs/fess/es/config/exentity/CrawlingConfigWrapper.java Normal file
View file

@ -0,0 +1,54 @@
package org.codelibs.fess.es.config.exentity;
import java.util.Map;
import org.codelibs.fess.crawler.client.CrawlerClientFactory;
public class CrawlingConfigWrapper implements CrawlingConfig {
private CrawlingConfig crawlingConfig;
public CrawlingConfigWrapper(final CrawlingConfig crawlingConfig) {
this.crawlingConfig = crawlingConfig;
}
public String getId() {
return crawlingConfig.getId();
}
public String getName() {
return crawlingConfig.getName();
}
public String[] getPermissions() {
return crawlingConfig.getPermissions();
}
public String[] getLabelTypeValues() {
return crawlingConfig.getLabelTypeValues();
}
public String getDocumentBoost() {
return crawlingConfig.getDocumentBoost();
}
public String getIndexingTarget(String input) {
return crawlingConfig.getIndexingTarget(input);
}
public String getConfigId() {
return crawlingConfig.getConfigId();
}
public Integer getTimeToLive() {
return crawlingConfig.getTimeToLive();
}
public Map<String, Object> initializeClientFactory(CrawlerClientFactory crawlerClientFactory) {
return crawlingConfig.initializeClientFactory(crawlerClientFactory);
}
public Map<String, String> getConfigParameterMap(ConfigName name) {
return crawlingConfig.getConfigParameterMap(name);
}
}

3
src/main/java/org/codelibs/fess/es/config/exentity/DataConfig.java
View file

@ -182,7 +182,7 @@ public class DataConfig extends BsDataConfig implements CrawlingConfig {
}
@Override
public void initializeClientFactory(final CrawlerClientFactory crawlerClientFactory) {
public Map<String, Object> initializeClientFactory(final CrawlerClientFactory crawlerClientFactory) {
final Map<String, String> paramMap = getHandlerParameterMap();
final Map<String, Object> factoryParamMap = new HashMap<>();
@ -353,6 +353,7 @@ public class DataConfig extends BsDataConfig implements CrawlingConfig {
}
}
return factoryParamMap;
}
@Override

3
src/main/java/org/codelibs/fess/es/config/exentity/FileConfig.java
View file

@ -198,7 +198,7 @@ public class FileConfig extends BsFileConfig implements CrawlingConfig {
}
@Override
public void initializeClientFactory(final CrawlerClientFactory clientFactory) {
public Map<String, Object> initializeClientFactory(final CrawlerClientFactory clientFactory) {
final FileAuthenticationService fileAuthenticationService = ComponentUtil.getComponent(FileAuthenticationService.class);
// Parameters
@ -237,6 +237,7 @@ public class FileConfig extends BsFileConfig implements CrawlingConfig {
paramMap.put(SmbClient.SMB_AUTHENTICATIONS_PROPERTY, smbAuthList.toArray(new SmbAuthentication[smbAuthList.size()]));
paramMap.put(FtpClient.FTP_AUTHENTICATIONS_PROPERTY, ftpAuthList.toArray(new FtpAuthentication[ftpAuthList.size()]));
return paramMap;
}
@Override

3
src/main/java/org/codelibs/fess/es/config/exentity/WebConfig.java
View file

@ -197,7 +197,7 @@ public class WebConfig extends BsWebConfig implements CrawlingConfig {
}
@Override
public void initializeClientFactory(final CrawlerClientFactory clientFactory) {
public Map<String, Object> initializeClientFactory(final CrawlerClientFactory clientFactory) {
final WebAuthenticationService webAuthenticationService = ComponentUtil.getComponent(WebAuthenticationService.class);
final RequestHeaderService requestHeaderService = ComponentUtil.getComponent(RequestHeaderService.class);
final FessConfig fessConfig = ComponentUtil.getFessConfig();
@ -237,6 +237,7 @@ public class WebConfig extends BsWebConfig implements CrawlingConfig {
paramMap.put(HcHttpClient.REQUERT_HEADERS_PROPERTY,
rhList.toArray(new org.codelibs.fess.crawler.client.http.RequestHeader[rhList.size()]));
return paramMap;
}
@Override