0ct0pu5/fess
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix #1100 add http.proxy...

Browse source
This commit is contained in:
Shinsuke Sugaya 2017-06-10 15:04:10 +09:00
parent f84545574f
commit 82e8a5731c
5 changed files with 196 additions and 31 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

29
src/main/java/org/codelibs/fess/ds/impl/GitBucketDataStoreImpl.java
View file

@ -37,6 +37,7 @@ import org.codelibs.fess.es.config.exentity.CrawlingConfig;
import org.codelibs.fess.es.config.exentity.CrawlingConfigWrapper;
import org.codelibs.fess.es.config.exentity.DataConfig;
import org.codelibs.fess.helper.SystemHelper;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.util.ComponentUtil;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.xcontent.json.JsonXContent;
@ -175,8 +176,10 @@ public class GitBucketDataStoreImpl extends AbstractDataStoreImpl {
}
protected Map<String, String> getFessPluginInfo(final String rootURL, final String authToken) {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final String url = rootURL + "api/v3/fess/info";
try (CurlResponse curlResponse = Curl.get(url).header("Authorization", "token " + authToken).execute()) {
try (CurlResponse curlResponse =
Curl.get(url).proxy(fessConfig.getHttpProxy()).header("Authorization", "token " + authToken).execute()) {
@SuppressWarnings({ "rawtypes", "unchecked" })
final Map<String, String> map = (Map) curlResponse.getContentAsMap();
assert (map.containsKey("version"));
@ -190,6 +193,7 @@ public class GitBucketDataStoreImpl extends AbstractDataStoreImpl {
}
protected List<Map<String, Object>> getRepositoryList(final String rootURL, final String authToken) {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final String url = rootURL + "api/v3/fess/repos";
int totalCount = -1; // initialize with dummy value
final List<Map<String, Object>> repoList = new ArrayList<>();
@ -197,7 +201,8 @@ public class GitBucketDataStoreImpl extends AbstractDataStoreImpl {
do {
final String urlWithOffset = url + "?offset=" + repoList.size();
try (CurlResponse curlResponse = Curl.get(urlWithOffset).header("Authorization", "token " + authToken).execute()) {
try (CurlResponse curlResponse =
Curl.get(urlWithOffset).proxy(fessConfig.getHttpProxy()).header("Authorization", "token " + authToken).execute()) {
final Map<String, Object> map = curlResponse.getContentAsMap();
assert (map.containsKey("total_count"));
@ -224,9 +229,11 @@ public class GitBucketDataStoreImpl extends AbstractDataStoreImpl {
}
protected String getGitRef(final String rootURL, final String authToken, final String owner, final String name, final String branch) {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final String url = encode(rootURL, "api/v3/repos/" + owner + "/" + name + "/git/refs/heads/" + branch, null);
try (CurlResponse curlResponse = Curl.get(url).header("Authorization", "token " + authToken).execute()) {
try (CurlResponse curlResponse =
Curl.get(url).proxy(fessConfig.getHttpProxy()).header("Authorization", "token " + authToken).execute()) {
final Map<String, Object> map = curlResponse.getContentAsMap();
assert (map.containsKey("object"));
@SuppressWarnings("unchecked")
@ -294,6 +301,7 @@ public class GitBucketDataStoreImpl extends AbstractDataStoreImpl {
final String name, final Integer issueId, final List<String> roleList, final CrawlingConfig crawlingConfig,
final IndexUpdateCallback callback, final Map<String, String> paramMap, final Map<String, String> scriptMap,
final Map<String, Object> defaultDataMap) {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final String issueUrl = rootURL + "api/v3/repos/" + owner + "/" + name + "/issues/" + issueId.toString();
final String viewUrl = rootURL + owner + "/" + name + "/issues/" + issueId.toString();
@ -308,7 +316,8 @@ public class GitBucketDataStoreImpl extends AbstractDataStoreImpl {
// Get issue description
// FIXME: Use `ComponentUtil.getDocumentHelper().processRequest` instead of `Curl.get`
try (CurlResponse curlResponse = Curl.get(issueUrl).header("Authorization", "token " + authToken).execute()) {
try (CurlResponse curlResponse =
Curl.get(issueUrl).proxy(fessConfig.getHttpProxy()).header("Authorization", "token " + authToken).execute()) {
final Map<String, Object> map = curlResponse.getContentAsMap();
dataMap.put("title", map.getOrDefault("title", ""));
contentStr = (String) map.getOrDefault("body", "");
@ -332,10 +341,12 @@ public class GitBucketDataStoreImpl extends AbstractDataStoreImpl {
}
private List<String> getIssueComments(final String issueUrl, final String authToken) {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final String commentsUrl = issueUrl + "/comments";
final List<String> commentList = new ArrayList<>();
try (CurlResponse curlResponse = Curl.get(commentsUrl).header("Authorization", "token " + authToken).execute()) {
try (CurlResponse curlResponse =
Curl.get(commentsUrl).proxy(fessConfig.getHttpProxy()).header("Authorization", "token " + authToken).execute()) {
final String commentsJson = curlResponse.getContentAsString();
final List<Map<String, Object>> comments =
new ObjectMapper().readValue(commentsJson, new TypeReference<List<Map<String, Object>>>() {
@ -358,12 +369,14 @@ public class GitBucketDataStoreImpl extends AbstractDataStoreImpl {
final String name, final List<String> roleList, final CrawlingConfig crawlingConfig, final IndexUpdateCallback callback,
final Map<String, String> paramMap, final Map<String, String> scriptMap, final Map<String, Object> defaultDataMap,
final long readInterval) {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final String wikiUrl = rootURL + "api/v3/fess/" + owner + "/" + name + "/wiki";
List<String> pageList = Collections.emptyList();
// Get list of pages
try (CurlResponse curlResponse = Curl.get(wikiUrl).header("Authorization", "token " + authToken).execute()) {
try (CurlResponse curlResponse =
Curl.get(wikiUrl).proxy(fessConfig.getHttpProxy()).header("Authorization", "token " + authToken).execute()) {
final Map<String, Object> map = curlResponse.getContentAsMap();
pageList = (List<String>) map.get("pages");
} catch (final Exception e) {
@ -406,9 +419,11 @@ public class GitBucketDataStoreImpl extends AbstractDataStoreImpl {
return;
}
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final String url = encode(rootURL, "api/v3/repos/" + owner + "/" + name + "/contents/" + path, "ref=" + refStr);
try (CurlResponse curlResponse = Curl.get(url).header("Authorization", "token " + authToken).execute()) {
try (CurlResponse curlResponse =
Curl.get(url).proxy(fessConfig.getHttpProxy()).header("Authorization", "token " + authToken).execute()) {
final InputStream iStream = curlResponse.getContentAsStream();
final List<Object> fileList = parseList(iStream);

104
src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java
View file

@ -139,6 +139,18 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
/** The key of the configuration. e.g. */
String VIRTUAL_HOST_HEADERS = "virtual.host.headers";
/** The key of the configuration. e.g. */
String HTTP_PROXY_HOST = "http.proxy.host";
/** The key of the configuration. e.g. 8080 */
String HTTP_PROXY_PORT = "http.proxy.port";
/** The key of the configuration. e.g. */
String HTTP_PROXY_USERNAME = "http.proxy.username";
/** The key of the configuration. e.g. */
String HTTP_PROXY_PASSWORD = "http.proxy.password";
/** The key of the configuration. e.g. 50 */
String CRAWLER_DOCUMENT_MAX_SITE_LENGTH = "crawler.document.max.site.length";
@ -1501,6 +1513,66 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
*/
Integer getVirtualHostHeadersAsInteger();
/**
* Get the value for the key 'http.proxy.host'. <br>
* The value is, e.g. <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getHttpProxyHost();
/**
* Get the value for the key 'http.proxy.host' as {@link Integer}. <br>
* The value is, e.g. <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
* @throws NumberFormatException When the property is not integer.
*/
Integer getHttpProxyHostAsInteger();
/**
* Get the value for the key 'http.proxy.port'. <br>
* The value is, e.g. 8080 <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getHttpProxyPort();
/**
* Get the value for the key 'http.proxy.port' as {@link Integer}. <br>
* The value is, e.g. 8080 <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
* @throws NumberFormatException When the property is not integer.
*/
Integer getHttpProxyPortAsInteger();
/**
* Get the value for the key 'http.proxy.username'. <br>
* The value is, e.g. <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getHttpProxyUsername();
/**
* Get the value for the key 'http.proxy.username' as {@link Integer}. <br>
* The value is, e.g. <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
* @throws NumberFormatException When the property is not integer.
*/
Integer getHttpProxyUsernameAsInteger();
/**
* Get the value for the key 'http.proxy.password'. <br>
* The value is, e.g. <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
*/
String getHttpProxyPassword();
/**
* Get the value for the key 'http.proxy.password' as {@link Integer}. <br>
* The value is, e.g. <br>
* @return The value of found property. (NotNull: if not found, exception but basically no way)
* @throws NumberFormatException When the property is not integer.
*/
Integer getHttpProxyPasswordAsInteger();
/**
* Get the value for the key 'crawler.document.max.site.length'. <br>
* The value is, e.g. 50 <br>
@ -5181,6 +5253,38 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
return getAsInteger(FessConfig.VIRTUAL_HOST_HEADERS);
}
public String getHttpProxyHost() {
return get(FessConfig.HTTP_PROXY_HOST);
}
public Integer getHttpProxyHostAsInteger() {
return getAsInteger(FessConfig.HTTP_PROXY_HOST);
}
public String getHttpProxyPort() {
return get(FessConfig.HTTP_PROXY_PORT);
}
public Integer getHttpProxyPortAsInteger() {
return getAsInteger(FessConfig.HTTP_PROXY_PORT);
}
public String getHttpProxyUsername() {
return get(FessConfig.HTTP_PROXY_USERNAME);
}
public Integer getHttpProxyUsernameAsInteger() {
return getAsInteger(FessConfig.HTTP_PROXY_USERNAME);
}
public String getHttpProxyPassword() {
return get(FessConfig.HTTP_PROXY_PASSWORD);
}
public Integer getHttpProxyPasswordAsInteger() {
return getAsInteger(FessConfig.HTTP_PROXY_PASSWORD);
}
public String getCrawlerDocumentMaxSiteLength() {
return get(FessConfig.CRAWLER_DOCUMENT_MAX_SITE_LENGTH);
}

38
src/main/java/org/codelibs/fess/mylasta/direction/FessProp.java
View file

@ -18,6 +18,12 @@ package org.codelibs.fess.mylasta.direction;
import static org.codelibs.core.stream.StreamUtil.split;
import static org.codelibs.core.stream.StreamUtil.stream;
import java.net.Authenticator;
import java.net.InetSocketAddress;
import java.net.PasswordAuthentication;
import java.net.Proxy;
import java.net.Proxy.Type;
import java.net.SocketAddress;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
@ -62,6 +68,8 @@ import org.lastaflute.web.validation.theme.typed.LongTypeValidator;
public interface FessProp {
public static final String HTML_PROXY = "httpProxy";
public static final String CRAWLER_FAILURE_URL_STATUS_CODES = "crawlerFailureUrlStatusCodes";
public static final String VIRTUAL_HOST_HEADERS = "virtualHostHeaders";
@ -1627,4 +1635,34 @@ public interface FessProp {
return true;
}
String getHttpProxyHost();
Integer getHttpProxyHostAsInteger();
String getHttpProxyUsername();
String getHttpProxyPassword();
public default Proxy getHttpProxy() {
Proxy proxy = (Proxy) propMap.get(HTML_PROXY);
if (proxy == null) {
if (StringUtil.isNotBlank(getHttpProxyHost()) && getHttpProxyHostAsInteger() != null) {
final SocketAddress addr = new InetSocketAddress(getHttpProxyHost(), getHttpProxyHostAsInteger());
proxy = new Proxy(Type.HTTP, addr);
if (StringUtil.isNotBlank(getHttpProxyUsername())) {
Authenticator.setDefault(new Authenticator() {
@Override
protected PasswordAuthentication getPasswordAuthentication() {
return new PasswordAuthentication(getHttpProxyUsername(), getHttpProxyPassword().toCharArray());
}
});
}
} else {
proxy = Proxy.NO_PROXY;
}
propMap.put(HTML_PROXY, proxy);
}
return proxy;
}
}

51
src/main/java/org/codelibs/fess/thumbnail/impl/HtmlTagBasedGenerator.java
View file

@ -85,31 +85,34 @@ public class HtmlTagBasedGenerator extends BaseThumbnailGenerator {
return false;
}
Curl.get(url).execute(
con -> {
boolean created = false;
try (ImageInputStream input = ImageIO.createImageInputStream(con.getInputStream())) {
if (saveImage(input, outputFile)) {
created = true;
} else {
logger.warn("Failed to create thumbnail: " + thumbnailId + " -> " + url);
}
} catch (final Throwable t) {
if (logger.isDebugEnabled()) {
logger.warn("Failed to create thumbnail: " + thumbnailId + " -> " + url, t);
} else {
logger.warn("Failed to create thumbnail: " + thumbnailId + " -> " + url + " ("
+ t.getClass().getCanonicalName() + ": " + t.getMessage() + ")");
}
} finally {
if (!created) {
updateThumbnailField(thumbnailId, url, StringUtil.EMPTY);
if (outputFile.exists() && !outputFile.delete()) {
logger.warn("Failed to delete " + outputFile.getAbsolutePath());
final FessConfig fessConfig = ComponentUtil.getFessConfig();
Curl.get(url)
.proxy(fessConfig.getHttpProxy())
.execute(
con -> {
boolean created = false;
try (ImageInputStream input = ImageIO.createImageInputStream(con.getInputStream())) {
if (saveImage(input, outputFile)) {
created = true;
} else {
logger.warn("Failed to create thumbnail: " + thumbnailId + " -> " + url);
}
} catch (final Throwable t) {
if (logger.isDebugEnabled()) {
logger.warn("Failed to create thumbnail: " + thumbnailId + " -> " + url, t);
} else {
logger.warn("Failed to create thumbnail: " + thumbnailId + " -> " + url + " ("
+ t.getClass().getCanonicalName() + ": " + t.getMessage() + ")");
}
} finally {
if (!created) {
updateThumbnailField(thumbnailId, url, StringUtil.EMPTY);
if (outputFile.exists() && !outputFile.delete()) {
logger.warn("Failed to delete " + outputFile.getAbsolutePath());
}
}
}
}
}
});
});
return outputFile.exists();
}

5
src/main/resources/fess_config.properties
View file

@ -83,6 +83,11 @@ api.admin.access.permissions=Radmin-api
# Virtual Host: Host:fess.codelibs.org=fess
virtual.host.headers=
http.proxy.host=
http.proxy.port=8080
http.proxy.username=
http.proxy.password=
# ========================================================================================
# Index
# ====