Browse Source

fix #1100 add http.proxy...

Shinsuke Sugaya 8 years ago
parent
commit
82e8a5731c

+ 22 - 7
src/main/java/org/codelibs/fess/ds/impl/GitBucketDataStoreImpl.java

@@ -37,6 +37,7 @@ import org.codelibs.fess.es.config.exentity.CrawlingConfig;
 import org.codelibs.fess.es.config.exentity.CrawlingConfigWrapper;
 import org.codelibs.fess.es.config.exentity.CrawlingConfigWrapper;
 import org.codelibs.fess.es.config.exentity.DataConfig;
 import org.codelibs.fess.es.config.exentity.DataConfig;
 import org.codelibs.fess.helper.SystemHelper;
 import org.codelibs.fess.helper.SystemHelper;
+import org.codelibs.fess.mylasta.direction.FessConfig;
 import org.codelibs.fess.util.ComponentUtil;
 import org.codelibs.fess.util.ComponentUtil;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
@@ -175,8 +176,10 @@ public class GitBucketDataStoreImpl extends AbstractDataStoreImpl {
     }
     }
 
 
     protected Map<String, String> getFessPluginInfo(final String rootURL, final String authToken) {
     protected Map<String, String> getFessPluginInfo(final String rootURL, final String authToken) {
+        final FessConfig fessConfig = ComponentUtil.getFessConfig();
         final String url = rootURL + "api/v3/fess/info";
         final String url = rootURL + "api/v3/fess/info";
-        try (CurlResponse curlResponse = Curl.get(url).header("Authorization", "token " + authToken).execute()) {
+        try (CurlResponse curlResponse =
+                Curl.get(url).proxy(fessConfig.getHttpProxy()).header("Authorization", "token " + authToken).execute()) {
             @SuppressWarnings({ "rawtypes", "unchecked" })
             @SuppressWarnings({ "rawtypes", "unchecked" })
             final Map<String, String> map = (Map) curlResponse.getContentAsMap();
             final Map<String, String> map = (Map) curlResponse.getContentAsMap();
             assert (map.containsKey("version"));
             assert (map.containsKey("version"));
@@ -190,6 +193,7 @@ public class GitBucketDataStoreImpl extends AbstractDataStoreImpl {
     }
     }
 
 
     protected List<Map<String, Object>> getRepositoryList(final String rootURL, final String authToken) {
     protected List<Map<String, Object>> getRepositoryList(final String rootURL, final String authToken) {
+        final FessConfig fessConfig = ComponentUtil.getFessConfig();
         final String url = rootURL + "api/v3/fess/repos";
         final String url = rootURL + "api/v3/fess/repos";
         int totalCount = -1; // initialize with dummy value
         int totalCount = -1; // initialize with dummy value
         final List<Map<String, Object>> repoList = new ArrayList<>();
         final List<Map<String, Object>> repoList = new ArrayList<>();
@@ -197,7 +201,8 @@ public class GitBucketDataStoreImpl extends AbstractDataStoreImpl {
         do {
         do {
             final String urlWithOffset = url + "?offset=" + repoList.size();
             final String urlWithOffset = url + "?offset=" + repoList.size();
 
 
-            try (CurlResponse curlResponse = Curl.get(urlWithOffset).header("Authorization", "token " + authToken).execute()) {
+            try (CurlResponse curlResponse =
+                    Curl.get(urlWithOffset).proxy(fessConfig.getHttpProxy()).header("Authorization", "token " + authToken).execute()) {
                 final Map<String, Object> map = curlResponse.getContentAsMap();
                 final Map<String, Object> map = curlResponse.getContentAsMap();
 
 
                 assert (map.containsKey("total_count"));
                 assert (map.containsKey("total_count"));
@@ -224,9 +229,11 @@ public class GitBucketDataStoreImpl extends AbstractDataStoreImpl {
     }
     }
 
 
     protected String getGitRef(final String rootURL, final String authToken, final String owner, final String name, final String branch) {
     protected String getGitRef(final String rootURL, final String authToken, final String owner, final String name, final String branch) {
+        final FessConfig fessConfig = ComponentUtil.getFessConfig();
         final String url = encode(rootURL, "api/v3/repos/" + owner + "/" + name + "/git/refs/heads/" + branch, null);
         final String url = encode(rootURL, "api/v3/repos/" + owner + "/" + name + "/git/refs/heads/" + branch, null);
 
 
-        try (CurlResponse curlResponse = Curl.get(url).header("Authorization", "token " + authToken).execute()) {
+        try (CurlResponse curlResponse =
+                Curl.get(url).proxy(fessConfig.getHttpProxy()).header("Authorization", "token " + authToken).execute()) {
             final Map<String, Object> map = curlResponse.getContentAsMap();
             final Map<String, Object> map = curlResponse.getContentAsMap();
             assert (map.containsKey("object"));
             assert (map.containsKey("object"));
             @SuppressWarnings("unchecked")
             @SuppressWarnings("unchecked")
@@ -294,6 +301,7 @@ public class GitBucketDataStoreImpl extends AbstractDataStoreImpl {
             final String name, final Integer issueId, final List<String> roleList, final CrawlingConfig crawlingConfig,
             final String name, final Integer issueId, final List<String> roleList, final CrawlingConfig crawlingConfig,
             final IndexUpdateCallback callback, final Map<String, String> paramMap, final Map<String, String> scriptMap,
             final IndexUpdateCallback callback, final Map<String, String> paramMap, final Map<String, String> scriptMap,
             final Map<String, Object> defaultDataMap) {
             final Map<String, Object> defaultDataMap) {
+        final FessConfig fessConfig = ComponentUtil.getFessConfig();
 
 
         final String issueUrl = rootURL + "api/v3/repos/" + owner + "/" + name + "/issues/" + issueId.toString();
         final String issueUrl = rootURL + "api/v3/repos/" + owner + "/" + name + "/issues/" + issueId.toString();
         final String viewUrl = rootURL + owner + "/" + name + "/issues/" + issueId.toString();
         final String viewUrl = rootURL + owner + "/" + name + "/issues/" + issueId.toString();
@@ -308,7 +316,8 @@ public class GitBucketDataStoreImpl extends AbstractDataStoreImpl {
 
 
         // Get issue description
         // Get issue description
         // FIXME: Use `ComponentUtil.getDocumentHelper().processRequest` instead of `Curl.get`
         // FIXME: Use `ComponentUtil.getDocumentHelper().processRequest` instead of `Curl.get`
-        try (CurlResponse curlResponse = Curl.get(issueUrl).header("Authorization", "token " + authToken).execute()) {
+        try (CurlResponse curlResponse =
+                Curl.get(issueUrl).proxy(fessConfig.getHttpProxy()).header("Authorization", "token " + authToken).execute()) {
             final Map<String, Object> map = curlResponse.getContentAsMap();
             final Map<String, Object> map = curlResponse.getContentAsMap();
             dataMap.put("title", map.getOrDefault("title", ""));
             dataMap.put("title", map.getOrDefault("title", ""));
             contentStr = (String) map.getOrDefault("body", "");
             contentStr = (String) map.getOrDefault("body", "");
@@ -332,10 +341,12 @@ public class GitBucketDataStoreImpl extends AbstractDataStoreImpl {
     }
     }
 
 
     private List<String> getIssueComments(final String issueUrl, final String authToken) {
     private List<String> getIssueComments(final String issueUrl, final String authToken) {
+        final FessConfig fessConfig = ComponentUtil.getFessConfig();
         final String commentsUrl = issueUrl + "/comments";
         final String commentsUrl = issueUrl + "/comments";
         final List<String> commentList = new ArrayList<>();
         final List<String> commentList = new ArrayList<>();
 
 
-        try (CurlResponse curlResponse = Curl.get(commentsUrl).header("Authorization", "token " + authToken).execute()) {
+        try (CurlResponse curlResponse =
+                Curl.get(commentsUrl).proxy(fessConfig.getHttpProxy()).header("Authorization", "token " + authToken).execute()) {
             final String commentsJson = curlResponse.getContentAsString();
             final String commentsJson = curlResponse.getContentAsString();
             final List<Map<String, Object>> comments =
             final List<Map<String, Object>> comments =
                     new ObjectMapper().readValue(commentsJson, new TypeReference<List<Map<String, Object>>>() {
                     new ObjectMapper().readValue(commentsJson, new TypeReference<List<Map<String, Object>>>() {
@@ -358,12 +369,14 @@ public class GitBucketDataStoreImpl extends AbstractDataStoreImpl {
             final String name, final List<String> roleList, final CrawlingConfig crawlingConfig, final IndexUpdateCallback callback,
             final String name, final List<String> roleList, final CrawlingConfig crawlingConfig, final IndexUpdateCallback callback,
             final Map<String, String> paramMap, final Map<String, String> scriptMap, final Map<String, Object> defaultDataMap,
             final Map<String, String> paramMap, final Map<String, String> scriptMap, final Map<String, Object> defaultDataMap,
             final long readInterval) {
             final long readInterval) {
+        final FessConfig fessConfig = ComponentUtil.getFessConfig();
         final String wikiUrl = rootURL + "api/v3/fess/" + owner + "/" + name + "/wiki";
         final String wikiUrl = rootURL + "api/v3/fess/" + owner + "/" + name + "/wiki";
 
 
         List<String> pageList = Collections.emptyList();
         List<String> pageList = Collections.emptyList();
 
 
         // Get list of pages
         // Get list of pages
-        try (CurlResponse curlResponse = Curl.get(wikiUrl).header("Authorization", "token " + authToken).execute()) {
+        try (CurlResponse curlResponse =
+                Curl.get(wikiUrl).proxy(fessConfig.getHttpProxy()).header("Authorization", "token " + authToken).execute()) {
             final Map<String, Object> map = curlResponse.getContentAsMap();
             final Map<String, Object> map = curlResponse.getContentAsMap();
             pageList = (List<String>) map.get("pages");
             pageList = (List<String>) map.get("pages");
         } catch (final Exception e) {
         } catch (final Exception e) {
@@ -406,9 +419,11 @@ public class GitBucketDataStoreImpl extends AbstractDataStoreImpl {
             return;
             return;
         }
         }
 
 
+        final FessConfig fessConfig = ComponentUtil.getFessConfig();
         final String url = encode(rootURL, "api/v3/repos/" + owner + "/" + name + "/contents/" + path, "ref=" + refStr);
         final String url = encode(rootURL, "api/v3/repos/" + owner + "/" + name + "/contents/" + path, "ref=" + refStr);
 
 
-        try (CurlResponse curlResponse = Curl.get(url).header("Authorization", "token " + authToken).execute()) {
+        try (CurlResponse curlResponse =
+                Curl.get(url).proxy(fessConfig.getHttpProxy()).header("Authorization", "token " + authToken).execute()) {
             final InputStream iStream = curlResponse.getContentAsStream();
             final InputStream iStream = curlResponse.getContentAsStream();
             final List<Object> fileList = parseList(iStream);
             final List<Object> fileList = parseList(iStream);
 
 

+ 104 - 0
src/main/java/org/codelibs/fess/mylasta/direction/FessConfig.java

@@ -139,6 +139,18 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
     /** The key of the configuration. e.g.  */
     /** The key of the configuration. e.g.  */
     String VIRTUAL_HOST_HEADERS = "virtual.host.headers";
     String VIRTUAL_HOST_HEADERS = "virtual.host.headers";
 
 
+    /** The key of the configuration. e.g.  */
+    String HTTP_PROXY_HOST = "http.proxy.host";
+
+    /** The key of the configuration. e.g. 8080 */
+    String HTTP_PROXY_PORT = "http.proxy.port";
+
+    /** The key of the configuration. e.g.  */
+    String HTTP_PROXY_USERNAME = "http.proxy.username";
+
+    /** The key of the configuration. e.g.  */
+    String HTTP_PROXY_PASSWORD = "http.proxy.password";
+
     /** The key of the configuration. e.g. 50 */
     /** The key of the configuration. e.g. 50 */
     String CRAWLER_DOCUMENT_MAX_SITE_LENGTH = "crawler.document.max.site.length";
     String CRAWLER_DOCUMENT_MAX_SITE_LENGTH = "crawler.document.max.site.length";
 
 
@@ -1501,6 +1513,66 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
      */
      */
     Integer getVirtualHostHeadersAsInteger();
     Integer getVirtualHostHeadersAsInteger();
 
 
+    /**
+     * Get the value for the key 'http.proxy.host'. <br>
+     * The value is, e.g.  <br>
+     * @return The value of found property. (NotNull: if not found, exception but basically no way)
+     */
+    String getHttpProxyHost();
+
+    /**
+     * Get the value for the key 'http.proxy.host' as {@link Integer}. <br>
+     * The value is, e.g.  <br>
+     * @return The value of found property. (NotNull: if not found, exception but basically no way)
+     * @throws NumberFormatException When the property is not integer.
+     */
+    Integer getHttpProxyHostAsInteger();
+
+    /**
+     * Get the value for the key 'http.proxy.port'. <br>
+     * The value is, e.g. 8080 <br>
+     * @return The value of found property. (NotNull: if not found, exception but basically no way)
+     */
+    String getHttpProxyPort();
+
+    /**
+     * Get the value for the key 'http.proxy.port' as {@link Integer}. <br>
+     * The value is, e.g. 8080 <br>
+     * @return The value of found property. (NotNull: if not found, exception but basically no way)
+     * @throws NumberFormatException When the property is not integer.
+     */
+    Integer getHttpProxyPortAsInteger();
+
+    /**
+     * Get the value for the key 'http.proxy.username'. <br>
+     * The value is, e.g.  <br>
+     * @return The value of found property. (NotNull: if not found, exception but basically no way)
+     */
+    String getHttpProxyUsername();
+
+    /**
+     * Get the value for the key 'http.proxy.username' as {@link Integer}. <br>
+     * The value is, e.g.  <br>
+     * @return The value of found property. (NotNull: if not found, exception but basically no way)
+     * @throws NumberFormatException When the property is not integer.
+     */
+    Integer getHttpProxyUsernameAsInteger();
+
+    /**
+     * Get the value for the key 'http.proxy.password'. <br>
+     * The value is, e.g.  <br>
+     * @return The value of found property. (NotNull: if not found, exception but basically no way)
+     */
+    String getHttpProxyPassword();
+
+    /**
+     * Get the value for the key 'http.proxy.password' as {@link Integer}. <br>
+     * The value is, e.g.  <br>
+     * @return The value of found property. (NotNull: if not found, exception but basically no way)
+     * @throws NumberFormatException When the property is not integer.
+     */
+    Integer getHttpProxyPasswordAsInteger();
+
     /**
     /**
      * Get the value for the key 'crawler.document.max.site.length'. <br>
      * Get the value for the key 'crawler.document.max.site.length'. <br>
      * The value is, e.g. 50 <br>
      * The value is, e.g. 50 <br>
@@ -5181,6 +5253,38 @@ public interface FessConfig extends FessEnv, org.codelibs.fess.mylasta.direction
             return getAsInteger(FessConfig.VIRTUAL_HOST_HEADERS);
             return getAsInteger(FessConfig.VIRTUAL_HOST_HEADERS);
         }
         }
 
 
+        public String getHttpProxyHost() {
+            return get(FessConfig.HTTP_PROXY_HOST);
+        }
+
+        public Integer getHttpProxyHostAsInteger() {
+            return getAsInteger(FessConfig.HTTP_PROXY_HOST);
+        }
+
+        public String getHttpProxyPort() {
+            return get(FessConfig.HTTP_PROXY_PORT);
+        }
+
+        public Integer getHttpProxyPortAsInteger() {
+            return getAsInteger(FessConfig.HTTP_PROXY_PORT);
+        }
+
+        public String getHttpProxyUsername() {
+            return get(FessConfig.HTTP_PROXY_USERNAME);
+        }
+
+        public Integer getHttpProxyUsernameAsInteger() {
+            return getAsInteger(FessConfig.HTTP_PROXY_USERNAME);
+        }
+
+        public String getHttpProxyPassword() {
+            return get(FessConfig.HTTP_PROXY_PASSWORD);
+        }
+
+        public Integer getHttpProxyPasswordAsInteger() {
+            return getAsInteger(FessConfig.HTTP_PROXY_PASSWORD);
+        }
+
         public String getCrawlerDocumentMaxSiteLength() {
         public String getCrawlerDocumentMaxSiteLength() {
             return get(FessConfig.CRAWLER_DOCUMENT_MAX_SITE_LENGTH);
             return get(FessConfig.CRAWLER_DOCUMENT_MAX_SITE_LENGTH);
         }
         }

+ 38 - 0
src/main/java/org/codelibs/fess/mylasta/direction/FessProp.java

@@ -18,6 +18,12 @@ package org.codelibs.fess.mylasta.direction;
 import static org.codelibs.core.stream.StreamUtil.split;
 import static org.codelibs.core.stream.StreamUtil.split;
 import static org.codelibs.core.stream.StreamUtil.stream;
 import static org.codelibs.core.stream.StreamUtil.stream;
 
 
+import java.net.Authenticator;
+import java.net.InetSocketAddress;
+import java.net.PasswordAuthentication;
+import java.net.Proxy;
+import java.net.Proxy.Type;
+import java.net.SocketAddress;
 import java.util.ArrayList;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Collections;
 import java.util.Date;
 import java.util.Date;
@@ -62,6 +68,8 @@ import org.lastaflute.web.validation.theme.typed.LongTypeValidator;
 
 
 public interface FessProp {
 public interface FessProp {
 
 
+    public static final String HTML_PROXY = "httpProxy";
+
     public static final String CRAWLER_FAILURE_URL_STATUS_CODES = "crawlerFailureUrlStatusCodes";
     public static final String CRAWLER_FAILURE_URL_STATUS_CODES = "crawlerFailureUrlStatusCodes";
 
 
     public static final String VIRTUAL_HOST_HEADERS = "virtualHostHeaders";
     public static final String VIRTUAL_HOST_HEADERS = "virtualHostHeaders";
@@ -1627,4 +1635,34 @@ public interface FessProp {
 
 
         return true;
         return true;
     }
     }
+
+    String getHttpProxyHost();
+
+    Integer getHttpProxyHostAsInteger();
+
+    String getHttpProxyUsername();
+
+    String getHttpProxyPassword();
+
+    public default Proxy getHttpProxy() {
+        Proxy proxy = (Proxy) propMap.get(HTML_PROXY);
+        if (proxy == null) {
+            if (StringUtil.isNotBlank(getHttpProxyHost()) && getHttpProxyHostAsInteger() != null) {
+                final SocketAddress addr = new InetSocketAddress(getHttpProxyHost(), getHttpProxyHostAsInteger());
+                proxy = new Proxy(Type.HTTP, addr);
+                if (StringUtil.isNotBlank(getHttpProxyUsername())) {
+                    Authenticator.setDefault(new Authenticator() {
+                        @Override
+                        protected PasswordAuthentication getPasswordAuthentication() {
+                            return new PasswordAuthentication(getHttpProxyUsername(), getHttpProxyPassword().toCharArray());
+                        }
+                    });
+                }
+            } else {
+                proxy = Proxy.NO_PROXY;
+            }
+            propMap.put(HTML_PROXY, proxy);
+        }
+        return proxy;
+    }
 }
 }

+ 27 - 24
src/main/java/org/codelibs/fess/thumbnail/impl/HtmlTagBasedGenerator.java

@@ -85,31 +85,34 @@ public class HtmlTagBasedGenerator extends BaseThumbnailGenerator {
             return false;
             return false;
         }
         }
 
 
-        Curl.get(url).execute(
-                con -> {
-                    boolean created = false;
-                    try (ImageInputStream input = ImageIO.createImageInputStream(con.getInputStream())) {
-                        if (saveImage(input, outputFile)) {
-                            created = true;
-                        } else {
-                            logger.warn("Failed to create thumbnail: " + thumbnailId + " -> " + url);
-                        }
-                    } catch (final Throwable t) {
-                        if (logger.isDebugEnabled()) {
-                            logger.warn("Failed to create thumbnail: " + thumbnailId + " -> " + url, t);
-                        } else {
-                            logger.warn("Failed to create thumbnail: " + thumbnailId + " -> " + url + " ("
-                                    + t.getClass().getCanonicalName() + ": " + t.getMessage() + ")");
-                        }
-                    } finally {
-                        if (!created) {
-                            updateThumbnailField(thumbnailId, url, StringUtil.EMPTY);
-                            if (outputFile.exists() && !outputFile.delete()) {
-                                logger.warn("Failed to delete " + outputFile.getAbsolutePath());
+        final FessConfig fessConfig = ComponentUtil.getFessConfig();
+        Curl.get(url)
+                .proxy(fessConfig.getHttpProxy())
+                .execute(
+                        con -> {
+                            boolean created = false;
+                            try (ImageInputStream input = ImageIO.createImageInputStream(con.getInputStream())) {
+                                if (saveImage(input, outputFile)) {
+                                    created = true;
+                                } else {
+                                    logger.warn("Failed to create thumbnail: " + thumbnailId + " -> " + url);
+                                }
+                            } catch (final Throwable t) {
+                                if (logger.isDebugEnabled()) {
+                                    logger.warn("Failed to create thumbnail: " + thumbnailId + " -> " + url, t);
+                                } else {
+                                    logger.warn("Failed to create thumbnail: " + thumbnailId + " -> " + url + " ("
+                                            + t.getClass().getCanonicalName() + ": " + t.getMessage() + ")");
+                                }
+                            } finally {
+                                if (!created) {
+                                    updateThumbnailField(thumbnailId, url, StringUtil.EMPTY);
+                                    if (outputFile.exists() && !outputFile.delete()) {
+                                        logger.warn("Failed to delete " + outputFile.getAbsolutePath());
+                                    }
+                                }
                             }
                             }
-                        }
-                    }
-                });
+                        });
 
 
         return outputFile.exists();
         return outputFile.exists();
     }
     }

+ 5 - 0
src/main/resources/fess_config.properties

@@ -83,6 +83,11 @@ api.admin.access.permissions=Radmin-api
 # Virtual Host: Host:fess.codelibs.org=fess
 # Virtual Host: Host:fess.codelibs.org=fess
 virtual.host.headers=
 virtual.host.headers=
 
 
+http.proxy.host=
+http.proxy.port=8080
+http.proxy.username=
+http.proxy.password=
+
 # ========================================================================================
 # ========================================================================================
 #                                                                                   Index
 #                                                                                   Index
 #                                                                                     ====
 #                                                                                     ====