#1810 replace with HTTP

pom.xml

@@ -60,7 +60,7 @@
 		<utflute.version>0.8.6</utflute.version>
 
 		<!-- Crawler -->
-		<crawler.version>2.2.0</crawler.version>
+		<crawler.version>2.3.0-SNAPSHOT</crawler.version>
 
 		<!-- Suggest -->
 		<suggest.version>6.3.0</suggest.version>
@@ -1200,6 +1200,11 @@
 				</exclusion>
 			</exclusions>
 		</dependency>
+		<dependency>
+			<groupId>org.codelibs</groupId>
+			<artifactId>elasticsearch-httpclient</artifactId>
+			<version>0.1.0-SNAPSHOT</version>
+		</dependency>
 		<dependency>
 			<groupId>org.ow2.asm</groupId>
 			<artifactId>asm</artifactId>
@@ -1272,7 +1277,7 @@
 		<dependency>
 			<groupId>org.codelibs</groupId>
 			<artifactId>curl4j</artifactId>
-			<version>1.0.0</version>
+			<version>1.2.0-SNAPSHOT</version>
 		</dependency>
 		<dependency>
 			<groupId>org.codelibs</groupId>

src/main/assemblies/files/fess.in.bat

@@ -117,5 +117,4 @@ set FESS_JAVA_OPTS=%FESS_JAVA_OPTS% -Dtomcat.config.path=tomcat_config.propertie
 
 REM External elasticsearch cluster
 REM set FESS_JAVA_OPTS=%FESS_JAVA_OPTS% -Dfess.es.http_address=http://localhost:9200
-REM set FESS_JAVA_OPTS=%FESS_JAVA_OPTS% -Dfess.es.transport_addresses=localhost:9300
 REM set FESS_JAVA_OPTS=%FESS_JAVA_OPTS% -Dfess.dictionary.path=%ES_HOME%/config/

src/main/assemblies/files/fess.in.sh

@@ -116,9 +116,6 @@ fi
 if [ "x$ES_HTTP_URL" != "x" ]; then
   FESS_JAVA_OPTS="$FESS_JAVA_OPTS -Dfess.es.http_address=$ES_HTTP_URL"
 fi
-if [ "x$ES_TRANSPORT_URL" != "x" ]; then
-  FESS_JAVA_OPTS="$FESS_JAVA_OPTS -Dfess.es.transport_addresses=$ES_TRANSPORT_URL"
-fi
 if [ "x$FESS_DICTIONARY_PATH" != "x" ]; then
   FESS_JAVA_OPTS="$FESS_JAVA_OPTS -Dfess.dictionary.path=$FESS_DICTIONARY_PATH"
 fi

src/main/java/org/codelibs/fess/Constants.java

@@ -317,9 +317,7 @@ public class Constants extends CoreLibConstants {
 
     public static final String MATCHES_ALL_QUERY = "*:*";
 
-    public static final String FESS_ES_TRANSPORT_ADDRESSES = "fess.es.transport_addresses";
-
-    public static final String FESS_ES_CLUSTER_NAME = "fess.es.cluster_name";
+    public static final String FESS_ES_HTTP_ADDRESS = "fess.es.http_address";
 
     public static final int DEFAULT_PAGE_SIZE = 20;
 

src/main/java/org/codelibs/fess/api/es/EsApiManager.java

@@ -36,6 +36,7 @@ import org.codelibs.core.io.CopyUtil;
 import org.codelibs.core.lang.StringUtil;
 import org.codelibs.curl.Curl.Method;
 import org.codelibs.curl.CurlRequest;
+import org.codelibs.curl.CurlResponse;
 import org.codelibs.fess.Constants;
 import org.codelibs.fess.api.BaseApiManager;
 import org.codelibs.fess.exception.FessSystemException;
@@ -136,7 +137,7 @@ public class EsApiManager extends BaseApiManager {
                 curlRequest.param(entry.getKey(), entry.getValue()[0]);
             }
         });
-        curlRequest.onConnect((req, con) -> {
+        final CurlResponse curlResponse = curlRequest.onConnect((req, con) -> {
             con.setDoOutput(true);
             if (httpMethod != Method.GET) {
                 try (ServletInputStream in = request.getInputStream(); OutputStream out = con.getOutputStream()) {
@@ -145,27 +146,20 @@ public class EsApiManager extends BaseApiManager {
                     throw new WebApiException(HttpServletResponse.SC_BAD_REQUEST, e);
                 }
             }
-        }).execute(con -> {
-            try (ServletOutputStream out = response.getOutputStream()) {
-                try (InputStream in = con.getInputStream()) {
-                    response.setStatus(con.getResponseCode());
-                    CopyUtil.copy(in, out);
-                } catch (final Exception e) {
-                    response.setStatus(con.getResponseCode());
-                    try (InputStream err = con.getErrorStream()) {
-                        CopyUtil.copy(err, out);
-                    }
-                }
-            } catch (final ClientAbortException e) {
+        }).execute();
+
+        try (ServletOutputStream out = response.getOutputStream(); InputStream in = curlResponse.getContentAsStream()) {
+            response.setStatus(curlResponse.getHttpStatusCode());
+            CopyUtil.copy(in, out);
+        } catch (final ClientAbortException e) {
+            logger.debug("Client aborts this request.", e);
+        } catch (final Exception e) {
+            if (e.getCause() instanceof ClientAbortException) {
                 logger.debug("Client aborts this request.", e);
-            } catch (final Exception e) {
-                if (e.getCause() instanceof ClientAbortException) {
-                    logger.debug("Client aborts this request.", e);
-                } else {
-                    throw new WebApiException(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, e);
-                }
+            } else {
+                throw new WebApiException(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, e);
             }
-        });
+        }
     }
 
     protected void processPluginRequest(final HttpServletRequest request, final HttpServletResponse response, final String path) {

src/main/java/org/codelibs/fess/es/client/FessEsClient.java

@@ -44,6 +44,7 @@ import org.codelibs.core.io.FileUtil;
 import org.codelibs.core.io.ResourceUtil;
 import org.codelibs.core.lang.StringUtil;
 import org.codelibs.curl.CurlResponse;
+import org.codelibs.elasticsearch.client.HttpClient;
 import org.codelibs.elasticsearch.runner.ElasticsearchClusterRunner;
 import org.codelibs.elasticsearch.runner.ElasticsearchClusterRunner.Configs;
 import org.codelibs.fess.Constants;
@@ -130,13 +131,10 @@ import org.elasticsearch.action.update.UpdateRequestBuilder;
 import org.elasticsearch.action.update.UpdateResponse;
 import org.elasticsearch.client.AdminClient;
 import org.elasticsearch.client.Client;
-import org.elasticsearch.client.transport.TransportClient;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.document.DocumentField;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.settings.Settings.Builder;
-import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.InnerHitBuilder;
@@ -150,7 +148,6 @@ import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilde
 import org.elasticsearch.search.collapse.CollapseBuilder;
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.client.PreBuiltTransportClient;
 import org.lastaflute.core.message.UserMessages;
 import org.lastaflute.di.exception.ContainerInitFailureException;
 import org.slf4j.Logger;
@@ -158,7 +155,6 @@ import org.slf4j.LoggerFactory;
 
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.collect.Lists;
 import com.google.common.io.BaseEncoding;
 
 public class FessEsClient implements Client {
@@ -166,7 +162,7 @@ public class FessEsClient implements Client {
 
     protected ElasticsearchClusterRunner runner;
 
-    protected List<TransportAddress> transportAddressList = new ArrayList<>();
+    protected String httpAddress;
 
     protected Client client;
 
@@ -218,10 +214,6 @@ public class FessEsClient implements Client {
         return this.runner != null;
     }
 
-    public void addTransportAddress(final String host, final int port) {
-        transportAddressList.add(new TransportAddress(getInetAddressByName(host), port));
-    }
-
     protected InetAddress getInetAddressByName(final String host) {
         try {
             return InetAddress.getByName(host);
@@ -234,24 +226,8 @@ public class FessEsClient implements Client {
     public void open() {
         final FessConfig fessConfig = ComponentUtil.getFessConfig();
 
-        final String transportAddressesValue = System.getProperty(Constants.FESS_ES_TRANSPORT_ADDRESSES);
-        if (StringUtil.isNotBlank(transportAddressesValue)) {
-            for (final String transportAddressValue : transportAddressesValue.split(",")) {
-                final String[] addressPair = transportAddressValue.trim().split(":");
-                if (addressPair.length < 3) {
-                    final String host = addressPair[0];
-                    int port = 9300;
-                    if (addressPair.length == 2) {
-                        port = Integer.parseInt(addressPair[1]);
-                    }
-                    addTransportAddress(host, port);
-                } else {
-                    logger.warn("Invalid address format: " + transportAddressValue);
-                }
-            }
-        }
-
-        if (transportAddressList.isEmpty()) {
+        String httpAddress = System.getProperty(Constants.FESS_ES_HTTP_ADDRESS);
+        if (StringUtil.isBlank(httpAddress)) {
             if (runner == null) {
                 runner = new ElasticsearchClusterRunner();
                 final Configs config = newConfigs().clusterName(fessConfig.getElasticsearchClusterName()).numOfNode(1).useLogger();
@@ -273,27 +249,14 @@ public class FessEsClient implements Client {
                 });
                 runner.build(config);
             }
-            final int port = runner.node().settings().getAsInt("transport.tcp.port", 9300);
-            client = createTransportClient(fessConfig, Lists.newArrayList(new TransportAddress(getInetAddressByName("localhost"), port)));
-            addTransportAddress("localhost", port);
+            final int port = runner.node().settings().getAsInt("http.port", 9200);
+            httpAddress = "http://localhost:" + port;
             logger.warn("Embedded Elasticsearch is running. This configuration is not recommended for production use.");
-        } else {
-            client = createTransportClient(fessConfig, transportAddressList);
         }
+        client = createHttpClient(fessConfig, httpAddress);
 
-        if (StringUtil.isBlank(transportAddressesValue)) {
-            final StringBuilder buf = new StringBuilder();
-            for (final TransportAddress transportAddress : transportAddressList) {
-                if (buf.length() > 0) {
-                    buf.append(',');
-                }
-                buf.append(transportAddress.address().getHostName());
-                buf.append(':');
-                buf.append(transportAddress.address().getPort());
-            }
-            if (buf.length() > 0) {
-                System.setProperty(Constants.FESS_ES_TRANSPORT_ADDRESSES, buf.toString());
-            }
+        if (StringUtil.isNotBlank(httpAddress)) {
+            System.setProperty(Constants.FESS_ES_HTTP_ADDRESS, httpAddress);
         }
 
         waitForYellowStatus(fessConfig);
@@ -352,18 +315,9 @@ public class FessEsClient implements Client {
         });
     }
 
-    protected Client createTransportClient(final FessConfig fessConfig, final List<TransportAddress> transportAddressList) {
-        final Builder settingsBuilder = Settings.builder();
-        settingsBuilder.put("cluster.name", fessConfig.getElasticsearchClusterName());
-        settingsBuilder.put("client.transport.sniff", fessConfig.isElasticsearchTransportSniff());
-        settingsBuilder.put("client.transport.ping_timeout", fessConfig.getElasticsearchTransportPingTimeout());
-        settingsBuilder.put("client.transport.nodes_sampler_interval", fessConfig.getElasticsearchTransportNodesSamplerInterval());
-        final Settings settings = settingsBuilder.build();
-        final TransportClient transportClient = new PreBuiltTransportClient(settings);
-        for (final TransportAddress address : transportAddressList) {
-            transportClient.addTransportAddress(address);
-        }
-        return transportClient;
+    protected Client createHttpClient(final FessConfig fessConfig, final String host) {
+        final Settings settings = Settings.builder().putList("http.hosts", host).build();
+        return new HttpClient(settings, null);
     }
 
     public boolean existsIndex(final String indexName) {
@@ -631,7 +585,7 @@ public class FessEsClient implements Client {
             }
         }
         final String message =
-                "Elasticsearch (" + System.getProperty(Constants.FESS_ES_TRANSPORT_ADDRESSES)
+                "Elasticsearch (" + System.getProperty(Constants.FESS_ES_HTTP_ADDRESS)
                         + ") is not available. Check the state of your Elasticsearch cluster (" + fessConfig.getElasticsearchClusterName()
                         + ").";
         throw new ContainerInitFailureException(message, cause);

src/main/java/org/codelibs/fess/exec/Crawler.java

@@ -203,13 +203,9 @@ public class Crawler {
 
         initializeProbes();
 
-        final String transportAddresses = System.getProperty(Constants.FESS_ES_TRANSPORT_ADDRESSES);
-        if (StringUtil.isNotBlank(transportAddresses)) {
-            System.setProperty(EsClient.TRANSPORT_ADDRESSES, transportAddresses);
-        }
-        final String clusterName = System.getProperty(Constants.FESS_ES_CLUSTER_NAME);
-        if (StringUtil.isNotBlank(clusterName)) {
-            System.setProperty(EsClient.CLUSTER_NAME, clusterName);
+        final String httpAddress = System.getProperty(Constants.FESS_ES_HTTP_ADDRESS);
+        if (StringUtil.isNotBlank(httpAddress)) {
+            System.setProperty(EsClient.HTTP_ADDRESS, httpAddress);
         }
 
         TimeoutTask systemMonitorTask = null;

src/main/java/org/codelibs/fess/exec/SuggestCreator.java

@@ -105,13 +105,9 @@ public class SuggestCreator {
             }
         }
 
-        final String transportAddresses = System.getProperty(Constants.FESS_ES_TRANSPORT_ADDRESSES);
-        if (StringUtil.isNotBlank(transportAddresses)) {
-            System.setProperty(EsClient.TRANSPORT_ADDRESSES, transportAddresses);
-        }
-        final String clusterName = System.getProperty(Constants.FESS_ES_CLUSTER_NAME);
-        if (StringUtil.isNotBlank(clusterName)) {
-            System.setProperty(EsClient.CLUSTER_NAME, clusterName);
+        final String httpAddress = System.getProperty(Constants.FESS_ES_HTTP_ADDRESS);
+        if (StringUtil.isNotBlank(httpAddress)) {
+            System.setProperty(EsClient.HTTP_ADDRESS, httpAddress);
         }
 
         TimeoutTask systemMonitorTask = null;

src/main/java/org/codelibs/fess/exec/ThumbnailGenerator.java

@@ -110,13 +110,9 @@ public class ThumbnailGenerator {
             }
         }
 
-        final String transportAddresses = System.getProperty(Constants.FESS_ES_TRANSPORT_ADDRESSES);
-        if (StringUtil.isNotBlank(transportAddresses)) {
-            System.setProperty(EsClient.TRANSPORT_ADDRESSES, transportAddresses);
-        }
-        final String clusterName = System.getProperty(Constants.FESS_ES_CLUSTER_NAME);
-        if (StringUtil.isNotBlank(clusterName)) {
-            System.setProperty(EsClient.CLUSTER_NAME, clusterName);
+        final String httpAddress = System.getProperty(Constants.FESS_ES_HTTP_ADDRESS);
+        if (StringUtil.isNotBlank(httpAddress)) {
+            System.setProperty(EsClient.HTTP_ADDRESS, httpAddress);
         }
 
         TimeoutTask systemMonitorTask = null;

src/main/java/org/codelibs/fess/job/CrawlJob.java

@@ -266,19 +266,12 @@ public class CrawlJob {
         cmdList.add(buf.toString());
 
         if (useLocalElasticsearch) {
-            final String transportAddresses = System.getProperty(Constants.FESS_ES_TRANSPORT_ADDRESSES);
-            if (StringUtil.isNotBlank(transportAddresses)) {
-                cmdList.add("-D" + Constants.FESS_ES_TRANSPORT_ADDRESSES + "=" + transportAddresses);
+            final String httpAddress = System.getProperty(Constants.FESS_ES_HTTP_ADDRESS);
+            if (StringUtil.isNotBlank(httpAddress)) {
+                cmdList.add("-D" + Constants.FESS_ES_HTTP_ADDRESS + "=" + httpAddress);
             }
         }
 
-        final String clusterName = System.getProperty(Constants.FESS_ES_CLUSTER_NAME);
-        if (StringUtil.isNotBlank(clusterName)) {
-            cmdList.add("-D" + Constants.FESS_ES_CLUSTER_NAME + "=" + clusterName);
-        } else {
-            cmdList.add("-D" + Constants.FESS_ES_CLUSTER_NAME + "=" + fessConfig.getElasticsearchClusterName());
-        }
-
         final String systemLastaEnv = System.getProperty("lasta.env");
         if (StringUtil.isNotBlank(systemLastaEnv)) {
             if (systemLastaEnv.equals("web")) {

src/main/java/org/codelibs/fess/job/GenerateThumbnailJob.java

@@ -197,19 +197,12 @@ public class GenerateThumbnailJob {
         cmdList.add(buf.toString());
 
         if (useLocaleElasticsearch) {
-            final String transportAddresses = System.getProperty(Constants.FESS_ES_TRANSPORT_ADDRESSES);
-            if (StringUtil.isNotBlank(transportAddresses)) {
-                cmdList.add("-D" + Constants.FESS_ES_TRANSPORT_ADDRESSES + "=" + transportAddresses);
+            final String httpAddress = System.getProperty(Constants.FESS_ES_HTTP_ADDRESS);
+            if (StringUtil.isNotBlank(httpAddress)) {
+                cmdList.add("-D" + Constants.FESS_ES_HTTP_ADDRESS + "=" + httpAddress);
             }
         }
 
-        final String clusterName = System.getProperty(Constants.FESS_ES_CLUSTER_NAME);
-        if (StringUtil.isNotBlank(clusterName)) {
-            cmdList.add("-D" + Constants.FESS_ES_CLUSTER_NAME + "=" + clusterName);
-        } else {
-            cmdList.add("-D" + Constants.FESS_ES_CLUSTER_NAME + "=" + fessConfig.getElasticsearchClusterName());
-        }
-
         final String systemLastaEnv = System.getProperty("lasta.env");
         if (StringUtil.isNotBlank(systemLastaEnv)) {
             if (systemLastaEnv.equals("web")) {

src/main/java/org/codelibs/fess/job/SuggestJob.java

@@ -183,19 +183,12 @@ public class SuggestJob {
         cmdList.add(buf.toString());
 
         if (useLocaleElasticsearch) {
-            final String transportAddresses = System.getProperty(Constants.FESS_ES_TRANSPORT_ADDRESSES);
-            if (StringUtil.isNotBlank(transportAddresses)) {
-                cmdList.add("-D" + Constants.FESS_ES_TRANSPORT_ADDRESSES + "=" + transportAddresses);
+            final String httpAddress = System.getProperty(Constants.FESS_ES_HTTP_ADDRESS);
+            if (StringUtil.isNotBlank(httpAddress)) {
+                cmdList.add("-D" + Constants.FESS_ES_HTTP_ADDRESS + "=" + httpAddress);
             }
         }
 
-        final String clusterName = System.getProperty(Constants.FESS_ES_CLUSTER_NAME);
-        if (StringUtil.isNotBlank(clusterName)) {
-            cmdList.add("-D" + Constants.FESS_ES_CLUSTER_NAME + "=" + clusterName);
-        } else {
-            cmdList.add("-D" + Constants.FESS_ES_CLUSTER_NAME + "=" + fessConfig.getElasticsearchClusterName());
-        }
-
         final String systemLastaEnv = System.getProperty("lasta.env");
         if (StringUtil.isNotBlank(systemLastaEnv)) {
             if (systemLastaEnv.equals("web")) {

src/main/resources/fess_config.properties

@@ -8,13 +8,6 @@
 # The title of domain the application for logging
 domain.title = Fess
 
-# Elasticsearch
-elasticsearch.cluster.name=elasticsearch
-elasticsearch.http.url=http://localhost:9201
-elasticsearch.transport.sniff=false
-elasticsearch.transport.ping_timeout=1m
-elasticsearch.transport.nodes_sampler_interval=5s
-
 # Cryptographer
 app.cipher.algorism=aes
 app.cipher.key=___change__me___