TLS enabled zookeeper (#722)

* Switch zk client. Resolves #683

* Add an example docker compose file with TLS enabled zookeeper

* Update readme a bit

* Fix annoying sonar boy

* Apply review suggestion

* Rename zookeeper ssl options

README.md

@@ -120,6 +120,8 @@ To be continued
 
 # Configuration
 
+We have a plenty of docker-compose files as examples. Please check them out in ``docker`` directory.
+
 ## Configuration File
 Example of how to configure clusters in the [application-local.yml](https://github.com/provectus/kafka-ui/blob/master/kafka-ui-api/src/main/resources/application-local.yml) configuration file:
 

docker/kafka-ui-zookeeper-ssl.yml

@@ -0,0 +1,145 @@
+---
+version: '2'
+services:
+
+  kafka-ui:
+    container_name: kafka-ui
+    image: provectuslabs/kafka-ui:latest
+    ports:
+      - 8080:8080
+      - 5005:5005
+    volumes:
+      - /tmp/kafka/secrets/kafka.kafka1.keystore.jks:/etc/kafka/secrets/kafka.zookeeper.keystore.jks
+      - /tmp/kafka/secrets/kafka.zookeeper.truststore.jks:/etc/kafka/secrets/kafka.zookeeper.truststore.jks
+    depends_on:
+      - zookeeper0
+      - kafka0
+      - schemaregistry0
+      - kafka-connect0
+    environment:
+      KAFKA_CLUSTERS_0_NAME: local
+      KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka0:29092
+      KAFKA_CLUSTERS_0_ZOOKEEPER: zookeeper0:2182
+      KAFKA_CLUSTERS_0_JMXPORT: 9997
+      KAFKA_CLUSTERS_0_SCHEMAREGISTRY: http://schemaregistry0:8085
+      KAFKA_CLUSTERS_0_KAFKACONNECT_0_NAME: first
+      KAFKA_CLUSTERS_0_KAFKACONNECT_0_ADDRESS: http://kafka-connect0:8083
+      KAFKA_CLUSTERS_0_ZOOKEEPER_CLIENTCNXNSOCKET: org.apache.zookeeper.ClientCnxnSocketNetty
+      KAFKA_CLUSTERS_0_ZOOKEEPER_CLIENT_SECURE: 'true'
+      KAFKA_CLUSTERS_0_ZOOKEEPER_SSL_KEYSTORE_LOCATION: /etc/kafka/secrets/kafka.zookeeper.keystore.jks
+      KAFKA_CLUSTERS_0_ZOOKEEPER_SSL_KEYSTORE_PASSWORD: 12345678
+      KAFKA_CLUSTERS_0_ZOOKEEPER_SSL_TRUSTSTORE_LOCATION: /etc/kafka/secrets/kafka.zookeeper.truststore.jks
+      KAFKA_CLUSTERS_0_ZOOKEEPER_SSL_TRUSTSTORE_PASSWORD: 12345678
+
+  zookeeper0:
+    image: confluentinc/cp-zookeeper:5.2.4
+    volumes:
+      - /tmp/kafka/secrets/kafka.kafka1.keystore.jks:/etc/kafka/secrets/kafka.zookeeper.keystore.jks
+      - /tmp/kafka/secrets/kafka.zookeeper.truststore.jks:/etc/kafka/secrets/kafka.zookeeper.truststore.jks
+    environment:
+      ZOOKEEPER_CLIENT_PORT: 2182
+      ZOOKEEPER_TICK_TIME: 2000
+
+      ZOOKEEPER_SECURE_CLIENT_PORT: 2182
+      ZOOKEEPER_SERVER_CNXN_FACTORY: org.apache.zookeeper.server.NettyServerCnxnFactory
+      ZOOKEEPER_SSL_KEYSTORE_LOCATION: /etc/kafka/secrets/kafka.zookeeper.keystore.jks
+      ZOOKEEPER_SSL_KEYSTORE_PASSWORD: 12345678
+      ZOOKEEPER_SSL_KEYSTORE_TYPE: PKCS12
+      ZOOKEEPER_SSL_TRUSTSTORE_LOCATION: /etc/kafka/secrets/kafka.zookeeper.truststore.jks
+      ZOOKEEPER_SSL_TRUSTSTORE_PASSWORD: 12345678
+      ZOOKEEPER_SSL_TRUSTSTORE_TYPE: JKS
+      # TLS 1.2 is the tested-default - TLS 1.3 has not been tested for production
+      # You can evaluate TLS 1.3 for ZooKeeper by uncommenting the following two properties
+      # and setting KAFKA_ZOOKEEPER_SSL_PROTOCOL on brokers
+      ZOOKEEPER_SSL_ENABLED_PROTOCOLS: TLSv1.3,TLSv1.2
+      ZOOKEEPER_SSL_QUORUM_ENABLED_PROTOCOLS: TLSv1.3,TLSv1.2
+      ZOOKEEPER_SSL_CIPHER_SUITES: TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,TLS_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256
+      ZOOKEEPER_SSL_CLIENT_AUTH: need
+      ZOOKEEPER_AUTH_PROVIDER_X509: org.apache.zookeeper.server.auth.X509AuthenticationProvider
+      ZOOKEEPER_AUTH_PROVIDER_SASL: org.apache.zookeeper.server.auth.SASLAuthenticationProvider
+    ports:
+      - 2182:2182
+
+  kafka0:
+    image: confluentinc/cp-kafka:5.2.4
+    depends_on:
+      - zookeeper0
+    ports:
+      - 9092:9092
+      - 9997:9997
+    volumes:
+      - /tmp/kafka/secrets/kafka.kafka1.keystore.jks:/etc/kafka/secrets/kafka.kafka1.keystore.jks
+      - /tmp/kafka/secrets/kafka.server.truststore.jks:/etc/kafka/secrets/kafka.kafka1.truststore.jks
+    environment:
+      KAFKA_BROKER_ID: 1
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper0:2182
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka0:29092,PLAINTEXT_HOST://localhost:9092
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
+      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+      JMX_PORT: 9997
+      KAFKA_JMX_OPTS: -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=kafka0 -Dcom.sun.management.jmxremote.rmi.port=9997
+      KAFKA_ZOOKEEPER_SSL_CLIENT_ENABLE: 'true'
+      KAFKA_ZOOKEEPER_SSL_CIPHER_SUITES: TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,TLS_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256
+      KAFKA_ZOOKEEPER_CLIENT_CNXN_SOCKET: org.apache.zookeeper.ClientCnxnSocketNetty
+      KAFKA_ZOOKEEPER_SSL_KEYSTORE_LOCATION: /etc/kafka/secrets/kafka.kafka1.keystore.jks
+      KAFKA_ZOOKEEPER_SSL_KEYSTORE_PASSWORD: 12345678
+      KAFKA_ZOOKEEPER_SSL_KEYSTORE_TYPE: PKCS12
+      KAFKA_ZOOKEEPER_SSL_TRUSTSTORE_LOCATION: /etc/kafka/secrets/kafka.kafka1.truststore.jks
+      KAFKA_ZOOKEEPER_SSL_TRUSTSTORE_PASSWORD: 12345678
+      KAFKA_ZOOKEEPER_SSL_TRUSTSTORE_TYPE: JKS
+
+  schemaregistry0:
+    image: confluentinc/cp-schema-registry:5.2.4
+    ports:
+      - 8085:8085
+    depends_on:
+      - zookeeper0
+      - kafka0
+    environment:
+      SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: PLAINTEXT://kafka0:29092
+      SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: zookeeper0:2182
+      SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL: PLAINTEXT
+      SCHEMA_REGISTRY_HOST_NAME: schemaregistry0
+      SCHEMA_REGISTRY_LISTENERS: http://schemaregistry0:8085
+
+      SCHEMA_REGISTRY_SCHEMA_REGISTRY_INTER_INSTANCE_PROTOCOL: "http"
+      SCHEMA_REGISTRY_LOG4J_ROOT_LOGLEVEL: INFO
+      SCHEMA_REGISTRY_KAFKASTORE_TOPIC: _schemas
+
+  kafka-connect0:
+    image: confluentinc/cp-kafka-connect:5.2.4
+    ports:
+      - 8083:8083
+    depends_on:
+      - kafka0
+      - schemaregistry0
+    environment:
+      CONNECT_BOOTSTRAP_SERVERS: kafka0:29092
+      CONNECT_GROUP_ID: compose-connect-group
+      CONNECT_CONFIG_STORAGE_TOPIC: _connect_configs
+      CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
+      CONNECT_OFFSET_STORAGE_TOPIC: _connect_offset
+      CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
+      CONNECT_STATUS_STORAGE_TOPIC: _connect_status
+      CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
+      CONNECT_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter
+      CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: http://schemaregistry0:8085
+      CONNECT_VALUE_CONVERTER: org.apache.kafka.connect.storage.StringConverter
+      CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schemaregistry0:8085
+      CONNECT_INTERNAL_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter
+      CONNECT_INTERNAL_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
+      CONNECT_REST_ADVERTISED_HOST_NAME: kafka-connect0
+      CONNECT_PLUGIN_PATH: "/usr/share/java,/usr/share/confluent-hub-components"
+
+  kafka-init-topics:
+    image: confluentinc/cp-kafka:5.2.4
+    volumes:
+      - ./message.json:/data/message.json
+    depends_on:
+      - kafka0
+    command: "bash -c 'echo Waiting for Kafka to be ready... && \
+               cub kafka-ready -b kafka0:29092 1 30 && \
+               kafka-topics --create --topic second.users --partitions 3 --replication-factor 1 --if-not-exists --zookeeper zookeeper0:2182 && \
+               kafka-topics --create --topic first.messages --partitions 2 --replication-factor 1 --if-not-exists --zookeeper zookeeper0:2182 && \
+               kafka-console-producer --broker-list kafka0:29092 -topic second.users < /data/message.json'"

kafka-ui-api/pom.xml

@@ -62,9 +62,9 @@
             <version>${kafka.version}</version>
         </dependency>
         <dependency>
-            <groupId>com.101tec</groupId>
-            <artifactId>zkclient</artifactId>
-            <version>${zkclient.version}</version>
+            <groupId>org.apache.zookeeper</groupId>
+            <artifactId>zookeeper</artifactId>
+            <version>${zookeper.version}</version>
         </dependency>
         <dependency>
             <groupId>org.projectlombok</groupId>

kafka-ui-api/src/main/java/com/provectus/kafka/ui/exception/ZooKeeperException.java

@@ -0,0 +1,9 @@
+package com.provectus.kafka.ui.exception;
+
+public class ZooKeeperException extends RuntimeException {
+
+  public ZooKeeperException(Throwable cause) {
+    super(cause);
+  }
+
+}

kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/KafkaService.java

@@ -159,8 +159,9 @@ public class KafkaService {
     ServerStatus zookeeperStatus = ServerStatus.OFFLINE;
     Throwable zookeeperException = null;
     try {
-      zookeeperStatus = zookeeperService.isZookeeperOnline(currentCluster) ? ServerStatus.ONLINE :
-          ServerStatus.OFFLINE;
+      zookeeperStatus = zookeeperService.isZookeeperOnline(currentCluster)
+              ? ServerStatus.ONLINE
+              : ServerStatus.OFFLINE;
     } catch (Throwable e) {
       zookeeperException = e;
     }

kafka-ui-api/src/main/java/com/provectus/kafka/ui/service/ZookeeperService.java

@@ -1,11 +1,15 @@
 package com.provectus.kafka.ui.service;
 
+import com.provectus.kafka.ui.exception.ZooKeeperException;
 import com.provectus.kafka.ui.model.KafkaCluster;
+import java.io.IOException;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.log4j.Log4j2;
-import org.I0Itec.zkclient.ZkClient;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.ZooKeeper;
+import org.jetbrains.annotations.Nullable;
 import org.springframework.stereotype.Service;
 import org.springframework.util.StringUtils;
 
@@ -14,7 +18,7 @@ import org.springframework.util.StringUtils;
 @Log4j2
 public class ZookeeperService {
 
-  private final Map<String, ZkClient> cachedZkClient = new ConcurrentHashMap<>();
+  private final Map<String, ZooKeeper> cachedZkClient = new ConcurrentHashMap<>();
 
   public boolean isZookeeperOnline(KafkaCluster kafkaCluster) {
     var isConnected = false;
@@ -28,20 +32,41 @@ public class ZookeeperService {
     return isConnected;
   }
 
-  private boolean isZkClientConnected(ZkClient zkClient) {
-    zkClient.getChildren("/brokers/ids");
+  private boolean isZkClientConnected(ZooKeeper zkClient) {
+    try {
+      zkClient.getChildren("/brokers/ids", null);
+    } catch (KeeperException e) {
+      log.error("A zookeeper exception has occurred", e);
+      return false;
+    } catch (InterruptedException e) {
+      log.error("Interrupted: ", e);
+      Thread.currentThread().interrupt();
+    }
     return true;
   }
 
-  private ZkClient getOrCreateZkClient(KafkaCluster cluster) {
+  @Nullable
+  private ZooKeeper getOrCreateZkClient(KafkaCluster cluster) {
+    final var clusterName = cluster.getName();
+    final var client = cachedZkClient.get(clusterName);
+    if (client != null && client.getState() != ZooKeeper.States.CONNECTED) {
+      cachedZkClient.remove(clusterName);
+    }
     try {
-      return cachedZkClient.computeIfAbsent(
-          cluster.getName(),
-          (n) -> new ZkClient(cluster.getZookeeper(), 1000)
-      );
+      return cachedZkClient.computeIfAbsent(clusterName, n -> createClient(cluster));
     } catch (Exception e) {
-      log.error("Error while creating zookeeper client for cluster {}", cluster.getName());
+      log.error("Error while creating zookeeper client for cluster {}", clusterName);
       return null;
     }
   }
+
+  private ZooKeeper createClient(KafkaCluster cluster) {
+    try {
+      return new ZooKeeper(cluster.getZookeeper(), 60 * 1000, watchedEvent -> {});
+    } catch (IOException e) {
+      log.error("Error while creating a zookeeper client for cluster [{}]",
+              cluster.getName());
+      throw new ZooKeeperException(e);
+    }
+  }
 }

pom.xml

@@ -1,68 +1,90 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
-	<modelVersion>4.0.0</modelVersion>
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
     <packaging>pom</packaging>
     <modules>
         <module>kafka-ui-contract</module>
-		<module>kafka-ui-api</module>
-		<module>kafka-ui-e2e-checks</module>
-	</modules>
+        <module>kafka-ui-api</module>
+        <module>kafka-ui-e2e-checks</module>
+    </modules>
 
-	<properties>
-		<maven.compiler.source>13</maven.compiler.source>
-		<maven.compiler.target>13</maven.compiler.target>
-		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    <properties>
+        <maven.compiler.source>13</maven.compiler.source>
+        <maven.compiler.target>13</maven.compiler.target>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
 
-		<spring-boot.version>2.2.4.RELEASE</spring-boot.version>
-		<jackson-databind-nullable.version>0.2.1</jackson-databind-nullable.version>
-		<org.mapstruct.version>1.3.1.Final</org.mapstruct.version>
-		<org.projectlombok.version>1.18.10</org.projectlombok.version>
+        <spring-boot.version>2.2.4.RELEASE</spring-boot.version>
+        <jackson-databind-nullable.version>0.2.1</jackson-databind-nullable.version>
+        <org.mapstruct.version>1.3.1.Final</org.mapstruct.version>
+        <org.projectlombok.version>1.18.10</org.projectlombok.version>
 		<org.projectlombok.e2e-checks.version>1.18.20</org.projectlombok.e2e-checks.version>
-		<git.revision>latest</git.revision>
-		<zkclient.version>0.11</zkclient.version>
-		<kafka-clients.version>2.4.1</kafka-clients.version>
-		<node.version>v14.17.1</node.version>
-		<dockerfile-maven-plugin.version>1.4.10</dockerfile-maven-plugin.version>
-		<frontend-maven-plugin.version>1.8.0</frontend-maven-plugin.version>
-		<maven-compiler-plugin.version>3.5.1</maven-compiler-plugin.version>
-		<maven-clean-plugin.version>3.1.0</maven-clean-plugin.version>
-		<maven-resources-plugin.version>3.1.0</maven-resources-plugin.version>
-		<maven-surefire-plugin.version>2.22.0</maven-surefire-plugin.version>
-		<openapi-generator-maven-plugin.version>4.3.0</openapi-generator-maven-plugin.version>
-		<swagger-annotations.version>1.6.0</swagger-annotations.version>
-		<springdoc-openapi-webflux-ui.version>1.2.32</springdoc-openapi-webflux-ui.version>
-		<kafka.version>2.4.1</kafka.version>
-		<avro.version>1.9.2</avro.version>
-		<confluent.version>5.5.1</confluent.version>
-		<apache.commons.version>2.2</apache.commons.version>
-		<test.containers.version>1.15.1</test.containers.version>
-		<junit-jupiter-engine.version>5.4.0</junit-jupiter-engine.version>
-		<mockito.version>2.21.0</mockito.version>
-		<assertj.version>3.19.0</assertj.version>
+        <git.revision>latest</git.revision>
+        <zookeper.version>3.5.7</zookeper.version>
+        <kafka-clients.version>2.4.1</kafka-clients.version>
+        <node.version>v14.17.1</node.version>
+        <dockerfile-maven-plugin.version>1.4.10</dockerfile-maven-plugin.version>
+        <frontend-maven-plugin.version>1.8.0</frontend-maven-plugin.version>
+        <maven-compiler-plugin.version>3.5.1</maven-compiler-plugin.version>
+        <maven-clean-plugin.version>3.1.0</maven-clean-plugin.version>
+        <maven-resources-plugin.version>3.1.0</maven-resources-plugin.version>
+        <maven-surefire-plugin.version>2.22.0</maven-surefire-plugin.version>
+        <openapi-generator-maven-plugin.version>4.3.0</openapi-generator-maven-plugin.version>
+        <swagger-annotations.version>1.6.0</swagger-annotations.version>
+        <springdoc-openapi-webflux-ui.version>1.2.32</springdoc-openapi-webflux-ui.version>
+        <kafka.version>2.4.1</kafka.version>
+        <avro.version>1.9.2</avro.version>
+        <confluent.version>5.5.1</confluent.version>
+        <apache.commons.version>2.2</apache.commons.version>
+        <test.containers.version>1.15.1</test.containers.version>
+        <junit-jupiter-engine.version>5.4.0</junit-jupiter-engine.version>
+        <mockito.version>2.21.0</mockito.version>
+        <assertj.version>3.19.0</assertj.version>
 
-		<frontend-generated-sources-directory>..//kafka-ui-react-app/src/generated-sources</frontend-generated-sources-directory>
-		<sonar.organization>provectus</sonar.organization>
-		<sonar.host.url>https://sonarcloud.io</sonar.host.url>
-	</properties>
+        <frontend-generated-sources-directory>..//kafka-ui-react-app/src/generated-sources
+        </frontend-generated-sources-directory>
+        <sonar.organization>provectus</sonar.organization>
+        <sonar.host.url>https://sonarcloud.io</sonar.host.url>
+    </properties>
 
-	<repositories>
-		<repository>
-			<id>confluent</id>
-			<url>https://packages.confluent.io/maven/</url>
-		</repository>
-	</repositories>
+    <repositories>
+        <repository>
+            <id>confluent</id>
+            <url>https://packages.confluent.io/maven/</url>
+        </repository>
+        <repository>
+            <id>central</id>
+            <name>Central Repository</name>
+            <url>https://repo.maven.apache.org/maven2</url>
+            <layout>default</layout>
+            <snapshots>
+                <enabled>false</enabled>
+            </snapshots>
+        </repository>
+    </repositories>
 
-	<pluginRepositories>
-		<pluginRepository>
-			<id>confluent</id>
-			<url>https://packages.confluent.io/maven/</url>
-		</pluginRepository>
-	</pluginRepositories>
+    <pluginRepositories>
+        <pluginRepository>
+            <id>confluent</id>
+            <url>https://packages.confluent.io/maven/</url>
+        </pluginRepository>
+        <pluginRepository>
+            <id>central</id>
+            <name>Central Repository</name>
+            <url>https://repo.maven.apache.org/maven2</url>
+            <layout>default</layout>
+            <snapshots>
+                <enabled>false</enabled>
+            </snapshots>
+            <releases>
+                <updatePolicy>never</updatePolicy>
+            </releases>
+        </pluginRepository>
+    </pluginRepositories>
 
-	<groupId>com.provectus</groupId>
-	<artifactId>kafka-ui</artifactId>
-	<version>0.1.1-SNAPSHOT</version>
-	<name>kafka-ui</name>
-	<description>Kafka metrics for UI panel</description>
+    <groupId>com.provectus</groupId>
+    <artifactId>kafka-ui</artifactId>
+    <version>0.1.1-SNAPSHOT</version>
+    <name>kafka-ui</name>
+    <description>Kafka metrics for UI panel</description>
 </project>