diff --git a/README.md b/README.md index 10e61d65c0..55deb5f4a0 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,18 @@ To run UI for Apache Kafka, you can use a pre-built Docker image or build it loc We have plenty of [docker-compose files](documentation/compose/DOCKER_COMPOSE.md) as examples. They're built for various configuration stacks. +# Guides + +- [SSO configuration](documentation/guides/SSO.md) +- [AWS IAM configuration](documentation/guides/AWS_IAM.md) +- [Docker-compose files](documentation/guides/yaml-description.md) +- [Connection to a secure broker]() + +## Connecting to a Secure Broker + +The app supports TLS (SSL) and SASL connections for [encryption and authentication](http://kafka.apache.org/090/documentation.html#security).
+An example is located [here](documentation/compose/kafka-ssl.yml). + ### Configuration File Example of how to configure clusters in the [application-local.yml](https://github.com/provectus/kafka-ui/blob/master/kafka-ui-api/src/main/resources/application-local.yml) configuration file: @@ -97,7 +109,7 @@ kafka: Configure as many clusters as you need by adding their configs below separated with `-`. -## Running From Docker Image +## Running a Docker Image The official Docker image for UI for Apache Kafka is hosted here: [hub.docker.com/r/provectuslabs/kafka-ui](https://hub.docker.com/r/provectuslabs/kafka-ui). Launch Docker container in the background: @@ -131,18 +143,6 @@ Check [building.md](documentation/project/contributing/building.md) Check [running.md](documentation/project/contributing/running.md) -# Guides - -- [SSO configuration](documentation/guides/SSO.md) -- [AWS IAM configuration](documentation/guides/AWS_IAM.md) -- [Docker-compose files](documentation/guides/yaml-description.md) - -## Connecting to a Secure Broker - -UI for Apache Kafka supports TLS (SSL) and SASL connections for [encryption and authentication](http://kafka.apache.org/090/documentation.html#security). This can be configured by providing a combination of the following files (placed into the Kafka root directory): - -To be continued - ## Liveliness and readiness probes Liveliness and readiness endpoint is at `/actuator/health`. Info endpoint (build info) is located at `/actuator/info`. diff --git a/documentation/compose/DOCKER_COMPOSE.md b/documentation/compose/DOCKER_COMPOSE.md index c7379efe9e..02a3a6e157 100644 --- a/documentation/compose/DOCKER_COMPOSE.md +++ b/documentation/compose/DOCKER_COMPOSE.md @@ -1,12 +1,13 @@ # Descriptions of docker-compose configurations (*.yaml) -1. [kafka-ui.yaml](https://github.com/provectus/kafka-ui/blob/master/docker/kafka-ui.yaml) - Default configuration with 2 kafka clusters with two nodes of Schema Registry, one kafka-connect and a few dummy topics. -2. [kafka-clusters-only.yaml](https://github.com/provectus/kafka-ui/blob/master/docker/kafka-clusters-only.yaml) - A configuration for development purposes, everything besides `kafka-ui` itself (to be run locally). -3. [kafka-cluster-sr-auth.yaml](https://github.com/provectus/kafka-ui/blob/master/docker/kafka-cluster-sr-auth.yaml) - Schema registry with authentication. -4. [kafka-ui-auth-context.yaml](https://github.com/provectus/kafka-ui/blob/master/docker/kafka-ui-auth-context.yaml) - Basic (username/password) authentication with custom path (URL) (issue 861). -5. [kafka-ui-connectors.yaml](https://github.com/provectus/kafka-ui/blob/master/docker/kafka-ui-connectors.yaml) - Configuration with different connectors (github-source, s3, sink-activities, source-activities) and Ksql functionality. -6. [kafka-ui-jmx-secured.yml](https://github.com/provectus/kafka-ui/blob/master/docker/kafka-ui-jmx-secured.yml) - Kafka’s JMX with SSL and authentication. -7. [kafka-ui-reverse-proxy.yaml](https://github.com/provectus/kafka-ui/blob/master/docker/kafka-ui-reverse-proxy.yaml) - An example for using the app behind a proxy (like nginx). -8. [kafka-ui-sasl.yaml](https://github.com/provectus/kafka-ui/blob/master/docker/kafka-ui-sasl.yaml) - SASL auth for Kafka. -9. [kafka-ui-traefik-proxy.yaml](https://github.com/provectus/kafka-ui/blob/master/docker/kafka-ui-traefik-proxy.yaml) - Traefik specific proxy configuration. -10. [kafka-ui-zookeeper-ssl.yml](https://github.com/provectus/kafka-ui/blob/master/docker/kafka-ui-zookeeper-ssl.yml) - SSL enabled zookeeper. +1. [kafka-ui.yaml](./kafka-ui.yaml) - Default configuration with 2 kafka clusters with two nodes of Schema Registry, one kafka-connect and a few dummy topics. +2. [kafka-clusters-only.yaml](./kafka-clusters-only.yaml) - A configuration for development purposes, everything besides `kafka-ui` itself (to be run locally). +3. [kafka-ui-ssl.yml](./kafka-ssl.yml) - Connect to Kafka via TLS/SSL +4. [kafka-cluster-sr-auth.yaml](./kafka-cluster-sr-auth.yaml) - Schema registry with authentication. +5. [kafka-ui-auth-context.yaml](./kafka-ui-auth-context.yaml) - Basic (username/password) authentication with custom path (URL) (issue 861). +6. [kafka-ui-connectors.yaml](./kafka-ui-connectors.yaml) - Configuration with different connectors (github-source, s3, sink-activities, source-activities) and Ksql functionality. +7. [kafka-ui-jmx-secured.yml](./kafka-ui-jmx-secured.yml) - Kafka’s JMX with SSL and authentication. +8. [kafka-ui-reverse-proxy.yaml](./kafka-ui-reverse-proxy.yaml) - An example for using the app behind a proxy (like nginx). +9. [kafka-ui-sasl.yaml](./kafka-ui-sasl.yaml) - SASL auth for Kafka. +10. [kafka-ui-traefik-proxy.yaml](./kafka-ui-traefik-proxy.yaml) - Traefik specific proxy configuration. +11. [kafka-ui-zookeeper-ssl.yml](./kafka-ui-zookeeper-ssl.yml) - SSL enabled zookeeper. diff --git a/documentation/compose/kafka-ssl.yml b/documentation/compose/kafka-ssl.yml new file mode 100644 index 0000000000..b298cdc9d6 --- /dev/null +++ b/documentation/compose/kafka-ssl.yml @@ -0,0 +1,58 @@ +--- +version: '3.4' +services: + + kafka-ui: + container_name: kafka-ui + image: provectuslabs/kafka-ui:latest + ports: + - 8080:8080 + depends_on: + - zookeeper0 + - kafka0 + environment: + KAFKA_CLUSTERS_0_NAME: local + KAFKA_CLUSTERS_0_PROPERTIES_SECURITY_PROTOCOL: SSL + KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka0:29092 # SSL LISTENER! + KAFKA_CLUSTERS_0_ZOOKEEPER: zookeeper0:2181 + KAFKA_CLUSTERS_0_PROPERTIES_SSL_TRUSTSTORE_LOCATION: /kafka.truststore.jks + KAFKA_CLUSTERS_0_PROPERTIES_SSL_TRUSTSTORE_PASSWORD: secret # A FILE WITH THE TRUSTSTORE PASSWORD + volumes: + - ./ssl/kafka.truststore.jks:/kafka.truststore.jks + + zookeeper0: + image: confluentinc/cp-zookeeper:6.0.1 + environment: + ZOOKEEPER_CLIENT_PORT: 2181 + ZOOKEEPER_TICK_TIME: 2000 + ports: + - 2181:2181 + + kafka0: + image: confluentinc/cp-kafka:6.0.1 + hostname: kafka0 + depends_on: + - zookeeper0 + ports: + - '9092:9092' + environment: + KAFKA_BROKER_ID: 1 + KAFKA_ZOOKEEPER_CONNECT: zookeeper0:2181 + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_ADVERTISED_LISTENERS: SSL://kafka0:29092,PLAINTEXT_HOST://localhost:9092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: SSL:SSL,PLAINTEXT_HOST:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: SSL + KAFKA_SECURITY_PROTOCOL: SSL + KAFKA_SSL_ENABLED_MECHANISMS: PLAIN,SSL + KAFKA_SSL_KEYSTORE_FILENAME: kafka.keystore.jks + KAFKA_SSL_KEYSTORE_CREDENTIALS: creds + KAFKA_SSL_KEY_CREDENTIALS: creds + KAFKA_SSL_TRUSTSTORE_FILENAME: kafka.truststore.jks + KAFKA_SSL_TRUSTSTORE_CREDENTIALS: creds + #KAFKA_SSL_CLIENT_AUTH: 'required' + KAFKA_SSL_CLIENT_AUTH: "requested" + KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: '' # COMMON NAME VERIFICATION IS DISABLED SERVER-SIDE + volumes: + - ./ssl/creds:/etc/kafka/secrets/creds + - ./ssl/kafka.truststore.jks:/etc/kafka/secrets/kafka.truststore.jks + - ./ssl/kafka.keystore.jks:/etc/kafka/secrets/kafka.keystore.jks \ No newline at end of file diff --git a/documentation/compose/ssl/creds b/documentation/compose/ssl/creds new file mode 100644 index 0000000000..536aca34db --- /dev/null +++ b/documentation/compose/ssl/creds @@ -0,0 +1 @@ +secret \ No newline at end of file diff --git a/documentation/compose/ssl/generate_certs.sh b/documentation/compose/ssl/generate_certs.sh new file mode 100644 index 0000000000..ebb916657b --- /dev/null +++ b/documentation/compose/ssl/generate_certs.sh @@ -0,0 +1,175 @@ +#!/usr/bin/env bash + +set -eu + +KEYSTORE_FILENAME="kafka.keystore.jks" +VALIDITY_IN_DAYS=3650 +DEFAULT_TRUSTSTORE_FILENAME="kafka.truststore.jks" +TRUSTSTORE_WORKING_DIRECTORY="truststore" +KEYSTORE_WORKING_DIRECTORY="keystore" +CA_CERT_FILE="ca-cert" +KEYSTORE_SIGN_REQUEST="cert-file" +KEYSTORE_SIGN_REQUEST_SRL="ca-cert.srl" +KEYSTORE_SIGNED_CERT="cert-signed" + +export COUNTRY=US +export STATE=IL +export ORGANIZATION_UNIT=SE +export CITY=Chicago +export PASSWORD=secret + +COUNTRY=$COUNTRY +STATE=$STATE +OU=$ORGANIZATION_UNIT +CN=kafka0 # COMMON NAME VERIFICATION GOES BRR +LOCATION=$CITY +PASS=$PASSWORD + +function file_exists_and_exit() { + echo "'$1' cannot exist. Move or delete it before" + echo "re-running this script." + exit 1 +} + +if [ -e "$KEYSTORE_WORKING_DIRECTORY" ]; then + file_exists_and_exit $KEYSTORE_WORKING_DIRECTORY +fi + +if [ -e "$CA_CERT_FILE" ]; then + file_exists_and_exit $CA_CERT_FILE +fi + +if [ -e "$KEYSTORE_SIGN_REQUEST" ]; then + file_exists_and_exit $KEYSTORE_SIGN_REQUEST +fi + +if [ -e "$KEYSTORE_SIGN_REQUEST_SRL" ]; then + file_exists_and_exit $KEYSTORE_SIGN_REQUEST_SRL +fi + +if [ -e "$KEYSTORE_SIGNED_CERT" ]; then + file_exists_and_exit $KEYSTORE_SIGNED_CERT +fi + +echo "Welcome to the Kafka SSL keystore and trust store generator script." + +trust_store_file="" +trust_store_private_key_file="" + + if [ -e "$TRUSTSTORE_WORKING_DIRECTORY" ]; then + file_exists_and_exit $TRUSTSTORE_WORKING_DIRECTORY + fi + + mkdir $TRUSTSTORE_WORKING_DIRECTORY + echo + echo "OK, we'll generate a trust store and associated private key." + echo + echo "First, the private key." + echo + + openssl req -new -x509 -keyout $TRUSTSTORE_WORKING_DIRECTORY/ca-key \ + -out $TRUSTSTORE_WORKING_DIRECTORY/ca-cert -days $VALIDITY_IN_DAYS -nodes \ + -subj "/C=$COUNTRY/ST=$STATE/L=$LOCATION/O=$OU/CN=$CN" + + trust_store_private_key_file="$TRUSTSTORE_WORKING_DIRECTORY/ca-key" + + echo + echo "Two files were created:" + echo " - $TRUSTSTORE_WORKING_DIRECTORY/ca-key -- the private key used later to" + echo " sign certificates" + echo " - $TRUSTSTORE_WORKING_DIRECTORY/ca-cert -- the certificate that will be" + echo " stored in the trust store in a moment and serve as the certificate" + echo " authority (CA). Once this certificate has been stored in the trust" + echo " store, it will be deleted. It can be retrieved from the trust store via:" + echo " $ keytool -keystore -export -alias CARoot -rfc" + + echo + echo "Now the trust store will be generated from the certificate." + echo + + keytool -keystore $TRUSTSTORE_WORKING_DIRECTORY/$DEFAULT_TRUSTSTORE_FILENAME \ + -alias CARoot -import -file $TRUSTSTORE_WORKING_DIRECTORY/ca-cert \ + -noprompt -dname "C=$COUNTRY, ST=$STATE, L=$LOCATION, O=$OU, CN=$CN" -keypass $PASS -storepass $PASS -storetype JKS + + trust_store_file="$TRUSTSTORE_WORKING_DIRECTORY/$DEFAULT_TRUSTSTORE_FILENAME" + + echo + echo "$TRUSTSTORE_WORKING_DIRECTORY/$DEFAULT_TRUSTSTORE_FILENAME was created." + + # don't need the cert because it's in the trust store. + rm $TRUSTSTORE_WORKING_DIRECTORY/$CA_CERT_FILE + +echo +echo "Continuing with:" +echo " - trust store file: $trust_store_file" +echo " - trust store private key: $trust_store_private_key_file" + +mkdir $KEYSTORE_WORKING_DIRECTORY + +echo +echo "Now, a keystore will be generated. Each broker and logical client needs its own" +echo "keystore. This script will create only one keystore. Run this script multiple" +echo "times for multiple keystores." +echo +echo " NOTE: currently in Kafka, the Common Name (CN) does not need to be the FQDN of" +echo " this host. However, at some point, this may change. As such, make the CN" +echo " the FQDN. Some operating systems call the CN prompt 'first / last name'" + +# To learn more about CNs and FQDNs, read: +# https://docs.oracle.com/javase/7/docs/api/javax/net/ssl/X509ExtendedTrustManager.html + +keytool -keystore $KEYSTORE_WORKING_DIRECTORY/$KEYSTORE_FILENAME \ + -alias localhost -validity $VALIDITY_IN_DAYS -genkey -keyalg RSA \ + -noprompt -dname "C=$COUNTRY, ST=$STATE, L=$LOCATION, O=$OU, CN=$CN" -keypass $PASS -storepass $PASS -storetype JKS + +echo +echo "'$KEYSTORE_WORKING_DIRECTORY/$KEYSTORE_FILENAME' now contains a key pair and a" +echo "self-signed certificate. Again, this keystore can only be used for one broker or" +echo "one logical client. Other brokers or clients need to generate their own keystores." + +echo +echo "Fetching the certificate from the trust store and storing in $CA_CERT_FILE." +echo + +keytool -keystore $trust_store_file -export -alias CARoot -rfc -file $CA_CERT_FILE -keypass $PASS -storepass $PASS + +echo +echo "Now a certificate signing request will be made to the keystore." +echo +keytool -keystore $KEYSTORE_WORKING_DIRECTORY/$KEYSTORE_FILENAME -alias localhost \ + -certreq -file $KEYSTORE_SIGN_REQUEST -keypass $PASS -storepass $PASS + +echo +echo "Now the trust store's private key (CA) will sign the keystore's certificate." +echo +openssl x509 -req -CA $CA_CERT_FILE -CAkey $trust_store_private_key_file \ + -in $KEYSTORE_SIGN_REQUEST -out $KEYSTORE_SIGNED_CERT \ + -days $VALIDITY_IN_DAYS -CAcreateserial +# creates $KEYSTORE_SIGN_REQUEST_SRL which is never used or needed. + +echo +echo "Now the CA will be imported into the keystore." +echo +keytool -keystore $KEYSTORE_WORKING_DIRECTORY/$KEYSTORE_FILENAME -alias CARoot \ + -import -file $CA_CERT_FILE -keypass $PASS -storepass $PASS -noprompt +rm $CA_CERT_FILE # delete the trust store cert because it's stored in the trust store. + +echo +echo "Now the keystore's signed certificate will be imported back into the keystore." +echo +keytool -keystore $KEYSTORE_WORKING_DIRECTORY/$KEYSTORE_FILENAME -alias localhost -import \ + -file $KEYSTORE_SIGNED_CERT -keypass $PASS -storepass $PASS + +echo +echo "All done!" +echo +echo "Deleting intermediate files. They are:" +echo " - '$KEYSTORE_SIGN_REQUEST_SRL': CA serial number" +echo " - '$KEYSTORE_SIGN_REQUEST': the keystore's certificate signing request" +echo " (that was fulfilled)" +echo " - '$KEYSTORE_SIGNED_CERT': the keystore's certificate, signed by the CA, and stored back" +echo " into the keystore" + + rm $KEYSTORE_SIGN_REQUEST_SRL + rm $KEYSTORE_SIGN_REQUEST + rm $KEYSTORE_SIGNED_CERT diff --git a/documentation/compose/ssl/kafka.keystore.jks b/documentation/compose/ssl/kafka.keystore.jks new file mode 100644 index 0000000000..54b3f3d1bc Binary files /dev/null and b/documentation/compose/ssl/kafka.keystore.jks differ diff --git a/documentation/compose/ssl/kafka.truststore.jks b/documentation/compose/ssl/kafka.truststore.jks new file mode 100644 index 0000000000..eff350ce4e Binary files /dev/null and b/documentation/compose/ssl/kafka.truststore.jks differ