feat(dbmaster,replication-manager): remove MySQL replication

.env.default

@@ -9,7 +9,6 @@ DESECSTACK_IPV6_ADDRESS=fda8:7213:9e5e:1::0642:ac10:0080
 
 # certificates
 DESECSTACK_WWW_CERTS=./certs
-DESECSTACK_DBMASTER_CERTS=./certs
 
 # API-related
 DESECSTACK_API_ADMIN=
@@ -33,14 +32,10 @@ DESECSTACK_NSLORD_DEFAULT_TTL=3600
 
 # nsmaster-related
 DESECSTACK_DBMASTER_PASSWORD_pdns=
-DESECSTACK_DBMASTER_PASSWORD_replication_manager=
 DESECSTACK_NSMASTER_APIKEY=
 DESECSTACK_NSMASTER_CARBONSERVER=
 DESECSTACK_NSMASTER_CARBONOURNAME=
 
-# replication-manager
-DESECSTACK_REPLICATION_MANAGER_CERTS=
-
 # monitoring
 DESECSTACK_WATCHDOG_SLAVES=ns1.example.org ns2.example.net
 DESECSTACK_PROMETHEUS_PASSWORD=

.env.dev

@@ -9,7 +9,6 @@ DESECSTACK_IPV6_ADDRESS=fda8:7213:9e5e:1::0642:ac10:0080
 
 # certificates
 DESECSTACK_WWW_CERTS=./certs
-DESECSTACK_DBMASTER_CERTS=./certs
 
 # API-related
 DESECSTACK_API_ADMIN=admin@example.com
@@ -33,14 +32,10 @@ DESECSTACK_NSLORD_DEFAULT_TTL=3600
 
 # nsmaster-related
 DESECSTACK_DBMASTER_PASSWORD_pdns=insecure
-DESECSTACK_DBMASTER_PASSWORD_replication_manager=insecure
 DESECSTACK_NSMASTER_APIKEY=insecure
 DESECSTACK_NSMASTER_CARBONSERVER=
 DESECSTACK_NSMASTER_CARBONOURNAME=
 
-# replication-manager
-DESECSTACK_REPLICATION_MANAGER_CERTS=./replication-certs
-
 # monitoring
 DESECSTACK_WATCHDOG_SLAVES=
 DESECSTACK_PROMETHEUS_PASSWORD=insecure

.gitignore

@@ -11,7 +11,6 @@ api/desecapi.sqlite
 
 # local certificates
 /certs/
-/replication-certs/
 
 # Webapp development files
 node_modules

.travis.yml

@@ -28,7 +28,6 @@ env:
    - DESECSTACK_IPV6_SUBNET=bade:affe:dead:beef:b011::/80
    - DESECSTACK_IPV6_ADDRESS=bade:affe:dead:beef:b011:0642:ac10:0080
    - DESECSTACK_WWW_CERTS=./certs
-   - DESECSTACK_DBMASTER_CERTS=./dbmastercerts
    - DESECSTACK_MINIMUM_TTL_DEFAULT=3600
    - DESECSTACK_PROMETHEUS_PASSWORD=Je9NNkqbULsg
 

README.md

@@ -6,7 +6,7 @@ This is a docker-compose application providing the basic stack for deSEC name se
 - `nslord`: Eventually authoritative DNS server (PowerDNS). DNSSEC keying material is generated here.
 - `nsmaster`: Stealth authoritative DNS server (PowerDNS). Receives fully signed AXFR zone transfers from `nslord`. No access to keys.
 - `api`: RESTful API to create deSEC users and domains, see [documentation](https://desec.readthedocs.io/).
-- `dbapi`, `dblord`, `dbmaster`: MariaDB database services for `api`, `nslord`, and `nsmaster`, respectively. The `dbmaster` database is exposed at 3306 for TLS-secured replication.
+- `dbapi`, `dblord`, `dbmaster`: MariaDB database services for `api`, `nslord`, and `nsmaster`, respectively.
 - `www`: nginx instance serving static web site content and proxying to `api`
 - `celery`: A shadow instance of the `api` code for performing asynchronous tasks (email delivery).
 - `rabbitmq`: `celery`'s queue
@@ -21,9 +21,7 @@ Although most configuration is contained in this repository, some external depen
 
 1.  We run this software with the `--userland-proxy=false` flag of the `dockerd` daemon, and recommend you do the same.
 
-2.  Set up TLS-secured replication of the `nsmaster` database to feed your PowerDNS slaves. To generate the necessary keys and certificates, follow the instructions at https://dev.mysql.com/doc/refman/5.7/en/creating-ssl-files-using-openssl.html. In the `openssl req -newkey` steps, consider switching to a bigger key size, and add `-subj '/CN=slave.hostname.example'`. (It turned out that StartSSL and Let's Encrypt certificates do not work out of the box.)
-
-    Also, configure certificates for `openvpn-server`:
+2.  Also, configure certificates for `openvpn-server`:
 
     - [Get easy-rsa](https://github.com/OpenVPN/easy-rsa) and follow [this tutorial](https://github.com/OpenVPN/easy-rsa/blob/master/README.quickstart.md).
     - Then, copy `ca.crt`, `server.crt`, and `server.key` to `openvpn-server/secrets/`.
@@ -43,7 +41,6 @@ Although most configuration is contained in this repository, some external depen
       - `DESECSTACK_IPV6_ADDRESS`: IPv6 address of frontend container, ideally 0642:ac10:0080 in within the above subnet (see below)
     - certificates
       - `DESECSTACK_WWW_CERTS`: `./path/to/certificates` for `www` container. This directory is monitored for changes so that nginx can reload when new keys/certificates are provided. **Note:** The reload is done any time something changes in the directory. The relevant files are **not** watched individually.
-      - `DESECSTACK_DBMASTER_CERTS`: `./path/to/certificates` for `dbmaster` container
     - API-related
       - `DESECSTACK_API_ADMIN`: white-space separated list of Django admin email addresses
       - `DESECSTACK_API_DEBUG`: Django debug setting. Must be True (default in `docker-compose.dev.yml`) or False (default otherwise)
@@ -64,12 +61,9 @@ Although most configuration is contained in this repository, some external depen
       - `DESECSTACK_NSLORD_DEFAULT_TTL`: TTL to use by default, including for default NS records
     - nsmaster-related
       - `DESECSTACK_DBMASTER_PASSWORD_pdns`: mysql password for pdns on nsmaster
-      - `DESECSTACK_DBMASTER_PASSWORD_replication_manager`: mysql password for `replication-master` user (sets up permissions for replication slaves)
       - `DESECSTACK_NSMASTER_APIKEY`: pdns API key on nsmaster (required so that we can execute zone deletions on nsmaster, which replicates to the slaves)
       - `DESECSTACK_NSMASTER_CARBONSERVER`: pdns `carbon-server` setting on nsmaster (optional)
       - `DESECSTACK_NSMASTER_CARBONOURNAME`: pdns `carbon-ourname` setting on nsmaster (optional)
-    - replication-manager related
-      - `DESECSTACK_REPLICATION_MANAGER_CERTS`: a directory where `replication-manager` (to configure slave replication) will dump the slave's TLS key and certificate
     - monitoring-related
       - `DESECSTACK_WATCHDOG_SLAVES`: space-separated list of slave hostnames; used to check correct replication of recent DNS changes
       - `DESECSTACK_PROMETHEUS_PASSWORD`: basic auth password for user `prometheus` at `https://${DESECSTACK_DOMAIN}/prometheus/`

dbmaster/51-server.cnf

@@ -1,14 +1,2 @@
 [mysqld]
-ssl-ca     = /etc/ssl/private/db/ca.pem
-ssl-cert   = /etc/ssl/private/db/dev.desec.io-cert.pem
-ssl-key    = /etc/ssl/private/db/dev.desec.io-key.pem
-ssl-cipher = EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH
-
-server-id               = 1
-log_bin                 = /var/log/mysql/mysql-bin.log
-binlog_format=ROW
-log-basename=dbmaster
-binlog_do_db=pdns
-expire_logs_days        = 1
-
 wait_timeout = 28800

dbmaster/Dockerfile

@@ -12,9 +12,5 @@ RUN chown -R mysql:mysql /docker-entrypoint-initdb.d/
 # Additional configuration
 COPY ./51-server.cnf /etc/mysql/conf.d/51-server.cnf
 
-# mountable ssl certificate and key directory
-# (we don't want any keys in this repository)
-VOLUME /etc/ssl/private/
-
 # mountable storage
 VOLUME /var/lib/mysql

dbmaster/initdb.d/00-init.sql.var

@@ -2,16 +2,3 @@
 CREATE DATABASE pdns;
 CREATE USER 'pdns'@'${DESECSTACK_IPV4_REAR_PREFIX16}.4.%' IDENTIFIED BY '${DESECSTACK_DBMASTER_PASSWORD_pdns}';
 GRANT SELECT, INSERT, UPDATE, DELETE ON pdns.* TO 'pdns'@'${DESECSTACK_IPV4_REAR_PREFIX16}.4.%';
-
--- Replication Manager
-CREATE USER 'replication-manager'@'${DESECSTACK_IPV4_REAR_PREFIX16}.4.%' IDENTIFIED BY '${DESECSTACK_DBMASTER_PASSWORD_replication_manager}';
-
--- privileges without GRANT OPTION
-GRANT CREATE USER ON  *.* TO 'replication-manager'@'${DESECSTACK_IPV4_REAR_PREFIX16}.4.%';
--- The following mysql.* is needed so that this user can GRANT anything to the users it creates. Replacing the wildcard with all (!) specific table names does not work.
-GRANT SELECT, UPDATE ON mysql.* TO 'replication-manager'@'${DESECSTACK_IPV4_REAR_PREFIX16}.4.%';
-
--- privileges with GRANT OPTION
-GRANT RELOAD, REPLICATION CLIENT, REPLICATION SLAVE ON  *.* TO 'replication-manager'@'${DESECSTACK_IPV4_REAR_PREFIX16}.4.%' WITH GRANT OPTION;
-GRANT SELECT ON pdns.* TO 'replication-manager'@'${DESECSTACK_IPV4_REAR_PREFIX16}.4.%' WITH GRANT OPTION;
-

docker-compose.replication-manager.yml

@@ -1,18 +0,0 @@
-version: '2.2'
-
-services:
-  replication-manager:
-    build: replication-manager
-    image: desec/replication-manager:latest
-    restart: "no"
-    depends_on:
-    - dbmaster
-    volumes:
-    - ${DESECSTACK_REPLICATION_MANAGER_CERTS}:/usr/src/app/certs
-    environment:
-    - DESECSTACK_DBMASTER_PASSWORD_replication_manager
-    networks:
-    - rearmaster
-    logging:
-      driver: "json-file"
-

docker-compose.yml

@@ -93,15 +93,11 @@ services:
     image: desec/dedyn-dbmaster:latest
     init: true
     user: mysql:mysql
-    ports:
-    - "${DESECSTACK_DBMASTER_PORT:-3306}:3306"
     volumes:
-    - ${DESECSTACK_DBMASTER_CERTS}:/etc/ssl/private:ro
     - dbmaster_mysql:/var/lib/mysql
     environment:
     - DESECSTACK_IPV4_REAR_PREFIX16
     - DESECSTACK_DBMASTER_PASSWORD_pdns
-    - DESECSTACK_DBMASTER_PASSWORD_replication_manager
     networks:
     - rearmaster
     logging:

replication-manager-build.sh

@@ -1,3 +0,0 @@
-#!/usr/bin/env bash
-
-docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.replication-manager.yml build replication-manager

replication-manager.sh

@@ -1,3 +0,0 @@
-#!/usr/bin/env bash
-
-docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.replication-manager.yml run replication-manager "$@"

replication-manager/Dockerfile

@@ -1,15 +0,0 @@
-FROM python:3-alpine
-
-WORKDIR /usr/src/app
-COPY requirements.txt ./
-
-RUN apk add --no-cache build-base && \
-    apk add --no-cache gcc musl-dev python3-dev libffi-dev openssl-dev && \
-    apk add --no-cache mariadb-connector-c-dev && \
-    pip install --no-cache-dir -r requirements.txt && \
-    apk del gcc musl-dev python3-dev libffi-dev openssl-dev && \
-    apk del build-base
-
-COPY . .
-
-ENTRYPOINT [ "python", "./replication-manager.py" ]

replication-manager/pki_utils.py

@@ -1,103 +0,0 @@
-import datetime
-import uuid
-from collections import OrderedDict
-from cryptography import x509
-from cryptography.hazmat.backends import default_backend
-from cryptography.hazmat.primitives import asymmetric, hashes, serialization
-from cryptography.x509.oid import NameOID
-
-
-class PKI:
-    key = None
-    crt = None
-
-    def __init__(self, ca_crt_pem, ca_key_pem, ca_key_password=None, key=None):
-        self.ca_crt = x509.load_pem_x509_certificate(
-            ca_crt_pem,
-            default_backend(),
-        )
-
-        self.ca_pkey = serialization.load_pem_private_key(
-            ca_key_pem,
-            password=ca_key_password,
-            backend=default_backend(),
-        )
-
-    def initialize_key(self, key=None):
-        self.key = key or self._generate_key()
-
-    @staticmethod
-    def _generate_key():
-        return asymmetric.rsa.generate_private_key(
-            public_exponent=65537,
-            key_size=4096,
-            backend=default_backend(),
-        )
-
-    @property
-    def key_pem(self):
-        assert self.key is not None, 'You must call `initialize_key()` before accessing the key.'
-
-        return self.key.private_bytes(
-            encoding=serialization.Encoding.PEM,
-            format=serialization.PrivateFormat.TraditionalOpenSSL,
-            encryption_algorithm=serialization.NoEncryption(),
-        )
-
-    @property
-    def crt_pem(self):
-        assert self.crt is not None, 'You must call `create_certificate()` before accessing the certificate.'
-
-        return self.crt.public_bytes(encoding=serialization.Encoding.PEM)
-
-    @property
-    def subject_attributes(self):
-        assert self.crt is not None, 'You must call `create_certificate()` before accessing the certificate.'
-
-        oids = OrderedDict()
-        oids[NameOID.COMMON_NAME] = 'CN'
-        oids[NameOID.X500_UNIQUE_IDENTIFIER] = 'x500UniqueIdentifier'
-
-        attrs = OrderedDict()
-        for oid, label in oids.items():
-            assert label not in attrs
-            attrs[label] = [attribute.value for attribute in self.crt.subject.get_attributes_for_oid(oid)]
-
-        return attrs
-
-    def _generate_csr(self, common_name):
-        assert self.key is not None, 'You must call `initialize_key()` before requesting a certificate.'
-
-        # Copy attributes from CA certificate, except for CN
-        attributes = [
-            x509.NameAttribute(NameOID.COMMON_NAME, common_name),
-            x509.NameAttribute(NameOID.X500_UNIQUE_IDENTIFIER, str(uuid.uuid4())),
-        ]
-
-        # Initialize and set attributes
-        csr = x509.CertificateSigningRequestBuilder().subject_name(x509.Name(attributes))
-
-        # Sign and return
-        return csr.sign(self.key, hashes.SHA256(), default_backend())
-
-    def create_certificate(self, common_name, days):
-        csr = self._generate_csr(common_name)
-
-        self.crt = x509.CertificateBuilder().subject_name(
-            csr.subject
-        ).issuer_name(
-            self.ca_crt.subject
-        ).public_key(
-            csr.public_key()
-        ).serial_number(
-            x509.random_serial_number()
-        ).not_valid_before(
-            datetime.datetime.utcnow()
-        ).not_valid_after(
-            datetime.datetime.utcnow() + datetime.timedelta(days=days)
-        ).sign(
-            private_key=self.ca_pkey,
-            algorithm=hashes.SHA256(),
-            backend=default_backend()
-        )
-

replication-manager/replication-manager.py

@@ -1,120 +0,0 @@
-import argparse
-import MySQLdb
-import os
-import re
-import secrets
-import string
-import sys
-from collections import OrderedDict
-
-from pki_utils import PKI
-
-host = 'dbmaster'
-user = 'replication-manager'
-
-
-def validate_name(value):
-    pattern = re.compile("^([a-z0-9._-]+[a-z0-9]+)$")
-    if not pattern.match(value):
-        raise ValueError(f'Name does not match pattern {pattern}')
-
-    return value
-
-def add_slave(cursor, **kwargs):
-    name = validate_name(kwargs['name'])
-
-    alphabet = string.ascii_letters + string.digits
-    passwd = ''.join(secrets.choice(alphabet) for i in range(32))
-
-    # Create key and certificate
-    with open('certs/ca.pem', 'r') as ca_crt_file, open('certs/ca-key.pem', 'r') as ca_key_file:
-        ca_crt_pem = ca_crt_file.read().encode('ascii')
-        ca_key_pem = ca_key_file.read().encode('ascii')
-
-    pki = PKI(ca_crt_pem=ca_crt_pem, ca_key_pem=ca_key_pem)
-    pki.initialize_key()
-    pki.create_certificate(common_name=name, days=365*10)
-
-    subject = ''
-    for label, attributes in pki.subject_attributes.items():
-        assert len(attributes) == 1
-        value = attributes[0]
-        subject += f'/{label}={value}'
-
-    # Configure slave in database
-    username = name.replace('.', '_')  # allows using username as binlog on the slave
-    print(f'Creating slave user {username} with subject {subject} ...')
-    cursor.execute("CREATE USER %s@'%%' IDENTIFIED BY %s", (username, passwd,))
-    cursor.execute("GRANT RELOAD, REPLICATION CLIENT, REPLICATION SLAVE ON *.* TO %s@'%%' REQUIRE SUBJECT %s", (username, subject,))
-    cursor.execute("GRANT SELECT ON pdns.* TO %s@'%%' REQUIRE SUBJECT %s", (username, subject,))
-    cursor.execute("FLUSH PRIVILEGES")
-    print(f'Password is {passwd}')
-
-    # Write key and certificate
-    umask = os.umask(0o077)
-    key_filename = f'certs/{name}-key.pem'
-    crt_filename = f'certs/{name}-crt.pem'
-    with open(key_filename, 'wb') as key_file, open(crt_filename, 'wb') as crt_file:
-        key_file.write(pki.key_pem)
-        crt_file.write(pki.crt_pem)
-    os.umask(umask)
-
-    print(key_filename, '(key)')
-    print(crt_filename, '(certificate)')
-
-
-def list_slaves(cursor):
-    cursor.execute("SELECT User, x509_subject FROM mysql.user WHERE x509_subject != ''")
-    for row in cursor.fetchall():
-        print(row[0], row[1].decode('utf-8'))
-
-
-def remove_slave(cursor, **kwargs):
-    slavename = validate_name(kwargs['name'])
-
-    cursor.execute("DROP USER %s@'%%'", (slavename,))
-    cursor.execute("FLUSH PRIVILEGES")
-
-
-def main():
-    parser = argparse.ArgumentParser(description='List, add, and remove pdns database replication slaves.')
-    subparsers = parser.add_subparsers(dest='action', required=True)
-
-    actions = {}
-
-    # add
-    description = 'Add a slave and generate TLS key/certificate. The slave replication password is read from stdin (first line).'
-    subparser = subparsers.add_parser('add', help='Add a slave and generate TLS key/certificate', description=description)
-    subparser.add_argument('--name', type=str, help='Slave identifier (usually hostname)', required=True)
-    actions['add'] = add_slave
-
-    # list
-    subparser = subparsers.add_parser('list', help='List slaves', description='List slaves.')
-    actions['list'] = list_slaves
-
-    # remove
-    subparser = subparsers.add_parser('remove', help='Remove a slave', description='Remove a slave.')
-    subparser.add_argument('--name', type=str, help='Slave identifier (usually hostname)', required=True)
-    actions['remove'] = remove_slave
-
-    # Validate and extract arguments (errors out if insufficient arguments are given)
-    args = parser.parse_args()
-    kwargs = vars(args).copy()
-
-    # Initialize database
-    db = MySQLdb.connect(host=host, user=user, passwd=os.environ['DESECSTACK_DBMASTER_PASSWORD_replication_manager'])
-
-    # Action!
-    action = kwargs.pop('action')
-    action_func = actions[action]
-    try:
-        action_func(db.cursor(), **kwargs)
-    except Exception as e:
-        raise e
-    finally:
-        db.close()
-
-
-if __name__ == "__main__":
-    main()
-

replication-manager/requirements.txt

@@ -1,2 +0,0 @@
-cryptography~=2.7
-mysqlclient~=1.4.2.post1