feat(): add replication-manager container, fixes #214

.env.default

@@ -34,6 +34,10 @@ DESECSTACK_NSLORD_DEFAULT_TTL=3600
 
 # nsmaster-related
 DESECSTACK_DBMASTER_PASSWORD_pdns=
+DESECSTACK_DBMASTER_PASSWORD_replication_manager=
 DESECSTACK_NSMASTER_APIKEY=
 DESECSTACK_NSMASTER_CARBONSERVER=
 DESECSTACK_NSMASTER_CARBONOURNAME=
+
+# replication-manager
+DESECSTACK_REPLICATION_MANAGER_CERTS=

.env.dev

@@ -34,6 +34,10 @@ DESECSTACK_NSLORD_DEFAULT_TTL=3600
 
 # nsmaster-related
 DESECSTACK_DBMASTER_PASSWORD_pdns=insecure
+DESECSTACK_DBMASTER_PASSWORD_replication_manager=insecure
 DESECSTACK_NSMASTER_APIKEY=insecure
 DESECSTACK_NSMASTER_CARBONSERVER=
 DESECSTACK_NSMASTER_CARBONOURNAME=
+
+# replication-manager
+DESECSTACK_REPLICATION_MANAGER_CERTS=./replication-certs

.gitignore

@@ -11,3 +11,4 @@ api/desecapi.sqlite
 
 # local certificates
 /certs/
+/replication-certs/

README.md

@@ -51,9 +51,12 @@ Although most configuration is contained in this repository, some external depen
       - `DESECSTACK_NSLORD_DEFAULT_TTL`: TTL to use by default, including for default NS records
     - nsmaster-related
       - `DESECSTACK_DBMASTER_PASSWORD_pdns`: mysql password for pdns on nsmaster
+      - `DESECSTACK_DBMASTER_PASSWORD_replication_manager`: mysql password for `replication-master` user (sets up permissions for replication slaves)
       - `DESECSTACK_NSMASTER_APIKEY`: pdns API key on nsmaster (required so that we can execute zone deletions on nsmaster, which replicates to the slaves)
       - `DESECSTACK_NSMASTER_CARBONSERVER`: pdns `carbon-server` setting on nsmaster (optional)
       - `DESECSTACK_NSMASTER_CARBONOURNAME`: pdns `carbon-ourname` setting on nsmaster (optional)
+    - replication-manager related
+      - `DESECSTACK_REPLICATION_MANAGER_CERTS`: a directory where `replication-manager` (to configure slave replication) will dump the slave's TLS key and certificate
 
 Running the standard stack will also fire up an instance of the `www` proxy service (see `desec-www` repository), assuming that the `desec-static` project is located under the `static` directory/symlink.
 

dbmaster/initdb.d/00-init.sql.var

@@ -2,3 +2,16 @@
 CREATE DATABASE pdns;
 CREATE USER 'pdns'@'${DESECSTACK_IPV4_REAR_PREFIX16}.4.%' IDENTIFIED BY '${DESECSTACK_DBMASTER_PASSWORD_pdns}';
 GRANT SELECT, INSERT, UPDATE, DELETE ON pdns.* TO 'pdns'@'${DESECSTACK_IPV4_REAR_PREFIX16}.4.%';
+
+-- Replication Manager
+CREATE USER 'replication-manager'@'${DESECSTACK_IPV4_REAR_PREFIX16}.4.%' IDENTIFIED BY '${DESECSTACK_DBMASTER_PASSWORD_replication_manager}';
+
+-- privileges without GRANT OPTION
+GRANT CREATE USER ON  *.* TO 'replication-manager'@'${DESECSTACK_IPV4_REAR_PREFIX16}.4.%';
+-- The following mysql.* is needed so that this user can GRANT anything to the users it creates. Replacing the wildcard with all (!) specific table names does not work.
+GRANT SELECT, UPDATE ON mysql.* TO 'replication-manager'@'${DESECSTACK_IPV4_REAR_PREFIX16}.4.%';
+
+-- privileges with GRANT OPTION
+GRANT RELOAD, REPLICATION CLIENT, REPLICATION SLAVE ON  *.* TO 'replication-manager'@'${DESECSTACK_IPV4_REAR_PREFIX16}.4.%' WITH GRANT OPTION;
+GRANT SELECT ON pdns.* TO 'replication-manager'@'${DESECSTACK_IPV4_REAR_PREFIX16}.4.%' WITH GRANT OPTION;
+

docker-compose.replication-manager.yml

@@ -0,0 +1,18 @@
+version: '2.2'
+
+services:
+  replication-manager:
+    build: replication-manager
+    image: desec/replication-manager:latest
+    restart: "no"
+    depends_on:
+    - dbmaster
+    volumes:
+    - ${DESECSTACK_REPLICATION_MANAGER_CERTS}:/usr/src/app/certs
+    environment:
+    - DESECSTACK_DBMASTER_PASSWORD_replication_manager
+    networks:
+    - rearmaster
+    logging:
+      driver: "json-file"
+

docker-compose.yml

@@ -87,6 +87,7 @@ services:
     environment:
     - DESECSTACK_IPV4_REAR_PREFIX16
     - DESECSTACK_DBMASTER_PASSWORD_pdns
+    - DESECSTACK_DBMASTER_PASSWORD_replication_manager
     networks:
     - rearmaster
     logging:

replication-manager-build.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.replication-manager.yml build replication-manager

replication-manager.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.replication-manager.yml run replication-manager "$@"

replication-manager/Dockerfile

@@ -0,0 +1,15 @@
+FROM python:3-alpine
+
+WORKDIR /usr/src/app
+COPY requirements.txt ./
+
+RUN apk add --no-cache build-base && \
+    apk add --no-cache gcc musl-dev python3-dev libffi-dev openssl-dev && \
+    apk add --no-cache mariadb-connector-c-dev && \
+    pip install --no-cache-dir -r requirements.txt && \
+    apk del gcc musl-dev python3-dev libffi-dev openssl-dev && \
+    apk del build-base
+
+COPY . .
+
+ENTRYPOINT [ "python", "./replication-manager.py" ]

replication-manager/pki_utils.py

@@ -0,0 +1,103 @@
+import datetime
+import uuid
+from collections import OrderedDict
+from cryptography import x509
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives import asymmetric, hashes, serialization
+from cryptography.x509.oid import NameOID
+
+
+class PKI:
+    key = None
+    crt = None
+
+    def __init__(self, ca_crt_pem, ca_key_pem, ca_key_password=None, key=None):
+        self.ca_crt = x509.load_pem_x509_certificate(
+            ca_crt_pem,
+            default_backend(),
+        )
+
+        self.ca_pkey = serialization.load_pem_private_key(
+            ca_key_pem,
+            password=ca_key_password,
+            backend=default_backend(),
+        )
+
+    def initialize_key(self, key=None):
+        self.key = key or self._generate_key()
+
+    @staticmethod
+    def _generate_key():
+        return asymmetric.rsa.generate_private_key(
+            public_exponent=65537,
+            key_size=4096,
+            backend=default_backend(),
+        )
+
+    @property
+    def key_pem(self):
+        assert self.key is not None, 'You must call `initialize_key()` before accessing the key.'
+
+        return self.key.private_bytes(
+            encoding=serialization.Encoding.PEM,
+            format=serialization.PrivateFormat.TraditionalOpenSSL,
+            encryption_algorithm=serialization.NoEncryption(),
+        )
+
+    @property
+    def crt_pem(self):
+        assert self.crt is not None, 'You must call `create_certificate()` before accessing the certificate.'
+
+        return self.crt.public_bytes(encoding=serialization.Encoding.PEM)
+
+    @property
+    def subject_attributes(self):
+        assert self.crt is not None, 'You must call `create_certificate()` before accessing the certificate.'
+
+        oids = OrderedDict()
+        oids[NameOID.COMMON_NAME] = 'CN'
+        oids[NameOID.X500_UNIQUE_IDENTIFIER] = 'x500UniqueIdentifier'
+
+        attrs = OrderedDict()
+        for oid, label in oids.items():
+            assert label not in attrs
+            attrs[label] = [attribute.value for attribute in self.crt.subject.get_attributes_for_oid(oid)]
+
+        return attrs
+
+    def _generate_csr(self, common_name):
+        assert self.key is not None, 'You must call `initialize_key()` before requesting a certificate.'
+
+        # Copy attributes from CA certificate, except for CN
+        attributes = [
+            x509.NameAttribute(NameOID.COMMON_NAME, common_name),
+            x509.NameAttribute(NameOID.X500_UNIQUE_IDENTIFIER, str(uuid.uuid4())),
+        ]
+
+        # Initialize and set attributes
+        csr = x509.CertificateSigningRequestBuilder().subject_name(x509.Name(attributes))
+
+        # Sign and return
+        return csr.sign(self.key, hashes.SHA256(), default_backend())
+
+    def create_certificate(self, common_name, days):
+        csr = self._generate_csr(common_name)
+
+        self.crt = x509.CertificateBuilder().subject_name(
+            csr.subject
+        ).issuer_name(
+            self.ca_crt.subject
+        ).public_key(
+            csr.public_key()
+        ).serial_number(
+            x509.random_serial_number()
+        ).not_valid_before(
+            datetime.datetime.utcnow()
+        ).not_valid_after(
+            datetime.datetime.utcnow() + datetime.timedelta(days=days)
+        ).sign(
+            private_key=self.ca_pkey,
+            algorithm=hashes.SHA256(),
+            backend=default_backend()
+        )
+

replication-manager/replication-manager.py

@@ -0,0 +1,120 @@
+import argparse
+import MySQLdb
+import os
+import re
+import secrets
+import string
+import sys
+from collections import OrderedDict
+
+from pki_utils import PKI
+
+host = 'dbmaster'
+user = 'replication-manager'
+
+
+def validate_name(value):
+    pattern = re.compile("^([a-z0-9.-]+\.[a-z0-9]+)$")
+    if not pattern.match(value):
+        raise ValueError(f'Name does not match pattern {pattern}')
+
+    return value
+
+def add_slave(cursor, **kwargs):
+    name = validate_name(kwargs['name'])
+
+    alphabet = string.ascii_letters + string.digits
+    passwd = ''.join(secrets.choice(alphabet) for i in range(32))
+
+    # Create key and certificate
+    with open('certs/ca.pem', 'r') as ca_crt_file, open('certs/ca-key.pem', 'r') as ca_key_file:
+        ca_crt_pem = ca_crt_file.read().encode('ascii')
+        ca_key_pem = ca_key_file.read().encode('ascii')
+
+    pki = PKI(ca_crt_pem=ca_crt_pem, ca_key_pem=ca_key_pem)
+    pki.initialize_key()
+    pki.create_certificate(common_name=name, days=365*10)
+
+    subject = ''
+    for label, attributes in pki.subject_attributes.items():
+        assert len(attributes) == 1
+        value = attributes[0]
+        subject += f'/{label}={value}'
+
+    # Configure slave in database
+    username = name.replace('.', '_')  # allows using username as binlog on the slave
+    print(f'Creating slave user {username} with subject {subject} ...')
+    cursor.execute("CREATE USER %s@'%%' IDENTIFIED BY %s", (username, passwd,))
+    cursor.execute("GRANT RELOAD, REPLICATION CLIENT, REPLICATION SLAVE ON *.* TO %s@'%%' REQUIRE SUBJECT %s", (username, subject,))
+    cursor.execute("GRANT SELECT ON pdns.* TO %s@'%%' REQUIRE SUBJECT %s", (username, subject,))
+    cursor.execute("FLUSH PRIVILEGES")
+    print(f'Password is {passwd}')
+
+    # Write key and certificate
+    umask = os.umask(0o077)
+    key_filename = f'certs/{name}-key.pem'
+    crt_filename = f'certs/{name}-crt.pem'
+    with open(key_filename, 'wb') as key_file, open(crt_filename, 'wb') as crt_file:
+        key_file.write(pki.key_pem)
+        crt_file.write(pki.crt_pem)
+    os.umask(umask)
+
+    print(key_filename, '(key)')
+    print(crt_filename, '(certificate)')
+
+
+def list_slaves(cursor):
+    cursor.execute("SELECT User, x509_subject FROM mysql.user WHERE x509_subject != ''")
+    for row in cursor.fetchall():
+        print(row[0], row[1].decode('utf-8'))
+
+
+def remove_slave(cursor, **kwargs):
+    slavename = validate_name(kwargs['name'])
+
+    cursor.execute("DROP USER %s@'%%'", (slavename,))
+    cursor.execute("FLUSH PRIVILEGES")
+
+
+def main():
+    parser = argparse.ArgumentParser(description='List, add, and remove pdns database replication slaves.')
+    subparsers = parser.add_subparsers(dest='action', required=True)
+
+    actions = {}
+
+    # add
+    description = 'Add a slave and generate TLS key/certificate. The slave replication password is read from stdin (first line).'
+    subparser = subparsers.add_parser('add', help='Add a slave and generate TLS key/certificate', description=description)
+    subparser.add_argument('--name', type=str, help='Slave identifier (usually hostname)', required=True)
+    actions['add'] = add_slave
+
+    # list
+    subparser = subparsers.add_parser('list', help='List slaves', description='List slaves.')
+    actions['list'] = list_slaves
+
+    # remove
+    subparser = subparsers.add_parser('remove', help='Remove a slave', description='Remove a slave.')
+    subparser.add_argument('--name', type=str, help='Slave identifier (usually hostname)', required=True)
+    actions['remove'] = remove_slave
+
+    # Validate and extract arguments (errors out if insufficient arguments are given)
+    args = parser.parse_args()
+    kwargs = vars(args).copy()
+
+    # Initialize database
+    db = MySQLdb.connect(host=host, user=user, passwd=os.environ['DESECSTACK_DBMASTER_PASSWORD_replication_manager'])
+
+    # Action!
+    action = kwargs.pop('action')
+    action_func = actions[action]
+    try:
+        action_func(db.cursor(), **kwargs)
+    except Exception as e:
+        raise e
+    finally:
+        db.close()
+
+
+if __name__ == "__main__":
+    main()
+

replication-manager/requirements.txt

@@ -0,0 +1,2 @@
+cryptography~=2.7
+mysqlclient~=1.4.2.post1