feat(ns): improve provisioning on nsmaster, fixes #190

Previously, zones were provisioned on nsmaster using the supermaster
feature of pdns, followed by a NOTIFY from nslord, triggering an AXFR
by nsmaster.

This mechanism relied on various configuration in database and pdns
containers, and further was subject to a race condition: If a user
created a new zone and immediately deleted the supermaster hostname
from the NS RRset, the AXFR would fail and the domain would not be
provisioned on nsmaster.

This commit removes the dependency on specific supermaster hostnames,
reduces configuration complexity and makes sure that new domains are
created on and AXFR'ed by nsmaster regardless of circumstances.

api/desecapi/pdns.py

@@ -1,20 +1,33 @@
 import json
 import random
+import socket
 
 import requests
 
-from api import settings
+from api import settings as api_settings
 from desecapi.exceptions import PdnsException
 
-headers_nslord = {
-    'Accept': 'application/json',
-    'User-Agent': 'desecapi',
-    'X-API-Key': settings.NSLORD_PDNS_API_TOKEN,
-}
+NSLORD = object()
+NSMASTER = object()
+
+settings = {
+    NSLORD: {
+        'base_url': api_settings.NSLORD_PDNS_API,
+        'headers': {
+            'Accept': 'application/json',
+            'User-Agent': 'desecapi',
+            'X-API-Key': api_settings.NSLORD_PDNS_API_TOKEN,
+        }
+    },
+    NSMASTER: {
+        'base_url': api_settings.NSMASTER_PDNS_API,
+        'headers': {
+            'Accept': 'application/json',
+            'User-Agent': 'desecapi',
+            'X-API-Key': api_settings.NSMASTER_PDNS_API_TOKEN,
+        }
+    }
 
-headers_nsmaster = {
-    'User-Agent': 'desecapi',
-    'X-API-Key': settings.NSMASTER_PDNS_API_TOKEN,
 }
 
 
@@ -24,7 +37,7 @@ def _pdns_delete_zone(domain):
     # We first delete the zone from nslord, the main authoritative source of our DNS data.
     # However, we do not want to wait for the zone to expire on the slave ("nsmaster").
     # We thus issue a second delete request on nsmaster to delete the zone there immediately.
-    r1 = requests.delete(settings.NSLORD_PDNS_API + path, headers=headers_nslord)
+    r1 = requests.delete(settings[NSLORD]['base_url'] + path, headers=settings[NSLORD]['headers'])
     if r1.status_code < 200 or r1.status_code >= 300:
         # Deletion technically does not fail if the zone didn't exist in the first place
         if r1.status_code == 422 and 'Could not find domain' in r1.text:
@@ -33,7 +46,7 @@ def _pdns_delete_zone(domain):
             raise PdnsException(r1)
 
     # Delete from nsmaster as well
-    r2 = requests.delete(settings.NSMASTER_PDNS_API + path, headers=headers_nsmaster)
+    r2 = requests.delete(settings[NSMASTER]['base_url'] + path, headers=settings[NSMASTER]['headers'])
     if r2.status_code < 200 or r2.status_code >= 300:
         # Deletion technically does not fail if the zone didn't exist in the first place
         if r2.status_code == 422 and 'Could not find domain' in r2.text:
@@ -44,32 +57,32 @@ def _pdns_delete_zone(domain):
     return r1, r2
 
 
-def _pdns_request(method, path, body=None, acceptable_range=range(200, 300)):
+def _pdns_request(method, *, server, path, body=None, acceptable_range=range(200, 300)):
     data = json.dumps(body) if body else None
-    if data is not None and len(data) > settings.PDNS_MAX_BODY_SIZE:
+    if data is not None and len(data) > api_settings.PDNS_MAX_BODY_SIZE:
         raise PdnsException(detail='Payload too large', status=413)
 
-    r = requests.request(method, settings.NSLORD_PDNS_API + path, data=data, headers=headers_nslord)
+    r = requests.request(method, settings[server]['base_url'] + path, data=data, headers=settings[server]['headers'])
     if r.status_code not in acceptable_range:
         raise PdnsException(r)
 
     return r
 
 
-def _pdns_post(path, body):
-    return _pdns_request('post', path, body)
+def _pdns_post(server, path, body):
+    return _pdns_request('post', server=server, path=path, body=body)
 
 
-def _pdns_patch(path, body):
-    return _pdns_request('patch', path, body)
+def _pdns_patch(server, path, body):
+    return _pdns_request('patch', server=server, path=path, body=body)
 
 
-def _pdns_get(path):
-    return _pdns_request('get', path, acceptable_range=range(200, 400))
+def _pdns_get(server, path):
+    return _pdns_request('get', server=server, path=path, acceptable_range=range(200, 400))
 
 
-def _pdns_put(path):
-    return _pdns_request('put', path, acceptable_range=range(200, 500))
+def _pdns_put(server, path):
+    return _pdns_request('put', server=server, path=path, acceptable_range=range(200, 500))
 
 
 def create_zone(domain, nameservers):
@@ -83,9 +96,12 @@ def create_zone(domain, nameservers):
     salt = '%016x' % random.randrange(16**16)
     payload = {'name': name, 'kind': 'MASTER', 'dnssec': True,
                'nsec3param': '1 0 127 %s' % salt, 'nameservers': nameservers}
-    _pdns_post('/zones', payload)
+    _pdns_post(NSLORD, '/zones', payload)
+
+    payload = {'name': name, 'kind': 'SLAVE', 'masters': [socket.gethostbyname('nslord')]}
+    _pdns_post(NSMASTER, '/zones', payload)
 
-    notify_zone(domain)
+    axfr_zone(domain)
 
 
 def delete_zone(domain):
@@ -99,7 +115,7 @@ def get_keys(domain):
     """
     Retrieves a dict representation of the DNSSEC key information
     """
-    r = _pdns_get('/zones/%s/cryptokeys' % domain.pdns_id)
+    r = _pdns_get(NSLORD, '/zones/%s/cryptokeys' % domain.pdns_id)
     return [{k: key[k] for k in ('dnskey', 'ds', 'flags', 'keytype')}
             for key in r.json()
             if key['active'] and key['keytype'] in ['csk', 'ksk']]
@@ -109,7 +125,7 @@ def get_zone(domain):
     """
     Retrieves a dict representation of the zone from pdns
     """
-    r = _pdns_get('/zones/' + domain.pdns_id)
+    r = _pdns_get(NSLORD, '/zones/' + domain.pdns_id)
 
     return r.json()
 
@@ -126,7 +142,7 @@ def get_rrset_datas(domain):
             for rrset in get_zone(domain)['rrsets']]
 
 
-def set_rrsets(domain, rrsets, notify=True):
+def set_rrsets(domain, rrsets, axfr=True):
     data = {
         'rrsets':
         [
@@ -141,14 +157,14 @@ def set_rrsets(domain, rrsets, notify=True):
             for rrset in rrsets
         ]
     }
-    _pdns_patch('/zones/' + domain.pdns_id, data)
+    _pdns_patch(NSLORD, '/zones/' + domain.pdns_id, data)
 
-    if notify:
-        notify_zone(domain)
+    if axfr:
+        axfr_zone(domain)
 
 
-def notify_zone(domain):
+def axfr_zone(domain):
     """
-    Commands pdns to notify the zone to the pdns slaves.
+    Commands nsmaster to retrieve the zone from nslord.
     """
-    _pdns_put('/zones/%s/notify' % domain.pdns_id)
+    _pdns_put(NSMASTER, '/zones/%s/axfr-retrieve' % domain.pdns_id)

api/desecapi/tests/base.py

@@ -213,7 +213,7 @@ class MockPDNSTestCase(APITestCase):
     PDNS_ZONES = r'/zones'
     PDNS_ZONE_CRYPTO_KEYS = r'/zones/(?P<id>[^/]+)/cryptokeys'
     PDNS_ZONE = r'/zones/(?P<id>[^/]+)'
-    PDNS_ZONE_NOTIFY = r'/zones/(?P<id>[^/]+)/notify'
+    PDNS_ZONE_AXFR = r'/zones/(?P<id>[^/]+)/axfr-retrieve'
 
     @classmethod
     def get_full_pdns_url(cls, path_regex, ns='LORD', **kwargs):
@@ -260,17 +260,17 @@ class MockPDNSTestCase(APITestCase):
             return [x.rstrip('.') + '.' for x in arg]
 
     @classmethod
-    def request_pdns_zone_create(cls):
+    def request_pdns_zone_create(cls, ns):
         return {
             'method': 'POST',
-            'uri': cls.get_full_pdns_url(cls.PDNS_ZONES),
+            'uri': cls.get_full_pdns_url(cls.PDNS_ZONES, ns=ns),
             'status': 201,
             'body': None,
         }
 
     @classmethod
     def request_pdns_zone_create_422(cls):
-        request = cls.request_pdns_zone_create()
+        request = cls.request_pdns_zone_create(ns='LORD')
         request['status'] = 422
         return request
 
@@ -371,10 +371,10 @@ class MockPDNSTestCase(APITestCase):
         }
 
     @classmethod
-    def request_pdns_zone_notify(cls, name=None):
+    def request_pdns_zone_axfr(cls, name=None):
         return {
             'method': 'PUT',
-            'uri': cls.get_full_pdns_url(cls.PDNS_ZONE_NOTIFY, id=cls._pdns_zone_id_heuristic(name)),
+            'uri': cls.get_full_pdns_url(cls.PDNS_ZONE_AXFR, ns='MASTER', id=cls._pdns_zone_id_heuristic(name)),
             'status': 200,
             'body': None,
         }
@@ -414,7 +414,8 @@ class MockPDNSTestCase(APITestCase):
         return AssertRequestsContextManager(
             test_case=self,
             expected_requests=[
-                self.request_pdns_zone_create()
+                self.request_pdns_zone_create(ns='LORD'),
+                self.request_pdns_zone_create(ns='MASTER')
             ],
         )
 
@@ -454,8 +455,9 @@ class MockPDNSTestCase(APITestCase):
         httpretty.reset()
         hr_core.POTENTIAL_HTTP_PORTS.add(8081)  # FIXME static dependency on settings variable
         for request in [
-            cls.request_pdns_zone_create(),
-            cls.request_pdns_zone_notify(),
+            cls.request_pdns_zone_create(ns='LORD'),
+            cls.request_pdns_zone_create(ns='MASTER'),
+            cls.request_pdns_zone_axfr(),
             cls.request_pdns_zone_update(),
             cls.request_pdns_zone_retrieve_crypto_keys(),
             cls.request_pdns_zone_retrieve()
@@ -620,8 +622,9 @@ class DesecTestCase(MockPDNSTestCase):
     @classmethod
     def requests_desec_domain_creation(cls, name=None):
         return [
-            cls.request_pdns_zone_create(),
-            cls.request_pdns_zone_notify(name=name),
+            cls.request_pdns_zone_create(ns='LORD'),
+            cls.request_pdns_zone_create(ns='MASTER'),
+            cls.request_pdns_zone_axfr(name=name),
             cls.request_pdns_zone_retrieve(name=name),
             cls.request_pdns_zone_retrieve_crypto_keys(name=name),
         ]
@@ -638,7 +641,7 @@ class DesecTestCase(MockPDNSTestCase):
         delegate_at = cls._find_auto_delegation_zone(name)
         return cls.requests_desec_domain_creation(name=name) + [
             cls.request_pdns_zone_update(name=delegate_at),
-            cls.request_pdns_zone_notify(name=delegate_at),
+            cls.request_pdns_zone_axfr(name=delegate_at),
             cls.request_pdns_zone_retrieve_crypto_keys(name=name),
         ]
 
@@ -647,7 +650,7 @@ class DesecTestCase(MockPDNSTestCase):
         delegate_at = cls._find_auto_delegation_zone(name)
         return [
             cls.request_pdns_zone_update(name=delegate_at),
-            cls.request_pdns_zone_notify(name=delegate_at),
+            cls.request_pdns_zone_axfr(name=delegate_at),
             cls.request_pdns_zone_delete(name=name, ns='LORD'),
             cls.request_pdns_zone_delete(name=name, ns='MASTER'),
         ]
@@ -656,7 +659,7 @@ class DesecTestCase(MockPDNSTestCase):
     def requests_desec_rr_sets_update(cls, name=None):
         return [
             cls.request_pdns_zone_update(name=name),
-            cls.request_pdns_zone_notify(name=name),
+            cls.request_pdns_zone_axfr(name=name),
         ]
 
 
@@ -717,8 +720,8 @@ class DynDomainOwnerTestCase(DomainOwnerTestCase):
     DYN = True
 
     @classmethod
-    def request_pdns_zone_notify(cls, name=None):
-        return super().request_pdns_zone_notify(name.lower() if name else None)
+    def request_pdns_zone_axfr(cls, name=None):
+        return super().request_pdns_zone_axfr(name.lower() if name else None)
 
     @classmethod
     def request_pdns_zone_update(cls, name=None):
@@ -734,7 +737,7 @@ class DynDomainOwnerTestCase(DomainOwnerTestCase):
     def assertDynDNS12Update(self, domain_name=None, mock_remote_addr='', **kwargs):
         pdns_name = self._normalize_name(domain_name).lower() if domain_name else None
         return self._assertDynDNS12Update(
-            [self.request_pdns_zone_update(name=pdns_name), self.request_pdns_zone_notify(name=pdns_name)],
+            [self.request_pdns_zone_update(name=pdns_name), self.request_pdns_zone_axfr(name=pdns_name)],
             mock_remote_addr,
             **kwargs
         )

api/desecapi/tests/test_dyndns12update.py

@@ -43,7 +43,7 @@ class DynDNS12UpdateTest(DynDomainOwnerTestCase):
         """
         with self.assertPdnsRequests(
             self.request_pdns_zone_update(self.my_domain.name),
-            self.request_pdns_zone_notify(self.my_domain.name),
+            self.request_pdns_zone_axfr(self.my_domain.name),
         ):
             response = self.client_token_authorized.patch_rr_set(
                 self.my_domain.name.lower(), subname='', type_='A', ttl=3600)
@@ -58,7 +58,7 @@ class DynDNS12UpdateTest(DynDomainOwnerTestCase):
         # /nic/dyndns?action=edit&started=1&hostname=YES&host_id=foobar.dedyn.io&myip=10.1.2.3
         with self.assertPdnsRequests(
                 self.request_pdns_zone_update(self.my_domain.name),
-                self.request_pdns_zone_notify(self.my_domain.name),
+                self.request_pdns_zone_axfr(self.my_domain.name),
         ):
             response = self.client.get(
                 self.reverse('v1:dyndns12update'),

dbmaster/initdb.d/11-pdns-master-supermasters.sql

@@ -1,2 +0,0 @@
--- This file is required to exist and will be overriden by 00-init.sh.
--- If it is created only by 00-init.sh, the entrypoint script will miss it.

dbmaster/initdb.d/11-pdns-master-supermasters.sql.var

@@ -1,3 +0,0 @@
-USE pdns;
-
-INSERT INTO supermasters SET ip="${DESECSTACK_IPV4_REAR_PREFIX16}.1.11", nameserver="ns1.desec.io", account="";

docker-compose.test-e2e.yml

@@ -17,6 +17,10 @@ services:
     logging:
       driver: "json-file"
 
+  nsmaster:
+    logging:
+      driver: "json-file"
+
   static:
     build: test/e2e/mock-static # build a mock static to save time executing tests
     logging:
@@ -34,6 +38,7 @@ services:
     depends_on:
     - www
     - nslord
+    - nsmaster
     networks:
       front:
         ipv4_address: ${DESECSTACK_IPV4_REAR_PREFIX16}.0.127

nslord/conf/pdns.conf.var

@@ -1,5 +1,4 @@
 allow-axfr-ips=${DESECSTACK_IPV4_REAR_PREFIX16}.1.0/24
-also-notify=${DESECSTACK_IPV4_REAR_PREFIX16}.1.12
 api=yes
 api-key=${DESECSTACK_NSLORD_APIKEY}
 default-soa-edit=INCREMENT-WEEKS

nsmaster/conf/pdns.conf.var

@@ -1,5 +1,3 @@
-allow-unsigned-notify=yes
-allow-unsigned-supermaster=yes
 api=yes
 api-key=${DESECSTACK_NSMASTER_APIKEY}
 disable-axfr=yes