feat(api): add catalog zone, closes #270

.travis.yml

@@ -49,7 +49,7 @@ before_install:
   - docker-compose build api
   - docker-compose build www
 
-  # test images
+  # build images
   - docker-compose -f docker-compose.yml -f docker-compose.test-e2e.yml build test-e2e
 
 script:

api/api/settings.py

@@ -164,11 +164,12 @@ DEFAULT_NS_TTL = os.environ['DESECSTACK_NSLORD_DEFAULT_TTL']
 PSL_RESOLVER = os.environ.get('DESECSTACK_API_PSL_RESOLVER')
 LOCAL_PUBLIC_SUFFIXES = {'dedyn.%s' % os.environ['DESECSTACK_DOMAIN']}
 
-# PowerDNS API access
+# PowerDNS-related
 NSLORD_PDNS_API = 'http://nslord:8081/api/v1/servers/localhost'
 NSLORD_PDNS_API_TOKEN = os.environ['DESECSTACK_NSLORD_APIKEY']
 NSMASTER_PDNS_API = 'http://nsmaster:8081/api/v1/servers/localhost'
 NSMASTER_PDNS_API_TOKEN = os.environ['DESECSTACK_NSMASTER_APIKEY']
+CATALOG_ZONE = 'catalog.internal'
 
 # Celery
 CELERY_BROKER_URL = 'amqp://rabbitmq'

api/desecapi/management/commands/align-catalog-zone.py

@@ -0,0 +1,58 @@
+from django.conf import settings
+from django.core.management import BaseCommand
+
+from desecapi.exceptions import PDNSException
+from desecapi.pdns import _pdns_delete, _pdns_get, _pdns_post, NSLORD, NSMASTER, pdns_id, construct_catalog_rrset
+
+
+class Command(BaseCommand):
+    # https://tools.ietf.org/html/draft-muks-dnsop-dns-catalog-zones-04
+    help = 'Generate a catalog zone on nsmaster, based on zones known on nslord.'
+
+    def add_arguments(self, parser):
+        pass
+
+    def handle(self, *args, **options):
+        catalog_zone_id = pdns_id(settings.CATALOG_ZONE)
+
+        # Fetch zones from NSLORD
+        response = _pdns_get(NSLORD, '/zones').json()
+        zones = {zone['name'] for zone in response}
+
+        # Retrieve catalog zone serial (later reused for recreating the catalog zone, for allow for smooth rollover)
+        try:
+            response = _pdns_get(NSMASTER, f'/zones/{catalog_zone_id}')
+            serial = response.json()['serial']
+        except PDNSException as e:
+            if e.response.status_code == 404:
+                serial = None
+            else:
+                raise e
+
+        # Purge catalog zone if exists
+        try:
+            _pdns_delete(NSMASTER, f'/zones/{catalog_zone_id}')
+        except PDNSException as e:
+            if e.response.status_code != 404:
+                raise e
+
+        # Create new catalog zone
+        rrsets = [
+            construct_catalog_rrset(subname='', qtype='NS', rdata='invalid.'),  # as per the specification
+            construct_catalog_rrset(subname='version', qtype='TXT', rdata='"2"'),  # as per the specification
+            *(construct_catalog_rrset(zone=zone) for zone in zones)
+        ]
+
+        data = {
+            'name': settings.CATALOG_ZONE + '.',
+            'kind': 'MASTER',
+            'dnssec': False,  # as per the specification
+            'nameservers': [],
+            'rrsets': rrsets,
+        }
+
+        if serial is not None:
+            data['serial'] = serial + 1  # actually, pdns does increase this as well, but let's not rely on this
+
+        _pdns_post(NSMASTER, '/zones', data)
+        print(f'Aligned catalog zone ({len(zones)} member zones).')

api/desecapi/pdns.py

@@ -1,3 +1,4 @@
+from hashlib import sha1
 import json
 import re
 
@@ -104,3 +105,26 @@ def get_rrset_datas(domain):
              'records': [record['content'] for record in rrset['records']],
              'ttl': rrset['ttl']}
             for rrset in get_zone(domain)['rrsets']]
+
+
+def construct_catalog_rrset(zone=None, delete=False, subname=None, qtype='PTR', rdata=None):
+    # subname can be generated from zone for convenience; exactly one needs to be given
+    assert (zone is None) ^ (subname is None)
+    # sanity check: one can't delete an rrset and give record data at the same time
+    assert not (delete and rdata)
+
+    if subname is None:
+        zone = zone.rstrip('.') + '.'
+        m_unique = sha1(zone.encode()).hexdigest()
+        subname = f'{m_unique}.zones'
+
+    if rdata is None:
+        rdata = zone
+
+    return {
+        'name': f'{subname}.{settings.CATALOG_ZONE}'.strip('.') + '.',
+        'type': qtype,
+        'ttl': 0,  # as per the specification
+        'changetype': 'REPLACE',
+        'records': [] if delete else [{'content': rdata, 'disabled': False}],
+    }

api/desecapi/pdns_change_tracker.py

@@ -8,7 +8,8 @@ from django.utils import timezone
 
 from desecapi.exceptions import PDNSValidationError
 from desecapi.models import RRset, RR, Domain
-from desecapi.pdns import _pdns_post, NSLORD, NSMASTER, _pdns_delete, _pdns_patch, _pdns_put, pdns_id
+from desecapi.pdns import _pdns_post, NSLORD, NSMASTER, _pdns_delete, _pdns_patch, _pdns_put, pdns_id, \
+    construct_catalog_rrset
 
 
 class PDNSChangeTracker:
@@ -71,6 +72,10 @@ class PDNSChangeTracker:
         def api_do(self):
             raise NotImplementedError()
 
+        def update_catalog(self, delete=False):
+            return _pdns_patch(NSMASTER, '/zones/' + pdns_id(settings.CATALOG_ZONE),
+                               {'rrsets': [construct_catalog_rrset(zone=self.domain_name, delete=delete)]})
+
     class CreateDomain(PDNSChange):
         @property
         def axfr_required(self):
@@ -98,6 +103,8 @@ class PDNSChangeTracker:
                 }
             )
 
+            self.update_catalog()
+
         def api_do(self):
             rr_set = RRset(
                 domain=Domain.objects.get(name=self.domain_name),
@@ -120,6 +127,7 @@ class PDNSChangeTracker:
         def pdns_do(self):
             _pdns_delete(NSLORD, '/zones/' + self.domain_pdns_id)
             _pdns_delete(NSMASTER, '/zones/' + self.domain_pdns_id)
+            self.update_catalog(delete=True)
 
         def api_do(self):
             pass

api/desecapi/tests/base.py

@@ -483,6 +483,16 @@ class MockPDNSTestCase(APITestCase):
             'body': None,
         }
 
+    @classmethod
+    def request_pdns_update_catalog(cls):
+        return {
+            'method': 'PATCH',
+            'uri': cls.get_full_pdns_url(cls.PDNS_ZONE, ns='MASTER', id=cls._pdns_zone_id_heuristic('catalog.internal')),
+            'status': 204,
+            'body': None,
+            'priority': 1,  # avoid collision with DELETE zones/(?P<id>[^/]+)$ (httpretty does not match the method)
+        }
+
     def assertPdnsRequests(self, *expected_requests, expect_order=True):
         """
         Assert the given requests are made. To build requests, use the `MockPDNSTestCase.request_*` functions.
@@ -756,6 +766,7 @@ class DesecTestCase(MockPDNSTestCase):
         return [
             cls.request_pdns_zone_create(ns='LORD'),
             cls.request_pdns_zone_create(ns='MASTER'),
+            cls.request_pdns_update_catalog(),
             cls.request_pdns_zone_axfr(name=name),
             cls.request_pdns_zone_retrieve_crypto_keys(name=name),
         ]
@@ -765,6 +776,7 @@ class DesecTestCase(MockPDNSTestCase):
         return [
             cls.request_pdns_zone_delete(name=name, ns='LORD'),
             cls.request_pdns_zone_delete(name=name, ns='MASTER'),
+            cls.request_pdns_update_catalog(),
         ]
 
     @classmethod
@@ -781,6 +793,7 @@ class DesecTestCase(MockPDNSTestCase):
         return [
             cls.request_pdns_zone_delete(name=name, ns='LORD'),
             cls.request_pdns_zone_delete(name=name, ns='MASTER'),
+            cls.request_pdns_update_catalog(),
             cls.request_pdns_zone_update(name=delegate_at),
             cls.request_pdns_zone_axfr(name=delegate_at),
         ]

api/desecapi/tests/test_pdns_change_tracker.py

@@ -477,6 +477,7 @@ class DomainTestCase(PdnsChangeTrackerTestCase):
                 [
                     self.request_pdns_zone_create('LORD'),
                     self.request_pdns_zone_create('MASTER'),
+                    self.request_pdns_update_catalog(),
                     self.request_pdns_zone_axfr(name)
                 ]), PDNSChangeTracker():
             Domain.objects.create(name=name, owner=self.user)

api/entrypoint.sh

@@ -13,5 +13,8 @@ echo "waiting for dependencies ..."
 # migrate database
 python manage.py migrate || exit 1
 
-echo Finished migrations, starting API server ...
+# Prepare catalog zone
+python manage.py align-catalog-zone
+
+echo Starting API server ...
 exec uwsgi --ini uwsgi.ini