feat(api): contact PCH API for domain creation and deletion

.env.default

@@ -22,6 +22,8 @@ DESECSTACK_API_EMAIL_HOST_PASSWORD=
 DESECSTACK_API_EMAIL_PORT=
 DESECSTACK_API_SECRETKEY=
 DESECSTACK_API_PSL_RESOLVER=
+DESECSTACK_API_PCH_API=
+DESECSTACK_API_PCH_API_TOKEN=
 DESECSTACK_DBAPI_PASSWORD_desec=
 DESECSTACK_MINIMUM_TTL_DEFAULT=900
 

.env.dev

@@ -21,6 +21,8 @@ DESECSTACK_API_EMAIL_HOST_PASSWORD=
 DESECSTACK_API_EMAIL_PORT=
 DESECSTACK_API_SECRETKEY=insecure
 DESECSTACK_API_PSL_RESOLVER=9.9.9.9
+DESECSTACK_API_PCH_API=https://localhost/pch/api
+DESECSTACK_API_PCH_API_TOKEN=insecure
 DESECSTACK_DBAPI_PASSWORD_desec=insecure
 DESECSTACK_MINIMUM_TTL_DEFAULT=3600
 

.github/workflows/test.yml

@@ -24,6 +24,8 @@ env:
   DESECSTACK_API_EMAIL_HOST_PASSWORD: password
   DESECSTACK_API_EMAIL_PORT: 25
   DESECSTACK_API_SECRETKEY: 9Fn33T5yGuds
+  DESECSTACK_API_PCH_API: http://pch
+  DESECSTACK_API_PCH_API_TOKEN: insecure
   DESECSTACK_API_PSL_RESOLVER: 8.8.8.8
   DESECSTACK_DBAPI_PASSWORD_desec: 9Fn33T5yGueeee
   DESECSTACK_NSLORD_APIKEY: 9Fn33T5yGukjekwjew

api/api/settings.py

@@ -232,6 +232,10 @@ CAPTCHA_VALIDITY_PERIOD = timedelta(hours=24)
 WATCHDOG_SECONDARIES = os.environ.get("DESECSTACK_WATCHDOG_SECONDARIES", "").split()
 WATCHDOG_WINDOW_SEC = 600
 
+# PCH
+PCH_API = os.environ.get("DESECSTACK_API_PCH_API", "")
+PCH_API_TOKEN = os.environ.get("DESECSTACK_API_PCH_API_TOKEN", "")
+
 # Prometheus (see https://github.com/korfuri/django-prometheus/blob/master/documentation/exports.md)
 #  TODO Switch to PROMETHEUS_METRICS_EXPORT_PORT_RANGE instead of this workaround, which currently necessary to due
 #  https://github.com/korfuri/django-prometheus/issues/215

api/desecapi/exceptions.py

@@ -8,7 +8,7 @@ class RequestEntityTooLarge(APIException):
     default_code = "too_large"
 
 
-class PDNSException(APIException):
+class ExternalAPIException(APIException):
     def __init__(self, response=None):
         self.response = response
         detail = (
@@ -19,6 +19,14 @@ class PDNSException(APIException):
         return super().__init__(detail)
 
 
+class PDNSException(ExternalAPIException):
+    pass
+
+
+class PCHException(ExternalAPIException):
+    pass
+
+
 class ConcurrencyException(APIException):
     status_code = status.HTTP_429_TOO_MANY_REQUESTS
     default_detail = "Too many concurrent requests."

api/desecapi/metrics.py

@@ -74,6 +74,19 @@ set_counter(
 )
 set_counter("desecapi_pdns_keys_fetched", "number of times pdns keys were fetched")
 
+# pch.py metrics
+set_counter(
+    "desecapi_pch_request_success",
+    "number of times PCH request was successful",
+    ["method", "status"],
+)
+set_counter(
+    "desecapi_pch_request_failure",
+    "number of times PCH request failed",
+    ["method", "path", "status"],
+)
+
+
 # pdns_change_tracker.py metrics
 set_counter(
     "desecapi_pdns_catalog_updated",

api/desecapi/pch.py

@@ -0,0 +1,54 @@
+import json
+
+import requests
+from django.conf import settings
+
+from desecapi import metrics
+from desecapi.exceptions import PCHException
+
+_config = {
+    "base_url": settings.PCH_API,
+    "token": settings.PCH_API_TOKEN,
+}
+
+
+def _pch_request(
+    method,
+    *,
+    path,
+    expect_status,
+    data=None,
+    accept="application/json",
+):
+    if data is not None:
+        data = json.dumps(data)
+
+    headers = {
+        "Accept": accept,
+        "User-Agent": "desecapi",
+        "Authorization": _config["token"],
+    }
+    r = requests.request(method, _config["base_url"] + path, data=data, headers=headers)
+    if r.status_code not in expect_status:
+        metrics.get("desecapi_pch_request_failure").labels(
+            method, path, r.status_code
+        ).inc()
+        raise PCHException(response=r)
+    metrics.get("desecapi_pch_request_success").labels(method, r.status_code).inc()
+    return r
+
+
+def _post(path, data, **kwargs):
+    return _pch_request("post", path=path, data=data, **kwargs)
+
+
+def _delete(path, data, **kwargs):
+    return _pch_request("delete", path=path, data=data, **kwargs)
+
+
+def create_domains(domains):
+    _post("/zones", {"zones": domains}, expect_status=[201])
+
+
+def delete_domains(domains):
+    _delete("/zones", {"zones": domains}, expect_status=[200])

api/desecapi/pdns_change_tracker.py

@@ -5,7 +5,7 @@ from django.db.models.signals import post_save, post_delete
 from django.db.transaction import atomic
 from django.utils import timezone
 
-from desecapi import metrics
+from desecapi import metrics, pch
 from desecapi.models import RRset, RR, Domain
 from desecapi.pdns import (
     _pdns_post,
@@ -79,6 +79,9 @@ class PDNSChangeTracker:
         def api_do(self):
             raise NotImplementedError()
 
+        def pch_do(self):
+            raise NotImplementedError()
+
         def update_catalog(self, delete=False):
             content = _pdns_patch(
                 NSMASTER,
@@ -153,6 +156,9 @@ class PDNSChangeTracker:
             rrs = [RR(rrset=rr_set, content=ns) for ns in settings.DEFAULT_NS]
             RR.objects.bulk_create(rrs)  # One INSERT
 
+        def pch_do(self):
+            pch.create_domains([self.domain_name])
+
         def __str__(self):
             return "Create Domain %s" % self.domain_name
 
@@ -169,6 +175,9 @@ class PDNSChangeTracker:
         def api_do(self):
             pass
 
+        def pch_do(self):
+            pch.delete_domains([self.domain_name])
+
         def __str__(self):
             return "Delete Domain %s" % self.domain_name
 
@@ -223,6 +232,9 @@ class PDNSChangeTracker:
         def api_do(self):
             pass
 
+        def pch_do(self):
+            pass
+
         def __str__(self):
             return (
                 "Update RRsets of %s: additions=%s, modifications=%s, deletions=%s"
@@ -304,6 +316,8 @@ class PDNSChangeTracker:
             try:
                 change.pdns_do()
                 change.api_do()
+                if settings.PCH_API and not settings.DEBUG:
+                    change.pch_do()
                 if change.axfr_required:
                     axfr_required.add(change.domain_name)
             except Exception as e:

api/desecapi/tests/base.py

@@ -304,6 +304,8 @@ class MockPDNSTestCase(APITestCase):
     PDNS_ZONE = r"/zones/(?P<id>[^/]+)"
     PDNS_ZONE_AXFR = r"/zones/(?P<id>[^/]+)/axfr-retrieve"
     PDNS_ZONE_EXPORT = r"/zones/(?P<id>[^/]+)/export"
+    PCH_ZONE_CREATE = r"/zones"
+    PCH_ZONE_DELETE = r"/zones"
 
     @classmethod
     def get_full_pdns_url(cls, path_regex, ns="LORD", **kwargs):
@@ -609,6 +611,62 @@ class MockPDNSTestCase(APITestCase):
             "priority": 1,  # avoid collision with DELETE zones/(?P<id>[^/]+)$ (httpretty does not match the method)
         }
 
+    def request_pch_zone_create(self, name, **kwargs):
+        def request_callback(r, _, response_headers):
+            try:
+                self.assertEqual(
+                    r.parsed_body,
+                    {"zones": [name]},
+                    f"Expected PCH zone creation request for {name}, but got '{r.parsed_body}'.",
+                )
+            finally:
+                return [
+                    201,
+                    response_headers,
+                    json.dumps(
+                        {
+                            "status": True,
+                            "message": "Zone(s) ADDED",
+                            "zones": [name],
+                        }
+                    ),
+                ]
+
+        return {
+            "method": "POST",
+            "uri": re.compile("^" + settings.PCH_API + self.PCH_ZONE_CREATE),
+            "body": request_callback,
+            **kwargs,
+        }
+
+    def request_pch_zone_delete(self, name, **kwargs):
+        def request_callback(r, _, response_headers):
+            try:
+                self.assertEqual(
+                    r.parsed_body,
+                    {"zones": [name]},
+                    f"Expected PCH zone deletion request for {name}, but got '{r.parsed_body}'.",
+                )
+            finally:
+                return [
+                    200,
+                    response_headers,
+                    json.dumps(
+                        {
+                            "status": True,
+                            "message": "Zone(s) deleted",
+                            "zones": [name],
+                        }
+                    ),
+                ]
+
+        return {
+            "method": "DELETE",
+            "uri": re.compile("^" + settings.PCH_API + self.PCH_ZONE_DELETE),
+            "body": request_callback,
+            **kwargs,
+        }
+
     def assertRequests(self, *expected_requests, expect_order=True, exit_hook=None):
         """
         Assert the given requests are made. To build requests, use the `MockPDNSTestCase.request_*` functions.
@@ -639,29 +697,30 @@ class MockPDNSTestCase(APITestCase):
             expect_order=False,
         )
 
-    def assertZoneCreation(self):
+    def assertZoneCreation(self, name):
         """
-        Asserts that nslord is contact and a zone is created.
+        Asserts that nslord, nsmaster and PCH are contacted for zone creation.
+        Name is only asserted for requests to PCH.
         """
         return AssertRequestsContextManager(
             test_case=self,
             expected_requests=[
                 self.request_pdns_zone_create(ns="LORD"),
                 self.request_pdns_zone_create(ns="MASTER"),
+                self.request_pch_zone_create(name=name),
             ],
         )
 
-    def assertZoneDeletion(self, name=None):
+    def assertZoneDeletion(self, name):
         """
-        Asserts that nslord and nsmaster are contacted to delete a zone.
-        Args:
-            name: If given, the test is restricted to the name of this zone.
+        Asserts that nslord, nsmaster and PCH are contacted for zone deletion.
         """
         return AssertRequestsContextManager(
             test_case=self,
             expected_requests=[
                 self.request_pdns_zone_delete(ns="LORD", name=name),
                 self.request_pdns_zone_delete(ns="MASTER", name=name),
+                self.request_pch_zone_delete(name=name),
             ],
         )
 
@@ -714,6 +773,7 @@ class MockPDNSTestCase(APITestCase):
             8081
         )  # FIXME static dependency on settings variable
         for request in [
+            # TODO delete not in this list - is this even needed?
             cls.request_pdns_zone_create(ns="LORD"),
             cls.request_pdns_zone_create(ns="MASTER"),
             cls.request_pdns_zone_axfr(),
@@ -786,6 +846,13 @@ class MockPDNSTestCase(APITestCase):
                     status=599,
                     priority=-100,
                 )
+            httpretty.register_uri(
+                method,
+                re.compile("^" + settings.PCH_API + ".*"),
+                body=request_callback,
+                status=599,
+                priority=-100,
+            )
 
 
 class DesecTestCase(MockPDNSTestCase):
@@ -980,43 +1047,42 @@ class DesecTestCase(MockPDNSTestCase):
             )
         return parents[0]
 
-    @classmethod
-    def requests_desec_domain_creation(cls, name=None, axfr=True, keys=True):
+    def requests_desec_domain_creation(self, name=None, axfr=True, keys=True):
         soa_content = "get.desec.io. get.desec.io. 1 86400 3600 2419200 3600"
         requests = [
-            cls.request_pdns_zone_create(ns="LORD", payload=soa_content),
-            cls.request_pdns_zone_create(ns="MASTER"),
-            cls.request_pdns_update_catalog(),
+            self.request_pdns_zone_create(ns="LORD", payload=soa_content),
+            self.request_pdns_zone_create(ns="MASTER"),
+            self.request_pdns_update_catalog(),
+            self.request_pch_zone_create(name=name),
         ]
         if axfr:
-            requests.append(cls.request_pdns_zone_axfr(name=name))
+            requests.append(self.request_pdns_zone_axfr(name=name))
         if keys:
-            requests.append(cls.request_pdns_zone_retrieve_crypto_keys(name=name))
+            requests.append(self.request_pdns_zone_retrieve_crypto_keys(name=name))
         return requests
 
-    @classmethod
-    def requests_desec_domain_deletion(cls, domain):
+    def requests_desec_domain_deletion(self, domain):
         requests = [
-            cls.request_pdns_zone_delete(name=domain.name, ns="LORD"),
-            cls.request_pdns_zone_delete(name=domain.name, ns="MASTER"),
-            cls.request_pdns_update_catalog(),
+            self.request_pdns_zone_delete(name=domain.name, ns="LORD"),
+            self.request_pdns_zone_delete(name=domain.name, ns="MASTER"),
+            self.request_pdns_update_catalog(),
+            self.request_pch_zone_delete(name=domain.name),
         ]
 
         if domain.is_locally_registrable:
-            delegate_at = cls._find_auto_delegation_zone(domain.name)
+            delegate_at = self._find_auto_delegation_zone(domain.name)
             requests += [
-                cls.request_pdns_zone_update(name=delegate_at),
-                cls.request_pdns_zone_axfr(name=delegate_at),
+                self.request_pdns_zone_update(name=delegate_at),
+                self.request_pdns_zone_axfr(name=delegate_at),
             ]
 
         return requests
 
-    @classmethod
-    def requests_desec_domain_creation_auto_delegation(cls, name=None):
-        delegate_at = cls._find_auto_delegation_zone(name)
-        return cls.requests_desec_domain_creation(name=name) + [
-            cls.request_pdns_zone_update(name=delegate_at),
-            cls.request_pdns_zone_axfr(name=delegate_at),
+    def requests_desec_domain_creation_auto_delegation(self, name=None):
+        delegate_at = self._find_auto_delegation_zone(name)
+        return self.requests_desec_domain_creation(name=name) + [
+            self.request_pdns_zone_update(name=delegate_at),
+            self.request_pdns_zone_axfr(name=delegate_at),
         ]
 
     @classmethod

api/desecapi/tests/test_pdns_change_tracker.py

@@ -580,12 +580,7 @@ class DomainTestCase(PdnsChangeTrackerTestCase):
     def test_create(self):
         name = self.random_domain_name()
         with self.assertRequests(
-            [
-                self.request_pdns_zone_create("LORD"),
-                self.request_pdns_zone_create("MASTER"),
-                self.request_pdns_update_catalog(),
-                self.request_pdns_zone_axfr(name),
-            ]
+            self.requests_desec_domain_creation(keys=False)
         ), PDNSChangeTracker():
             Domain.objects.create(name=name, owner=self.user)
 

docker-compose.yml

@@ -147,6 +147,8 @@ services:
     - DESECSTACK_API_EMAIL_PORT
     - DESECSTACK_API_SECRETKEY
     - DESECSTACK_API_PSL_RESOLVER
+    - DESECSTACK_API_PCH_API
+    - DESECSTACK_API_PCH_API_TOKEN
     - DESECSTACK_API_AUTHACTION_VALIDITY
     - DESECSTACK_DBAPI_PASSWORD_desec
     - DESECSTACK_IPV4_REAR_PREFIX16