feat(api): move RR set content validation to API realm

Fixes #211, fixes #285

api/desecapi/dns.py

@@ -0,0 +1,47 @@
+import struct
+
+import dns
+import dns.rdtypes.txtbase
+import dns.rdtypes.ANY.OPENPGPKEY
+
+
+class LongQuotedTXT(dns.rdtypes.txtbase.TXTBase):
+    """
+    A TXT record like RFC 1035, but
+    - allows arbitrarily long tokens, and
+    - all tokens must be quoted.
+    """
+
+    @classmethod
+    def from_text(cls, rdclass, rdtype, tok, origin=None, relativize=True):
+        strings = []
+        while 1:
+            token = tok.get().unescape()
+            if token.is_eol_or_eof():
+                break
+            if not token.is_quoted_string():
+                raise dns.exception.SyntaxError("Content must be quoted.")
+            value = token.value
+            if isinstance(value, bytes):
+                strings.append(value)
+            else:
+                strings.append(value.encode())
+        if len(strings) == 0:
+            raise dns.exception.UnexpectedEnd
+        return cls(rdclass, rdtype, strings)
+
+    def to_wire(self, file, compress=None, origin=None):
+        for long_s in self.strings:
+            for s in [long_s[i:i+255] for i in range(0, max(len(long_s), 1), 255)]:
+                l = len(s)
+                assert l < 256
+                file.write(struct.pack('!B', l))
+                file.write(s)
+
+
+class OPENPGPKEY(dns.rdtypes.ANY.OPENPGPKEY.OPENPGPKEY):
+    # TODO remove when https://github.com/rthalley/dnspython/commit/d6a95982fcd454a10467260bfb874c3c9d31d06f was
+    #  released
+
+    def to_text(self, origin=None, relativize=True, **kw):
+        return super().to_text(origin, relativize, **kw).replace(' ', '')

api/desecapi/exceptions.py

@@ -1,7 +1,5 @@
-import json
-
 from rest_framework import status
-from rest_framework.exceptions import APIException, ValidationError
+from rest_framework.exceptions import APIException
 
 
 class RequestEntityTooLarge(APIException):
@@ -10,18 +8,6 @@ class RequestEntityTooLarge(APIException):
     default_code = 'too_large'
 
 
-class PDNSValidationError(ValidationError):
-    pdns_code = 422
-
-    def __init__(self, response=None):
-        try:
-            detail = json.loads(response.text)['error']
-        except json.JSONDecodeError:
-            detail = response.text
-
-        return super().__init__(detail={'detail': detail}, code='invalid')
-
-
 class PDNSException(APIException):
     def __init__(self, response=None):
         self.response = response

api/desecapi/management/commands/sync-from-pdns.py

@@ -2,7 +2,7 @@ from django.core.management import BaseCommand, CommandError
 from django.db import transaction
 
 from desecapi import pdns
-from desecapi.models import Domain, RRset, RR
+from desecapi.models import Domain, RRset, RR, RR_SET_TYPES_AUTOMATIC
 
 
 class Command(BaseCommand):
@@ -40,7 +40,7 @@ class Command(BaseCommand):
         rrsets = []
         rrs = []
         for rrset_data in pdns.get_rrset_datas(domain):
-            if rrset_data['type'] in RRset.RESTRICTED_TYPES:
+            if rrset_data['type'] in RR_SET_TYPES_AUTOMATIC:
                 continue
             records = rrset_data.pop('records')
             rrset = RRset(**rrset_data)

api/desecapi/migrations/0003_rr_content.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.1 on 2020-08-26 07:46
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('desecapi', '0002_unmanaged_donations'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='rr',
+            name='content',
+            field=models.TextField(),
+        ),
+    ]

api/desecapi/models.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import binascii
 import json
 import logging
 import re
@@ -11,6 +12,7 @@ from datetime import timedelta
 from functools import cached_property
 from hashlib import sha256
 
+import dns
 import psl_dns
 import rest_framework.authtoken.models
 from django.conf import settings
@@ -24,13 +26,15 @@ from django.db.models import Manager, Q
 from django.template.loader import get_template
 from django.utils import timezone
 from django_prometheus.models import ExportModelOperationsMixin
+from dns import rdata, rdataclass, rdatatype
 from dns.exception import Timeout
+from dns.rdtypes import ANY, IN
 from dns.resolver import NoNameservers
 from rest_framework.exceptions import APIException
 
 from desecapi import metrics
 from desecapi import pdns
-
+from desecapi.dns import LongQuotedTXT, OPENPGPKEY
 
 logger = logging.getLogger(__name__)
 psl = psl_dns.PSL(resolver=settings.PSL_RESOLVER, timeout=.5)
@@ -415,6 +419,31 @@ class Donation(ExportModelOperationsMixin('Donation'), models.Model):
         managed = False
 
 
+# RR set types: the good, the bad, and the ugly
+# known, but unsupported types
+RR_SET_TYPES_UNSUPPORTED = {
+    'ALIAS',  # Requires signing at the frontend, hence unsupported in desec-stack
+    'DNAME',  # "do not combine with DNSSEC", https://doc.powerdns.com/authoritative/settings.html#dname-processing
+    'IPSECKEY',  # broken in pdns, https://github.com/PowerDNS/pdns/issues/9055 TODO enable support
+    'KEY',  # Application use restricted by RFC 3445, DNSSEC use replaced by DNSKEY and handled automatically
+    'WKS',  # General usage not recommended, "SHOULD NOT" be used in SMTP (RFC 1123)
+}
+# restricted types are managed in use by the API, and cannot directly be modified by the API client
+RR_SET_TYPES_AUTOMATIC = {
+    # corresponding functionality is automatically managed:
+    'CDNSKEY', 'CDS', 'DNSKEY', 'KEY', 'NSEC', 'NSEC3', 'OPT', 'RRSIG',
+    # automatically managed by the API:
+    'NSEC3PARAM', 'SOA'
+}
+# backend types are types that are the types supported by the backend(s)
+RR_SET_TYPES_BACKEND = pdns.SUPPORTED_RRSET_TYPES
+# validation types are types supported by the validation backend, currently: dnspython
+RR_SET_TYPES_VALIDATION = set(ANY.__all__) | set(IN.__all__)
+# manageable types are directly managed by the API client
+RR_SET_TYPES_MANAGEABLE = \
+        (RR_SET_TYPES_BACKEND & RR_SET_TYPES_VALIDATION) - RR_SET_TYPES_UNSUPPORTED - RR_SET_TYPES_AUTOMATIC
+
+
 class RRsetManager(Manager):
     def create(self, contents=None, **kwargs):
         rrset = super().create(**kwargs)
@@ -456,9 +485,6 @@ class RRset(ExportModelOperationsMixin('RRset'), models.Model):
 
     objects = RRsetManager()
 
-    DEAD_TYPES = ('ALIAS', 'DNAME')
-    RESTRICTED_TYPES = ('SOA', 'RRSIG', 'DNSKEY', 'NSEC3PARAM', 'OPT')
-
     class Meta:
         unique_together = (("domain", "subname", "type"),)
 
@@ -474,6 +500,91 @@ class RRset(ExportModelOperationsMixin('RRset'), models.Model):
         self.full_clean(validate_unique=False)
         super().save(*args, **kwargs)
 
+    def clean_records(self, records_presentation_format):
+        """
+        Validates the records belonging to this set. Validation rules follow the DNS specification; some types may
+        incur additional validation rules.
+
+        Raises ValidationError if violation of DNS specification is found.
+
+        Returns a set of records in canonical presentation format.
+
+        :param records_presentation_format: iterable of records in presentation format
+        """
+        rdtype = rdatatype.from_text(self.type)
+        errors = []
+
+        def _error_msg(record, detail):
+            return f'Record content of {self.type} {self.name} invalid: \'{record}\': {detail}'
+
+        records_canonical_format = set()
+        for r in records_presentation_format:
+            try:
+                r_canonical_format = RR.canonical_presentation_format(r, rdtype)
+            except binascii.Error:
+                # e.g., odd-length string
+                errors.append(_error_msg(r, 'Cannot parse hexadecimal or base64 record contents'))
+            except dns.exception.SyntaxError as e:
+                # e.g., A/127.0.0.999
+                if 'quote' in e.args[0]:
+                    errors.append(_error_msg(r, f'Data for {self.type} records must be given using quotation marks.'))
+                else:
+                    errors.append(_error_msg(r, f'Record content malformed: {",".join(e.args)}'))
+            except dns.name.NeedAbsoluteNameOrOrigin:
+                errors.append(_error_msg(r, 'Hostname must be fully qualified (i.e., end in a dot: "example.com.")'))
+            except ValueError:
+                # e.g., string ("asdf") cannot be parsed into int on base 10
+                errors.append(_error_msg(r, 'Cannot parse record contents'))
+            except AssertionError:
+                # e.g., HINFO token too long, cf. https://github.com/rthalley/dnspython/issues/493
+                errors.append(_error_msg(r, 'Invalid record content'))
+            except Exception as e:
+                # TODO see what exceptions raise here for faulty input
+                raise e
+            else:
+                if r_canonical_format in records_canonical_format:
+                    errors.append(_error_msg(r, f'Duplicate record content: this is identical to '
+                                                f'\'{r_canonical_format}\''))
+                else:
+                    records_canonical_format.add(r_canonical_format)
+
+        if any(errors):
+            raise ValidationError(errors)
+
+        return records_canonical_format
+
+    def save_records(self, records):
+        """
+        Updates this RR set's resource records, discarding any old values.
+
+        Records are expected in presentation format and are converted to canonical
+        presentation format (e.g., 127.00.0.1 will be converted to 127.0.0.1).
+        Raises if a invalid set of records is provided.
+
+        This method triggers the following database queries:
+        - two SELECT queries for comparison of old with new records
+        - for each removed record, one DELETE query
+        - if one or more records were added, a total of one INSERT query
+
+        Changes are saved to the database immediately.
+
+        :param records: list of records in presentation format
+        """
+        records = self.clean_records(records)
+
+        # Remove RRs that we didn't see in the new list
+        removed_rrs = self.records.exclude(content__in=records)  # one SELECT
+        for rr in removed_rrs:
+            rr.delete()  # one DELETE query
+
+        # Figure out which entries in records have not changed
+        unchanged_rrs = self.records.filter(content__in=records)  # one SELECT
+        unchanged_content = [unchanged_rr.content for unchanged_rr in unchanged_rrs]
+        added_content = filter(lambda c: c not in unchanged_content, records)
+
+        rrs = [RR(rrset=self, content=content) for content in added_content]
+        RR.objects.bulk_create(rrs)  # One INSERT
+
     def __str__(self):
         return '<RRSet %s domain=%s type=%s subname=%s>' % (self.pk, self.domain.name, self.type, self.subname)
 
@@ -493,20 +604,50 @@ class RRManager(Manager):
 class RR(ExportModelOperationsMixin('RR'), models.Model):
     created = models.DateTimeField(auto_now_add=True)
     rrset = models.ForeignKey(RRset, on_delete=models.CASCADE, related_name='records')
-    # The pdns lmdb backend used on our slaves does not only store the record contents itself, but other metadata (such
-    # as type etc.) Both together have to fit into the lmdb backend's current total limit of 512 bytes per RR, see
-    # https://github.com/PowerDNS/pdns/issues/8012
-    # I found the additional data to be 12 bytes (by trial and error). I believe these are the 12 bytes mentioned here:
-    # https://lists.isc.org/pipermail/bind-users/2008-April/070137.html So we can use 500 bytes for the actual content.
-    # Note: This is a conservative estimate, as record contents may be stored more efficiently depending on their type,
-    # effectively allowing a longer length in "presentation format". For example, A record contents take up 4 bytes,
-    # although the presentation format (usual IPv4 representation) takes up to 15 bytes. Similarly, OPENPGPKEY contents
-    # are base64-decoded before storage in binary format, so a "presentation format" value (which is the value our API
-    # sees) can have up to 668 bytes. Instead of introducing per-type limits, setting it to 500 should always work.
-    content = models.CharField(max_length=500)  #
+    content = models.TextField()
 
     objects = RRManager()
 
+    @staticmethod
+    def canonical_presentation_format(any_presentation_format, type_):
+        """
+        Converts any valid presentation format for a RR into it's canonical presentation format.
+        Raises if provided presentation format is invalid.
+        """
+        if type_ in (dns.rdatatype.TXT, dns.rdatatype.SPF):
+            # for TXT record, we slightly deviate from RFC 1035 and allow tokens that are longer than 255 byte.
+            cls = LongQuotedTXT
+        elif type_ == dns.rdatatype.OPENPGPKEY:
+            cls = OPENPGPKEY
+        else:
+            # For all other record types, let dnspython decide
+            cls = rdata
+
+        wire = cls.from_text(
+            rdclass=rdataclass.IN,
+            rdtype=type_,
+            tok=dns.tokenizer.Tokenizer(any_presentation_format),
+            relativize=False
+        ).to_digestable()
+
+        # The pdns lmdb backend used on our frontends does not only store the record contents itself, but other metadata
+        # (such as type etc.) Both together have to fit into the lmdb backend's current total limit of 512 bytes per RR.
+        # I found the additional data to be 12 bytes (by trial and error). I believe these are the 12 bytes mentioned
+        # here: https://lists.isc.org/pipermail/bind-users/2008-April/070137.html So we can use 500 bytes for the actual
+        # content stored in wire format.
+        # This check can be relaxed as soon as lmdb supports larger records,
+        # cf. https://github.com/desec-io/desec-slave/issues/34 and https://github.com/PowerDNS/pdns/issues/8012
+        if len(wire) > 500:
+            raise ValidationError(f'Ensure this value has no more than 500 byte in wire format (it has {len(wire)}).')
+
+        return cls.from_wire(
+            rdclass=rdataclass.IN,
+            rdtype=rdatatype.from_text(type_),
+            wire=dns.wiredata.maybe_wrap(wire),
+            current=0,
+            rdlen=len(wire)
+        ).to_text()
+
     def __str__(self):
         return '<RR %s %s rr_set=%s>' % (self.pk, self.content, self.rrset.pk)
 

api/desecapi/pdns.py

@@ -1,14 +1,24 @@
-from hashlib import sha1
 import json
 import re
+from hashlib import sha1
 
 import requests
 from django.conf import settings
 from django.core.exceptions import SuspiciousOperation
 
 from desecapi import metrics
-from desecapi.exceptions import PDNSException, PDNSValidationError, RequestEntityTooLarge
+from desecapi.exceptions import PDNSException, RequestEntityTooLarge
 
+SUPPORTED_RRSET_TYPES = {
+    # https://doc.powerdns.com/authoritative/appendices/types.html
+    # "major" types
+    'A', 'AAAA', 'AFSDB', 'ALIAS', 'CAA', 'CERT', 'CDNSKEY', 'CDS', 'CNAME', 'DNSKEY', 'DNAME', 'DS', 'HINFO', 'KEY',
+    'LOC', 'MX', 'NAPTR', 'NS', 'NSEC', 'NSEC3', 'NSEC3PARAM', 'OPENPGPKEY', 'PTR', 'RP', 'RRSIG', 'SOA', 'SPF',
+    'SSHFP', 'SRV', 'TLSA', 'SMIMEA', 'TXT', 'URI',
+
+    # "additional" types, without obsolete ones
+    'DHCID', 'DLV', 'EUI48', 'EUI64', 'IPSECKEY', 'KX', 'MINFO', 'MR', 'RKEY', 'WKS',
+}
 
 NSLORD = object()
 NSMASTER = object()
@@ -41,9 +51,7 @@ def _pdns_request(method, *, server, path, data=None):
         raise RequestEntityTooLarge
 
     r = requests.request(method, _config[server]['base_url'] + path, data=data, headers=_config[server]['headers'])
-    if r.status_code == PDNSValidationError.pdns_code:
-        raise PDNSValidationError(response=r)
-    elif r.status_code not in range(200, 300):
+    if r.status_code not in range(200, 300):
         raise PDNSException(response=r)
     metrics.get('desecapi_pdns_request_success').labels(method, r.status_code).inc()
     return r

api/desecapi/pdns_change_tracker.py

@@ -7,7 +7,6 @@ from django.db.transaction import atomic
 from django.utils import timezone
 
 from desecapi import metrics
-from desecapi.exceptions import PDNSValidationError
 from desecapi.models import RRset, RR, Domain
 from desecapi.pdns import _pdns_post, NSLORD, NSMASTER, _pdns_delete, _pdns_patch, _pdns_put, pdns_id, \
     construct_catalog_rrset
@@ -262,9 +261,6 @@ class PDNSChangeTracker:
                 change.api_do()
                 if change.axfr_required:
                     axfr_required.add(change.domain_name)
-            except PDNSValidationError as e:
-                self.transaction.__exit__(type(e), e, e.__traceback__)
-                raise e
             except Exception as e:
                 self.transaction.__exit__(type(e), e, e.__traceback__)
                 exc = ValueError(f'For changes {list(map(str, changes))}, {type(e)} occurred during {change}: {str(e)}')

api/desecapi/renderers.py

@@ -25,9 +25,15 @@ class PlainTextRenderer(renderers.BaseRenderer):
             except (TypeError, AttributeError):
                 pass
 
+            try:
+                return '; '.join([f'{err.code}: {err}' for err in data])
+            except (TypeError, AttributeError):
+                pass
+
             raise ValueError('Expected response.data to be one of the following:\n'
                              '- a dict with error details in response.data[\'detail\'],\n'
                              '- a list with at least one element that has error details in element[\'detail\'];\n'
+                             '- a list with all elements being ErrorDetail instances;\n'
                              'but got %s:\n\n%s' % (type(response.data), response.data))
 
         return data

api/desecapi/serializers.py

@@ -6,6 +6,7 @@ from base64 import urlsafe_b64decode, urlsafe_b64encode, b64encode
 from captcha.image import ImageCaptcha
 from django.contrib.auth.models import AnonymousUser
 from django.contrib.auth.password_validation import validate_password
+import django.core.exceptions
 from django.core.validators import MinValueValidator
 from django.db import IntegrityError, OperationalError
 from django.db.models import Model, Q
@@ -245,12 +246,15 @@ class RRsetSerializer(ConditionalExistenceModelSerializer):
 
     @staticmethod
     def validate_type(value):
-        if value in models.RRset.DEAD_TYPES:
-            raise serializers.ValidationError(f'The {value} RRset type is currently unsupported.')
-        if value in models.RRset.RESTRICTED_TYPES:
-            raise serializers.ValidationError(f'You cannot tinker with the {value} RRset.')
-        if value.startswith('TYPE'):
-            raise serializers.ValidationError('Generic type format is not supported.')
+        if value not in models.RR_SET_TYPES_MANAGEABLE:
+            # user cannot manage this type, let's try to tell her the reason
+            if value in models.RR_SET_TYPES_AUTOMATIC:
+                raise serializers.ValidationError(f'You cannot tinker with the {value} RR set. It is managed '
+                                                  f'automatically.')
+            elif value.startswith('TYPE'):
+                raise serializers.ValidationError('Generic type format is not supported.')
+            else:
+                raise serializers.ValidationError(f'The {value} RR set type is currently unsupported.')
         return value
 
     def validate_records(self, value):
@@ -317,27 +321,14 @@ class RRsetSerializer(ConditionalExistenceModelSerializer):
         """
         Updates this RR set's resource records, discarding any old values.
 
-        To do so, two large select queries and one query per changed (added or removed) resource record are needed.
-
-        Changes are saved to the database immediately.
-
         :param rrset: the RRset at which we overwrite all RRs
         :param rrs: list of RR representations
         """
         record_contents = [rr['content'] for rr in rrs]
-
-        # Remove RRs that we didn't see in the new list
-        removed_rrs = rrset.records.exclude(content__in=record_contents)  # one SELECT
-        for rr in removed_rrs:
-            rr.delete()  # one DELETE query
-
-        # Figure out which entries in record_contents have not changed
-        unchanged_rrs = rrset.records.filter(content__in=record_contents)  # one SELECT
-        unchanged_content = [unchanged_rr.content for unchanged_rr in unchanged_rrs]
-        added_content = filter(lambda c: c not in unchanged_content, record_contents)
-
-        rrs = [models.RR(rrset=rrset, content=content) for content in added_content]
-        models.RR.objects.bulk_create(rrs)  # One INSERT
+        try:
+            rrset.save_records(record_contents)
+        except django.core.exceptions.ValidationError as e:
+            raise serializers.ValidationError(e.messages, code='record-content')
 
 
 class RRsetListSerializer(serializers.ListSerializer):

api/desecapi/tests/base.py

@@ -17,7 +17,8 @@ from rest_framework.reverse import reverse
 from rest_framework.test import APITestCase, APIClient
 from rest_framework.utils import json
 
-from desecapi.models import User, Domain, Token, RRset, RR, psl
+from desecapi.models import User, Domain, Token, RRset, RR, psl, RR_SET_TYPES_AUTOMATIC, RR_SET_TYPES_UNSUPPORTED, \
+    RR_SET_TYPES_MANAGEABLE
 
 
 class DesecAPIClient(APIClient):
@@ -343,38 +344,6 @@ class MockPDNSTestCase(APITestCase):
             'body': None,
         }
 
-    @classmethod
-    def request_pdns_zone_update_unknown_type(cls, name=None, unknown_types=None):
-        def request_callback(r, _, response_headers):
-            body = json.loads(r.parsed_body)
-            if not unknown_types or body['rrsets'][0]['type'] in unknown_types:
-                return [
-                    422, response_headers,
-                    json.dumps({'error': 'Mocked error. Unknown RR type %s.' % body['rrsets'][0]['type']})
-                ]
-            else:
-                return [200, response_headers, None]
-
-        request = cls.request_pdns_zone_update(name)
-        # noinspection PyTypeChecker
-        request['body'] = request_callback
-        request.pop('status')
-        return request
-
-    @classmethod
-    def request_pdns_zone_update_invalid_rr(cls, name=None):
-        def request_callback(r, _, response_headers):
-            return [
-                422, response_headers,
-                json.dumps({'error': 'Mocked error. Considering RR content invalid.'})
-            ]
-
-        request = cls.request_pdns_zone_update(name)
-        # noinspection PyTypeChecker
-        request['body'] = request_callback
-        request.pop('status')
-        return request
-
     def request_pdns_zone_update_assert_body(self, name: str = None, updated_rr_sets: Union[List[RRset], Dict] = None):
         if updated_rr_sets is None:
             updated_rr_sets = []
@@ -665,6 +634,9 @@ class DesecTestCase(MockPDNSTestCase):
     AUTO_DELEGATION_DOMAINS = settings.LOCAL_PUBLIC_SUFFIXES
     PUBLIC_SUFFIXES = {'de', 'com', 'io', 'gov.cd', 'edu.ec', 'xxx', 'pinb.gov.pl', 'valer.ostfold.no',
                        'kota.aichi.jp', 's3.amazonaws.com', 'wildcard.ck'}
+    SUPPORTED_RR_SET_TYPES = {'A', 'AAAA', 'AFSDB', 'CAA', 'CERT', 'CNAME', 'DHCID', 'DLV', 'DS', 'EUI48', 'EUI64',
+                              'HINFO', 'KX', 'LOC', 'MX', 'NAPTR', 'NS', 'OPENPGPKEY', 'PTR', 'RP', 'SPF', 'SRV',
+                              'SSHFP', 'TLSA', 'TXT', 'URI'}
 
     admin = None
     auto_delegation_domains = None
@@ -859,8 +831,14 @@ class DesecTestCase(MockPDNSTestCase):
     def assertContains(self, response, text, count=None, status_code=200, msg_prefix='', html=False):
         # convenience method to check the status separately, which yields nicer error messages
         self.assertStatus(response, status_code)
+        # same for the substring check
+        self.assertIn(text, response.content.decode(response.charset),
+                      f'Could not find {text} in the following response:\n{response.content.decode(response.charset)}')
         return super().assertContains(response, text, count, status_code, msg_prefix, html)
 
+    def assertAllSupportedRRSetTypes(self, types):
+        self.assertEqual(types, self.SUPPORTED_RR_SET_TYPES, 'Either some RR types given are unsupported, or not all '
+                                                             'supported RR types were in the given set.')
 
 class PublicSuffixMockMixin():
     def _mock_get_public_suffix(self, domain_name, public_suffixes=None):
@@ -988,16 +966,9 @@ class DynDomainOwnerTestCase(DomainOwnerTestCase):
 
 
 class AuthenticatedRRSetBaseTestCase(DomainOwnerTestCase):
-    DEAD_TYPES = ['ALIAS', 'DNAME']
-    RESTRICTED_TYPES = ['SOA', 'RRSIG', 'DNSKEY', 'NSEC3PARAM', 'OPT']
-
-    # see https://doc.powerdns.com/md/types/
-    PDNS_RR_TYPES = ['A', 'AAAA', 'AFSDB', 'ALIAS', 'CAA', 'CERT', 'CDNSKEY', 'CDS', 'CNAME', 'DNSKEY', 'DNAME', 'DS',
-                     'HINFO', 'KEY', 'LOC', 'MX', 'NAPTR', 'NS', 'NSEC', 'NSEC3', 'NSEC3PARAM', 'OPENPGPKEY', 'PTR',
-                     'RP', 'RRSIG', 'SOA', 'SPF', 'SSHFP', 'SRV', 'TKEY', 'TSIG', 'TLSA', 'SMIMEA', 'TXT', 'URI']
-    ALLOWED_TYPES = ['A', 'AAAA', 'AFSDB', 'CAA', 'CERT', 'CDNSKEY', 'CDS', 'CNAME', 'DS', 'HINFO', 'KEY', 'LOC', 'MX',
-                     'NAPTR', 'NS', 'NSEC', 'NSEC3', 'OPENPGPKEY', 'PTR', 'RP', 'SPF', 'SSHFP', 'SRV', 'TKEY', 'TSIG',
-                     'TLSA', 'SMIMEA', 'TXT', 'URI']
+    UNSUPPORTED_TYPES = RR_SET_TYPES_UNSUPPORTED
+    AUTOMATIC_TYPES = RR_SET_TYPES_AUTOMATIC
+    ALLOWED_TYPES = RR_SET_TYPES_MANAGEABLE
 
     SUBNAMES = ['foo', 'bar.baz', 'q.w.e.r.t', '*', '*.foobar', '_', '-foo.test', '_bar']
 

api/desecapi/tests/test_domains.py

@@ -347,7 +347,8 @@ class DomainOwnerTestCase1(DomainOwnerTestCase):
     def test_create_domain_atomicity(self):
         name = self.random_domain_name()
         with self.assertPdnsRequests(self.request_pdns_zone_create_422()):
-            self.client.post(self.reverse('v1:domain-list'), {'name': name})
+            with self.assertRaises(ValueError):
+                self.client.post(self.reverse('v1:domain-list'), {'name': name})
             self.assertFalse(Domain.objects.filter(name=name).exists())
 
     def test_create_domain_punycode(self):

api/desecapi/tests/test_dyndns12update.py

@@ -130,10 +130,9 @@ class DynDNS12UpdateTest(DynDomainOwnerTestCase):
             'hostname': self.my_domain.name,
             'myip': '10.2.3.4asdf',
         }
-        with self.assertPdnsRequests(self.request_pdns_zone_update_invalid_rr()):
-            response = self.client.get(self.reverse('v1:dyndns12update'), params)
+        response = self.client.get(self.reverse('v1:dyndns12update'), params)
         self.assertStatus(response, status.HTTP_400_BAD_REQUEST)
-        self.assertIn('Mocked error. Considering RR content invalid.', str(response.data))
+        self.assertIn('Record content malformed', str(response.data))
 
     def test_ddclient_dyndns2_v6_success(self):
         # /nic/update?system=dyndns&hostname=foobar.dedyn.io&myipv6=::1338

api/desecapi/tests/test_pdns_change_tracker.py

@@ -75,8 +75,9 @@ class RRTestCase(PdnsChangeTrackerTestCase):
             RR(content=self.CONTENT_VALUES[1], rrset=self.simple_rr_set).save()
 
     def test_create_in_full_rr_set(self):
-        with self.assertPdnsFullRRSetUpdate(), PDNSChangeTracker():
-            RR(content=self.ALT_CONTENT_VALUES, rrset=self.full_rr_set).save()
+        for content in self.ALT_CONTENT_VALUES:
+            with self.assertPdnsFullRRSetUpdate(), PDNSChangeTracker():
+                RR(content=content, rrset=self.full_rr_set).save()
 
     def test_create_multiple_in_empty_rr_set(self):
         with self.assertPdnsEmptyRRSetUpdate(), PDNSChangeTracker():
@@ -268,8 +269,8 @@ class TXTRRTestCase(RRTestCase):
     TYPE = 'TXT'
     TTL = 876
     CONTENT_VALUES = ['"The quick brown fox jumps over the lazy dog"',
-                      '"main( ) {printf(\"hello, world\n\");}"',
-                      '“红色联合”对“四·二八兵团”总部大楼的攻击已持续了两天"']
+                      '"main( ) {printf(\\"hello, world\\010\\");}"',
+                      '"“红色联合”对“四·二八兵团”总部大楼的攻击已持续了两天"']
     ALT_CONTENT_VALUES = ['"🧥 👚 👕 👖 👔 👗 👙 👘 👠 👡 👢 👞 👟 🥾 🥿 🧦 🧤 🧣 🎩 🧢 👒 🎓 ⛑ 👑 👝 👛 👜 💼 🎒 👓 🕶 🥽 🥼 🌂 🧵"',
                           '"v=spf1 ip4:192.0.2.0/24 ip4:198.51.100.123 a -all"',
                           '"https://en.wikipedia.org/wiki/Domain_Name_System"']
@@ -278,7 +279,7 @@ class TXTRRTestCase(RRTestCase):
 class RRSetTestCase(PdnsChangeTrackerTestCase):
     TEST_DATA = {
         ('A', '_asdf', 123): ['1.2.3.4', '5.5.5.5'],
-        ('TXT', 'test', 455): ['ASDF', 'foobar', '92847'],
+        ('TXT', 'test', 455): ['"ASDF"', '"foobar"', '"92847"'],
         ('A', 'foo', 1010): ['1.2.3.4', '5.5.4.5'],
         ('AAAA', '*', 100023): ['::1', '::2', '::3', '::4'],
     }

api/desecapi/tests/test_rrsets.py

@@ -1,12 +1,13 @@
 from ipaddress import IPv4Network
 import re
+from itertools import product
 
 from django.conf import settings
 from django.core.exceptions import ValidationError
 from django.core.management import call_command
 from rest_framework import status
 
-from desecapi.models import Domain, RRset
+from desecapi.models import Domain, RRset, RR_SET_TYPES_AUTOMATIC, RR_SET_TYPES_UNSUPPORTED
 from desecapi.tests.base import DesecTestCase, AuthenticatedRRSetBaseTestCase
 
 
@@ -167,11 +168,11 @@ class AuthenticatedRRSetTestCase(AuthenticatedRRSetBaseTestCase):
                 {'subname': subname, 'records': ['10 example.com.'], 'ttl': 60, 'type': 'txt'}
             ] + [
                 {'subname': subname, 'records': ['10 example.com.'], 'ttl': 60, 'type': type_}
-                for type_ in self.DEAD_TYPES
+                for type_ in self.UNSUPPORTED_TYPES
             ] + [
                 {'subname': subname, 'records': ['set.an.example. get.desec.io. 2584 10800 3600 604800 60'],
                  'ttl': 60, 'type': type_}
-                for type_ in self.RESTRICTED_TYPES
+                for type_ in self.AUTOMATIC_TYPES
             ]:
                 response = self.client.post_rr_set(self.my_domain.name, **data)
                 self.assertStatus(response, status.HTTP_400_BAD_REQUEST)
@@ -201,19 +202,42 @@ class AuthenticatedRRSetTestCase(AuthenticatedRRSetBaseTestCase):
         response = self.client.post_rr_set(self.other_domain.name, **data)
         self.assertStatus(response, status.HTTP_404_NOT_FOUND)
 
+    @staticmethod
+    def _create_test_txt_record(record, type_='TXT'):
+        return {'records': [f'{record}'], 'ttl': 3600, 'type': type_, 'subname': f'name{len(record)}'}
+
+    def test_create_my_rr_sets_chunk_too_long(self):
+        for l, t in product([1, 255, 256, 498], ['TXT', 'SPF']):
+            with self.assertPdnsRequests(self.requests_desec_rr_sets_update(self.my_empty_domain.name)):
+                response = self.client.post_rr_set(
+                    self.my_empty_domain.name,
+                    **self._create_test_txt_record(f'"{"A" * l}"', t)
+                )
+                self.assertStatus(response, status.HTTP_201_CREATED)
+            with self.assertPdnsRequests(self.requests_desec_rr_sets_update(self.my_empty_domain.name)):
+                self.client.delete_rr_set(self.my_empty_domain.name, type_=t, subname=f'name{l+2}')
+
     def test_create_my_rr_sets_too_long_content(self):
-        def _create_data(length):
-            content_string = 'A' * (length - 2)  # we have two quotes
-            return {'records': [f'"{content_string}"'], 'ttl': 3600, 'type': 'TXT', 'subname': f'name{length}'}
+        for t in ['SPF', 'TXT']:
+            response = self.client.post_rr_set(
+                self.my_empty_domain.name,
+                # record of wire length 501 bytes in chunks of max 255 each (RFC 4408)
+                **self._create_test_txt_record(f'"{"A" * 255}" "{"A" * 244}"', t)
+            )
+            self.assertStatus(response, status.HTTP_400_BAD_REQUEST)
+            self.assertIn(
+                'Ensure this value has no more than 500 byte in wire format (it has 501).',
+                str(response.data)
+            )
 
         with self.assertPdnsRequests(self.requests_desec_rr_sets_update(self.my_empty_domain.name)):
-            response = self.client.post_rr_set(self.my_empty_domain.name, **_create_data(500))
+            response = self.client.post_rr_set(
+                self.my_empty_domain.name,
+                # record of wire length 500 bytes in chunks of max 255 each (RFC 4408)
+                ** self._create_test_txt_record(f'"{"A" * 255}" "{"A" * 243}"')
+            )
             self.assertStatus(response, status.HTTP_201_CREATED)
 
-        response = self.client.post_rr_set(self.my_empty_domain.name, **_create_data(501))
-        self.assertStatus(response, status.HTTP_400_BAD_REQUEST)
-        self.assertIn('Ensure this field has no more than 500 characters.', str(response.data))
-
     def test_create_my_rr_sets_too_large_rrset(self):
         network = IPv4Network('127.0.0.0/20')  # size: 4096 IP addresses
         data = {'records': [str(ip) for ip in network], 'ttl': 3600, 'type': 'A', 'subname': 'name'}
@@ -232,6 +256,15 @@ class AuthenticatedRRSetTestCase(AuthenticatedRRSetBaseTestCase):
         response = self.client.post_rr_set(self.my_empty_domain.name, **data)
         self.assertStatus(response, status.HTTP_400_BAD_REQUEST)
 
+    def test_create_my_rr_sets_duplicate_content(self):
+        for records in [
+            ['127.0.0.1', '127.00.0.1'],
+            # TODO add more examples
+        ]:
+            data = {'records': records, 'ttl': 3660, 'type': 'A'}
+            response = self.client.post_rr_set(self.my_empty_domain.name, **data)
+            self.assertContains(response, 'Duplicate', status_code=status.HTTP_400_BAD_REQUEST)
+
     def test_create_my_rr_sets_upper_case(self):
         for subname in ['asdF', 'cAse', 'asdf.FOO', '--F', 'ALLCAPS']:
             data = {'records': ['1.2.3.4'], 'ttl': 60, 'type': 'A', 'subname': subname}
@@ -243,13 +276,189 @@ class AuthenticatedRRSetTestCase(AuthenticatedRRSetBaseTestCase):
         response = self.client.post_rr_set(self.my_empty_domain.name)
         self.assertContains(response, 'No data provided', status_code=status.HTTP_400_BAD_REQUEST)
 
+    def test_create_my_rr_sets_canonical_content(self):
+        # TODO fill in more examples
+        datas = [
+            # record type: (non-canonical input, canonical output expectation)
+            ('A', ('127.0.000.1', '127.0.0.1')),
+            ('AAAA', ('0000::0000:0001', '::1')),
+            ('AFSDB', ('02 turquoise.FEMTO.edu.', '2 turquoise.femto.edu.')),
+            ('CAA', ('0128 "issue" "letsencrypt.org"', '128 issue "letsencrypt.org"')),
+            ('CERT', ('06 00 00 sadfdd==', '6 0 0 sadfdQ==')),
+            ('CNAME', ('EXAMPLE.COM.', 'example.com.')),
+            ('DHCID', ('xxxx', 'xxxx')),
+            ('DLV', ('6454 8 2 5CBA665A006F6487625C6218522F09BD3673C25FA10F25CB18459AA1 0DF1F520',
+                     '6454 8 2 5CBA665A006F6487625C6218522F09BD3673C25FA10F25CB18459AA10DF1F520'.lower())),
+            ('DLV', ('6454 8 2 5C BA665A006F6487625C6218522F09BD3673C25FA10F25CB18459AA1 0DF1F520',
+                     '6454 8 2 5CBA665A006F6487625C6218522F09BD3673C25FA10F25CB18459AA10DF1F520'.lower())),
+            ('DS', ('6454 8 2 5CBA665A006F6487625C6218522F09BD3673C25FA10F25CB18459AA1 0DF1F520',
+                    '6454 8 2 5CBA665A006F6487625C6218522F09BD3673C25FA10F25CB18459AA10DF1F520'.lower())),
+            ('DS', ('6454 8 2 5C BA665A006F6487625C6218522F09BD3673C25FA10F25CB18459AA1 0DF1F520',
+                    '6454 8 2 5CBA665A006F6487625C6218522F09BD3673C25FA10F25CB18459AA10DF1F520'.lower())),
+            ('EUI48', ('AA-BB-CC-DD-EE-FF', 'aa-bb-cc-dd-ee-ff')),
+            ('EUI64', ('AA-BB-CC-DD-EE-FF-aa-aa', 'aa-bb-cc-dd-ee-ff-aa-aa')),
+            ('HINFO', ('cpu os', '"cpu" "os"')),
+            ('HINFO', ('"cpu" "os"', '"cpu" "os"')),
+            # ('IPSECKEY', ('01 00 02 . ASDFAA==', '1 0 2 . ASDFAF==')),
+            # ('IPSECKEY', ('01 00 02 . 00000w==', '1 0 2 . 000000==')),
+            ('KX', ('010 example.com.', '10 example.com.')),
+            ('LOC', ('023 012 59 N 042 022 48.500 W 65.00m 20.00m 10.00m 10.00m',
+                     '23 12 59.000 N 42 22 48.500 W 65.00m 20.00m 10.00m 10.00m')),
+            ('MX', ('10 010.1.1.1.', '10 010.1.1.1.')),
+            ('MX', ('010 010.1.1.2.', '10 010.1.1.2.')),
+            ('NAPTR', ('100  50  "s"  "z3950+I2L+I2C"     ""  _z3950._tcp.gatech.edu.',
+                       '100 50 "s" "z3950+I2L+I2C" "" _z3950._tcp.gatech.edu.')),
+            ('NS', ('EXaMPLE.COM.', 'example.com.')),
+            ('OPENPGPKEY', ('mG8EXtVIsRMFK4EEACIDAwQSZPNqE4tS xLFJYhX+uabSgMrhOqUizJhkLx82',
+                            'mG8EXtVIsRMFK4EEACIDAwQSZPNqE4tSxLFJYhX+uabSgMrhOqUizJhkLx82')),
+            ('PTR', ('EXAMPLE.COM.', 'example.com.')),
+            ('RP', ('hostmaster.EXAMPLE.com. .', 'hostmaster.example.com. .')),
+            # ('SMIMEA', ('3 01 0 aaBBccddeeff', '3 1 0 aabbccddeeff')),
+            ('SPF', ('"v=spf1 ip4:10.1" ".1.1 ip4:127" ".0.0.0/16 ip4:192.168.0.0/27 include:example.com -all"',
+                     '"v=spf1 ip4:10.1" ".1.1 ip4:127" ".0.0.0/16 ip4:192.168.0.0/27 include:example.com -all"')),
+            ('SPF', ('"foo" "bar"', '"foo" "bar"')),
+            ('SPF', ('"foobar"', '"foobar"')),
+            ('SRV', ('0 000 0 .', '0 0 0 .')),
+            # ('SRV', ('100 1 5061 EXAMPLE.com.', '100 1 5061 example.com.')),  # TODO fixed in dnspython 5c58601
+            ('SRV', ('100 1 5061 example.com.', '100 1 5061 example.com.')),
+            ('SSHFP', ('2 2 aabbccEEddff', '2 2 aabbcceeddff')),
+            ('TLSA', ('3 0001 1 000AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA', '3 1 1 000aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')),
+            ('TXT', ('"foo" "bar"', '"foo" "bar"')),
+            ('TXT', ('"foobar"', '"foobar"')),
+            ('TXT', ('"foo" "" "bar"', '"foo" "" "bar"')),
+            ('TXT', ('"" "" "foo" "" "bar"', '"" "" "foo" "" "bar"')),
+            ('URI', ('10 01 "ftp://ftp1.example.com/public"', '10 1 "ftp://ftp1.example.com/public"')),
+        ]
+        for t, (record, canonical_record) in datas:
+            if not record:
+                continue
+            data = {'records': [record], 'ttl': 3660, 'type': t, 'subname': ''}
+            with self.assertPdnsRequests(self.requests_desec_rr_sets_update(name=self.my_empty_domain.name)):
+                response = self.client.post_rr_set(self.my_empty_domain.name, **data)
+                self.assertStatus(response, status.HTTP_201_CREATED)
+                self.assertEqual(canonical_record, response.data['records'][0],
+                                 f'For RR set type {t}, expected \'{canonical_record}\' to be the canonical form of '
+                                 f'\'{record}\', but saw \'{response.data["records"][0]}\'.')
+            with self.assertPdnsRequests(self.requests_desec_rr_sets_update(name=self.my_empty_domain.name)):
+                response = self.client.delete_rr_set(self.my_empty_domain.name, subname='', type_=t)
+                self.assertStatus(response, status.HTTP_204_NO_CONTENT)
+        self.assertAllSupportedRRSetTypes(set(t for t, _ in datas))
+
+    def test_create_my_rr_sets_known_type_benign(self):
+        # TODO fill in more examples
+        datas = {
+            'A': ['127.0.0.1', '127.0.0.2'],
+            'AAAA': ['::1', '::2'],
+            'AFSDB': ['2 turquoise.femto.edu.'],
+            'CAA': ['128 issue "letsencrypt.org"', '128 iodef "mailto:desec@example.com"', '1 issue "letsencrypt.org"'],
+            'CERT': ['6 0 0 sadfdd=='],
+            'CNAME': ['example.com.'],
+            'DHCID': ['aaaaaaaaaaaa', 'aa aaa  aaaa a a a'],
+            'DLV': ['39556 13 1 aabbccddeeff'],
+            'DS': ['39556 13 1 aabbccddeeff'],
+            'EUI48': ['aa-bb-cc-dd-ee-ff', 'AA-BB-CC-DD-EE-FF'],
+            'EUI64': ['aa-bb-cc-dd-ee-ff-00-11', 'AA-BB-CC-DD-EE-FF-00-11'],
+            'HINFO': ['"ARMv8-A" "Linux"'],
+            # 'IPSECKEY': ['12 0 2 . asdfdf==', '03 1 1 127.0.00.1 asdfdf==', '12 3 1 example.com. asdfdf==',],
+            'KX': ['4 example.com.', '28 io.'],
+            'LOC': ['23 12 59.000 N 42 22 48.500 W 65.00m 20.00m 10.00m 10.00m'],
+            'MX': ['10 example.com.', '20 1.1.1.1.'],
+            'NAPTR': ['100  50  "s"  "z3950+I2L+I2C"     ""  _z3950._tcp.gatech.edu.'],
+            'NS': ['ns1.example.com.'],
+            'OPENPGPKEY': [
+                'mG8EXtVIsRMFK4EEACIDAwQSZPNqE4tSxLFJYhX+uabSgMrhOqUizJhkLx82',  # key incomplete
+                'YWFh\xf0\x9f\x92\xa9YWFh',  # valid as non-alphabet bytes will be ignored
+            ],
+            'PTR': ['example.com.', '*.example.com.'],
+            'RP': ['hostmaster.example.com. .'],
+            # 'SMIMEA': ['3 1 0 aabbccddeeff'],
+            'SPF': ['"v=spf1 include:example.com ~all"',
+                    '"v=spf1 ip4:10.1.1.1 ip4:127.0.0.0/16 ip4:192.168.0.0/27 include:example.com -all"',
+                    '"spf2.0/pra,mfrom ip6:2001:558:fe14:76:68:87:28:0/120 -all"'],
+            'SRV': ['0 0 0 .', '100 1 5061 example.com.'],
+            'SSHFP': ['2 2 aabbcceeddff'],
+            'TLSA': ['3 1 1 AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA'],
+            'TXT': ['"foobar"', '"foo" "bar"', '"“红色联合”对“四·二八兵团”总部大楼的攻击已持续了两天"', '"new\\010line"'
+                    '"🧥 👚 👕 👖 👔 👗 👙 👘 👠 👡 👢 👞 👟 🥾 🥿  🧦 🧤 🧣 🎩 🧢 👒 🎓 ⛑ 👑 👝 👛 👜 💼 🎒 👓 🕶 🥽 🥼 🌂 🧵"'],
+            'URI': ['10 1 "ftp://ftp1.example.com/public"'],
+        }
+        self.assertAllSupportedRRSetTypes(set(datas.keys()))
+        for t, records in datas.items():
+            for r in records:
+                data = {'records': [r], 'ttl': 3660, 'type': t, 'subname': ''}
+                with self.assertPdnsRequests(self.requests_desec_rr_sets_update(name=self.my_empty_domain.name)):
+                    response = self.client.post_rr_set(self.my_empty_domain.name, **data)
+                    self.assertStatus(response, status.HTTP_201_CREATED)
+                with self.assertPdnsRequests(self.requests_desec_rr_sets_update(name=self.my_empty_domain.name)):
+                    response = self.client.delete_rr_set(self.my_empty_domain.name, subname='', type_=t)
+                    self.assertStatus(response, status.HTTP_204_NO_CONTENT)
+
+    def test_create_my_rr_sets_known_type_invalid(self):
+        # TODO fill in more examples
+        datas = {
+            # recordtype: [list of examples expected to be rejected, individually]
+            'A': ['127.0.0.999', '127.0.0.256', '::1', 'foobar', '10.0.1', '10!'],
+            'AAAA': ['::g', '1:1:1:1:1:1:1:1:', '1:1:1:1:1:1:1:1:1'],
+            'AFSDB': ['example.com.', '1 1', '1 de'],
+            'CAA': ['43235 issue "letsencrypt.org"'],
+            'CERT': ['6 0 sadfdd=='],
+            'CNAME': ['example.com', '10 example.com.'],
+            'DHCID': ['x', 'xx', 'xxx'],
+            'DLV': ['-34 13 1 aabbccddeeff'],
+            'DS': ['-34 13 1 aabbccddeeff'],
+            'EUI48': ['aa-bb-ccdd-ee-ff', 'AA-BB-CC-DD-EE-GG'],
+            'EUI64': ['aa-bb-cc-dd-ee-ff-gg-11', 'AA-BB-C C-DD-EE-FF-00-11'],
+            'HINFO': ['"ARMv8-A"', f'"a" "{"b"*256}"'],
+            # 'IPSECKEY': [],
+            'KX': ['-1 example.com', '10 example.com'],
+            'LOC': ['23 12 61.000 N 42 22 48.500 W 65.00m 20.00m 10.00m 10.00m', 'foo', '1.1.1.1'],
+            'MX': ['10 example.com', 'example.com.', '-5 asdf.', '65537 asdf.'],
+            'NAPTR': ['100  50  "s"  "z3950+I2L+I2C"     ""  _z3950._tcp.gatech.edu',
+                      '100  50  "s"     ""  _z3950._tcp.gatech.edu.',
+                      '100  50  3 2  "z3950+I2L+I2C"     ""  _z3950._tcp.gatech.edu.'],
+            'NS': ['ns1.example.com', '127.0.0.1'],
+            'OPENPGPKEY': ['1 2 3'],
+            'PTR': ['"example.com."', '10 *.example.com.'],
+            'RP': ['hostmaster.example.com.', '10 foo.'],
+            # 'SMIMEA': ['3 1 0 aGVsbG8gd29ybGQh'],
+            'SPF': ['"v=spf1', 'v=spf1 include:example.com ~all'],
+            'SRV': ['0 0 0 0', '100 5061 example.com.'],
+            'SSHFP': ['aabbcceeddff'],
+            'TLSA': ['3 1 1 AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA'],
+            'TXT': ['foob"ar', 'v=spf1 include:example.com ~all', '"foo\nbar"', '"\x00" "NUL byte yo"'],
+            'URI': ['"1" "2" "3"'],
+        }
+        self.assertAllSupportedRRSetTypes(set(datas.keys()))
+        for t, records in datas.items():
+            for r in records:
+                data = {'records': [r], 'ttl': 3660, 'type': t, 'subname': ''}
+                response = self.client.post_rr_set(self.my_empty_domain.name, **data)
+                self.assertNotContains(response, 'Duplicate', status_code=status.HTTP_400_BAD_REQUEST)
+
+    def test_create_my_rr_sets_txt_splitting(self):
+        for t in ['TXT', 'SPF']:
+            for l in [200, 255, 256, 300, 400]:
+                data = {'records': [f'"{"a"*l}"'], 'ttl': 3660, 'type': t, 'subname': f'x{l}'}
+                with self.assertPdnsRequests(self.requests_desec_rr_sets_update(name=self.my_empty_domain.name)):
+                    response = self.client.post_rr_set(self.my_empty_domain.name, **data)
+                    self.assertStatus(response, status.HTTP_201_CREATED)
+                response = self.client.get_rr_set(self.my_empty_domain.name, f'x{l}', t)
+                num_tokens = response.data['records'][0].count(' ') + 1
+                num_tokens_expected = l // 256 + 1
+                self.assertEqual(num_tokens, num_tokens_expected,
+                                 f'For a {t} record with a token of length of {l}, expected to see '
+                                 f'{num_tokens_expected} tokens in the canonical format, but saw {num_tokens}.')
+                self.assertEqual("".join(r.strip('" ') for r in response.data['records'][0]), 'a'*l)
+
     def test_create_my_rr_sets_unknown_type(self):
-        for _type in ['AA', 'ASDF']:
-            with self.assertPdnsRequests(
-                    self.request_pdns_zone_update_unknown_type(name=self.my_domain.name, unknown_types=_type)
-            ):
-                response = self.client.post_rr_set(self.my_domain.name, records=['1234'], ttl=3660, type=_type)
-                self.assertStatus(response, status.HTTP_400_BAD_REQUEST)
+        for _type in ['AA', 'ASDF'] + list(RR_SET_TYPES_AUTOMATIC | RR_SET_TYPES_UNSUPPORTED):
+            response = self.client.post_rr_set(self.my_domain.name, records=['1234'], ttl=3660, type=_type)
+            self.assertContains(
+                response,
+                text='managed automatically' if _type in RR_SET_TYPES_AUTOMATIC else 'type is currently unsupported',
+                status_code=status.HTTP_400_BAD_REQUEST
+            )
+
 
     def test_create_my_rr_sets_insufficient_ttl(self):
         ttl = settings.MINIMUM_TTL_DEFAULT - 1
@@ -270,7 +479,7 @@ class AuthenticatedRRSetTestCase(AuthenticatedRRSetBaseTestCase):
         self.assertEqual(response.data['ttl'], 3620)
 
     def test_retrieve_my_rr_sets_restricted_types(self):
-        for type_ in self.RESTRICTED_TYPES:
+        for type_ in self.AUTOMATIC_TYPES:
             response = self.client.get_rr_sets(self.my_domain.name, type=type_)
             self.assertStatus(response, status.HTTP_403_FORBIDDEN)
             response = self.client.get_rr_sets(self.my_domain.name, type=type_, subname='')

api/desecapi/tests/test_rrsets_bulk.py

@@ -136,8 +136,8 @@ class AuthenticatedRRSetBulkTestCase(AuthenticatedRRSetBaseTestCase):
                 {},
                 {'ttl': ['This field is required.']},
                 {'records': ['This field is required.']},
-                {'type': ['You cannot tinker with the SOA RRset.']},
-                {'type': ['You cannot tinker with the OPT RRset.']},
+                {'type': ['You cannot tinker with the SOA RR set. It is managed automatically.']},
+                {'type': ['You cannot tinker with the OPT RR set. It is managed automatically.']},
                 {'type': ['Generic type format is not supported.']},
             ]
         )

api/desecapi/views.py

@@ -200,7 +200,7 @@ class RRsetList(EmptyPayloadMixin, DomainViewMixin, generics.ListCreateAPIView,
 
             if value is not None:
                 # TODO consider moving this
-                if filter_field == 'type' and value in models.RRset.RESTRICTED_TYPES:
+                if filter_field == 'type' and value in models.RR_SET_TYPES_AUTOMATIC:
                     raise PermissionDenied("You cannot tinker with the %s RRset." % value)
 
                 rrsets = rrsets.filter(**{'%s__exact' % filter_field: value})

docs/dns/rrsets.rst

@@ -90,6 +90,12 @@ Field details:
     The maximum number of array elements is 4091, and the maximum length of
     the array is 64,000 (after JSON encoding).
 
+    Records must be given in presentation format (a.k.a. "BIND" or zone file
+    format). Record values that are not given in canonical form, such as
+    ``0:0000::1`` will be converted by the API into canonical form, e.g.
+    ``::1``. Exact validation and canonicalization depend on the record
+    type.
+
 ``subname``
     :Access mode: read, write-once (upon RRset creation)
 

test/e2e/spec/api_spec.js

@@ -480,8 +480,8 @@ describe("API v1", function () {
                             { subname: [ 'This field is required.' ] },
                             { ttl: [ 'This field is required.' ] },
                             { records: [ 'This field is required.' ] },
-                            { type: [ 'You cannot tinker with the SOA RRset.' ] },
-                            { type: [ 'You cannot tinker with the OPT RRset.' ] },
+                            { type: [ 'You cannot tinker with the SOA RR set. It is managed automatically.' ] },
+                            { type: [ 'You cannot tinker with the OPT RR set. It is managed automatically.' ] },
                             { type: [ 'Generic type format is not supported.' ] },
                         ]);
 

test/e2e2/README.md

@@ -0,0 +1,28 @@
+# End-to-End Tests
+
+A collection of tests against the stack written in python and pytest.
+
+## Running
+
+The tests can be run from the **CLI** using
+
+    docker-compose -f docker-compose.yml -f docker-compose.test-e2e2.yml build test-e2e2
+    docker-compose -f docker-compose.yml -f docker-compose.test-e2e2.yml up test-e2e2
+
+If you had any changes to other containers (say `api`), then also rebuild them.
+
+To run the test in **pycharm**, make sure that pytest is installed in the Python environment that pycharm is using.
+Then add the docker-compose environment to your pycharm build configuration. Note that you need to update the pycharm
+environment configuration when you update the corresponding variables in your `.env`.
+
+When run from pycharm, the tests are not run inside a docker container and have no access to any self-signed 
+certificates that www may be using. Hence, self-signed certificates cannot be verified when running the tests in 
+pycharm. Make sure to use publicly valid certificate!
+
+Note that tests running in pycharm currently have no access to the nslord dns query interface, hence some tests will
+fail.
+
+## Troubleshooting
+
+**KeyError: 'content'.** If the content field of captchas is missing, make sure you started the API using the 
+configuration of `docker-compose.test-e2e2.yml`.

test/e2e2/conftest.py

@@ -2,10 +2,45 @@ import json
 import os
 import random
 import string
-from typing import Optional, Tuple
+from json import JSONDecodeError
+from typing import Optional, Tuple, Iterable, Callable
 
+import dns
+import dns.name
+import dns.query
 import pytest
 import requests
+from requests.exceptions import SSLError
+
+
+@pytest.fixture()
+def random_email() -> Callable[[], str]:
+    return lambda: (
+        "".join(random.choice(string.ascii_letters) for _ in range(10))
+        + "@desec.test"
+    )
+
+
+@pytest.fixture()
+def random_password() -> Callable[[], str]:
+    return lambda: "".join(random.choice(string.ascii_letters) for _ in range(16))
+
+
+@pytest.fixture()
+def random_domainname() -> Callable[[], str]:
+    return lambda: (
+        "".join(random.choice(string.ascii_lowercase) for _ in range(16))
+        + ".test"
+    )
+
+
+@pytest.fixture()
+def random_local_public_suffix_domainname() -> Callable[[], str]:
+    return lambda: (
+        "".join(random.choice(string.ascii_lowercase) for _ in range(16))
+        + ".dedyn."
+        + os.environ['DESECSTACK_DOMAIN']
+    )
 
 
 class DeSECAPIV1Client:
@@ -16,41 +51,72 @@ class DeSECAPIV1Client:
         "User-Agent": "e2e2",
     }
 
-    @staticmethod
-    def random_email() -> str:
-        return (
-            "".join(random.choice(string.ascii_letters) for _ in range(10))
-            + "@desec.test"
-        )
-
-    @staticmethod
-    def random_password() -> str:
-        return "".join(random.choice(string.ascii_letters) for _ in range(16))
-
     def __init__(self) -> None:
         super().__init__()
         self.email = None
         self.password = None
+        self.domains = []
+        self.verify = True
+
+        # We support two certificate verification methods
+        # (1) against self-signed certificates, if /autocert path is present
+        # (this is usually the case when run inside a docker container)
+        # (2) against the default certificate store, if /autocert is not available
+        # (this is usually the case when run outside a docker container)
+        self.verify = True
+        self.verify_alt = f'/autocert/desec.{os.environ["DESECSTACK_DOMAIN"]}.cer'
+
+    @staticmethod
+    def _filter_response_output(output: dict) -> dict:
+        try:
+            output['challenge'] = output['challenge'][:10] + '...'
+        except (KeyError, TypeError):
+            pass
+        return output
+
+    def _do_request(self, *args, **kwargs):
+        try:
+            return requests.request(*args, **kwargs, verify=self.verify)
+        except SSLError:
+            self.verify, self.verify_alt = self.verify_alt, self.verify
+            return requests.request(*args, **kwargs, verify=self.verify)  # if problem persists, let it raise
 
     def _request(self, method: str, *, path: str, data: Optional[dict] = None, **kwargs) -> requests.Response:
         if data is not None:
             data = json.dumps(data)
 
-        return requests.request(
+        url = self.base_url + path
+
+        print(f"API >>> {method} {url}")
+        if data:
+            print(f"API >>> {type(data)}: {data}")
+
+        response = self._do_request(
             method,
-            self.base_url + path,
+            url,
             data=data,
             headers=self.headers,
-            verify=f'/autocert/desec.{os.environ["DESECSTACK_DOMAIN"]}.cer',
             **kwargs,
         )
 
+        print(f"API <<< {response.status_code}")
+        if response.text:
+            try:
+                print(f"API <<< {self._filter_response_output(response.json())}")
+            except JSONDecodeError:
+                print(f"API <<< {response.text}")
+
+        return response
+
     def get(self, path: str, **kwargs) -> requests.Response:
         return self._request("GET", path=path, **kwargs)
 
     def post(self, path: str, data: Optional[dict] = None, **kwargs) -> requests.Response:
         return self._request("POST", path=path, data=data, **kwargs)
 
+    def delete(self, path: str, **kwargs) -> requests.Response:
+        return self._request("DELETE", path=path, **kwargs)
+
     def register(self, email: str, password: str) -> Tuple[requests.Response, requests.Response]:
         self.email = email
         self.password = password
@@ -76,6 +142,34 @@ class DeSECAPIV1Client:
         self.headers["Authorization"] = f'Token {token.json()["token"]}'
         return token
 
+    def domain_list(self) -> requests.Response:
+        return self.get("/domains/")
+
+    def domain_create(self, name) -> requests.Response:
+        self.domains.append(name)
+        return self.post(
+            "/domains/",
+            data={
+                "name": name,
+            }
+        )
+
+    def domain_destroy(self, name) -> requests.Response:
+        self.domains.remove(name)
+        return self.delete(f"/domains/{name}/")
+
+    def rr_set_create(self, domain_name: str, rr_type: str, records: Iterable[str], subname: str = '',
+                      ttl: int = 3600) -> requests.Response:
+        return self.post(
+            f"/domains/{domain_name}/rrsets/",
+            data={
+                "subname": subname,
+                "type": rr_type,
+                "ttl": ttl,
+                "records": records,
+            }
+        )
+
 
 @pytest.fixture
 def api_anon() -> DeSECAPIV1Client:
@@ -86,14 +180,55 @@ def api_anon() -> DeSECAPIV1Client:
 
 
 @pytest.fixture()
-def api_user(api_anon) -> DeSECAPIV1Client:
+def api_user(api_anon, random_email, random_password) -> DeSECAPIV1Client:
     """
     Access to the API with a fresh user account (zero domains, one token). Authorization header
     is preconfigured, email address and password are randomly chosen.
     """
     api = api_anon
-    email = api.random_email()
-    password = api.random_password()
+    email = random_email()
+    password = random_password()
     api.register(email, password)
     api.login(email, password)
     return api
+
+
+@pytest.fixture()
+def api_user_domain(api_user, random_domainname) -> DeSECAPIV1Client:
+    """
+    Access to the API with a fresh user account that owns a domain with random name. The domain has
+    no records other than the default ones.
+    """
+    api_user.domain_create(random_domainname())
+    return api_user
+
+
+class NSClient:
+    where = None
+
+    def query(self, qname: str, qtype: str):
+        print(f'DNS >>> {qname}/{qtype} @{self.where}')
+        qname = dns.name.from_text(qname)
+        qtype = dns.rdatatype.from_text(qtype)
+        answer = dns.query.tcp(
+            q=dns.message.make_query(qname, qtype),
+            where=self.where,
+            timeout=2
+        )
+        try:
+            section = dns.message.AUTHORITY if qtype == dns.rdatatype.from_text('NS') else dns.message.ANSWER
+            response = answer.find_rrset(section, qname, dns.rdataclass.IN, qtype)
+            print(f'DNS <<< {response}')
+            return {i.to_text() for i in response.items}
+        except KeyError:
+            print('DNS <<< !!! not found !!! Complete Answer below:\n' + answer.to_text())
+            return {}
+
+
+class NSLordClient(NSClient):
+    where = os.environ["DESECSTACK_IPV4_REAR_PREFIX16"] + '.0.129'
+
+
+@pytest.fixture()
+def ns_lord() -> NSLordClient:
+    return NSLordClient()

test/e2e2/requirements.txt

@@ -1,2 +1,4 @@
 pytest
+pytest-xdist
 requests
+dnspython

test/e2e2/spec/test_api_domains.py

@@ -0,0 +1,11 @@
+from conftest import DeSECAPIV1Client
+
+
+def test_create(api_user: DeSECAPIV1Client, random_domainname):
+    assert api_user.domain_create(random_domainname()).status_code == 201
+
+
+def test_destroy(api_user_domain: DeSECAPIV1Client):
+    n = len(api_user_domain.domain_list().json())
+    assert api_user_domain.domain_destroy(api_user_domain.domains[0]).status_code == 204
+    assert len(api_user_domain.domain_list().json()) == n - 1

test/e2e2/spec/test_api_rr_validation.py

@@ -0,0 +1,182 @@
+from time import sleep
+from typing import List, Tuple
+
+import pytest
+
+from conftest import DeSECAPIV1Client, NSClient
+
+
+def generate_params(dict_value_lists_by_type: dict) -> List[Tuple[str, str]]:
+    return [
+        (rr_type, value)
+        for rr_type in dict_value_lists_by_type.keys()
+        for value in dict_value_lists_by_type[rr_type]
+    ]
+
+
+VALID_RECORDS_CANONICAL = {
+    'A': ['127.0.0.1', '127.0.0.2'],
+    'AAAA': ['::1', '::2'],
+    'AFSDB': ['2 turquoise.femto.edu.'],
+    'CAA': [
+        '128 issue "letsencrypt.org"', '128 iodef "mailto:desec@example.com"',
+        '1 issue "letsencrypt.org"'
+    ],
+    'CERT': ['6 0 0 sadfdQ=='],
+    'CNAME': ['example.com.'],
+    'DHCID': ['aaaaaaaaaaaa', 'xxxx'],
+    'DLV': [
+        '39556 13 1 aabbccddeeff',
+    ],
+    'DS': [
+        '39556 13 1 aabbccddeeff',
+    ],
+    'EUI48': ['aa-bb-cc-dd-ee-ff'],
+    'EUI64': ['aa-bb-cc-dd-ee-ff-00-11'],
+    'HINFO': ['"ARMv8-A" "Linux"'],
+    # 'IPSECKEY': ['12 0 2 . asdfdf==', '03 1 1 127.0.00.1 asdfdf==', '12 3 1 example.com. asdfdf==',],
+    'KX': ['4 example.com.', '28 io.'],
+    'LOC': [
+        '23 12 59.000 N 42 22 48.500 W 65.00m 20.00m 10.00m 10.00m',
+    ],
+    'MX': ['10 example.com.', '20 1.1.1.1.'],
+    'NAPTR': [
+        '100 50 "s" "z3950+I2L+I2C" "" _z3950._tcp.gatech.edu.',
+    ],
+    'NS': ['ns1.example.com.'],
+    'OPENPGPKEY': [
+        'mG8EXtVIsRMFK4EEACIDAwQSZPNqE4tS xLFJYhX+uabSgMrhOqUizJhkLx82',  # key incomplete due to 500 byte limit
+    ],
+    'PTR': ['example.com.', '*.example.com.'],
+    'RP': ['hostmaster.example.com. .'],
+    # 'SMIMEA': ['3 1 0 aabbccddeeff'],
+    'SPF': [
+        '"v=spf1 ip4:10.1" ".1.1 ip4:127" ".0.0.0/16 ip4:192.168.0.0/27 include:example.com -all"',
+        '"v=spf1 include:example.com ~all"',
+        '"v=spf1 ip4:10.1.1.1 ip4:127.0.0.0/16 ip4:192.168.0.0/27 include:example.com -all"',
+        '"spf2.0/pra,mfrom ip6:2001:558:fe14:76:68:87:28:0/120 -all"',
+    ],
+    'SRV': ['0 0 0 .', '100 1 5061 example.com.'],
+    'SSHFP': ['2 2 aabbcceeddff'],
+    'TLSA': ['3 1 1 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',],
+    'TXT': [
+        '"foobar"',
+        '"foo" "bar"',
+        '"foo" "" "bar"',
+        '"" "" "foo" "" "bar"',
+        '"new\\010line"',
+        f'"{"a" * 255}" "{"a" * 243}"',  # 500 byte total wire length
+    ],
+    'URI': ['10 1 "ftp://ftp1.example.com/public"'],
+}
+
+
+VALID_RECORDS_NON_CANONICAL = {
+    'A': ['127.0.000.3'],
+    'AAAA': ['0000::0000:0003'],
+    'AFSDB': ['03 turquoise.FEMTO.edu.'],
+    'CAA': ['0128 "issue" "letsencrypt.org"'],
+    'CERT': ['06 00 00 sadfee=='],
+    'CNAME': ['EXAMPLE.TEST.'],
+    'DHCID': ['aa aaa  aaaa a a a', 'xxxx'],
+    'DLV': [
+        '6454 8 2 5CBA665A006F6487625C6218522F09BD3673C25FA10F25CB18459AA1 0DF1F520',
+        '6454 8 2 5C BA665A006F6487625C6218522F09BD3673C25FA10F25CB18459AA1 0DF1F520',
+    ],
+    'DS': [
+        '6454 8 2 5CBA665A006F6487625C6218522F09BD3673C25FA10F25CB18459AA1 0DF1F520',
+        '6454 8 2 5C BA665A006F6487625C6218522F09BD3673C25FA10F25CB18459AA1 0DF1F520',
+    ],
+    'EUI48': ['AA-BB-CC-DD-EE-F1'],
+    'EUI64': ['AA-BB-CC-DD-EE-FF-00-12'],
+    'HINFO': ['cpu os'],
+    # 'IPSECKEY': ['12 0 2 . asdfdf==', '03 1 1 127.0.00.1 asdfdf==', '12 3 1 example.com. asdfdf==',],
+    'KX': ['012 example.TEST.'],
+    'LOC': [
+        '023 012 59 N 042 022 48.500 W 65.00m 20.00m 10.00m 10.00m',
+    ],
+    'MX': ['10 010.1.1.1.'],
+    'NAPTR': [
+        '100  50  "s"  "z3950+I2L+I2C"     ""  _z3950._tcp.gatech.edu.',
+    ],
+    'NS': ['EXaMPLE.COM.'],
+    'OPENPGPKEY': [
+        'mG8EXtVIsRMFK4EEAC==',
+        'mG8EXtVIsRMFK4EEACIDAwQSZPNqE4tSxLFJYhX+uabSgMrhOqUizJhkLx82',  # key incomplete due to 500 byte limit
+    ],
+    'PTR': ['EXAMPLE.TEST.'],
+    'RP': ['hostmaster.EXAMPLE.com. .'],
+    # 'SMIMEA': ['3 01 0 aabbccDDeeff'],
+    'SPF': [],
+    'SRV': ['100 01 5061 example.com.'],
+    'SSHFP': ['02 2 aabbcceeddff'],
+    'TLSA': ['3 0001 1 000AAAAAAABBAAAAAAAAAAAAAAAAAAAAAAAA',],
+    'TXT': [
+        f'"{"a" * 498}"',
+        '"🧥 👚 👕 👖 👔 👗 👙 👘 👠 👡 👢 👞 👟 🥾 🥿  🧦 🧤 🧣 🎩 🧢 👒 🎓 ⛑ 👑 👝 👛 👜 💼 🎒 "',
+        '"🧥 👚 👕 👖 👔 👗 👙 👘 👠 👡 👢 👞 👟 🥾 🥿  🧦 🧤 🧣 🎩 🧢 👒 🎓 ⛑ 👑 👝 👛 👜 💼 🎒 👓 🕶 🥽 🥼 🌂 🧵"',
+    ],
+    'URI': ['10 01 "ftp://ftp1.example.test/public"',],
+}
+
+
+INVALID_RECORDS = {
+    'A': ['127.0.0.999', '127.0.0.256', '::1', 'foobar', '10.0.1', '10!'],
+    'AAAA': ['::g', '1:1:1:1:1:1:1:1:', '1:1:1:1:1:1:1:1:1'],
+    'AFSDB': ['example.com.', '1 1', '1 de'],
+    'CAA': ['43235 issue "letsencrypt.org"'],
+    'CERT': ['6 0 sadfdd=='],
+    'CNAME': ['example.com', '10 example.com.'],
+    'DHCID': ['x', 'xx', 'xxx'],
+    'DLV': ['-34 13 1 aabbccddeeff'],
+    'DS': ['-34 13 1 aabbccddeeff'],
+    'EUI48': ['aa-bb-ccdd-ee-ff', 'AA-BB-CC-DD-EE-GG'],
+    'EUI64': ['aa-bb-cc-dd-ee-ff-gg-11', 'AA-BB-C C-DD-EE-FF-00-11'],
+    'HINFO': ['"ARMv8-A"', f'"a" "{"b" * 256}"'],
+    # 'IPSECKEY': [],
+    'KX': ['-1 example.com', '10 example.com'],
+    'LOC': ['23 12 61.000 N 42 22 48.500 W 65.00m 20.00m 10.00m 10.00m', 'foo', '1.1.1.1'],
+    'MX': ['10 example.com', 'example.com.', '-5 asdf.', '65537 asdf.'],
+    'NAPTR': ['100  50  "s"  "z3950+I2L+I2C"     ""  _z3950._tcp.gatech.edu',
+              '100  50  "s"     ""  _z3950._tcp.gatech.edu.',
+              '100  50  3 2  "z3950+I2L+I2C"     ""  _z3950._tcp.gatech.edu.'],
+    'NS': ['ns1.example.com', '127.0.0.1'],
+    'OPENPGPKEY': ['1 2 3'],
+    'PTR': ['"example.com."', '10 *.example.com.'],
+    'RP': ['hostmaster.example.com.', '10 foo.'],
+    # 'SMIMEA': ['3 1 0 aGVsbG8gd29ybGQh', 'x 0 0 aabbccddeeff'],
+    'SPF': ['"v=spf1', 'v=spf1 include:example.com ~all'],
+    'SRV': ['0 0 0 0', '100 5061 example.com.'],
+    'SSHFP': ['aabbcceeddff'],
+    'TLSA': ['3 1 1 AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA'],
+    'TXT': [
+        'foob"ar',
+        'v=spf1 include:example.com ~all',
+        '"foo\nbar"',
+        '"' + 124 * '🧥' + '==="',  # 501 byte total length
+        '"\x00" "NUL byte yo"',
+    ],
+    'URI': ['"1" "2" "3"'],
+}
+INVALID_RECORDS_PARAMS = [(rr_type, value) for rr_type in INVALID_RECORDS.keys() for value in INVALID_RECORDS[rr_type]]
+
+
+def test_soundness():
+    assert INVALID_RECORDS.keys() == VALID_RECORDS_CANONICAL.keys() == VALID_RECORDS_NON_CANONICAL.keys()
+
+
+@pytest.mark.parametrize("rr_type,value", generate_params(VALID_RECORDS_CANONICAL))
+def test_create_valid_canonical(api_user_domain: DeSECAPIV1Client, ns_lord: NSClient, rr_type: str, value: str):
+    assert api_user_domain.rr_set_create(api_user_domain.domains[0], rr_type, [value], subname="a").status_code == 201
+    assert ns_lord.query(f"a.{api_user_domain.domains[0]}", rr_type) == {value}
+
+
+@pytest.mark.parametrize("rr_type,value", generate_params(VALID_RECORDS_NON_CANONICAL))
+def test_create_valid_non_canonical(api_user_domain: DeSECAPIV1Client, ns_lord: NSClient, rr_type: str, value: str):
+    assert api_user_domain.rr_set_create(api_user_domain.domains[0], rr_type, [value], subname="a").status_code == 201
+    assert len(ns_lord.query(f"a.{api_user_domain.domains[0]}", rr_type)) == 1
+
+
+@pytest.mark.parametrize("rr_type,value", INVALID_RECORDS_PARAMS)
+def test_create_invalid(api_user_domain: DeSECAPIV1Client, rr_type: str, value: str):
+    assert api_user_domain.rr_set_create(api_user_domain.domains[0], rr_type, [value]).status_code == 400