fix(api): lift 500 byte length restriction for RR.content, fixes #334

api/desecapi/models.py

@@ -684,15 +684,8 @@ class RR(ExportModelOperationsMixin('RR'), models.Model):
             relativize=False
         ).to_digestable()
 
-        # The pdns lmdb backend used on our frontends does not only store the record contents itself, but other metadata
-        # (such as type etc.) Both together have to fit into the lmdb backend's current total limit of 512 bytes per RR.
-        # I found the additional data to be 12 bytes (by trial and error). I believe these are the 12 bytes mentioned
-        # here: https://lists.isc.org/pipermail/bind-users/2008-April/070137.html So we can use 500 bytes for the actual
-        # content stored in wire format.
-        # This check can be relaxed as soon as lmdb supports larger records,
-        # cf. https://github.com/desec-io/desec-slave/issues/34 and https://github.com/PowerDNS/pdns/issues/8012
-        if len(wire) > 500:
-            raise ValidationError(f'Ensure this value has no more than 500 byte in wire format (it has {len(wire)}).')
+        if len(wire) > 64000:
+            raise ValidationError(f'Ensure this value has no more than 64000 byte in wire format (it has {len(wire)}).')
 
         parser = dns.wire.Parser(wire, current=0)
         with parser.restrict_to(len(wire)):

api/desecapi/tests/test_rrsets.py

@@ -1,6 +1,7 @@
 from ipaddress import IPv4Network
 import re
 from itertools import product
+from math import ceil, floor
 
 from django.conf import settings
 from django.core.exceptions import ValidationError
@@ -238,15 +239,35 @@ class AuthenticatedRRSetTestCase(AuthenticatedRRSetBaseTestCase):
                 self.client.delete_rr_set(self.my_empty_domain.name, type_=t, subname=f'name{l+2}')
 
     def test_create_my_rr_sets_too_long_content(self):
+        def token(length):
+            if length == 0:
+                return ''
+            if length <= 255:
+                return f'"{"A" * length}"'
+            return f'{token(255)} ' * (length // 255) + token(length % 255)
+
+        def p2w_length(length):
+            return ceil(length / 255 * 256)
+
+        def w2p_length(length):
+            return floor(length / 256 * 255)
+
+        max_wirelength = 64000
+        max_preslength = w2p_length(max_wirelength)
+
+        assert max_preslength == 63750
+        assert p2w_length(max_preslength) == 64000
+        assert p2w_length(max_preslength + 1) == 64002
+
         for t in ['SPF', 'TXT']:
             response = self.client.post_rr_set(
                 self.my_empty_domain.name,
                 # record of wire length 501 bytes in chunks of max 255 each (RFC 4408)
-                **self._create_test_txt_record(f'"{"A" * 255}" "{"A" * 244}"', t)
+                **self._create_test_txt_record(token(max_preslength + 1), t)
             )
             self.assertStatus(response, status.HTTP_400_BAD_REQUEST)
             self.assertIn(
-                'Ensure this value has no more than 500 byte in wire format (it has 501).',
+                f'Ensure this value has no more than {max_wirelength} byte in wire format (it has {p2w_length(max_preslength + 1)}).',
                 str(response.data)
             )
 
@@ -254,7 +275,7 @@ class AuthenticatedRRSetTestCase(AuthenticatedRRSetBaseTestCase):
             response = self.client.post_rr_set(
                 self.my_empty_domain.name,
                 # record of wire length 500 bytes in chunks of max 255 each (RFC 4408)
-                ** self._create_test_txt_record(f'"{"A" * 255}" "{"A" * 243}"')
+                **self._create_test_txt_record(token(max_preslength))
             )
             self.assertStatus(response, status.HTTP_201_CREATED)
 

test/e2e2/spec/test_api_rr_validation.py

@@ -1,4 +1,3 @@
-from time import sleep
 from typing import List, Tuple
 
 import pytest
@@ -45,7 +44,56 @@ VALID_RECORDS_CANONICAL = {
     ],
     'NS': ['ns1.example.com.'],
     'OPENPGPKEY': [
-        'mG8EXtVIsRMFK4EEACIDAwQSZPNqE4tSxLFJYhX+uabSgMrhOqUizJhkLx82',  # key incomplete due to 500 byte limit
+        'mQINBF3yev8BEADR9GxB6OJ5AJlXBWc3nWyWZ+yNNVBiy73XjgOs0uowbxph'
+        'dIw6l75M6xw3i9xAlcjAGG2710FJaye7EZHot3RTIgHpn4FrErQSpNPuJKjD'
+        'IedZZ4av5SRtz5FfnXhNkQGs7jAVi6FmjR9/0GWMxj0BdbcOmeePCUfIIH7T'
+        'ujQJ2c3XHOu/kZ1h4zsFVSslcLEi4KXy0I52pEz0E2CyJrxCLdBd7uU7wDCg'
+        'G8KrIP3UJ5EtukP/LMq4D1eZ4FmtVqzkuDYlJJo70XQytEK9UqDdaDvlUeS5'
+        'FrVj4Zf7OaC5YcSvQemVV4VYSBgJIPb+iFY21/1mXAxyYaunqaR0j5qNaMjr'
+        'E2g3ADRxJiLExhhzlqwJU8+Lc+0QajF/s3lc+dB5usSPqGk6Eb4hBEMaqQvg'
+        '5I0W8pFtHINYipNW5xGSrsX0pyWVai6EkoTXfjbBMC7khwmwsycJ8pYj3ipe'
+        'aNQuUP+XXqJKepoVOY2475Z7YT1NRRbGGEp743mbqKo4SnEKxS2kApo1UPd1'
+        'FbI50TZ62Vsv4tne3bR25eCycjdvIOp6zPm/Pf9LFVm5KF8Wd2U3vRi/uo4v'
+        'HPUK1RoIzjmirp3XUBGBgHd/mhlOADPWB9dE96eXK4yEHlbfomfFiKAisHDc'
+        'vUa0E/UbklYBhJjdWBaw1fDDyiSxsBCTsq4ObQARAQABtBFzdXBwb3J0QHBv'
+        'c3Rlby5kZYkCVAQTAQgAPhYhBJZxyBhcZRmrtOitn6TrgtJXP3x3BQJd8nr/'
+        'AhsDBQkDw7iABQsJCAcCBhUKCQgLAgQWAgMBAh4BAheAAAoJEKTrgtJXP3x3'
+        '+UIP/jpw6Nkp5hLbXxpPRSL2TyyWDfEHPKkBQfU+jnAUIN+WgAV27HpOa+vZ'
+        '/hmTKOG6SlTOxHWACmDiUVfhLOYMV8QPDD3yPFCZWo4UxBKPZaai6GQwr44u'
+        'zCcU+E6AdFnb2nbzYSgACrErU5o5JoU2lPgleMI3FYsG8wb/kQAD7XGDX+Ev'
+        'tAbAQGK5EgevycJzot/hsR/S6EM/l0VsW74DIje3fbp3gaJY2fUG9fTdQu7a'
+        'gj6f9HuZAvXHIuSFeA/kwhUWuZfTcct8PV78gwQB4d6AOFMzoxLaFQAzxuTR'
+        '60kZxsyyi4U5km6D/XzI9rTd228PD8xkGr/2Kx1YRU0ixZnohv9xNc4GP/69'
+        'GNWbbOZcyJcSL+kvych+ddbP5VjHea+b4vT35KV++PMndj+78BE1u5sdqWir'
+        'X9pi09go7SW1BlaJsMHrkR0P8yFCaFWLyCmIC7C/KcSuHVwcjVYWHynLq6CK'
+        'kkv4r8BNM/QFzPCeozXjMk7zq9TkJjLVxsUVNcZaNqzlWO0JzCfE6ICpHhyI'
+        'g/1bO/VJQyk+6llyX1LwRKCeKQCp6KcLx4qnjgZ8g1ArNvazNot9fAssgAUz'
+        'yoyOBF1SYJxWnzu9GE1F47zU1iD6FB8mjspvE00voDs8t2e+xtZoqsM12WtC'
+        '8R4VbCY0LmTPGiWyxD9y7TnUlDfHuQINBF3yev8BEAC4dyN2BPiHCmwtKV/3'
+        '9ZUMVCjb39wnsAA8CH7WAAM5j+k8/uXKUmTcFoZ7+9ya6PZCLXbPC64FIAwl'
+        'YalzCEP5Jx25Ct/DPhVJPIFWHMOYbyUbLJ8tlC1vnnDhd8czeGmozkuyofMh'
+        '39QzR3SLzOqucJO3GC6Fx7eFNasajJsaAXaQToKx8YqKCGG4nHxn0Ucb79+G'
+        '/0wQhtR0Mk3CxcajYJAsTV2ulW05P9xqovblXImXDZpgv0bQ2TX43SdR17yk'
+        'QzL33HRNCT7clLblHLMPQVxYy1yGS6hOAQj/Rmp+BO7d3S082+oyAFWeb7a9'
+        'fwzedbxPeiE2VOLtZizQUWIHHqwKP0tNEWRvSfCbc6ktvZQnHCIKyhmTC8N7'
+        'kvS4T6WjWzpc1M+GOMlOqhtW6t3zV1i2tkcpujduBGRIZ8ZQY+yo/i1HSL5t'
+        'N98606YXN1s2JyqwAkBJfPYiMp67J2uaFsML3YQEKAxR64GhkjFR/OqYtlIB'
+        'cx1PvcrPbVWQzXZBfFyjbAd55MnWVk6GrbM3y1QATN3NNhXfbMzLLU6cw/8p'
+        'sJw0+hxv1W2bJTftrs/5PyLryNOKYHbPEtC6aIyuzbIFFKWxkNshUiasd82Q'
+        'Jafgx3pFNnCtB61UV46QeqPI7sVueLslurqVgEGb2dS6unKYWXedoIMELm3C'
+        'g0XdJQARAQABiQI8BBgBCAAmFiEElnHIGFxlGau06K2fpOuC0lc/fHcFAl3y'
+        'ev8CGwwFCQPDuIAACgkQpOuC0lc/fHc/PxAAj29SBqW6ZRG8zOOw0Dmg1sg4'
+        'ONYtJ4hEzqPv2WbtOKxgtdcjQS1gMadtfcrH0omZPn8YmeojdbJCd5b9UBYr'
+        'h4Km3usURy79ouqvyQdZOIBOCUuvNcAUX2xvgUEHQW+rDpkd2mxdASsay1I7'
+        'yx2S0xE/QP/L2dH0470JWJ+tCIz3WuW2BEi+wijy2tqJfzIkIWA5ND2jwl4n'
+        'roY7srmAwZfXlh97/T5oOPIUsupIp+vmtMd4B0qa1wLGFDch+VwVvklLN5/Q'
+        'Vfbedy1Y8yHYiRWSrd3pHvkdtE5rI8qCOWaU/271plT9MZiwHe5WzCWESbKi'
+        'dwHQanM0Y6+Y8rrvUWGXrlPDvVd3Gd6TjqNhA8+AEiG+BHsw7Azc5in97/yW'
+        '9cAYEldWv1tUjxgqvWWbGA8E6M/EuE3FuM48HNODfEh/b0ut+b2UAtuz3LzK'
+        'NVpqYZ9NIebpIMlUuJoQc9rPCWzMDNX37iGRBA016L7VizeJRpJ8VPRAQWHe'
+        'L5eC85dx9wcdK152fqlOUj729J2TZ5JYQdm9vF2cA6bsIB9m48j/UzNEeV3W'
+        'NZ3nuZqQ9VjVLYiPURbdkYxWfUvFdVawfqUZ4PGKbVWrFfod8WwHa+gsP4UJ'
+        'hLN/nxCalBbc3HnyYo0Inlytu4fumElS7kuUVNielOsJlyUr8kfxU3c6MPk=',
     ],
     'PTR': ['example.com.', '*.example.com.'],
     'RP': ['hostmaster.example.com. .'],
@@ -103,7 +151,56 @@ VALID_RECORDS_NON_CANONICAL = {
     'NS': ['EXaMPLE.COM.'],
     'OPENPGPKEY': [
         'mG8EXtVIsRMFK4EEAC==',
-        'mG8EXtVIsRMFK4EEACIDAwQSZPNqE4tSxLFJYhX+uabSgMrhOqUizJhkLx82',  # key incomplete due to 500 byte limit
+        'mQINBF3yev8BEADR9GxB6OJ5AJlXBWc3nWyWZ+yNNVBiy73XjgOs0uowbxph '
+        'dIw6l75M6xw3i9xAlcjAGG2710FJaye7EZHot3RTIgHpn4FrErQSpNPuJKjD '
+        'IedZZ4av5SRtz5FfnXhNkQGs7jAVi6FmjR9/0GWMxj0BdbcOmeePCUfIIH7T '
+        'ujQJ2c3XHOu/kZ1h4zsFVSslcLEi4KXy0I52pEz0E2CyJrxCLdBd7uU7wDCg '
+        'G8KrIP3UJ5EtukP/LMq4D1eZ4FmtVqzkuDYlJJo70XQytEK9UqDdaDvlUeS5 '
+        'FrVj4Zf7OaC5YcSvQemVV4VYSBgJIPb+iFY21/1mXAxyYaunqaR0j5qNaMjr '
+        'E2g3ADRxJiLExhhzlqwJU8+Lc+0QajF/s3lc+dB5usSPqGk6Eb4hBEMaqQvg '
+        '5I0W8pFtHINYipNW5xGSrsX0pyWVai6EkoTXfjbBMC7khwmwsycJ8pYj3ipe '
+        'aNQuUP+XXqJKepoVOY2475Z7YT1NRRbGGEp743mbqKo4SnEKxS2kApo1UPd1 '
+        'FbI50TZ62Vsv4tne3bR25eCycjdvIOp6zPm/Pf9LFVm5KF8Wd2U3vRi/uo4v '
+        'HPUK1RoIzjmirp3XUBGBgHd/mhlOADPWB9dE96eXK4yEHlbfomfFiKAisHDc '
+        'vUa0E/UbklYBhJjdWBaw1fDDyiSxsBCTsq4ObQARAQABtBFzdXBwb3J0QHBv '
+        'c3Rlby5kZYkCVAQTAQgAPhYhBJZxyBhcZRmrtOitn6TrgtJXP3x3BQJd8nr/ '
+        'AhsDBQkDw7iABQsJCAcCBhUKCQgLAgQWAgMBAh4BAheAAAoJEKTrgtJXP3x3 '
+        '+UIP/jpw6Nkp5hLbXxpPRSL2TyyWDfEHPKkBQfU+jnAUIN+WgAV27HpOa+vZ '
+        '/hmTKOG6SlTOxHWACmDiUVfhLOYMV8QPDD3yPFCZWo4UxBKPZaai6GQwr44u '
+        'zCcU+E6AdFnb2nbzYSgACrErU5o5JoU2lPgleMI3FYsG8wb/kQAD7XGDX+Ev '
+        'tAbAQGK5EgevycJzot/hsR/S6EM/l0VsW74DIje3fbp3gaJY2fUG9fTdQu7a '
+        'gj6f9HuZAvXHIuSFeA/kwhUWuZfTcct8PV78gwQB4d6AOFMzoxLaFQAzxuTR '
+        '60kZxsyyi4U5km6D/XzI9rTd228PD8xkGr/2Kx1YRU0ixZnohv9xNc4GP/69 '
+        'GNWbbOZcyJcSL+kvych+ddbP5VjHea+b4vT35KV++PMndj+78BE1u5sdqWir '
+        'X9pi09go7SW1BlaJsMHrkR0P8yFCaFWLyCmIC7C/KcSuHVwcjVYWHynLq6CK '
+        'kkv4r8BNM/QFzPCeozXjMk7zq9TkJjLVxsUVNcZaNqzlWO0JzCfE6ICpHhyI '
+        'g/1bO/VJQyk+6llyX1LwRKCeKQCp6KcLx4qnjgZ8g1ArNvazNot9fAssgAUz '
+        'yoyOBF1SYJxWnzu9GE1F47zU1iD6FB8mjspvE00voDs8t2e+xtZoqsM12WtC '
+        '8R4VbCY0LmTPGiWyxD9y7TnUlDfHuQINBF3yev8BEAC4dyN2BPiHCmwtKV/3 '
+        '9ZUMVCjb39wnsAA8CH7WAAM5j+k8/uXKUmTcFoZ7+9ya6PZCLXbPC64FIAwl '
+        'YalzCEP5Jx25Ct/DPhVJPIFWHMOYbyUbLJ8tlC1vnnDhd8czeGmozkuyofMh '
+        '39QzR3SLzOqucJO3GC6Fx7eFNasajJsaAXaQToKx8YqKCGG4nHxn0Ucb79+G '
+        '/0wQhtR0Mk3CxcajYJAsTV2ulW05P9xqovblXImXDZpgv0bQ2TX43SdR17yk '
+        'QzL33HRNCT7clLblHLMPQVxYy1yGS6hOAQj/Rmp+BO7d3S082+oyAFWeb7a9 '
+        'fwzedbxPeiE2VOLtZizQUWIHHqwKP0tNEWRvSfCbc6ktvZQnHCIKyhmTC8N7 '
+        'kvS4T6WjWzpc1M+GOMlOqhtW6t3zV1i2tkcpujduBGRIZ8ZQY+yo/i1HSL5t '
+        'N98606YXN1s2JyqwAkBJfPYiMp67J2uaFsML3YQEKAxR64GhkjFR/OqYtlIB '
+        'cx1PvcrPbVWQzXZBfFyjbAd55MnWVk6GrbM3y1QATN3NNhXfbMzLLU6cw/8p '
+        'sJw0+hxv1W2bJTftrs/5PyLryNOKYHbPEtC6aIyuzbIFFKWxkNshUiasd82Q '
+        'Jafgx3pFNnCtB61UV46QeqPI7sVueLslurqVgEGb2dS6unKYWXedoIMELm3C '
+        'g0XdJQARAQABiQI8BBgBCAAmFiEElnHIGFxlGau06K2fpOuC0lc/fHcFAl3y '
+        'ev8CGwwFCQPDuIAACgkQpOuC0lc/fHc/PxAAj29SBqW6ZRG8zOOw0Dmg1sg4 '
+        'ONYtJ4hEzqPv2WbtOKxgtdcjQS1gMadtfcrH0omZPn8YmeojdbJCd5b9UBYr '
+        'h4Km3usURy79ouqvyQdZOIBOCUuvNcAUX2xvgUEHQW+rDpkd2mxdASsay1I7 '
+        'yx2S0xE/QP/L2dH0470JWJ+tCIz3WuW2BEi+wijy2tqJfzIkIWA5ND2jwl4n '
+        'roY7srmAwZfXlh97/T5oOPIUsupIp+vmtMd4B0qa1wLGFDch+VwVvklLN5/Q '
+        'Vfbedy1Y8yHYiRWSrd3pHvkdtE5rI8qCOWaU/271plT9MZiwHe5WzCWESbKi '
+        'dwHQanM0Y6+Y8rrvUWGXrlPDvVd3Gd6TjqNhA8+AEiG+BHsw7Azc5in97/yW '
+        '9cAYEldWv1tUjxgqvWWbGA8E6M/EuE3FuM48HNODfEh/b0ut+b2UAtuz3LzK '
+        'NVpqYZ9NIebpIMlUuJoQc9rPCWzMDNX37iGRBA016L7VizeJRpJ8VPRAQWHe '
+        'L5eC85dx9wcdK152fqlOUj729J2TZ5JYQdm9vF2cA6bsIB9m48j/UzNEeV3W '
+        'NZ3nuZqQ9VjVLYiPURbdkYxWfUvFdVawfqUZ4PGKbVWrFfod8WwHa+gsP4UJ '
+        'hLN/nxCalBbc3HnyYo0Inlytu4fumElS7kuUVNielOsJlyUr8kfxU3c6MPk=',
     ],
     'PTR': ['EXAMPLE.TEST.'],
     'RP': ['hostmaster.EXAMPLE.com. .'],
@@ -114,6 +211,7 @@ VALID_RECORDS_NON_CANONICAL = {
     'TLSA': ['003 00 002 696B8F6B92A913560b23ef5720c378881faffe74432d04eb35db957c0a93987b47adf26abb5dac10ba482597ae16edb069b511bec3e26010d1927bf6392760dd',],
     'TXT': [
         f'"{"a" * 498}"',
+        '"' + 124 * '🧥' + '==="',  # 501 byte total length
         '"🧥 👚 👕 👖 👔 👗 👙 👘 👠 👡 👢 👞 👟 🥾 🥿  🧦 🧤 🧣 🎩 🧢 👒 🎓 ⛑ 👑 👝 👛 👜 💼 🎒 "',
         '"🧥 👚 👕 👖 👔 👗 👙 👘 👠 👡 👢 👞 👟 🥾 🥿  🧦 🧤 🧣 🎩 🧢 👒 🎓 ⛑ 👑 👝 👛 👜 💼 🎒 👓 🕶 🥽 🥼 🌂 🧵"',
     ],
@@ -154,7 +252,6 @@ INVALID_RECORDS = {
         'foob"ar',
         'v=spf1 include:example.com ~all',
         '"foo\nbar"',
-        '"' + 124 * '🧥' + '==="',  # 501 byte total length
         '"\x00" "Django rejects literal NUL byte"',
     ],
     'URI': ['"1" "2" "3"'],