feat(api): add cursor pagination based on Link header, fixes #228

api/api/settings.py

@@ -89,6 +89,8 @@ REST_FRAMEWORK = {
     'DEFAULT_AUTHENTICATION_CLASSES': (
         'desecapi.authentication.TokenAuthentication',
     ),
+    'DEFAULT_PAGINATION_CLASS': 'desecapi.pagination.LinkHeaderCursorPagination',
+    'PAGE_SIZE': 500,
     'TEST_REQUEST_DEFAULT_FORMAT': 'json',
     'EXCEPTION_HANDLER': 'desecapi.exception_handlers.exception_handler',
     'DEFAULT_VERSIONING_CLASS': 'rest_framework.versioning.NamespaceVersioning',

api/api/settings_quick_test.py

@@ -17,3 +17,5 @@ DATABASES = {
 PASSWORD_HASHERS = [
     'django.contrib.auth.hashers.MD5PasswordHasher',
 ]
+
+REST_FRAMEWORK['PAGE_SIZE'] = 20

api/desecapi/pagination.py

@@ -0,0 +1,44 @@
+from rest_framework import status
+from rest_framework.pagination import CursorPagination
+from rest_framework.response import Response
+from rest_framework.utils.urls import replace_query_param
+
+
+class LinkHeaderCursorPagination(CursorPagination):
+    """
+    Inform the user of pagination links via response headers, similar to what's
+    described in https://developer.github.com/v3/guides/traversing-with-pagination/
+    Inspired by the django-rest-framework-link-header-pagination package.
+    """
+    template = None
+
+    @staticmethod
+    def construct_headers(pagination_map):
+        links = [f'<{url}>; rel="{label}"' for label, url in pagination_map.items() if url is not None]
+        return {'Link': ', '.join(links)} if links else {}
+
+    def get_paginated_response(self, data):
+        pagination_required = self.has_next or self.has_previous
+        if not pagination_required:
+            return Response(data)
+
+        url = self.request.build_absolute_uri()
+        pagination_map = {'first': replace_query_param(url, self.cursor_query_param, '')}
+
+        if self.cursor_query_param not in self.request.query_params:
+            count = self.queryset.count()
+            data = {
+                'detail': f'Pagination required. You can query up to {self.page_size} items at a time ({count} total). '
+                          'Please use the `first` page link (see Link header).',
+            }
+            headers = self.construct_headers(pagination_map)
+            return Response(data, headers=headers, status=status.HTTP_400_BAD_REQUEST)
+
+        pagination_map.update(prev=self.get_previous_link(), next=self.get_next_link())
+        headers = self.construct_headers(pagination_map)
+        return Response(data, headers=headers)
+
+    def paginate_queryset(self, queryset, request, view=None):
+        self.request = request
+        self.queryset = queryset
+        return super().paginate_queryset(queryset, request, view)

api/desecapi/tests/base.py

@@ -73,7 +73,7 @@ class DesecAPIClient(APIClient):
 
     def get_rr_sets(self, domain_name, **kwargs):
         return self.get(
-            self.reverse('v1:rrsets', name=domain_name),
+            self.reverse('v1:rrsets', name=domain_name) + kwargs.pop('query', ''),
             kwargs
         )
 

api/desecapi/tests/test_domains.py

@@ -144,8 +144,10 @@ class DomainOwnerTestCase1(DomainOwnerTestCase):
             response = self.client.get(self.reverse('v1:domain-list'))
             self.assertStatus(response, status.HTTP_200_OK)
             self.assertEqual(len(response.data), self.NUM_OWNED_DOMAINS)
-            for i in range(self.NUM_OWNED_DOMAINS):
-                self.assertEqual(response.data[i]['name'], self.my_domains[i].name)
+
+            response_set = {data['name'] for data in response.data}
+            expected_set = {domain.name for domain in self.my_domains}
+            self.assertEqual(response_set, expected_set)
 
     def test_delete_my_domain(self):
         url = self.reverse('v1:domain-detail', name=self.my_domain.name)

api/desecapi/tests/test_rrsets.py

@@ -1,3 +1,5 @@
+import re
+
 from django.conf import settings
 from django.core.exceptions import ValidationError
 from django.core.management import call_command
@@ -42,15 +44,78 @@ class AuthenticatedRRSetTestCase(AuthenticatedRRSetBaseTestCase):
             self.assertStatus(response, status.HTTP_200_OK)
             self.assertEqual(len(response.data), 1, response.data)
 
+    def test_retrieve_my_rr_sets_pagination(self):
+        def convert_links(links):
+            mapping = {}
+            for link in links.split(', '):
+                _url, label = link.split('; ')
+                label = re.search('rel="(.*)"', label).group(1)
+                _url = _url[1:-1]
+                assert label not in mapping
+                mapping[label] = _url
+            return mapping
+
+        def assertPaginationResponse(response, expected_length, expected_directional_links=[]):
+            self.assertStatus(response, status.HTTP_200_OK)
+            self.assertEqual(len(response.data), expected_length)
+
+            _links = convert_links(response['Link'])
+            self.assertEqual(len(_links), len(expected_directional_links) + 1)  # directional links, plus "first"
+            self.assertTrue(_links['first'].endswith('/?cursor='))
+            for directional_link in expected_directional_links:
+                self.assertEqual(_links['first'].find('/?cursor='), _links[directional_link].find('/?cursor='))
+                self.assertTrue(len(_links[directional_link]) > len(_links['first']))
+
+        # Prepare extra records so that we get three pages (total: n + 1)
+        n = int(settings.REST_FRAMEWORK['PAGE_SIZE'] * 2.5)
+        RRset.objects.bulk_create(
+            [RRset(domain=self.my_domain, subname=str(i), ttl=123, type='A') for i in range(n)]
+        )
+
+        # No pagination
+        response = self.client.get_rr_sets(self.my_domain.name)
+        self.assertStatus(response, status.HTTP_400_BAD_REQUEST)
+        self.assertEqual(response.data['detail'],
+                         f'Pagination required. You can query up to {settings.REST_FRAMEWORK["PAGE_SIZE"]} items at a time ({n+1} total). '
+                         'Please use the `first` page link (see Link header).')
+        links = convert_links(response['Link'])
+        self.assertEqual(len(links), 1)
+        self.assertTrue(links['first'].endswith('/?cursor='))
+
+        # First page
+        url = links['first']
+        response = self.client.get(url)
+        assertPaginationResponse(response, settings.REST_FRAMEWORK['PAGE_SIZE'], ['next'])
+
+        # Next
+        url = convert_links(response['Link'])['next']
+        response = self.client.get(url)
+        assertPaginationResponse(response, settings.REST_FRAMEWORK['PAGE_SIZE'], ['next', 'prev'])
+        data_next = response.data.copy()
+
+        # Next-next (last) page
+        url = convert_links(response['Link'])['next']
+        response = self.client.get(url)
+        assertPaginationResponse(response, n/5 + 1, ['prev'])
+
+        # Prev
+        url = convert_links(response['Link'])['prev']
+        response = self.client.get(url)
+        assertPaginationResponse(response, settings.REST_FRAMEWORK['PAGE_SIZE'], ['next', 'prev'])
+
+        # Make sure that one step forward equals two steps forward and one step back
+        self.assertEqual(response.data, data_next)
+
     def test_retrieve_other_rr_sets(self):
         self.assertStatus(self.client.get_rr_sets(self.other_domain.name), status.HTTP_404_NOT_FOUND)
         self.assertStatus(self.client.get_rr_sets(self.other_domain.name, subname='test'), status.HTTP_404_NOT_FOUND)
         self.assertStatus(self.client.get_rr_sets(self.other_domain.name, type='A'), status.HTTP_404_NOT_FOUND)
 
     def test_retrieve_my_rr_sets_filter(self):
-        response = self.client.get_rr_sets(self.my_rr_set_domain.name)
+        response = self.client.get_rr_sets(self.my_rr_set_domain.name, query='?cursor=')
         self.assertStatus(response, status.HTTP_200_OK)
-        self.assertEqual(len(response.data), len(self._test_rr_sets()))
+        expected_number_of_rrsets = min(len(self._test_rr_sets()), settings.REST_FRAMEWORK['PAGE_SIZE'])
+        self.assertEqual(len(response.data), expected_number_of_rrsets)
 
         for subname in self.SUBNAMES:
             response = self.client.get_rr_sets(self.my_rr_set_domain.name, subname=subname)

docs/domains.rst

@@ -141,10 +141,15 @@ The ``/api/v1/domains/`` endpoint reponds to ``GET`` requests with an array of
     curl -X GET https://desec.io/api/v1/domains/ \
         --header "Authorization: Token {token}"
 
-to retrieve an overview of the domains you own.
+to retrieve an overview of the domains you own.  Domains are returned in
+reverse chronological order of their creation.
 
-The response status code is ``200 OK``.  This is true also if you do not own
-any domains; in this case, the response body will be an empty JSON array.
+The response status code in case of success is ``200 OK``.  This is true also
+if you do not own any domains; in this case, the response body will be an empty
+JSON array.
+
+Up to 500 items are returned at a time.  If you have a larger number of
+domains configured, the use of `pagination`_ is required.
 
 
 Retrieving a Specific Domain

docs/rrsets.rst

@@ -198,11 +198,32 @@ command::
     curl -X GET https://desec.io/api/v1/domains/:name/rrsets/ \
         --header "Authorization: Token {token}"
 
-to retrieve the contents of a zone that you own.
+to retrieve the contents of a zone that you own.  RRsets are returned in
+reverse chronological order of their creation.
 
-The response status code is ``200 OK``.  This is true also if there are no
-RRsets in the zone; in this case, the response body will be an empty JSON
-array.
+The response status code in case of success is ``200 OK``.  This is true also
+if there are no RRsets in the zone; in this case, the response body will be an
+empty JSON array.
+
+Pagination
+``````````
+Up to 500 items are returned at a time.  If more than 500 items would match the
+query, the use of the ``cursor`` query parameter is required.  The first page
+can be retrieved by sending an empty pagination parameter, ``cursor=``.
+
+Once in pagination mode, the URLs to retrieve the next (or previous) page are
+given in the ``Link:`` response header.  For example::
+
+    Link: <https://desec.io/api/v1/domains/:domain/rrsets/?cursor=>; rel="first",
+      <https://desec.io/api/v1/domains/:domain/rrsets/?cursor=:prev_cursor>; rel="prev",
+      <https://desec.io/api/v1/domains/:domain/rrsets/?cursor=:next_cursor>; rel="next"
+
+where ``:prev_cursor`` and ``:next_cursor`` are page identifiers that are to
+be treated opaque by clients.  On the first/last page, the ``Link:`` header
+will not contain a ``prev``/``next`` field, respectively.
+
+If no pagination parameter is given although pagination is required, the server
+will return ``400 Bad Request``, along with instructions for pagination.
 
 
 Filtering by Record Type
@@ -216,6 +237,8 @@ like::
     curl https://desec.io/api/v1/domains/:name/rrsets/?type=:type \
         --header "Authorization: Token {token}"
 
+Query parameters used for filtering are fully compatible with `pagination`_.
+
 
 Filtering by Subname
 ````````````````````
@@ -231,6 +254,8 @@ This approach also allows to retrieve all records associated with the zone
 apex (i.e. ``example.com`` where ``subname`` is empty), by querying
 ``rrsets/?subname=``.
 
+Query parameters used for filtering are fully compatible with `pagination`_.
+
 
 Retrieving a Specific RRset
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~