feat(api): add management command to handle inactive domain purge

api/desecapi/management/commands/scavenge-unused.py

@@ -0,0 +1,109 @@
+import datetime
+from functools import reduce
+import operator
+
+from django.conf import settings
+from django.core.mail import get_connection, mail_admins
+from django.core.management import BaseCommand
+from django.db import transaction
+from django.db.models import F, Q
+from django.test import RequestFactory
+from django.utils import timezone
+from rest_framework.reverse import reverse
+
+from desecapi import models, serializers, views
+from desecapi.pdns_change_tracker import PDNSChangeTracker
+
+
+fresh_days = 183
+notice_days_notify = 28
+notice_days_warn = 7
+
+
+class Command(BaseCommand):
+    base_queryset = models.Domain.objects.filter(reduce(
+        operator.or_,
+        (Q(name__endswith=f'.{lps}') for lps in settings.LOCAL_PUBLIC_SUFFIXES),
+        Q(pk__in=[])  # always-false default
+    ))
+
+    @classmethod
+    def renew_touched_domains(cls):
+        last_published_threshold = timezone.localdate() - datetime.timedelta(days=183)
+        recently_touched_domains = cls.base_queryset.filter(published__date__gte=last_published_threshold,
+                                                            renewal_changed__lt=F('published'))
+
+        print('Renewing domains:', *recently_touched_domains.values_list('name', flat=True))
+        recently_touched_domains.update(renewal_state=models.Domain.RenewalState.FRESH, renewal_changed=F('published'))
+
+    @classmethod
+    def warn_domain_deletion(cls, renewal_state, notice_days, inactive_days):
+        def confirmation_link(domain_name, user):
+            action = models.AuthenticatedRenewDomainBasicUserAction(domain=domain_name, user=user)
+            verification_code = serializers.AuthenticatedRenewDomainBasicUserActionSerializer(action).data['code']
+            request = RequestFactory().generic('', '', secure=True, HTTP_HOST=f'desec.{settings.DESECSTACK_DOMAIN}')
+            return reverse('v1:confirm-renew-domain', request=request, args=[verification_code])
+
+        # We act when `renewal_changed` is at least this date (or older)
+        inactive_threshold = timezone.localdate() - datetime.timedelta(days=inactive_days)
+        # Filter candidates which have the state of interest, at least since the calculated date
+        expiry_candidates = cls.base_queryset.filter(renewal_state=renewal_state,
+                                                     renewal_changed__date__lte=inactive_threshold)
+
+        # Group domains by user, so that we can send one message per user
+        domain_user_map = {}
+        for domain in expiry_candidates.order_by('name'):
+            if domain.owner not in domain_user_map:
+                domain_user_map[domain.owner] = []
+            domain_user_map[domain.owner].append(domain)
+
+        # Prepare and send emails, and keep renewal status in sync
+        deletion_date = timezone.localdate() + datetime.timedelta(days=notice_days)
+        for user, domains in domain_user_map.items():
+            with transaction.atomic():
+                # Update renewal status of the user's affected domains, but don't commit before sending the email
+                for domain in domains:
+                    domain.renewal_state += 1
+                    domain.renewal_changed = timezone.now()
+                    domain.save(update_fields=['renewal_state', 'renewal_changed'])
+                links = [{'name': domain, 'confirmation_link': confirmation_link(domain, user)} for domain in domains]
+                user.send_email('renew-domain', context={'domains': links, 'deletion_date': deletion_date})
+
+    @classmethod
+    def delete_domains(cls, inactive_days):
+        published_threshold = timezone.localdate() - datetime.timedelta(days=inactive_days)
+        renewal_changed_threshold = timezone.localdate() - datetime.timedelta(days=notice_days_warn)
+        expiry_candidates = cls.base_queryset.filter(renewal_state=models.Domain.RenewalState.WARNED)
+        domains = expiry_candidates.filter(renewal_changed__date__lte=renewal_changed_threshold,
+                                           published__date__lte=published_threshold)
+        for domain in domains:
+            with PDNSChangeTracker():
+                domain.delete()
+            if not domain.owner.domains.exists():
+                domain.owner.delete()
+        # Do one large delegation update
+        with PDNSChangeTracker():
+            for domain in domains:
+                views.DomainViewSet.auto_delegate(domain)
+
+    def handle(self, *args, **kwargs):
+        try:
+            # Reset renewal status for domains that have recently been touched
+            self.renew_touched_domains()
+
+            # Announce domain deletion in `notice_days_notice` days if not yet notified (FRESH) and inactive for
+            # `inactive_days` days. Updates status from FRESH to NOTIFIED.
+            self.warn_domain_deletion(models.Domain.RenewalState.FRESH, notice_days_notify, fresh_days)
+
+            # After `notice_days_notify - notice_days_warn` more days, warn again if the status has not changed
+            # Updates status from NOTIFIED to WARNED.
+            self.warn_domain_deletion(models.Domain.RenewalState.NOTIFIED, notice_days_warn,
+                                      notice_days_notify - notice_days_warn)
+
+            # Finally, delete domains inactive for `inactive_days + notice_days_notify` days if status has not changed
+            self.delete_domains(fresh_days + notice_days_notify)
+        except Exception as e:
+            subject = 'Renewal Exception!'
+            message = f'{type(e)}\n\n{str(e)}'
+            print(f'Chores exception: {type(e)}, {str(e)}')
+            mail_admins(subject, message, connection=get_connection('django.core.mail.backends.smtp.EmailBackend'))

api/desecapi/models.py

@@ -156,6 +156,7 @@ class User(ExportModelOperationsMixin('User'), AbstractBaseUser):
             'reset-password': fast_lane,
             'delete-user': fast_lane,
             'domain-dyndns': fast_lane,
+            'renew-domain': immediate_lane,
         }
         if reason not in lanes:
             raise ValueError(f'Cannot send email to user {self.pk} without a good reason: {reason}')

api/desecapi/templates/emails/renew-domain/content.txt

@@ -0,0 +1,32 @@
+Hi there,
+
+You are the owner of the following domain name(s):
+
+{% for domain in domains %}
+  * {{ domain.name }}
+{% endfor %}
+
+We noticed that the DNS information of the above domain(s) have not
+received any updates for more than 6 months.
+
+As explained in our Terms and Conditions (https://desec.io/terms),
+we regularly purge inactive domains (as indicated by the absence of
+DNS updates). This allows us to free up resources for active deSEC
+users.
+
+The above domain name(s) are scheduled for deletion on
+{{ deletion_date|date:"F j, Y" }}. If there are no other domains in
+your account, we will also delete your account on this date.
+
+To retain your domain name (and account), either change a DNS record
+before that date, or click the following link(s) (valid for {{ link_expiration_hours }} hours):
+
+{% for domain in domains %}
+  * {{ domain.name }}
+    {{ domain.confirmation_link }}
+{% endfor %}
+
+In case you have questions, feel free to contact us!
+
+Stay secure,
+The deSEC Team

api/desecapi/templates/emails/renew-domain/subject.txt

@@ -0,0 +1 @@
+[deSEC] Upcoming domain deletion

api/desecapi/tests/test_user_management.py

@@ -11,7 +11,10 @@ This involves testing five separate endpoints:
 (3) Change email address endpoint,
 (4) delete user endpoint, and
 (5) verify endpoint.
+
+Furthermore, domain renewals and unused domain/account scavenging are tested.
 """
+from datetime import timedelta
 import random
 import re
 import time
@@ -20,7 +23,9 @@ from urllib.parse import urlparse
 
 from django.contrib.auth.hashers import is_password_usable
 from django.core import mail
+from django.core.management import call_command
 from django.urls import resolve
+from django.utils import timezone
 from rest_framework import status
 from rest_framework.reverse import reverse
 from rest_framework.test import APIClient
@@ -28,7 +33,7 @@ from rest_framework.test import APIClient
 from api import settings
 from desecapi.models import Domain, User, Captcha
 from desecapi.serializers import AuthenticatedActionSerializer
-from desecapi.tests.base import DesecTestCase, PublicSuffixMockMixin
+from desecapi.tests.base import DesecTestCase, DomainOwnerTestCase, PublicSuffixMockMixin
 
 
 class UserManagementClient(APIClient):
@@ -144,7 +149,7 @@ class UserManagementTestCase(DesecTestCase, PublicSuffixMockMixin):
         self.assertFalse(mail.outbox, "Expected no email to be sent, but %i were sent. First subject line is '%s'." %
                          (len(mail.outbox), mail.outbox[0].subject if mail.outbox else '<n/a>'))
 
-    def assertEmailSent(self, subject_contains='', body_contains='', recipient=None, reset=True, pattern=None):
+    def assertEmailSent(self, subject_contains='', body_contains=None, recipient=None, reset=True, pattern=None):
         total = 1
         self.assertEqual(len(mail.outbox), total, "Expected %i message in the outbox, but found %i." %
                          (total, len(mail.outbox)))
@@ -152,9 +157,9 @@ class UserManagementTestCase(DesecTestCase, PublicSuffixMockMixin):
         self.assertTrue(subject_contains in email.subject,
                         "Expected '%s' in the email subject, but found '%s'" %
                         (subject_contains, email.subject))
-        self.assertTrue(body_contains in email.body,
-                        "Expected '%s' in the email body, but found '%s'" %
-                        (body_contains, email.body))
+        if type(body_contains) != list: body_contains = [] if body_contains is None else [body_contains]
+        for elem in body_contains:
+            self.assertTrue(elem in email.body, f"Expected '{elem}' in the email body, but found '{email.body}'")
         if recipient is not None:
             if isinstance(recipient, list):
                 self.assertListEqual(recipient, email.recipients())
@@ -870,3 +875,133 @@ class HasUserAccountTestCase(UserManagementTestCase):
         self.assertVerificationFailureInvalidCodeResponse(self.client.verify(delete_link))
         self.assertVerificationFailureInvalidCodeResponse(self.client.verify(reset_password_link,
                                                                              data={'new_password': 'dummy'}))
+
+
+class RenewTestCase(UserManagementTestCase, DomainOwnerTestCase):
+    DYN = True
+
+    def setUp(self):
+        super().setUp()
+        self.email, self.password = self._test_registration(password=self.random_password())
+
+    def assertRenewDomainEmail(self, recipient, body_contains, pattern, reset=True):
+        return self.assertEmailSent(
+            subject_contains='Upcoming domain deletion',
+            body_contains=body_contains,
+            recipient=[recipient],
+            reset=reset,
+            pattern=pattern,
+        )
+
+    def assertRenewDomainVerificationSuccessResponse(self, response):
+        return self.assertContains(
+            response=response,
+            text='We recorded that your domain ',
+            status_code=status.HTTP_200_OK
+        )
+
+    def test_renew_domain_non_local_public_child(self):
+        user = User.objects.get(email=self.email)
+        domain = self.create_domain(owner=user)
+        for days in [182, 184]:
+            domain.published = timezone.now() - timedelta(days=days)
+            domain.renewal_changed = domain.published
+            domain.save()
+
+            self.assertEqual(Domain.objects.get(pk=domain.pk).renewal_state, Domain.RenewalState.FRESH)
+            call_command('scavenge-unused')
+            self.assertEqual(Domain.objects.get(pk=domain.pk).renewal_state, Domain.RenewalState.FRESH)
+            self.assertEqual(Domain.objects.get(pk=domain.pk).renewal_changed, domain.renewal_changed)
+            self.assertEqual(Domain.objects.get(pk=domain.pk).published, domain.published)
+            self.assertEqual(len(mail.outbox), 0)
+
+    def test_renew_domain_recently_published(self):
+        domain = self.my_domains[0]
+        for days in [5, 182, 184]:
+            domain.published = timezone.now() - timedelta(days=1)
+            domain.renewal_changed = timezone.now() - timedelta(days=days)
+            for renewal_state in [Domain.RenewalState.FRESH, Domain.RenewalState.NOTIFIED, Domain.RenewalState.WARNED]:
+                domain.renewal_state = renewal_state
+                domain.save()
+
+                self.assertEqual(Domain.objects.get(pk=domain.pk).renewal_state, domain.renewal_state)
+                call_command('scavenge-unused')
+                self.assertEqual(Domain.objects.get(pk=domain.pk).renewal_state, Domain.RenewalState.FRESH)
+                self.assertEqual(Domain.objects.get(pk=domain.pk).renewal_changed, domain.published)
+                self.assertEqual(Domain.objects.get(pk=domain.pk).published, domain.published)
+                self.assertEqual(len(mail.outbox), 0)
+
+    def test_renew_domain_fresh_182_days(self):
+        domain = self.my_domains[0]
+        domain.published = timezone.now() - timedelta(days=182)
+        domain.renewal_changed = domain.published
+        domain.save()
+
+        self.assertEqual(Domain.objects.get(pk=domain.pk).renewal_state, Domain.RenewalState.FRESH)
+        call_command('scavenge-unused')
+        self.assertEqual(Domain.objects.get(pk=domain.pk).renewal_state, Domain.RenewalState.FRESH)
+        self.assertEqual(len(mail.outbox), 0)
+
+    def test_renew_domain_fresh_183_days(self):
+        domain = self.my_domains[0]
+        domain.published = timezone.now() - timedelta(days=183)
+        domain.renewal_changed = domain.published
+        domain.save()
+
+        self.assertEqual(Domain.objects.get(pk=domain.pk).renewal_state, Domain.RenewalState.FRESH)
+        call_command('scavenge-unused')
+        self.assertEqual(Domain.objects.get(pk=domain.pk).renewal_state, Domain.RenewalState.NOTIFIED)
+
+        deletion_date = timezone.localdate() + timedelta(days=28)
+        body_contains = [domain.name, deletion_date.strftime('%B %-d, %Y')]
+        pattern = r'following link[^:]*:\s+\* ' + domain.name.replace('.', r'\.') + r'\s+([^\s]*)'
+        confirmation_link = self.assertRenewDomainEmail(domain.owner.email, body_contains, pattern)
+        self.assertConfirmationLinkRedirect(confirmation_link)
+        self.assertRenewDomainVerificationSuccessResponse(self.client.verify(confirmation_link))
+        self.assertEqual(Domain.objects.get(pk=domain.pk).renewal_state, Domain.RenewalState.FRESH)
+        self.assertLess(timezone.now() - Domain.objects.get(pk=domain.pk).renewal_changed, timedelta(seconds=1))
+
+        for domain in self.my_domains[1:]:
+            self.assertEqual(Domain.objects.get(pk=domain.pk).renewal_state, Domain.RenewalState.FRESH)
+            self.assertEqual(Domain.objects.get(pk=domain.pk).renewal_changed, domain.renewal_changed)
+
+    def test_renew_domain_notified_21_days(self):
+        domain = self.my_domains[0]
+        domain.published = timezone.now() - timedelta(days=183+21)
+        domain.renewal_state = Domain.RenewalState.NOTIFIED
+        domain.renewal_changed = timezone.now() - timedelta(days=21)
+        domain.save()
+
+        call_command('scavenge-unused')
+        self.assertEqual(Domain.objects.get(pk=domain.pk).renewal_state, Domain.RenewalState.WARNED)
+
+        deletion_date = timezone.localdate() + timedelta(days=7)
+        body_contains = [domain.name, deletion_date.strftime('%B %-d, %Y')]
+        pattern = r'following link[^:]*:\s+\* ' + domain.name.replace('.', r'\.') + r'\s+([^\s]*)'
+        confirmation_link = self.assertRenewDomainEmail(domain.owner.email, body_contains, pattern)
+        self.assertConfirmationLinkRedirect(confirmation_link)
+        self.assertRenewDomainVerificationSuccessResponse(self.client.verify(confirmation_link))
+        self.assertEqual(Domain.objects.get(pk=domain.pk).renewal_state, Domain.RenewalState.FRESH)
+        self.assertLess(timezone.now() - Domain.objects.get(pk=domain.pk).renewal_changed, timedelta(seconds=1))
+
+        for domain in self.my_domains[1:]:
+            self.assertEqual(Domain.objects.get(pk=domain.pk).renewal_state, Domain.RenewalState.FRESH)
+            self.assertEqual(Domain.objects.get(pk=domain.pk).renewal_changed, domain.renewal_changed)
+
+    def test_renew_domain_warned_7_days(self):
+        while self.my_domains:
+            domain = self.my_domains.pop()
+            domain.published = timezone.now() - timedelta(days=183+28)
+            domain.renewal_state = Domain.RenewalState.WARNED
+            domain.renewal_changed = timezone.now() - timedelta(days=7)
+            domain.save()
+
+            with self.assertPdnsRequests(self.requests_desec_domain_deletion_auto_delegation(name=domain.name)):
+                 call_command('scavenge-unused')
+            self.assertFalse(Domain.objects.filter(pk=domain.pk).exists())
+
+            # User gets deleted when last domain is purged
+            self.assertEqual(User.objects.filter(pk=self.owner.pk).exists(), bool(self.my_domains))
+
+            for domain in self.my_domains:
+                self.assertEqual(Domain.objects.get(pk=domain.pk).renewal_state, Domain.RenewalState.FRESH)