desec-stack/api/desecapi/management/commands/check-secondaries.py

from datetime import timedelta
from socket import gethostbyname
from time import sleep

from django.conf import settings
from django.core.mail import get_connection, mail_admins
from django.core.management import BaseCommand
from django.utils import timezone
import dns.exception, dns.message, dns.query, dns.rdatatype

from desecapi import pdns
from desecapi.models import Domain


def query_serial(zone, server):
    """
    Checks a zone's serial on a server.
    :return: serial if received; None if the server did not know; False on error
    """
    query = dns.message.make_query(zone, 'SOA')
    try:
        response = dns.query.tcp(query, server, timeout=5)
    except dns.exception.Timeout:
        return False

    for rrset in response.answer:
        if rrset.rdtype == dns.rdatatype.SOA:
            return int(rrset[0].serial)
    return None


class Command(BaseCommand):
    help = 'Check secondaries for consistency with nsmaster.'

    def __init__(self, *args, **kwargs):
        self.servers = {gethostbyname(server): server for server in settings.WATCHDOG_SECONDARIES}
        super().__init__(*args, **kwargs)

    def add_arguments(self, parser):
        parser.add_argument('domain-name', nargs='*',
                            help='Domain name to check. If omitted, will check all recently published domains.')
        parser.add_argument('--delay', type=int, default=120, help='Delay SOA checks to allow pending AXFRs to finish.')
        parser.add_argument('--window', type=int, default=settings.WATCHDOG_WINDOW_SEC,
                            help='Check domains that were published no longer than this many seconds ago.')

    def find_outdated_servers(self, zone, local_serial):
        """
        Returns a dict, the key being the outdated secondary name, and the value being the node's current zone serial.
        """
        outdated = {}
        for server in self.servers:
            remote_serial = query_serial(zone, server)
            if not remote_serial or remote_serial < local_serial:
                outdated[self.servers[server]] = remote_serial

        return outdated

    def handle(self, *args, **options):
        threshold = timezone.now() - timedelta(seconds=options['window'])
        recent_domain_names = Domain.objects.filter(published__gt=threshold).values_list('name', flat=True)
        serials = {zone: s for zone, s in pdns.get_serials().items() if zone.rstrip('.') in recent_domain_names}

        if options['domain-name']:
            serials = {zone: serial for zone, serial in serials.items() if zone.rstrip('.') in options['domain-name']}

        print('Sleeping for {} seconds before checking {} domains ...'.format(options['delay'], len(serials)))
        sleep(options['delay'])

        outdated_zone_count = 0
        outdated_secondaries = set()

        output = []
        timeouts = {}
        for zone, local_serial in serials.items():
            outdated_serials = self.find_outdated_servers(zone, local_serial)
            for server, serial in outdated_serials.items():
                if serial is False:
                    timeouts.setdefault(server, [])
                    timeouts[server].append(zone)
            outdated_serials = {k: serial for k, serial in outdated_serials.items() if serial is not False}

            if outdated_serials:
                outdated_secondaries.update(outdated_serials.keys())
                output.append(f'{zone} ({local_serial}) is outdated on {outdated_serials}')
                print(output[-1])
                outdated_zone_count += 1
            else:
                print(f'{zone} ok')

        output.append(f'Checked {len(serials)} domains, {outdated_zone_count} were outdated.')
        print(output[-1])

        self.report(outdated_secondaries, output, timeouts)

    def report(self, outdated_secondaries, output, timeouts):
        if not outdated_secondaries and not timeouts:
            return

        subject = f'{timeouts and "CRITICAL ALERT" or "ALERT"} {len(outdated_secondaries)} secondaries out of sync'
        message = ''

        if timeouts:
            message += f'The following servers had timeouts:\n\n{timeouts}\n\n'

        if outdated_secondaries:
            message += f'The following {len(outdated_secondaries)} secondaries are out of sync:\n'
            for outdated_secondary in outdated_secondaries:
                message += f'* {outdated_secondary}\n'
            message += '\n'

        message += f'Current secondary IPs: {self.servers}\n'
        message += '\n'.join(output)

        mail_admins(subject, message, connection=get_connection('django.core.mail.backends.smtp.EmailBackend'))