desec-stack/api/desecapi/management/commands/check-secondaries.py

from datetime import timedelta
from socket import gethostbyname
from time import sleep

from django.conf import settings
from django.core.mail import get_connection, mail_admins
from django.core.management import BaseCommand
from django.utils import timezone
import dns.exception, dns.message, dns.query, dns.rdatatype

from desecapi import pdns
from desecapi.models import Domain


def query_serial(zone, server):
    """
    Checks a zone's serial on a server.
    :return: serial if received; None if the server did not know; False on error
    """
    query = dns.message.make_query(zone, "SOA")
    try:
        response = dns.query.tcp(query, server, timeout=5)
    except dns.exception.Timeout:
        return False

    for rrset in response.answer:
        if rrset.rdtype == dns.rdatatype.SOA:
            return int(rrset[0].serial)
    return None


class Command(BaseCommand):
    help = "Check secondaries for consistency with nsmaster."

    def __init__(self, *args, **kwargs):
        self.servers = {
            gethostbyname(server): server for server in settings.WATCHDOG_SECONDARIES
        }
        super().__init__(*args, **kwargs)

    def add_arguments(self, parser):
        parser.add_argument(
            "domain-name",
            nargs="*",
            help="Domain name to check. If omitted, will check all recently published domains.",
        )
        parser.add_argument(
            "--delay",
            type=int,
            default=300,
            help="Delay SOA checks to allow pending AXFRs to finish.",
        )
        parser.add_argument(
            "--window",
            type=int,
            default=1200,  # Should be sum of crontab interval and delay option (see above)
            help="Check domains that were published no longer than this many seconds ago.",
        )

    def find_outdated_servers(self, zone, local_serial):
        """
        Returns a dict, the key being the outdated secondary name, and the value being the node's current zone serial.
        """
        outdated = {}
        for server in self.servers:
            remote_serial = query_serial(zone, server)
            if not remote_serial or remote_serial < local_serial:
                outdated[self.servers[server]] = remote_serial

        return outdated

    def handle(self, *args, **options):
        threshold = timezone.now() - timedelta(seconds=options["window"])
        recent_domain_names = Domain.objects.filter(
            published__gt=threshold
        ).values_list("name", flat=True)
        serials = {
            zone: s
            for zone, s in pdns.get_serials().items()
            if zone.rstrip(".") in recent_domain_names
        }

        if options["domain-name"]:
            serials = {
                zone: serial
                for zone, serial in serials.items()
                if zone.rstrip(".") in options["domain-name"]
            }

        print(
            "Sleeping for {} seconds before checking {} domains ...".format(
                options["delay"], len(serials)
            )
        )
        sleep(options["delay"])

        outdated_zone_count = 0
        outdated_secondaries = set()

        output = []
        timeouts = {}
        for zone, local_serial in serials.items():
            outdated_serials = self.find_outdated_servers(zone, local_serial)
            for server, serial in outdated_serials.items():
                if serial is False:
                    timeouts.setdefault(server, [])
                    timeouts[server].append(zone)
            outdated_serials = {
                k: serial
                for k, serial in outdated_serials.items()
                if serial is not False
            }

            if outdated_serials:
                outdated_secondaries.update(outdated_serials.keys())
                output.append(
                    f"{zone} ({local_serial}) is outdated on {outdated_serials}"
                )
                print(output[-1])
                outdated_zone_count += 1
            else:
                print(f"{zone} ok")

        output.append(
            f"Checked {len(serials)} domains, {outdated_zone_count} were outdated."
        )
        print(output[-1])

        self.report(outdated_secondaries, output, timeouts)

    def report(self, outdated_secondaries, output, timeouts):
        # Do not report when timeouts occur, unless there's also replication out-of-sync somwhere.
        # Helps catch long-term unreachability, where subject will show timeouts for any emails.
        # Individual node downtimes should be tracked by external monitoring.
        if not outdated_secondaries:
            return

        subject = f"{len(timeouts)} timeouts, {len(outdated_secondaries)} secondaries out of sync"
        message = ""

        if timeouts:
            message += f"The following servers had timeouts:\n\n{timeouts}\n\n"

        if outdated_secondaries:
            message += f"The following {len(outdated_secondaries)} secondaries are out of sync:\n"
            for outdated_secondary in outdated_secondaries:
                message += f"* {outdated_secondary}\n"
            message += "\n"

        message += f"Current secondary IPs: {self.servers}\n"
        message += "\n".join(output)

        mail_admins(
            subject,
            message,
            connection=get_connection("django.core.mail.backends.smtp.EmailBackend"),
        )