from datetime import timedelta from socket import gethostbyname from time import sleep from django.conf import settings from django.core.mail import get_connection, mail_admins from django.core.management import BaseCommand from django.utils import timezone import dns.exception, dns.message, dns.query, dns.rdatatype from desecapi import pdns from desecapi.models import Domain def query_serial(zone, server): """ Checks a zone's serial on a server. :return: serial if received; None if the server did not know; False on error """ query = dns.message.make_query(zone, 'SOA') try: response = dns.query.tcp(query, server, timeout=5) except dns.exception.Timeout: return False for rrset in response.answer: if rrset.rdtype == dns.rdatatype.SOA: return int(rrset[0].serial) return None class Command(BaseCommand): help = 'Check secondaries for consistency with nsmaster.' def __init__(self, *args, **kwargs): self.servers = {gethostbyname(server): server for server in settings.WATCHDOG_SECONDARIES} super().__init__(*args, **kwargs) def add_arguments(self, parser): parser.add_argument('domain-name', nargs='*', help='Domain name to check. If omitted, will check all recently published domains.') parser.add_argument('--delay', type=int, default=120, help='Delay SOA checks to allow pending AXFRs to finish.') parser.add_argument('--window', type=int, default=settings.WATCHDOG_WINDOW_SEC, help='Check domains that were published no longer than this many seconds ago.') def find_outdated_servers(self, zone, local_serial): """ Returns a dict, the key being the outdated secondary name, and the value being the node's current zone serial. """ outdated = {} for server in self.servers: remote_serial = query_serial(zone, server) if not remote_serial or remote_serial < local_serial: outdated[self.servers[server]] = remote_serial return outdated def handle(self, *args, **options): threshold = timezone.now() - timedelta(seconds=options['window']) recent_domain_names = Domain.objects.filter(published__gt=threshold).values_list('name', flat=True) serials = {zone: s for zone, s in pdns.get_serials().items() if zone.rstrip('.') in recent_domain_names} if options['domain-name']: serials = {zone: serial for zone, serial in serials.items() if zone.rstrip('.') in options['domain-name']} print('Sleeping for {} seconds before checking {} domains ...'.format(options['delay'], len(serials))) sleep(options['delay']) outdated_zone_count = 0 outdated_secondaries = set() output = [] timeouts = {} for zone, local_serial in serials.items(): outdated_serials = self.find_outdated_servers(zone, local_serial) for server, serial in outdated_serials.items(): if serial is False: timeouts.setdefault(server, []) timeouts[server].append(zone) outdated_serials = {k: serial for k, serial in outdated_serials.items() if serial is not False} if outdated_serials: outdated_secondaries.update(outdated_serials.keys()) output.append(f'{zone} ({local_serial}) is outdated on {outdated_serials}') print(output[-1]) outdated_zone_count += 1 else: print(f'{zone} ok') output.append(f'Checked {len(serials)} domains, {outdated_zone_count} were outdated.') print(output[-1]) self.report(outdated_secondaries, output, timeouts) def report(self, outdated_secondaries, output, timeouts): if not outdated_secondaries and not timeouts: return subject = f'{timeouts and "CRITICAL ALERT" or "ALERT"} {len(outdated_secondaries)} secondaries out of sync' message = '' if timeouts: message += f'The following servers had timeouts:\n\n{timeouts}\n\n' if outdated_secondaries: message += f'The following {len(outdated_secondaries)} secondaries are out of sync:\n' for outdated_secondary in outdated_secondaries: message += f'* {outdated_secondary}\n' message += '\n' message += f'Current secondary IPs: {self.servers}\n' message += '\n'.join(output) mail_admins(subject, message, connection=get_connection('django.core.mail.backends.smtp.EmailBackend'))