import copy import django.core.exceptions import dns.name import dns.zone from django.core.validators import MinValueValidator from django.db.models import Q from django.utils import timezone from rest_framework import serializers from rest_framework.settings import api_settings from rest_framework.validators import UniqueTogetherValidator from api import settings from desecapi import models from desecapi.validators import ExclusionConstraintValidator, ReadOnlyOnUpdateValidator class ConditionalExistenceModelSerializer(serializers.ModelSerializer): """ Only considers data with certain condition as existing data. If the existence condition does not hold, given instances are deleted, and no new instances are created, respectively. Also, to_representation and data will return None. Contrary, if the existence condition holds, the behavior is the same as DRF's ModelSerializer. """ def exists(self, arg): """ Determine if arg is to be considered existing. :param arg: Either a model instance or (possibly invalid!) data object. :return: Whether we treat this as non-existing instance. """ raise NotImplementedError def to_representation(self, instance): return ( None if not self.exists(instance) else super().to_representation(instance) ) @property def data(self): try: return super().data except TypeError: return None def save(self, **kwargs): validated_data = {} validated_data.update(self.validated_data) validated_data.update(kwargs) known_instance = self.instance is not None data_exists = self.exists(validated_data) if known_instance and data_exists: self.instance = self.update(self.instance, validated_data) elif known_instance and not data_exists: self.delete() elif not known_instance and data_exists: self.instance = self.create(validated_data) elif not known_instance and not data_exists: pass # nothing to do return self.instance def delete(self): self.instance.delete() class NonBulkOnlyDefault: """ This class may be used to provide default values that are only used for non-bulk operations, but that do not return any value for bulk operations. Implementation inspired by CreateOnlyDefault. """ requires_context = True def __init__(self, default): self.default = default def __call__(self, serializer_field): is_many = getattr(serializer_field.root, "many", False) if is_many: raise serializers.SkipField() if callable(self.default): if getattr(self.default, "requires_context", False): return self.default(serializer_field) else: return self.default() return self.default def __repr__(self): return "%s(%s)" % (self.__class__.__name__, repr(self.default)) class RRSerializer(serializers.ModelSerializer): class Meta: model = models.RR fields = ("content",) def to_internal_value(self, data): if not isinstance(data, str): raise serializers.ValidationError( "Must be a string.", code="must-be-a-string" ) return super().to_internal_value({"content": data}) def to_representation(self, instance): return instance.content class RRsetListSerializer(serializers.ListSerializer): default_error_messages = { **serializers.Serializer.default_error_messages, **serializers.ListSerializer.default_error_messages, **{"not_a_list": "Expected a list of items but got {input_type}."}, } @staticmethod def _key(data_item): return data_item.get("subname"), data_item.get("type") @staticmethod def _types_by_position_string(conflicting_indices_by_type): types_by_position = {} for type_, conflict_positions in conflicting_indices_by_type.items(): for position in conflict_positions: types_by_position.setdefault(position, []).append(type_) # Sort by position, None at the end types_by_position = dict( sorted(types_by_position.items(), key=lambda x: (x[0] is None, x)) ) db_conflicts = types_by_position.pop(None, None) if db_conflicts: types_by_position["database"] = db_conflicts for position, types in types_by_position.items(): types_by_position[position] = ", ".join(sorted(types)) types_by_position = [ f"{position} ({types})" for position, types in types_by_position.items() ] return ", ".join(types_by_position) def to_internal_value(self, data): if not isinstance(data, list): message = self.error_messages["not_a_list"].format( input_type=type(data).__name__ ) raise serializers.ValidationError( {api_settings.NON_FIELD_ERRORS_KEY: [message]}, code="not_a_list" ) if not self.allow_empty and len(data) == 0: if self.parent and self.partial: raise serializers.SkipField() else: self.fail("empty") partial = self.partial # build look-up objects for instances and data, so we can look them up with their keys try: known_instances = {(x.subname, x.type): x for x in self.instance} except TypeError: # in case self.instance is None (as during POST) known_instances = {} errors = [{} for _ in data] indices = {} for idx, item in enumerate(data): # Validate data types before using anything from it if not isinstance(item, dict): errors[idx].update( non_field_errors=f"Expected a dictionary, but got {type(item).__name__}." ) continue s, t = self._key(item) # subname, type if not (isinstance(s, str) or s is None): errors[idx].update( subname=f"Expected a string, but got {type(s).__name__}." ) if not (isinstance(t, str) or t is None): errors[idx].update( type=f"Expected a string, but got {type(t).__name__}." ) if errors[idx]: continue # Construct an index of the RRsets in `data` by `s` and `t`. As (subname, type) may be given multiple times # (although invalid), we make indices[s][t] a set to properly keep track. We also check and record RRsets # which are known in the database (once per subname), using index `None` (for checking CNAME exclusivity). if s not in indices: types = self.child.domain.rrset_set.filter(subname=s).values_list( "type", flat=True ) indices[s] = {type_: {None} for type_ in types} items = indices[s].setdefault(t, set()) items.add(idx) collapsed_indices = copy.deepcopy(indices) for idx, item in enumerate(data): if errors[idx]: continue if item.get("records") == []: s, t = self._key(item) collapsed_indices[s][t] -= {idx, None} # Iterate over all rows in the data given ret = [] for idx, item in enumerate(data): if errors[idx]: continue try: # see if other rows have the same key s, t = self._key(item) data_indices = indices[s][t] - {None} if len(data_indices) > 1: raise serializers.ValidationError( { "non_field_errors": [ "Same subname and type as in position(s) %s, but must be unique." % ", ".join(map(str, data_indices - {idx})) ] } ) # see if other rows violate CNAME exclusivity if item.get("records") != []: conflicting_indices_by_type = { k: v for k, v in collapsed_indices[s].items() if (k == "CNAME") != (t == "CNAME") } if any(conflicting_indices_by_type.values()): types_by_position = self._types_by_position_string( conflicting_indices_by_type ) raise serializers.ValidationError( { "non_field_errors": [ f"RRset with conflicting type present: {types_by_position}." " (No other RRsets are allowed alongside CNAME.)" ] } ) # determine if this is a partial update (i.e. PATCH): # we allow partial update if a partial update method (i.e. PATCH) is used, as indicated by self.partial, # and if this is not actually a create request because it is unknown and nonempty unknown = self._key(item) not in known_instances.keys() nonempty = item.get("records", None) != [] self.partial = partial and not (unknown and nonempty) self.child.instance = known_instances.get(self._key(item), None) # with partial value and instance in place, let the validation begin! validated = self.child.run_validation(item) except serializers.ValidationError as exc: errors[idx].update(exc.detail) else: ret.append(validated) self.partial = partial if any(errors): raise serializers.ValidationError(errors) return ret def update(self, instance, validated_data): """ Creates, updates and deletes RRsets according to the validated_data given. Relevant instances must be passed as a queryset in the `instance` argument. RRsets that appear in `instance` are considered "known", other RRsets are considered "unknown". RRsets that appear in `validated_data` with records == [] are considered empty, otherwise non-empty. The update proceeds as follows: 1. All unknown, non-empty RRsets are created. 2. All known, non-empty RRsets are updated. 3. All known, empty RRsets are deleted. 4. Unknown, empty RRsets will not cause any action. Rationale: As both "known"/"unknown" and "empty"/"non-empty" are binary partitions on `everything`, the combination of both partitions `everything` in four disjoint subsets. Hence, every RRset in `everything` is taken care of. empty | non-empty ------- | -------- | ----------- known | delete | update unknown | no-op | create :param instance: QuerySet of relevant RRset objects, i.e. the Django.Model subclass instances. Relevant are all instances that are referenced in `validated_data`. If a referenced RRset is missing from instances, it will be considered unknown and hence be created. This may cause a database integrity error. If an RRset is given, but not relevant (i.e. not referred to by `validated_data`), a ValueError will be raised. :param validated_data: List of RRset data objects, i.e. dictionaries. :return: List of RRset objects (Django.Model subclass) that have been created or updated. """ def is_empty(data_item): return data_item.get("records", None) == [] query = Q( pk__in=[] ) # start out with an always empty query, see https://stackoverflow.com/q/35893867/6867099 for item in validated_data: query |= Q( type=item["type"], subname=item["subname"] ) # validation has ensured these fields exist instance = instance.filter(query) instance_index = {(rrset.subname, rrset.type): rrset for rrset in instance} data_index = {self._key(data): data for data in validated_data} if data_index.keys() | instance_index.keys() != data_index.keys(): raise ValueError( "Given set of known RRsets (`instance`) is not a subset of RRsets referred to in" " `validated_data`. While this would produce a correct result, this is illegal due to its" " inefficiency." ) everything = instance_index.keys() | data_index.keys() known = instance_index.keys() unknown = everything - known # noinspection PyShadowingNames empty = {self._key(data) for data in validated_data if is_empty(data)} nonempty = everything - empty # noinspection PyUnusedLocal noop = unknown & empty created = unknown & nonempty updated = known & nonempty deleted = known & empty ret = [] # The above algorithm makes sure that created, updated, and deleted are disjoint. Thus, no "override cases" # (such as: an RRset should be updated and delete, what should be applied last?) need to be considered. # We apply deletion first to get any possible CNAME exclusivity collisions out of the way. for subname, type_ in deleted: instance_index[(subname, type_)].delete() for subname, type_ in created: ret.append(self.child.create(validated_data=data_index[(subname, type_)])) for subname, type_ in updated: ret.append( self.child.update( instance=instance_index[(subname, type_)], validated_data=data_index[(subname, type_)], ) ) return ret def save(self, **kwargs): kwargs.setdefault("domain", self.child.domain) return super().save(**kwargs) class RRsetSerializer(ConditionalExistenceModelSerializer): domain = serializers.SlugRelatedField(read_only=True, slug_field="name") records = RRSerializer(many=True) ttl = serializers.IntegerField(max_value=settings.MAXIMUM_TTL) class Meta: model = models.RRset fields = ( "created", "domain", "subname", "name", "records", "ttl", "type", "touched", ) extra_kwargs = { "subname": {"required": False, "default": NonBulkOnlyDefault("")} } list_serializer_class = RRsetListSerializer def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) try: self.domain = self.context["domain"] except KeyError: raise ValueError( "RRsetSerializer() must be given a domain object (to validate uniqueness constraints)." ) self.minimum_ttl = self.context.get("minimum_ttl", self.domain.minimum_ttl) def get_fields(self): fields = super().get_fields() fields["subname"].validators.append(ReadOnlyOnUpdateValidator()) fields["type"].validators.append(ReadOnlyOnUpdateValidator()) fields["ttl"].validators.append(MinValueValidator(limit_value=self.minimum_ttl)) return fields def get_validators(self): return [ UniqueTogetherValidator( self.domain.rrset_set, ("subname", "type"), message="Another RRset with the same subdomain and type exists for this domain.", ), ExclusionConstraintValidator( self.domain.rrset_set, ("subname",), exclusion_condition=( "type", "CNAME", ), message="RRset with conflicting type present: database ({types})." " (No other RRsets are allowed alongside CNAME.)", ), ] @staticmethod def validate_type(value): if value not in models.RR_SET_TYPES_MANAGEABLE: # user cannot manage this type, let's try to tell her the reason if value in models.RR_SET_TYPES_AUTOMATIC: raise serializers.ValidationError( f"You cannot tinker with the {value} RR set. It is managed " f"automatically." ) elif value.startswith("TYPE"): raise serializers.ValidationError( "Generic type format is not supported." ) else: raise serializers.ValidationError( f"The {value} RR set type is currently unsupported." ) return value def validate_records(self, value): # `records` is usually allowed to be empty (for idempotent delete), except for POST requests which are intended # for RRset creation only. We use the fact that DRF generic views pass the request in the serializer context. request = self.context.get("request") if request and request.method == "POST" and not value: raise serializers.ValidationError( "This field must not be empty when using POST." ) return value def validate_subname(self, value): try: dns.name.from_text(value, dns.name.from_text(self.domain.name)) except dns.name.NameTooLong: raise serializers.ValidationError( "This field combined with the domain name must not exceed 255 characters.", code="name_too_long", ) return value def validate(self, attrs): if "records" in attrs: try: type_ = attrs["type"] except KeyError: # on the RRsetDetail endpoint, the type is not in attrs type_ = self.instance.type try: attrs["records"] = [ { "content": models.RR.canonical_presentation_format( rr["content"], type_ ) } for rr in attrs["records"] ] except ValueError as ex: raise serializers.ValidationError(str(ex)) # There is a 12 byte baseline requirement per record, c.f. # https://lists.isc.org/pipermail/bind-users/2008-April/070137.html # There also seems to be a 32 byte (?) baseline requirement per RRset, plus the qname length, see # https://lists.isc.org/pipermail/bind-users/2008-April/070148.html # The binary length of the record depends actually on the type, but it's never longer than vanilla len() qname = models.RRset.construct_name( attrs.get("subname", ""), self.domain.name ) conservative_total_length = ( 32 + len(qname) + sum(12 + len(rr["content"]) for rr in attrs["records"]) ) # Add some leeway for RRSIG record (really ~110 bytes) and other data we have not thought of conservative_total_length += 256 excess_length = conservative_total_length - 65535 # max response size if excess_length > 0: raise serializers.ValidationError( f"Total length of RRset exceeds limit by {excess_length} bytes.", code="max_length", ) return attrs def exists(self, arg): if isinstance(arg, models.RRset): return arg.records.exists() if arg.pk else False else: return bool(arg.get("records")) if "records" in arg.keys() else True def create(self, validated_data): rrs_data = validated_data.pop("records") rrset = models.RRset.objects.create(**validated_data) self._set_all_record_contents(rrset, rrs_data) return rrset def update(self, instance: models.RRset, validated_data): rrs_data = validated_data.pop("records", None) if rrs_data is not None: self._set_all_record_contents(instance, rrs_data) ttl = validated_data.pop("ttl", None) if ttl and instance.ttl != ttl: instance.ttl = ttl instance.save() # also updates instance.touched else: # Update instance.touched without triggering post-save signal (no pdns action required) models.RRset.objects.filter(pk=instance.pk).update(touched=timezone.now()) return instance def save(self, **kwargs): kwargs.setdefault("domain", self.domain) return super().save(**kwargs) @staticmethod def _set_all_record_contents(rrset: models.RRset, rrs): """ Updates this RR set's resource records, discarding any old values. :param rrset: the RRset at which we overwrite all RRs :param rrs: list of RR representations """ record_contents = [rr["content"] for rr in rrs] try: rrset.save_records(record_contents) except django.core.exceptions.ValidationError as e: raise serializers.ValidationError(e.messages, code="record-content")