From 1bad1e668b5344ce218b6fc38e76a2ece8ff284b Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Thu, 29 Jun 2023 15:40:00 -0700 Subject: [PATCH 01/12] WIP: full tinydns spec-compliant source implementation --- CHANGELOG.md | 2 + octodns/source/tinydns.py | 309 ++++++++++++++++++++++++++++---- tests/zones/tinydns/example.com | 16 +- 3 files changed, 288 insertions(+), 39 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a59c14..25c23ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ * octodns-report access --lenient flag to allow running reports with records sourced from providers with non-compliant record data. * Correctly handle FQDNs in TinyDNS config files that end with trailing .'s +* Complete rewrite of TinyDnsBaseSource to fully implement the spec and the ipv6 + extensions ## v1.0.0.rc0 - 2023-05-16 - First of the ones diff --git a/octodns/source/tinydns.py b/octodns/source/tinydns.py index 4d32e52..370d896 100755 --- a/octodns/source/tinydns.py +++ b/octodns/source/tinydns.py @@ -9,19 +9,26 @@ from collections import defaultdict from ipaddress import ip_address from os import listdir from os.path import join +from pprint import pprint from ..record import Record from ..zone import DuplicateRecordException, SubzoneRecordException from .base import BaseSource +def _decode_octal(s): + return re.sub(r'\\(\d\d\d)', lambda m: chr(int(m.group(1), 8)), s).replace( + ';', '\\;' + ) + + class TinyDnsBaseSource(BaseSource): + # spec https://cr.yp.to/djbdns/tinydns-data.html + # ipv6 addon spec https://docs.bytemark.co.uk/article/tinydns-format/ SUPPORTS_GEO = False SUPPORTS_DYNAMIC = False SUPPORTS = set(('A', 'CNAME', 'MX', 'NS', 'TXT', 'AAAA')) - split_re = re.compile(r':+') - def __init__(self, id, default_ttl=3600): super().__init__(id) self.default_ttl = default_ttl @@ -121,48 +128,282 @@ class TinyDnsBaseSource(BaseSource): 'populate: found %s records', len(zone.records) - before ) + def _records_for_at(self, zone, name, lines, in_addr=False, lenient=False): + # @fqdn:ip:x:dist:ttl:timestamp:lo + # MX (and optional A) + if in_addr: + return [] + + # see if we can find a ttl on any of the lines, first one wins + ttl = self.default_ttl + for line in lines: + try: + ttl = int(lines[0][4]) + break + except IndexError: + pass + + values = [] + for line in lines: + mx = line[2] + # if there's a . in the mx we hit a special case and use it as-is + if '.' not in mx: + # otherwise we treat it as the MX hostnam and construct the rest + mx = f'{mx}.ns.{zone.name}' + elif mx[-1] != '.': + mx = f'{mx}.' + + # default distance is 0 + try: + dist = line[3] or 0 + except IndexError: + dist = 0 + + # if we have an IP then we need to create an A for the MX + ip = line[1] + if ip: + mx_name = zone.hostname_from_fqdn(mx) + yield Record.new( + zone, mx_name, {'type': 'A', 'ttl': ttl, 'value': ip} + ) + + values.append({'preference': dist, 'exchange': mx}) + + yield Record.new( + zone, name, {'ttl': ttl, 'type': 'MX', 'values': values} + ) + + def _records_for_C(self, zone, name, lines, in_addr=False, lenient=False): + # Cfqdn:p:ttl:timestamp:lo + # CNAME + if in_addr: + return [] + + value = lines[0][1] + if value[-1] != '.': + value = f'{value}.' + + # see if we can find a ttl on any of the lines, first one wins + ttl = self.default_ttl + for line in lines: + try: + ttl = int(lines[0][2]) + break + except IndexError: + pass + + return [ + Record.new( + zone, name, {'ttl': ttl, 'type': 'CNAME', 'value': value} + ) + ] + + def _records_for_caret( + self, zone, name, lines, in_addr=False, lenient=False + ): + # .fqdn:ip:x:ttl:timestamp:lo + # NS (and optional A) + if not in_addr: + return [] + + raise NotImplementedError() + + def _records_for_equal( + self, zone, name, lines, in_addr=False, lenient=False + ): + # =fqdn:ip:ttl:timestamp:lo + # A (in_addr False) & PTR (in_addr True) + return self._records_for_plus( + zone, name, lines, in_addr, lenient + ) + self._records_for_caret(zone, name, lines, in_addr, lenient) + + def _records_for_dot(self, zone, name, lines, in_addr=False, lenient=False): + # .fqdn:ip:x:ttl:timestamp:lo + # NS (and optional A) + if not in_addr: + return [] + + # see if we can find a ttl on any of the lines, first one wins + ttl = self.default_ttl + for line in lines: + try: + ttl = int(lines[0][3]) + break + except IndexError: + pass + + values = [] + for line in lines: + ns = line[2] + # if there's a . in the ns we hit a special case and use it as-is + if '.' not in ns: + # otherwise we treat it as the NS hostnam and construct the rest + ns = f'{ns}.ns.{zone.name}' + elif ns[-1] != '.': + ns = f'{ns}.' + + # if we have an IP then we need to create an A for the MX + ip = line[1] + if ip: + ns_name = zone.hostname_from_fqdn(ns) + yield Record.new( + zone, ns_name, {'type': 'A', 'ttl': ttl, 'value': ip} + ) + + values.append(ns) + + yield Record.new( + zone, name, {'ttl': ttl, 'type': 'NS', 'values': values} + ) + + _records_for_amp = _records_for_dot + + def _records_for_plus( + self, zone, name, lines, in_addr=False, lenient=False + ): + # +fqdn:ip:ttl:timestamp:lo + # A + if in_addr: + return [] + + # collect our ip(s) + ips = [l[1] for l in lines if l[1] != '0.0.0.0'] + + if not ips: + # we didn't find any value ips so nothing to do + return [] + + # see if we can find a ttl on any of the lines, first one wins + ttl = self.default_ttl + for line in lines: + try: + ttl = int(lines[0][2]) + break + except IndexError: + pass + + return [ + Record.new(zone, name, {'ttl': ttl, 'type': 'A', 'values': ips}) + ] + + def _records_for_quote( + self, zone, name, lines, in_addr=False, lenient=False + ): + # 'fqdn:s:ttl:timestamp:lo + # TXT + if in_addr: + return [] + + # collect our ip(s) + values = [_decode_octal(l[1]) for l in lines] + + # see if we can find a ttl on any of the lines, first one wins + ttl = self.default_ttl + for line in lines: + try: + ttl = int(lines[0][2]) + break + except IndexError: + pass + + return [ + Record.new( + zone, name, {'ttl': ttl, 'type': 'TXT', 'values': values} + ) + ] + + def _records_for_three( + self, zone, name, lines, in_addr=False, lenient=False + ): + # 3fqdn:ip:ttl:timestamp:lo + # AAAA + if in_addr: + return [] + + # collect our ip(s) + ips = [] + for line in lines: + # TinyDNS files have the ipv6 address written in full, but with the + # colons removed. This inserts a colon every 4th character to make + # the address correct. + ips.append(u':'.join(textwrap.wrap(line[1], 4))) + + # see if we can find a ttl on any of the lines, first one wins + ttl = self.default_ttl + for line in lines: + try: + ttl = int(lines[0][2]) + break + except IndexError: + pass + + return [ + Record.new(zone, name, {'ttl': ttl, 'type': 'AAAA', 'values': ips}) + ] + + def _records_for_six(self, zone, name, lines, in_addr=False, lenient=False): + # 6fqdn:ip:ttl:timestamp:lo + # AAAA (in_addr False) & PTR (in_addr True) + return self._records_for_three( + zone, name, lines, in_addr, lenient + ) + self._records_for_caret(zone, name, lines, in_addr, lenient) + + TYPE_MAP = { + '=': _records_for_equal, # A + '^': _records_for_caret, # PTR + '.': _records_for_dot, # NS + 'C': _records_for_C, # CNAME + '+': _records_for_plus, # A + '@': _records_for_at, # MX + '&': _records_for_amp, # NS + '\'': _records_for_quote, # TXT + '3': _records_for_three, # AAAA + '6': _records_for_six, # AAAA + # TODO: + #'S': _records_for_S, # SRV + # Sfqdn:ip:x:port:priority:weight:ttl:timestamp:lo + #':': _record_for_semicolon # arbitrary + # :fqdn:n:rdata:ttl:timestamp:lo + } + def _populate_normal(self, zone, lenient): - type_map = { - '=': 'A', - '^': None, - '.': 'NS', - 'C': 'CNAME', - '+': 'A', - '@': 'MX', - '\'': 'TXT', - '3': 'AAAA', - '6': 'AAAA', - } name_re = re.compile(fr'((?P.+)\.)?{zone.name[:-1]}\.?$') data = defaultdict(lambda: defaultdict(list)) for line in self._lines(): _type = line[0] - if _type not in type_map: - # Something we don't care about - continue - _type = type_map[_type] - if not _type: - continue # Skip type, remove trailing comments, and omit newline line = line[1:].split('#', 1)[0] # Split on :'s including :: and strip leading/trailing ws - line = [p.strip() for p in self.split_re.split(line)] - match = name_re.match(line[0]) - if not match: + line = [p.strip() for p in line.split(':')] + # make sure the name portion matches the zone we're currently + # working on + name = line[0] + if not name_re.match(name): + self.log.info('skipping name %s, not a match, %s: ', name, line) continue - name = zone.hostname_from_fqdn(line[0]) - data[name][_type].append(line[1:]) - - for name, types in data.items(): - for _type, d in types.items(): - data_for = getattr(self, f'_data_for_{_type}') - data = data_for(_type, d) - if data: - record = Record.new( - zone, name, data, source=self, lenient=lenient - ) + # remove the zone name + name = zone.hostname_from_fqdn(name) + data[_type][name].append(line) + + pprint(data) + + for _type, names in data.items(): + records_for = self.TYPE_MAP.get(_type, None) + if _type not in self.TYPE_MAP: + # Something we don't care about + self.log.info( + 'skipping type %s, not supported/interested', _type + ) + continue + + print(_type) + for name, lines in names.items(): + for record in records_for( + self, zone, name, lines, lenient=lenient + ): + pprint({'record': record}) try: zone.add_record(record, lenient=lenient) except SubzoneRecordException: @@ -177,7 +418,7 @@ class TinyDnsBaseSource(BaseSource): for line in self._lines(): _type = line[0] # We're only interested in = (A+PTR), and ^ (PTR) records - if _type not in ('=', '^'): + if _type not in ('=', '^', '&'): continue # Skip type, remove trailing comments, and omit newline diff --git a/tests/zones/tinydns/example.com b/tests/zones/tinydns/example.com index a1f983f..e32bfed 100755 --- a/tests/zones/tinydns/example.com +++ b/tests/zones/tinydns/example.com @@ -27,20 +27,26 @@ Ccname.other.foo:www.other.foo # MX @example.com::smtp-1-host.example.com:10 @example.com.::smtp-2-host.example.com:20 -# MX with ttl -@smtp.example.com::smtp-1-host.example.com:30:1800 -@smtp.example.com.::smtp-2-host.example.com:40:1800 +# MX with ttl and ip +@smtp.example.com:21.22.23.24:smtp-1-host:30:1800 +@smtp.example.com.:22.23.24.25:smtp-2-host:40:1800 -# NS +# NS for sub .sub.example.com::ns3.ns.com:30 .sub.example.com.::ns4.ns.com:30 +# NS with ip +.other.example.com:14.15.16.17:ns5:30 +.other.example.com.:15.16.17.18:ns6:30 # A, under sub -+www.sub.example.com::1.2.3.4 ++www.sub.example.com:1.2.3.4 # Top-level NS .example.com::ns1.ns.com .example.com.::ns2.ns.com +# Top-level NS with automatic A +&example.com:42.43.44.45:a +&example.com.:43.44.45.46:b:31 # sub special cases +a1.blah-asdf.subtest.com:10.2.3.5 From 50df3387e68b37665a26867f6d272a84d65fb851 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 30 Jun 2023 12:25:06 -0700 Subject: [PATCH 02/12] tinydns lines to records isn't one-to-one, rework to handle that --- octodns/source/tinydns.py | 160 ++++++++++++++++++-------------------- 1 file changed, 77 insertions(+), 83 deletions(-) diff --git a/octodns/source/tinydns.py b/octodns/source/tinydns.py index 370d896..146d62a 100755 --- a/octodns/source/tinydns.py +++ b/octodns/source/tinydns.py @@ -16,12 +16,6 @@ from ..zone import DuplicateRecordException, SubzoneRecordException from .base import BaseSource -def _decode_octal(s): - return re.sub(r'\\(\d\d\d)', lambda m: chr(int(m.group(1), 8)), s).replace( - ';', '\\;' - ) - - class TinyDnsBaseSource(BaseSource): # spec https://cr.yp.to/djbdns/tinydns-data.html # ipv6 addon spec https://docs.bytemark.co.uk/article/tinydns-format/ @@ -128,7 +122,7 @@ class TinyDnsBaseSource(BaseSource): 'populate: found %s records', len(zone.records) - before ) - def _records_for_at(self, zone, name, lines, in_addr=False, lenient=False): + def _records_for_at(self, zone, name, lines, in_addr=False): # @fqdn:ip:x:dist:ttl:timestamp:lo # MX (and optional A) if in_addr: @@ -163,17 +157,13 @@ class TinyDnsBaseSource(BaseSource): ip = line[1] if ip: mx_name = zone.hostname_from_fqdn(mx) - yield Record.new( - zone, mx_name, {'type': 'A', 'ttl': ttl, 'value': ip} - ) + yield 'A', mx_name, [ip] values.append({'preference': dist, 'exchange': mx}) - yield Record.new( - zone, name, {'ttl': ttl, 'type': 'MX', 'values': values} - ) + yield 'MX', name, values - def _records_for_C(self, zone, name, lines, in_addr=False, lenient=False): + def _records_for_C(self, zone, name, lines, in_addr=False): # Cfqdn:p:ttl:timestamp:lo # CNAME if in_addr: @@ -192,15 +182,9 @@ class TinyDnsBaseSource(BaseSource): except IndexError: pass - return [ - Record.new( - zone, name, {'ttl': ttl, 'type': 'CNAME', 'value': value} - ) - ] + yield 'CNAME', name, [value] - def _records_for_caret( - self, zone, name, lines, in_addr=False, lenient=False - ): + def _records_for_caret(self, zone, name, lines, in_addr=False): # .fqdn:ip:x:ttl:timestamp:lo # NS (and optional A) if not in_addr: @@ -208,19 +192,16 @@ class TinyDnsBaseSource(BaseSource): raise NotImplementedError() - def _records_for_equal( - self, zone, name, lines, in_addr=False, lenient=False - ): + def _records_for_equal(self, zone, name, lines, in_addr=False): # =fqdn:ip:ttl:timestamp:lo # A (in_addr False) & PTR (in_addr True) - return self._records_for_plus( - zone, name, lines, in_addr, lenient - ) + self._records_for_caret(zone, name, lines, in_addr, lenient) + yield from self._records_for_plus(zone, name, lines, in_addr) + yield from self._records_for_caret(zone, name, lines, in_addr) - def _records_for_dot(self, zone, name, lines, in_addr=False, lenient=False): + def _records_for_dot(self, zone, name, lines, in_addr=False): # .fqdn:ip:x:ttl:timestamp:lo # NS (and optional A) - if not in_addr: + if in_addr: return [] # see if we can find a ttl on any of the lines, first one wins @@ -246,21 +227,15 @@ class TinyDnsBaseSource(BaseSource): ip = line[1] if ip: ns_name = zone.hostname_from_fqdn(ns) - yield Record.new( - zone, ns_name, {'type': 'A', 'ttl': ttl, 'value': ip} - ) + yield 'A', ns_name, [ip] values.append(ns) - yield Record.new( - zone, name, {'ttl': ttl, 'type': 'NS', 'values': values} - ) + yield 'NS', name, values _records_for_amp = _records_for_dot - def _records_for_plus( - self, zone, name, lines, in_addr=False, lenient=False - ): + def _records_for_plus(self, zone, name, lines, in_addr=False): # +fqdn:ip:ttl:timestamp:lo # A if in_addr: @@ -282,20 +257,19 @@ class TinyDnsBaseSource(BaseSource): except IndexError: pass - return [ - Record.new(zone, name, {'ttl': ttl, 'type': 'A', 'values': ips}) - ] + yield 'A', name, ips - def _records_for_quote( - self, zone, name, lines, in_addr=False, lenient=False - ): + def _records_for_quote(self, zone, name, lines, in_addr=False): # 'fqdn:s:ttl:timestamp:lo # TXT if in_addr: return [] # collect our ip(s) - values = [_decode_octal(l[1]) for l in lines] + values = [ + l[1].encode('latin1').decode('unicode-escape').replace(";", "\\;") + for l in lines + ] # see if we can find a ttl on any of the lines, first one wins ttl = self.default_ttl @@ -306,15 +280,9 @@ class TinyDnsBaseSource(BaseSource): except IndexError: pass - return [ - Record.new( - zone, name, {'ttl': ttl, 'type': 'TXT', 'values': values} - ) - ] + yield 'TXT', name, values - def _records_for_three( - self, zone, name, lines, in_addr=False, lenient=False - ): + def _records_for_three(self, zone, name, lines, in_addr=False): # 3fqdn:ip:ttl:timestamp:lo # AAAA if in_addr: @@ -337,18 +305,15 @@ class TinyDnsBaseSource(BaseSource): except IndexError: pass - return [ - Record.new(zone, name, {'ttl': ttl, 'type': 'AAAA', 'values': ips}) - ] + yield 'AAAA', name, ips - def _records_for_six(self, zone, name, lines, in_addr=False, lenient=False): + def _records_for_six(self, zone, name, lines, in_addr=False): # 6fqdn:ip:ttl:timestamp:lo # AAAA (in_addr False) & PTR (in_addr True) - return self._records_for_three( - zone, name, lines, in_addr, lenient - ) + self._records_for_caret(zone, name, lines, in_addr, lenient) + yield from self._records_for_three(zone, name, lines, in_addr) + yield from self._records_for_caret(zone, name, lines, in_addr) - TYPE_MAP = { + SYMBOL_MAP = { '=': _records_for_equal, # A '^': _records_for_caret, # PTR '.': _records_for_dot, # NS @@ -366,12 +331,12 @@ class TinyDnsBaseSource(BaseSource): # :fqdn:n:rdata:ttl:timestamp:lo } - def _populate_normal(self, zone, lenient): + def _process_lines(self, zone, lines): name_re = re.compile(fr'((?P.+)\.)?{zone.name[:-1]}\.?$') data = defaultdict(lambda: defaultdict(list)) - for line in self._lines(): - _type = line[0] + for line in lines: + symbol = line[0] # Skip type, remove trailing comments, and omit newline line = line[1:].split('#', 1)[0] @@ -385,32 +350,61 @@ class TinyDnsBaseSource(BaseSource): continue # remove the zone name name = zone.hostname_from_fqdn(name) - data[_type][name].append(line) + data[symbol][name].append(line) - pprint(data) + return data - for _type, names in data.items(): - records_for = self.TYPE_MAP.get(_type, None) - if _type not in self.TYPE_MAP: + def _process_symbols(self, zone, symbols): + types = defaultdict(lambda: defaultdict(list)) + ttls = defaultdict(lambda: defaultdict(lambda: self.default_ttl)) + for symbol, names in symbols.items(): + records_for = self.SYMBOL_MAP.get(symbol, None) + if not records_for: # Something we don't care about self.log.info( - 'skipping type %s, not supported/interested', _type + 'skipping type %s, not supported/interested', symbol ) continue - print(_type) for name, lines in names.items(): - for record in records_for( - self, zone, name, lines, lenient=lenient - ): - pprint({'record': record}) - try: - zone.add_record(record, lenient=lenient) - except SubzoneRecordException: - self.log.debug( - '_populate_normal: skipping subzone record=%s', - record, - ) + for _type, name, values in records_for(self, zone, name, lines): + types[_type][name].extend(values) + + return types, ttls + + def _populate_normal(self, zone, lenient): + # This is complicate b/c the mapping between tinydns line types (called + # symbols here) is not one to one with (octoDNS) records. Some lines + # create multiple types of records and multiple lines are often combined + # to make a single record (with multiple values.) Sometimes both happen. + # To deal with this we'll do things in 3 stages: + + # first group lines by their symbol and name + symbols = self._process_lines(zone, self._lines()) + pprint({'symbols': symbols}) + + # then work through those to group values by their _type and name + types, ttls = self._process_symbols(zone, symbols) + pprint({'types': types, 'ttls': ttls}) + + # now we finally have all the values for each (soon to be) record + # collected together, turn them into their coresponding record and add + # it to the zone + for _type, names in types.items(): + for name, values in names.items(): + data = {'ttl': ttls[_type][name], 'type': _type} + if len(values) > 1: + data['values'] = values + else: + data['value'] = values[0] + pprint({'name': name, 'data': data}) + record = Record.new(zone, name, data, lenient=lenient) + try: + zone.add_record(record, lenient=lenient) + except SubzoneRecordException: + self.log.debug( + '_populate_normal: skipping subzone record=%s', record + ) def _populate_in_addr_arpa(self, zone, lenient): name_re = re.compile(fr'(?P.+)\.{zone.name[:-1]}\.?$') From cdca6085a6f4c3d902884b712a4042b4829da069 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 30 Jun 2023 13:13:07 -0700 Subject: [PATCH 03/12] Correctly handle TTLs, remove dead code, arpa support --- octodns/source/tinydns.py | 221 +++++++++++---------------- tests/test_octodns_source_tinydns.py | 11 +- 2 files changed, 96 insertions(+), 136 deletions(-) diff --git a/octodns/source/tinydns.py b/octodns/source/tinydns.py index 146d62a..5572b04 100755 --- a/octodns/source/tinydns.py +++ b/octodns/source/tinydns.py @@ -17,8 +17,6 @@ from .base import BaseSource class TinyDnsBaseSource(BaseSource): - # spec https://cr.yp.to/djbdns/tinydns-data.html - # ipv6 addon spec https://docs.bytemark.co.uk/article/tinydns-format/ SUPPORTS_GEO = False SUPPORTS_DYNAMIC = False SUPPORTS = set(('A', 'CNAME', 'MX', 'NS', 'TXT', 'AAAA')) @@ -27,105 +25,10 @@ class TinyDnsBaseSource(BaseSource): super().__init__(id) self.default_ttl = default_ttl - def _data_for_A(self, _type, records): - values = [] - for record in records: - if record[0] != '0.0.0.0': - values.append(record[0]) - if len(values) == 0: - return - try: - ttl = records[0][1] - except IndexError: - ttl = self.default_ttl - return {'ttl': ttl, 'type': _type, 'values': values} - - def _data_for_AAAA(self, _type, records): - values = [] - for record in records: - # TinyDNS files have the ipv6 address written in full, but with the - # colons removed. This inserts a colon every 4th character to make - # the address correct. - values.append(u":".join(textwrap.wrap(record[0], 4))) - try: - ttl = records[0][1] - except IndexError: - ttl = self.default_ttl - return {'ttl': ttl, 'type': _type, 'values': values} - - def _data_for_TXT(self, _type, records): - values = [] - - for record in records: - new_value = ( - record[0] - .encode('latin1') - .decode('unicode-escape') - .replace(";", "\\;") - ) - values.append(new_value) - - try: - ttl = records[0][1] - except IndexError: - ttl = self.default_ttl - return {'ttl': ttl, 'type': _type, 'values': values} - - def _data_for_CNAME(self, _type, records): - first = records[0] - try: - ttl = first[1] - except IndexError: - ttl = self.default_ttl - return {'ttl': ttl, 'type': _type, 'value': f'{first[0]}.'} - - def _data_for_MX(self, _type, records): - try: - ttl = records[0][2] - except IndexError: - ttl = self.default_ttl - return { - 'ttl': ttl, - 'type': _type, - 'values': [ - {'preference': r[1], 'exchange': f'{r[0]}.'} for r in records - ], - } - - def _data_for_NS(self, _type, records): - try: - ttl = records[0][1] - except IndexError: - ttl = self.default_ttl - return { - 'ttl': ttl, - 'type': _type, - 'values': [f'{r[0]}.' for r in records], - } - - def populate(self, zone, target=False, lenient=False): - self.log.debug( - 'populate: name=%s, target=%s, lenient=%s', - zone.name, - target, - lenient, - ) - - before = len(zone.records) - - if zone.name.endswith('in-addr.arpa.'): - self._populate_in_addr_arpa(zone, lenient) - else: - self._populate_normal(zone, lenient) - - self.log.info( - 'populate: found %s records', len(zone.records) - before - ) - - def _records_for_at(self, zone, name, lines, in_addr=False): + def _records_for_at(self, zone, name, lines, arpa=False): # @fqdn:ip:x:dist:ttl:timestamp:lo # MX (and optional A) - if in_addr: + if arpa: return [] # see if we can find a ttl on any of the lines, first one wins @@ -157,16 +60,16 @@ class TinyDnsBaseSource(BaseSource): ip = line[1] if ip: mx_name = zone.hostname_from_fqdn(mx) - yield 'A', mx_name, [ip] + yield 'A', mx_name, ttl, [ip] values.append({'preference': dist, 'exchange': mx}) - yield 'MX', name, values + yield 'MX', name, ttl, values - def _records_for_C(self, zone, name, lines, in_addr=False): + def _records_for_C(self, zone, name, lines, arpa=False): # Cfqdn:p:ttl:timestamp:lo # CNAME - if in_addr: + if arpa: return [] value = lines[0][1] @@ -182,26 +85,47 @@ class TinyDnsBaseSource(BaseSource): except IndexError: pass - yield 'CNAME', name, [value] + yield 'CNAME', name, ttl, [value] - def _records_for_caret(self, zone, name, lines, in_addr=False): + def _records_for_caret(self, zone, name, lines, arpa=False): # .fqdn:ip:x:ttl:timestamp:lo # NS (and optional A) - if not in_addr: + if not arpa: + print('bailing') return [] - raise NotImplementedError() + print('here') + values = [] + for line in lines: + value = line[1] + if value[-1] != '.': + value = f'{value}.' + values.append(value) - def _records_for_equal(self, zone, name, lines, in_addr=False): + # see if we can find a ttl on any of the lines, first one wins + ttl = self.default_ttl + for line in lines: + try: + ttl = int(line[2]) + break + except IndexError: + pass + + pprint({'caret': values}) + + yield 'PTR', name, ttl, values + + def _records_for_equal(self, zone, name, lines, arpa=False): # =fqdn:ip:ttl:timestamp:lo - # A (in_addr False) & PTR (in_addr True) - yield from self._records_for_plus(zone, name, lines, in_addr) - yield from self._records_for_caret(zone, name, lines, in_addr) + # A (arpa False) & PTR (arpa True) + print(f'here for {name}: {lines}') + yield from self._records_for_plus(zone, name, lines, arpa) + yield from self._records_for_caret(zone, name, lines, arpa) - def _records_for_dot(self, zone, name, lines, in_addr=False): + def _records_for_dot(self, zone, name, lines, arpa=False): # .fqdn:ip:x:ttl:timestamp:lo # NS (and optional A) - if in_addr: + if arpa: return [] # see if we can find a ttl on any of the lines, first one wins @@ -227,18 +151,18 @@ class TinyDnsBaseSource(BaseSource): ip = line[1] if ip: ns_name = zone.hostname_from_fqdn(ns) - yield 'A', ns_name, [ip] + yield 'A', ns_name, ttl, [ip] values.append(ns) - yield 'NS', name, values + yield 'NS', name, ttl, values _records_for_amp = _records_for_dot - def _records_for_plus(self, zone, name, lines, in_addr=False): + def _records_for_plus(self, zone, name, lines, arpa=False): # +fqdn:ip:ttl:timestamp:lo # A - if in_addr: + if arpa: return [] # collect our ip(s) @@ -257,12 +181,12 @@ class TinyDnsBaseSource(BaseSource): except IndexError: pass - yield 'A', name, ips + yield 'A', name, ttl, ips - def _records_for_quote(self, zone, name, lines, in_addr=False): + def _records_for_quote(self, zone, name, lines, arpa=False): # 'fqdn:s:ttl:timestamp:lo # TXT - if in_addr: + if arpa: return [] # collect our ip(s) @@ -280,12 +204,12 @@ class TinyDnsBaseSource(BaseSource): except IndexError: pass - yield 'TXT', name, values + yield 'TXT', name, ttl, values - def _records_for_three(self, zone, name, lines, in_addr=False): + def _records_for_three(self, zone, name, lines, arpa=False): # 3fqdn:ip:ttl:timestamp:lo # AAAA - if in_addr: + if arpa: return [] # collect our ip(s) @@ -305,13 +229,13 @@ class TinyDnsBaseSource(BaseSource): except IndexError: pass - yield 'AAAA', name, ips + yield 'AAAA', name, ttl, ips - def _records_for_six(self, zone, name, lines, in_addr=False): + def _records_for_six(self, zone, name, lines, arpa=False): # 6fqdn:ip:ttl:timestamp:lo - # AAAA (in_addr False) & PTR (in_addr True) - yield from self._records_for_three(zone, name, lines, in_addr) - yield from self._records_for_caret(zone, name, lines, in_addr) + # AAAA (arpa False) & PTR (arpa True) + yield from self._records_for_three(zone, name, lines, arpa) + yield from self._records_for_caret(zone, name, lines, arpa) SYMBOL_MAP = { '=': _records_for_equal, # A @@ -354,7 +278,7 @@ class TinyDnsBaseSource(BaseSource): return data - def _process_symbols(self, zone, symbols): + def _process_symbols(self, zone, symbols, arpa): types = defaultdict(lambda: defaultdict(list)) ttls = defaultdict(lambda: defaultdict(lambda: self.default_ttl)) for symbol, names in symbols.items(): @@ -367,12 +291,25 @@ class TinyDnsBaseSource(BaseSource): continue for name, lines in names.items(): - for _type, name, values in records_for(self, zone, name, lines): + for _type, name, ttl, values in records_for( + self, zone, name, lines, arpa=arpa + ): types[_type][name].extend(values) + # last one wins + ttls[_type][name] = ttl return types, ttls - def _populate_normal(self, zone, lenient): + def populate(self, zone, target=False, lenient=False): + self.log.debug( + 'populate: name=%s, target=%s, lenient=%s', + zone.name, + target, + lenient, + ) + + before = len(zone.records) + # This is complicate b/c the mapping between tinydns line types (called # symbols here) is not one to one with (octoDNS) records. Some lines # create multiple types of records and multiple lines are often combined @@ -384,7 +321,11 @@ class TinyDnsBaseSource(BaseSource): pprint({'symbols': symbols}) # then work through those to group values by their _type and name - types, ttls = self._process_symbols(zone, symbols) + zone_name = zone.name + arpa = zone_name.endswith('in-addr.arpa.') or zone_name.endswith( + 'ip6.arpa.' + ) + types, ttls = self._process_symbols(zone, symbols, arpa) pprint({'types': types, 'ttls': ttls}) # now we finally have all the values for each (soon to be) record @@ -399,14 +340,19 @@ class TinyDnsBaseSource(BaseSource): data['value'] = values[0] pprint({'name': name, 'data': data}) record = Record.new(zone, name, data, lenient=lenient) + pprint({'lenient': lenient}) try: zone.add_record(record, lenient=lenient) except SubzoneRecordException: - self.log.debug( - '_populate_normal: skipping subzone record=%s', record + self.log.error( + 'populate: skipping subzone record=%s', record ) - def _populate_in_addr_arpa(self, zone, lenient): + self.log.info( + 'populate: found %s records', len(zone.records) - before + ) + + def _populate_arpa_arpa(self, zone, lenient): name_re = re.compile(fr'(?P.+)\.{zone.name[:-1]}\.?$') for line in self._lines(): @@ -467,6 +413,11 @@ class TinyDnsFileSource(TinyDnsBaseSource): default_ttl: 3600 NOTE: timestamps & lo fields are ignored if present. + + The source intends to conform to and fully support the official spec, + https://cr.yp.to/djbdns/tinydns-data.html and the common patch/extensions to + support IPv6 and a few other record types, + https://docs.bytemark.co.uk/article/tinydns-format/. ''' def __init__(self, id, directory, default_ttl=3600): diff --git a/tests/test_octodns_source_tinydns.py b/tests/test_octodns_source_tinydns.py index 40ce9f5..f571081 100644 --- a/tests/test_octodns_source_tinydns.py +++ b/tests/test_octodns_source_tinydns.py @@ -178,9 +178,18 @@ class TestTinyDnsFileSource(TestCase): expected.add_record(record) changes = expected.changes(got, SimpleProvider()) + from pprint import pprint + + pprint( + { + 'changes': changes, + 'expected': expected.records, + 'got': got.records, + } + ) self.assertEqual([], changes) def test_ignores_subs(self): got = Zone('example.com.', ['sub']) self.source.populate(got) - self.assertEqual(16, len(got.records)) + self.assertEqual(23, len(got.records)) From 49233e2da649b875e3281b424cfd54facb5893c7 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 30 Jun 2023 14:31:06 -0700 Subject: [PATCH 04/12] Add owns method to Zone --- octodns/zone.py | 22 ++++++++++++++++++++++ tests/test_octodns_zone.py | 19 +++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/octodns/zone.py b/octodns/zone.py index c35e94c..ba527a2 100644 --- a/octodns/zone.py +++ b/octodns/zone.py @@ -75,6 +75,28 @@ class Zone(object): # it has utf8 chars return self._utf8_name_re.sub('', fqdn) + def owns(self, _type, fqdn): + if fqdn[-1] != '.': + fqdn = f'{fqdn}.' + + # if we don't end with the zone's name we aren't owned by it + if not fqdn.endswith(self.name): + return False + + hostname = self.hostname_from_fqdn(fqdn) + if hostname in self.sub_zones: + # if our hostname matches a sub-zone exactly we have to be a NS + # record + return _type == 'NS' + + for sub_zone in self.sub_zones: + if hostname.endswith(f'.{sub_zone}'): + # this belongs under a sub-zone + return False + + # otherwise we own it + return True + def add_record(self, record, replace=False, lenient=False): if self._origin: self.hydrate() diff --git a/tests/test_octodns_zone.py b/tests/test_octodns_zone.py index 0ee01b0..b3f76e2 100644 --- a/tests/test_octodns_zone.py +++ b/tests/test_octodns_zone.py @@ -191,6 +191,25 @@ class TestZone(TestCase): Zone('space not allowed.', []) self.assertTrue('whitespace not allowed' in str(ctx.exception)) + def test_owns(self): + zone = Zone('unit.tests.', set(['sub'])) + + self.assertTrue(zone.owns('A', 'unit.tests')) + self.assertTrue(zone.owns('A', 'unit.tests.')) + self.assertTrue(zone.owns('A', 'www.unit.tests.')) + self.assertTrue(zone.owns('A', 'www.unit.tests.')) + # we do own our direct sub's delegation NS records + self.assertTrue(zone.owns('NS', 'sub.unit.tests.')) + + # we don't own the root of our sub + self.assertFalse(zone.owns('A', 'sub.unit.tests.')) + + # of anything under it + self.assertFalse(zone.owns('A', 'www.sub.unit.tests.')) + + # including subsequent delegatoin NS records + self.assertFalse(zone.owns('NS', 'below.sub.unit.tests.')) + def test_sub_zones(self): # NS for exactly the sub is allowed zone = Zone('unit.tests.', set(['sub', 'barred'])) From 4a7df31445c4cb355cb3a2bcdd26dcba51bb6001 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 30 Jun 2023 14:59:09 -0700 Subject: [PATCH 05/12] handle a couple zone.owns edge cases --- octodns/zone.py | 8 ++++++-- tests/test_octodns_zone.py | 6 ++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/octodns/zone.py b/octodns/zone.py index ba527a2..9cbc712 100644 --- a/octodns/zone.py +++ b/octodns/zone.py @@ -79,8 +79,12 @@ class Zone(object): if fqdn[-1] != '.': fqdn = f'{fqdn}.' - # if we don't end with the zone's name we aren't owned by it - if not fqdn.endswith(self.name): + # if we exactly match the zone name we own it + if fqdn == self.name: + return True + + # if we don't end with the zone's name on a boundary we aren't owned + if not fqdn.endswith(f'.{self.name}'): return False hostname = self.hostname_from_fqdn(fqdn) diff --git a/tests/test_octodns_zone.py b/tests/test_octodns_zone.py index b3f76e2..93d8a87 100644 --- a/tests/test_octodns_zone.py +++ b/tests/test_octodns_zone.py @@ -210,6 +210,12 @@ class TestZone(TestCase): # including subsequent delegatoin NS records self.assertFalse(zone.owns('NS', 'below.sub.unit.tests.')) + # edge cases + # we don't own something that ends with our name, but isn't a boundary + self.assertFalse(zone.owns('A', 'foo-unit.tests.')) + # we do something that ends with the sub-zone, but isn't at a boundary + self.assertTrue(zone.owns('A', 'foo-sub.unit.tests.')) + def test_sub_zones(self): # NS for exactly the sub is allowed zone = Zone('unit.tests.', set(['sub', 'barred'])) From ca3a4541a0b035f049a3b15375bd96928155d8f6 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 30 Jun 2023 14:59:48 -0700 Subject: [PATCH 06/12] TindyDNS rework passing tests --- octodns/source/tinydns.py | 162 +++++++++++++-------------- tests/test_octodns_source_tinydns.py | 50 ++++++--- tests/zones/tinydns/example.com | 4 +- 3 files changed, 114 insertions(+), 102 deletions(-) diff --git a/octodns/source/tinydns.py b/octodns/source/tinydns.py index 5572b04..27492c1 100755 --- a/octodns/source/tinydns.py +++ b/octodns/source/tinydns.py @@ -3,16 +3,14 @@ # import logging -import re import textwrap from collections import defaultdict from ipaddress import ip_address from os import listdir from os.path import join -from pprint import pprint from ..record import Record -from ..zone import DuplicateRecordException, SubzoneRecordException +from ..zone import SubzoneRecordException from .base import BaseSource @@ -28,9 +26,15 @@ class TinyDnsBaseSource(BaseSource): def _records_for_at(self, zone, name, lines, arpa=False): # @fqdn:ip:x:dist:ttl:timestamp:lo # MX (and optional A) + if arpa: + # no arpa return [] + if not zone.owns('MX', name): + # if name doesn't live under our zone there's nothing for us to do + return + # see if we can find a ttl on any of the lines, first one wins ttl = self.default_ttl for line in lines: @@ -46,7 +50,7 @@ class TinyDnsBaseSource(BaseSource): # if there's a . in the mx we hit a special case and use it as-is if '.' not in mx: # otherwise we treat it as the MX hostnam and construct the rest - mx = f'{mx}.ns.{zone.name}' + mx = f'{mx}.mx.{zone.name}' elif mx[-1] != '.': mx = f'{mx}.' @@ -59,8 +63,7 @@ class TinyDnsBaseSource(BaseSource): # if we have an IP then we need to create an A for the MX ip = line[1] if ip: - mx_name = zone.hostname_from_fqdn(mx) - yield 'A', mx_name, ttl, [ip] + yield 'A', mx, ttl, [ip] values.append({'preference': dist, 'exchange': mx}) @@ -69,9 +72,15 @@ class TinyDnsBaseSource(BaseSource): def _records_for_C(self, zone, name, lines, arpa=False): # Cfqdn:p:ttl:timestamp:lo # CNAME + if arpa: + # no arpa return [] + if not zone.owns('CNAME', name): + # if name doesn't live under our zone there's nothing for us to do + return + value = lines[0][1] if value[-1] != '.': value = f'{value}.' @@ -88,19 +97,35 @@ class TinyDnsBaseSource(BaseSource): yield 'CNAME', name, ttl, [value] def _records_for_caret(self, zone, name, lines, arpa=False): - # .fqdn:ip:x:ttl:timestamp:lo - # NS (and optional A) + # ^fqdn:p:ttl:timestamp:lo + # PTR, line may be a A/AAAA or straight PTR + if not arpa: - print('bailing') + # we only operate on arpa return [] - print('here') - values = [] + names = defaultdict(list) for line in lines: - value = line[1] + if line[0].endswith('in-addr.arpa') or line[0].endswith( + 'ip6.arpa.' + ): + # it's a straight PTR record, already in in-addr.arpa format, + # 2nd item is the name it points to + name = line[0] + value = line[1] + else: + # it's not a PTR we need to build up the PTR data from what + # we're given + value = line[0] + addr = line[1] + if '.' not in addr: + addr = u':'.join(textwrap.wrap(line[1], 4)) + addr = ip_address(addr) + name = addr.reverse_pointer + if value[-1] != '.': value = f'{value}.' - values.append(value) + names[name].append(value) # see if we can find a ttl on any of the lines, first one wins ttl = self.default_ttl @@ -111,23 +136,30 @@ class TinyDnsBaseSource(BaseSource): except IndexError: pass - pprint({'caret': values}) - - yield 'PTR', name, ttl, values + for name, values in names.items(): + if zone.owns('PTR', name): + yield 'PTR', name, ttl, values def _records_for_equal(self, zone, name, lines, arpa=False): # =fqdn:ip:ttl:timestamp:lo # A (arpa False) & PTR (arpa True) - print(f'here for {name}: {lines}') - yield from self._records_for_plus(zone, name, lines, arpa) - yield from self._records_for_caret(zone, name, lines, arpa) + if arpa: + yield from self._records_for_caret(zone, name, lines, arpa) + else: + yield from self._records_for_plus(zone, name, lines, arpa) def _records_for_dot(self, zone, name, lines, arpa=False): # .fqdn:ip:x:ttl:timestamp:lo # NS (and optional A) + if arpa: + # no arpa return [] + if not zone.owns('NS', name): + # if name doesn't live under our zone there's nothing for us to do + return + # see if we can find a ttl on any of the lines, first one wins ttl = self.default_ttl for line in lines: @@ -150,8 +182,7 @@ class TinyDnsBaseSource(BaseSource): # if we have an IP then we need to create an A for the MX ip = line[1] if ip: - ns_name = zone.hostname_from_fqdn(ns) - yield 'A', ns_name, ttl, [ip] + yield 'A', ns, ttl, [ip] values.append(ns) @@ -162,9 +193,15 @@ class TinyDnsBaseSource(BaseSource): def _records_for_plus(self, zone, name, lines, arpa=False): # +fqdn:ip:ttl:timestamp:lo # A + if arpa: + # no arpa return [] + if not zone.owns('A', name): + # if name doesn't live under our zone there's nothing for us to do + return + # collect our ip(s) ips = [l[1] for l in lines if l[1] != '0.0.0.0'] @@ -186,9 +223,15 @@ class TinyDnsBaseSource(BaseSource): def _records_for_quote(self, zone, name, lines, arpa=False): # 'fqdn:s:ttl:timestamp:lo # TXT + if arpa: + # no arpa return [] + if not zone.owns('TXT', name): + # if name doesn't live under our zone there's nothing for us to do + return + # collect our ip(s) values = [ l[1].encode('latin1').decode('unicode-escape').replace(";", "\\;") @@ -209,9 +252,15 @@ class TinyDnsBaseSource(BaseSource): def _records_for_three(self, zone, name, lines, arpa=False): # 3fqdn:ip:ttl:timestamp:lo # AAAA + if arpa: + # no arpa return [] + if not zone.owns('AAAA', name): + # if name doesn't live under our zone there's nothing for us to do + return + # collect our ip(s) ips = [] for line in lines: @@ -234,8 +283,10 @@ class TinyDnsBaseSource(BaseSource): def _records_for_six(self, zone, name, lines, arpa=False): # 6fqdn:ip:ttl:timestamp:lo # AAAA (arpa False) & PTR (arpa True) - yield from self._records_for_three(zone, name, lines, arpa) - yield from self._records_for_caret(zone, name, lines, arpa) + if arpa: + yield from self._records_for_caret(zone, name, lines, arpa) + else: + yield from self._records_for_three(zone, name, lines, arpa) SYMBOL_MAP = { '=': _records_for_equal, # A @@ -256,8 +307,6 @@ class TinyDnsBaseSource(BaseSource): } def _process_lines(self, zone, lines): - name_re = re.compile(fr'((?P.+)\.)?{zone.name[:-1]}\.?$') - data = defaultdict(lambda: defaultdict(list)) for line in lines: symbol = line[0] @@ -266,15 +315,7 @@ class TinyDnsBaseSource(BaseSource): line = line[1:].split('#', 1)[0] # Split on :'s including :: and strip leading/trailing ws line = [p.strip() for p in line.split(':')] - # make sure the name portion matches the zone we're currently - # working on - name = line[0] - if not name_re.match(name): - self.log.info('skipping name %s, not a match, %s: ', name, line) - continue - # remove the zone name - name = zone.hostname_from_fqdn(name) - data[symbol][name].append(line) + data[symbol][line[0]].append(line) return data @@ -294,6 +335,8 @@ class TinyDnsBaseSource(BaseSource): for _type, name, ttl, values in records_for( self, zone, name, lines, arpa=arpa ): + # remove the zone name + name = zone.hostname_from_fqdn(name) types[_type][name].extend(values) # last one wins ttls[_type][name] = ttl @@ -318,7 +361,6 @@ class TinyDnsBaseSource(BaseSource): # first group lines by their symbol and name symbols = self._process_lines(zone, self._lines()) - pprint({'symbols': symbols}) # then work through those to group values by their _type and name zone_name = zone.name @@ -326,7 +368,6 @@ class TinyDnsBaseSource(BaseSource): 'ip6.arpa.' ) types, ttls = self._process_symbols(zone, symbols, arpa) - pprint({'types': types, 'ttls': ttls}) # now we finally have all the values for each (soon to be) record # collected together, turn them into their coresponding record and add @@ -338,9 +379,7 @@ class TinyDnsBaseSource(BaseSource): data['values'] = values else: data['value'] = values[0] - pprint({'name': name, 'data': data}) record = Record.new(zone, name, data, lenient=lenient) - pprint({'lenient': lenient}) try: zone.add_record(record, lenient=lenient) except SubzoneRecordException: @@ -352,53 +391,6 @@ class TinyDnsBaseSource(BaseSource): 'populate: found %s records', len(zone.records) - before ) - def _populate_arpa_arpa(self, zone, lenient): - name_re = re.compile(fr'(?P.+)\.{zone.name[:-1]}\.?$') - - for line in self._lines(): - _type = line[0] - # We're only interested in = (A+PTR), and ^ (PTR) records - if _type not in ('=', '^', '&'): - continue - - # Skip type, remove trailing comments, and omit newline - line = line[1:].split('#', 1)[0] - # Split on :'s including :: and strip leading/trailing ws - line = [p.strip() for p in self.split_re.split(line)] - - if line[0].endswith('in-addr.arpa'): - # since it's already in in-addr.arpa format - match = name_re.match(line[0]) - value = line[1] - else: - addr = ip_address(line[1]) - match = name_re.match(addr.reverse_pointer) - value = line[0] - - if match: - try: - ttl = line[2] - except IndexError: - ttl = self.default_ttl - - if value[-1] != '.': - value = f'{value}.' - - name = match.group('name') - record = Record.new( - zone, - name, - {'ttl': ttl, 'type': 'PTR', 'value': value}, - source=self, - lenient=lenient, - ) - try: - zone.add_record(record, lenient=lenient) - except DuplicateRecordException: - self.log.warning( - f'Duplicate PTR record for {addr}, skipping' - ) - class TinyDnsFileSource(TinyDnsBaseSource): ''' diff --git a/tests/test_octodns_source_tinydns.py b/tests/test_octodns_source_tinydns.py index f571081..f4d7a05 100644 --- a/tests/test_octodns_source_tinydns.py +++ b/tests/test_octodns_source_tinydns.py @@ -17,7 +17,7 @@ class TestTinyDnsFileSource(TestCase): def test_populate_normal(self): got = Zone('example.com.', []) self.source.populate(got) - self.assertEqual(17, len(got.records)) + self.assertEqual(24, len(got.records)) expected = Zone('example.com.', []) for name, data in ( @@ -26,8 +26,13 @@ class TestTinyDnsFileSource(TestCase): '', { 'type': 'NS', - 'ttl': 3600, - 'values': ['ns1.ns.com.', 'ns2.ns.com.'], + 'ttl': 31, + 'values': [ + 'a.ns.example.com.', + 'b.ns.example.com.', + 'ns1.ns.com.', + 'ns2.ns.com.', + ], }, ), ( @@ -75,11 +80,11 @@ class TestTinyDnsFileSource(TestCase): 'values': [ { 'preference': 30, - 'exchange': 'smtp-1-host.example.com.', + 'exchange': 'smtp-3-host.mx.example.com.', }, { 'preference': 40, - 'exchange': 'smtp-2-host.example.com.', + 'exchange': 'smtp-4-host.mx.example.com.', }, ], }, @@ -111,6 +116,26 @@ class TestTinyDnsFileSource(TestCase): 'value': 'v=DKIM1\\; k=rsa\\; p=blah', }, ), + ('b.ns', {'type': 'A', 'ttl': 31, 'value': '43.44.45.46'}), + ('a.ns', {'type': 'A', 'ttl': 3600, 'value': '42.43.44.45'}), + ( + 'smtp-3-host.mx', + {'type': 'A', 'ttl': 1800, 'value': '21.22.23.24'}, + ), + ( + 'smtp-4-host.mx', + {'type': 'A', 'ttl': 1800, 'value': '22.23.24.25'}, + ), + ('ns5.ns', {'type': 'A', 'ttl': 30, 'value': '14.15.16.17'}), + ('ns6.ns', {'type': 'A', 'ttl': 30, 'value': '15.16.17.18'}), + ( + 'other', + { + 'type': 'NS', + 'ttl': 30, + 'values': ['ns5.ns.example.com.', 'ns6.ns.example.com.'], + }, + ), ): record = Record.new(expected, name, data) expected.add_record(record) @@ -162,7 +187,10 @@ class TestTinyDnsFileSource(TestCase): { 'type': 'PTR', 'ttl': 3600, - 'value': 'has-dup-def123.example.com.', + 'values': [ + 'has-dup-def123.example.com.', + 'has-dup-def456.example.com.', + ], }, ), ( @@ -178,18 +206,10 @@ class TestTinyDnsFileSource(TestCase): expected.add_record(record) changes = expected.changes(got, SimpleProvider()) - from pprint import pprint - - pprint( - { - 'changes': changes, - 'expected': expected.records, - 'got': got.records, - } - ) self.assertEqual([], changes) def test_ignores_subs(self): got = Zone('example.com.', ['sub']) self.source.populate(got) + # we don't see one www.sub.example.com. record b/c it's in a sub self.assertEqual(23, len(got.records)) diff --git a/tests/zones/tinydns/example.com b/tests/zones/tinydns/example.com index e32bfed..4d373e1 100755 --- a/tests/zones/tinydns/example.com +++ b/tests/zones/tinydns/example.com @@ -28,8 +28,8 @@ Ccname.other.foo:www.other.foo @example.com::smtp-1-host.example.com:10 @example.com.::smtp-2-host.example.com:20 # MX with ttl and ip -@smtp.example.com:21.22.23.24:smtp-1-host:30:1800 -@smtp.example.com.:22.23.24.25:smtp-2-host:40:1800 +@smtp.example.com:21.22.23.24:smtp-3-host:30:1800 +@smtp.example.com.:22.23.24.25:smtp-4-host:40:1800 # NS for sub .sub.example.com::ns3.ns.com:30 From 7920b815f31fe3f0c003842ceed5484373509788 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 30 Jun 2023 16:35:45 -0700 Subject: [PATCH 07/12] We now use .ownes, so no need to handle SubzoneRecordException --- octodns/source/tinydns.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/octodns/source/tinydns.py b/octodns/source/tinydns.py index 27492c1..1f864f7 100755 --- a/octodns/source/tinydns.py +++ b/octodns/source/tinydns.py @@ -10,7 +10,6 @@ from os import listdir from os.path import join from ..record import Record -from ..zone import SubzoneRecordException from .base import BaseSource @@ -380,12 +379,7 @@ class TinyDnsBaseSource(BaseSource): else: data['value'] = values[0] record = Record.new(zone, name, data, lenient=lenient) - try: - zone.add_record(record, lenient=lenient) - except SubzoneRecordException: - self.log.error( - 'populate: skipping subzone record=%s', record - ) + zone.add_record(record, lenient=lenient) self.log.info( 'populate: found %s records', len(zone.records) - before From 34980504b3626da98a480598496df8536f7dea1f Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 30 Jun 2023 16:36:19 -0700 Subject: [PATCH 08/12] Full TinyDNS rework test coverage --- tests/test_octodns_source_tinydns.py | 10 +++++++--- tests/zones/tinydns/example.com | 4 ++-- tests/zones/tinydns/other.foo | 1 + 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/tests/test_octodns_source_tinydns.py b/tests/test_octodns_source_tinydns.py index f4d7a05..47781f2 100644 --- a/tests/test_octodns_source_tinydns.py +++ b/tests/test_octodns_source_tinydns.py @@ -17,7 +17,7 @@ class TestTinyDnsFileSource(TestCase): def test_populate_normal(self): got = Zone('example.com.', []) self.source.populate(got) - self.assertEqual(24, len(got.records)) + self.assertEqual(25, len(got.records)) expected = Zone('example.com.', []) for name, data in ( @@ -48,6 +48,10 @@ class TestTinyDnsFileSource(TestCase): 'cname', {'type': 'CNAME', 'ttl': 3600, 'value': 'www.example.com.'}, ), + ( + 'cname2', + {'type': 'CNAME', 'ttl': 48, 'value': 'www2.example.com.'}, + ), ( 'some-host-abc123', {'type': 'A', 'ttl': 1800, 'value': '10.2.3.7'}, @@ -66,7 +70,7 @@ class TestTinyDnsFileSource(TestCase): 'exchange': 'smtp-1-host.example.com.', }, { - 'preference': 20, + 'preference': 0, 'exchange': 'smtp-2-host.example.com.', }, ], @@ -212,4 +216,4 @@ class TestTinyDnsFileSource(TestCase): got = Zone('example.com.', ['sub']) self.source.populate(got) # we don't see one www.sub.example.com. record b/c it's in a sub - self.assertEqual(23, len(got.records)) + self.assertEqual(24, len(got.records)) diff --git a/tests/zones/tinydns/example.com b/tests/zones/tinydns/example.com index 4d373e1..fcb38d4 100755 --- a/tests/zones/tinydns/example.com +++ b/tests/zones/tinydns/example.com @@ -26,14 +26,14 @@ Ccname.other.foo:www.other.foo # MX @example.com::smtp-1-host.example.com:10 -@example.com.::smtp-2-host.example.com:20 +@example.com.::smtp-2-host.example.com. # MX with ttl and ip @smtp.example.com:21.22.23.24:smtp-3-host:30:1800 @smtp.example.com.:22.23.24.25:smtp-4-host:40:1800 # NS for sub .sub.example.com::ns3.ns.com:30 -.sub.example.com.::ns4.ns.com:30 +.sub.example.com.::ns4.ns.com.:30 # NS with ip .other.example.com:14.15.16.17:ns5:30 .other.example.com.:15.16.17.18:ns6:30 diff --git a/tests/zones/tinydns/other.foo b/tests/zones/tinydns/other.foo index 82e010d..a96dee2 100644 --- a/tests/zones/tinydns/other.foo +++ b/tests/zones/tinydns/other.foo @@ -3,5 +3,6 @@ # CNAME with trailing comment Ccname.example.com:www.example.com # this is a comment +Ccname2.example.com:www2.example.com.:48 +www.other.foo:14.2.3.6 From 80adcc5d9bac71e4e102d9229baef7feeb4062bd Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 30 Jun 2023 18:31:09 -0700 Subject: [PATCH 09/12] TinyDNS SRV support and test coverage --- octodns/source/tinydns.py | 87 ++++++++++++++++++++++++---- tests/test_octodns_source_tinydns.py | 41 ++++++++++++- tests/zones/tinydns/example.com | 6 ++ 3 files changed, 122 insertions(+), 12 deletions(-) diff --git a/octodns/source/tinydns.py b/octodns/source/tinydns.py index 1f864f7..100f482 100755 --- a/octodns/source/tinydns.py +++ b/octodns/source/tinydns.py @@ -16,12 +16,16 @@ from .base import BaseSource class TinyDnsBaseSource(BaseSource): SUPPORTS_GEO = False SUPPORTS_DYNAMIC = False - SUPPORTS = set(('A', 'CNAME', 'MX', 'NS', 'TXT', 'AAAA')) def __init__(self, id, default_ttl=3600): super().__init__(id) self.default_ttl = default_ttl + @property + def SUPPORTS(self): + # All record types, including those registered by 3rd party modules + return set(Record.registered_types().keys()) + def _records_for_at(self, zone, name, lines, arpa=False): # @fqdn:ip:x:dist:ttl:timestamp:lo # MX (and optional A) @@ -38,7 +42,7 @@ class TinyDnsBaseSource(BaseSource): ttl = self.default_ttl for line in lines: try: - ttl = int(lines[0][4]) + ttl = int(line[4]) break except IndexError: pass @@ -61,7 +65,7 @@ class TinyDnsBaseSource(BaseSource): # if we have an IP then we need to create an A for the MX ip = line[1] - if ip: + if ip and zone.owns('A', mx): yield 'A', mx, ttl, [ip] values.append({'preference': dist, 'exchange': mx}) @@ -88,7 +92,7 @@ class TinyDnsBaseSource(BaseSource): ttl = self.default_ttl for line in lines: try: - ttl = int(lines[0][2]) + ttl = int(line[2]) break except IndexError: pass @@ -163,7 +167,7 @@ class TinyDnsBaseSource(BaseSource): ttl = self.default_ttl for line in lines: try: - ttl = int(lines[0][3]) + ttl = int(line[3]) break except IndexError: pass @@ -180,7 +184,7 @@ class TinyDnsBaseSource(BaseSource): # if we have an IP then we need to create an A for the MX ip = line[1] - if ip: + if ip and zone.owns('A', ns): yield 'A', ns, ttl, [ip] values.append(ns) @@ -212,7 +216,7 @@ class TinyDnsBaseSource(BaseSource): ttl = self.default_ttl for line in lines: try: - ttl = int(lines[0][2]) + ttl = int(line[2]) break except IndexError: pass @@ -241,7 +245,7 @@ class TinyDnsBaseSource(BaseSource): ttl = self.default_ttl for line in lines: try: - ttl = int(lines[0][2]) + ttl = int(line[2]) break except IndexError: pass @@ -272,13 +276,76 @@ class TinyDnsBaseSource(BaseSource): ttl = self.default_ttl for line in lines: try: - ttl = int(lines[0][2]) + ttl = int(line[2]) break except IndexError: pass yield 'AAAA', name, ttl, ips + def _records_for_S(self, zone, name, lines, arpa=False): + # Sfqdn:ip:x:port:priority:weight:ttl:timestamp:lo + # SRV + + if arpa: + # no arpa + return [] + + if not zone.owns('SRV', name): + # if name doesn't live under our zone there's nothing for us to do + return + + # see if we can find a ttl on any of the lines, first one wins + ttl = self.default_ttl + for line in lines: + try: + ttl = int(line[6]) + break + except IndexError: + pass + + values = [] + for line in lines: + target = line[2] + # if there's a . in the mx we hit a special case and use it as-is + if '.' not in target: + # otherwise we treat it as the MX hostnam and construct the rest + target = f'{target}.srv.{zone.name}' + elif target[-1] != '.': + target = f'{target}.' + + # if we have an IP then we need to create an A for the SRV + # has to be present, but can be empty + ip = line[1] + if ip and zone.owns('A', target): + yield 'A', target, ttl, [ip] + + # required + port = int(line[3]) + + # optional, default 0 + try: + priority = int(line[4] or 0) + except IndexError: + priority = 0 + + # optional, default 0 + try: + weight = int(line[5] or 0) + except IndexError: + weight = 0 + + values.append( + { + 'priority': priority, + 'weight': weight, + 'port': port, + 'target': target, + } + ) + + yield 'SRV', name, ttl, values + def _records_for_six(self, zone, name, lines, arpa=False): # 6fqdn:ip:ttl:timestamp:lo # AAAA (arpa False) & PTR (arpa True) @@ -297,9 +364,9 @@ class TinyDnsBaseSource(BaseSource): '&': _records_for_amp, # NS '\'': _records_for_quote, # TXT '3': _records_for_three, # AAAA + 'S': _records_for_S, # SRV '6': _records_for_six, # AAAA # TODO: - #'S': _records_for_S, # SRV # Sfqdn:ip:x:port:priority:weight:ttl:timestamp:lo #':': _record_for_semicolon # arbitrary # :fqdn:n:rdata:ttl:timestamp:lo diff --git a/tests/test_octodns_source_tinydns.py b/tests/test_octodns_source_tinydns.py index 47781f2..81129b0 100644 --- a/tests/test_octodns_source_tinydns.py +++ b/tests/test_octodns_source_tinydns.py @@ -17,7 +17,7 @@ class TestTinyDnsFileSource(TestCase): def test_populate_normal(self): got = Zone('example.com.', []) self.source.populate(got) - self.assertEqual(25, len(got.records)) + self.assertEqual(28, len(got.records)) expected = Zone('example.com.', []) for name, data in ( @@ -140,6 +140,43 @@ class TestTinyDnsFileSource(TestCase): 'values': ['ns5.ns.example.com.', 'ns6.ns.example.com.'], }, ), + ( + '_a._tcp', + { + 'type': 'SRV', + 'ttl': 43, + 'values': [ + { + 'priority': 0, + 'weight': 0, + 'port': 8888, + 'target': 'target.srv.example.com.', + }, + { + 'priority': 10, + 'weight': 50, + 'port': 8080, + 'target': 'target.somewhere.else.', + }, + ], + }, + ), + ('target.srv', {'type': 'A', 'ttl': 43, 'value': '56.57.58.59'}), + ( + '_b._tcp', + { + 'type': 'SRV', + 'ttl': 3600, + 'values': [ + { + 'priority': 0, + 'weight': 0, + 'port': 9999, + 'target': 'target.srv.example.com.', + } + ], + }, + ), ): record = Record.new(expected, name, data) expected.add_record(record) @@ -216,4 +253,4 @@ class TestTinyDnsFileSource(TestCase): got = Zone('example.com.', ['sub']) self.source.populate(got) # we don't see one www.sub.example.com. record b/c it's in a sub - self.assertEqual(24, len(got.records)) + self.assertEqual(27, len(got.records)) diff --git a/tests/zones/tinydns/example.com b/tests/zones/tinydns/example.com index fcb38d4..9cb1eb8 100755 --- a/tests/zones/tinydns/example.com +++ b/tests/zones/tinydns/example.com @@ -61,3 +61,9 @@ Ccname.other.foo:www.other.foo 6ipv6-6.example.com:2a021348017cd5d0002419fffef35743 'semicolon.example.com:v=DKIM1; k=rsa; p=blah:300 + +# SRV +S_a._tcp.example.com:56.57.58.59:target:8888 +S_a._tcp.example.com::target.somewhere.else:8080:10:50:43 +# TODO: add an IP so it tries to create a record that already exists +S_b._tcp.example.com::target.srv.example.com.:9999 From b1252f923e823ce959c974281c76eec21a6404de Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 30 Jun 2023 18:53:57 -0700 Subject: [PATCH 10/12] deal with duplicates and duplicate auto-creates cleanly --- octodns/source/tinydns.py | 26 +++++++++++++++++++++----- tests/zones/tinydns/example.com | 9 +++++++-- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/octodns/source/tinydns.py b/octodns/source/tinydns.py index 100f482..55a5eaf 100755 --- a/octodns/source/tinydns.py +++ b/octodns/source/tinydns.py @@ -13,6 +13,17 @@ from ..record import Record from .base import BaseSource +def _unique(values): + try: + # this will work if they're simple strings + return list(set(values)) + except TypeError: + pass + # if they're dictionaries it's a bit more involved since dict's aren't + # hashable, based on https://stackoverflow.com/a/38521207 + return [dict(s) for s in set(frozenset(v.items()) for v in values)] + + class TinyDnsBaseSource(BaseSource): SUPPORTS_GEO = False SUPPORTS_DYNAMIC = False @@ -387,7 +398,7 @@ class TinyDnsBaseSource(BaseSource): def _process_symbols(self, zone, symbols, arpa): types = defaultdict(lambda: defaultdict(list)) - ttls = defaultdict(lambda: defaultdict(lambda: self.default_ttl)) + ttls = defaultdict(dict) for symbol, names in symbols.items(): records_for = self.SYMBOL_MAP.get(symbol, None) if not records_for: @@ -404,8 +415,10 @@ class TinyDnsBaseSource(BaseSource): # remove the zone name name = zone.hostname_from_fqdn(name) types[_type][name].extend(values) - # last one wins - ttls[_type][name] = ttl + # first non-default wins, if we never see anything we'll + # just use the default below + if ttl != self.default_ttl: + ttls[_type][name] = ttl return types, ttls @@ -440,9 +453,12 @@ class TinyDnsBaseSource(BaseSource): # it to the zone for _type, names in types.items(): for name, values in names.items(): - data = {'ttl': ttls[_type][name], 'type': _type} + data = { + 'ttl': ttls[_type].get(name, self.default_ttl), + 'type': _type, + } if len(values) > 1: - data['values'] = values + data['values'] = _unique(values) else: data['value'] = values[0] record = Record.new(zone, name, data, lenient=lenient) diff --git a/tests/zones/tinydns/example.com b/tests/zones/tinydns/example.com index 9cb1eb8..301269f 100755 --- a/tests/zones/tinydns/example.com +++ b/tests/zones/tinydns/example.com @@ -5,6 +5,8 @@ # Multi-value A +example.com:10.2.3.4:30 +example.com.:10.2.3.5:30 +# duplicate value should be ignored ++example.com:10.2.3.4 Ccname.other.foo:www.other.foo @@ -65,5 +67,8 @@ Ccname.other.foo:www.other.foo # SRV S_a._tcp.example.com:56.57.58.59:target:8888 S_a._tcp.example.com::target.somewhere.else:8080:10:50:43 -# TODO: add an IP so it tries to create a record that already exists -S_b._tcp.example.com::target.srv.example.com.:9999 +# will try and re-create an already existing A with the same IP, should be a +# noop +S_b._tcp.example.com:56.57.58.59:target.srv.example.com.:9999 +# complete duplicate should be ignored +S_b._tcp.example.com:56.57.58.59:target.srv.example.com.:9999 From 59a8958226690410055368ac9fc6044e09c71223 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 1 Jul 2023 09:18:53 -0700 Subject: [PATCH 11/12] Record.parse_rdata_texts, TinyDns support for arbitrary record types --- octodns/record/base.py | 4 +++ octodns/source/tinydns.py | 43 +++++++++++++++++++++++++--- tests/test_octodns_record.py | 15 ++++++++++ tests/test_octodns_source_tinydns.py | 24 ++++++++++++++-- tests/zones/tinydns/example.com | 8 ++++++ 5 files changed, 88 insertions(+), 6 deletions(-) diff --git a/octodns/record/base.py b/octodns/record/base.py index b6e3ed3..23b9a5c 100644 --- a/octodns/record/base.py +++ b/octodns/record/base.py @@ -123,6 +123,10 @@ class Record(EqualityTupleMixin): return records + @classmethod + def parse_rdata_texts(cls, rdatas): + return [cls._value_type.parse_rdata_text(r) for r in rdatas] + def __init__(self, zone, name, data, source=None): self.zone = zone if name: diff --git a/octodns/source/tinydns.py b/octodns/source/tinydns.py index 55a5eaf..384e9e5 100755 --- a/octodns/source/tinydns.py +++ b/octodns/source/tinydns.py @@ -357,6 +357,44 @@ class TinyDnsBaseSource(BaseSource): yield 'SRV', name, ttl, values + def _records_for_colon(self, zone, name, lines, arpa=False): + # :fqdn:n:rdata:ttl:timestamp:lo + # ANY + + if arpa: + # no arpa + return [] + + if not zone.owns('SRV', name): + # if name doesn't live under our zone there's nothing for us to do + return + + # group by lines by the record type + types = defaultdict(list) + for line in lines: + types[line[1].upper()].append(line) + + classes = Record.registered_types() + for _type, lines in types.items(): + _class = classes.get(_type, None) + if not _class: + self.log.info( + '_records_for_colon: unrecognized type %s, %s', _type, line + ) + continue + + # see if we can find a ttl on any of the lines, first one wins + ttl = self.default_ttl + for line in lines: + try: + ttl = int(line[3]) + break + except IndexError: + pass + + rdatas = [l[2] for l in lines] + yield _type, name, ttl, _class.parse_rdata_texts(rdatas) + def _records_for_six(self, zone, name, lines, arpa=False): # 6fqdn:ip:ttl:timestamp:lo # AAAA (arpa False) & PTR (arpa True) @@ -376,11 +414,8 @@ class TinyDnsBaseSource(BaseSource): '\'': _records_for_quote, # TXT '3': _records_for_three, # AAAA 'S': _records_for_S, # SRV + ':': _records_for_colon, # arbitrary '6': _records_for_six, # AAAA - # TODO: - # Sfqdn:ip:x:port:priority:weight:ttl:timestamp:lo - #':': _record_for_semicolon # arbitrary - # :fqdn:n:rdata:ttl:timestamp:lo } def _process_lines(self, zone, lines): diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index 88a7dfc..becc802 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -8,6 +8,7 @@ from octodns.idna import idna_encode from octodns.record import ( AliasRecord, ARecord, + CnameRecord, Create, Delete, MxValue, @@ -176,6 +177,20 @@ class TestRecord(TestCase): # make sure there's nothing extra self.assertEqual(5, len(records)) + def test_parse_rdata_texts(self): + self.assertEqual(['2.3.4.5'], ARecord.parse_rdata_texts(['2.3.4.5'])) + self.assertEqual( + ['2.3.4.6', '3.4.5.7'], + ARecord.parse_rdata_texts(['2.3.4.6', '3.4.5.7']), + ) + self.assertEqual( + ['some.target.'], CnameRecord.parse_rdata_texts(['some.target.']) + ) + self.assertEqual( + ['some.target.', 'other.target.'], + CnameRecord.parse_rdata_texts(['some.target.', 'other.target.']), + ) + def test_values_mixin_data(self): # no values, no value or values in data a = ARecord(self.zone, '', {'type': 'A', 'ttl': 600, 'values': []}) diff --git a/tests/test_octodns_source_tinydns.py b/tests/test_octodns_source_tinydns.py index 81129b0..338c4d6 100644 --- a/tests/test_octodns_source_tinydns.py +++ b/tests/test_octodns_source_tinydns.py @@ -17,7 +17,7 @@ class TestTinyDnsFileSource(TestCase): def test_populate_normal(self): got = Zone('example.com.', []) self.source.populate(got) - self.assertEqual(28, len(got.records)) + self.assertEqual(30, len(got.records)) expected = Zone('example.com.', []) for name, data in ( @@ -177,6 +177,26 @@ class TestTinyDnsFileSource(TestCase): ], }, ), + ( + 'arbitrary-sshfp', + { + 'type': 'SSHFP', + 'ttl': 45, + 'values': [ + { + 'algorithm': 1, + 'fingerprint_type': 2, + 'fingerprint': '00479b27', + }, + { + 'algorithm': 2, + 'fingerprint_type': 2, + 'fingerprint': '00479a28', + }, + ], + }, + ), + ('arbitrary-a', {'type': 'A', 'ttl': 3600, 'value': '80.81.82.83'}), ): record = Record.new(expected, name, data) expected.add_record(record) @@ -253,4 +273,4 @@ class TestTinyDnsFileSource(TestCase): got = Zone('example.com.', ['sub']) self.source.populate(got) # we don't see one www.sub.example.com. record b/c it's in a sub - self.assertEqual(27, len(got.records)) + self.assertEqual(29, len(got.records)) diff --git a/tests/zones/tinydns/example.com b/tests/zones/tinydns/example.com index 301269f..3369a28 100755 --- a/tests/zones/tinydns/example.com +++ b/tests/zones/tinydns/example.com @@ -72,3 +72,11 @@ S_a._tcp.example.com::target.somewhere.else:8080:10:50:43 S_b._tcp.example.com:56.57.58.59:target.srv.example.com.:9999 # complete duplicate should be ignored S_b._tcp.example.com:56.57.58.59:target.srv.example.com.:9999 + +# arbitrary multi-value non-spec record +:arbitrary-sshfp.example.com:SSHFP:2 2 00479a28 +:arbitrary-sshfp.example.com:SSHFP:1 2 00479b27:45 +# does not make sense to do an A this way, but it'll work +:arbitrary-a.example.com:a:80.81.82.83 +# this should just be inored b/c the type is unknown +:arbitrary-invalid.example.com:invalid:does not matter:99 From 0b5dac30b65da9fbf84624470fdac618cd831a25 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Thu, 6 Jul 2023 08:00:51 -0400 Subject: [PATCH 12/12] DRY up tinydns ttl finding logic --- octodns/source/tinydns.py | 91 ++++++++------------------------------- 1 file changed, 19 insertions(+), 72 deletions(-) diff --git a/octodns/source/tinydns.py b/octodns/source/tinydns.py index 384e9e5..1faf2f7 100755 --- a/octodns/source/tinydns.py +++ b/octodns/source/tinydns.py @@ -37,6 +37,16 @@ class TinyDnsBaseSource(BaseSource): # All record types, including those registered by 3rd party modules return set(Record.registered_types().keys()) + def _ttl_for(self, lines, index): + # see if we can find a ttl on any of the lines, first one wins + for line in lines: + try: + return int(line[index]) + except IndexError: + pass + # and if we don't use the default + return self.default_ttl + def _records_for_at(self, zone, name, lines, arpa=False): # @fqdn:ip:x:dist:ttl:timestamp:lo # MX (and optional A) @@ -49,14 +59,7 @@ class TinyDnsBaseSource(BaseSource): # if name doesn't live under our zone there's nothing for us to do return - # see if we can find a ttl on any of the lines, first one wins - ttl = self.default_ttl - for line in lines: - try: - ttl = int(line[4]) - break - except IndexError: - pass + ttl = self._ttl_for(lines, 4) values = [] for line in lines: @@ -99,14 +102,7 @@ class TinyDnsBaseSource(BaseSource): if value[-1] != '.': value = f'{value}.' - # see if we can find a ttl on any of the lines, first one wins - ttl = self.default_ttl - for line in lines: - try: - ttl = int(line[2]) - break - except IndexError: - pass + ttl = self._ttl_for(lines, 2) yield 'CNAME', name, ttl, [value] @@ -141,14 +137,7 @@ class TinyDnsBaseSource(BaseSource): value = f'{value}.' names[name].append(value) - # see if we can find a ttl on any of the lines, first one wins - ttl = self.default_ttl - for line in lines: - try: - ttl = int(line[2]) - break - except IndexError: - pass + ttl = self._ttl_for(lines, 2) for name, values in names.items(): if zone.owns('PTR', name): @@ -174,14 +163,7 @@ class TinyDnsBaseSource(BaseSource): # if name doesn't live under our zone there's nothing for us to do return - # see if we can find a ttl on any of the lines, first one wins - ttl = self.default_ttl - for line in lines: - try: - ttl = int(line[3]) - break - except IndexError: - pass + ttl = self._ttl_for(lines, 3) values = [] for line in lines: @@ -223,14 +205,7 @@ class TinyDnsBaseSource(BaseSource): # we didn't find any value ips so nothing to do return [] - # see if we can find a ttl on any of the lines, first one wins - ttl = self.default_ttl - for line in lines: - try: - ttl = int(line[2]) - break - except IndexError: - pass + ttl = self._ttl_for(lines, 2) yield 'A', name, ttl, ips @@ -252,14 +227,7 @@ class TinyDnsBaseSource(BaseSource): for l in lines ] - # see if we can find a ttl on any of the lines, first one wins - ttl = self.default_ttl - for line in lines: - try: - ttl = int(line[2]) - break - except IndexError: - pass + ttl = self._ttl_for(lines, 2) yield 'TXT', name, ttl, values @@ -283,14 +251,7 @@ class TinyDnsBaseSource(BaseSource): # the address correct. ips.append(u':'.join(textwrap.wrap(line[1], 4))) - # see if we can find a ttl on any of the lines, first one wins - ttl = self.default_ttl - for line in lines: - try: - ttl = int(line[2]) - break - except IndexError: - pass + ttl = self._ttl_for(lines, 2) yield 'AAAA', name, ttl, ips @@ -306,14 +267,7 @@ class TinyDnsBaseSource(BaseSource): # if name doesn't live under our zone there's nothing for us to do return - # see if we can find a ttl on any of the lines, first one wins - ttl = self.default_ttl - for line in lines: - try: - ttl = int(line[6]) - break - except IndexError: - pass + ttl = self._ttl_for(lines, 6) values = [] for line in lines: @@ -383,14 +337,7 @@ class TinyDnsBaseSource(BaseSource): ) continue - # see if we can find a ttl on any of the lines, first one wins - ttl = self.default_ttl - for line in lines: - try: - ttl = int(line[3]) - break - except IndexError: - pass + ttl = self._ttl_for(lines, 3) rdatas = [l[2] for l in lines] yield _type, name, ttl, _class.parse_rdata_texts(rdatas)