From 3fe602546249b720179cf177ca4b0b701ba007f6 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Thu, 29 Sep 2022 15:33:01 -0700 Subject: [PATCH] POC of auto-arpa concept, think there's too many complications though --- octodns/auto_arpa.py | 51 +++++++++++ octodns/manager.py | 72 ++++++++++++--- tests/config/auto-arpa.yaml | 26 ++++++ tests/test_octodns_auto_arpa.py | 149 ++++++++++++++++++++++++++++++++ tests/test_octodns_manager.py | 10 +++ 5 files changed, 298 insertions(+), 10 deletions(-) create mode 100644 octodns/auto_arpa.py create mode 100644 tests/config/auto-arpa.yaml create mode 100644 tests/test_octodns_auto_arpa.py diff --git a/octodns/auto_arpa.py b/octodns/auto_arpa.py new file mode 100644 index 0000000..20f8a24 --- /dev/null +++ b/octodns/auto_arpa.py @@ -0,0 +1,51 @@ +# +# +# + +from collections import defaultdict +from ipaddress import ip_address +from logging import getLogger + +from .processor.base import BaseProcessor +from .record import Record +from .source.base import BaseSource + + +class AutoArpa(BaseProcessor, BaseSource): + SUPPORTS = set(('PTR',)) + SUPPORTS_GEO = False + + log = getLogger('AutoArpa') + + def __init__(self, ttl=3600): + super().__init__('auto-arpa') + self.ttl = ttl + + self._addrs = defaultdict(list) + + def process_source_zone(self, desired, sources): + for record in desired.records: + if record._type in ('A', 'AAAA'): + for value in record.values: + addr = ip_address(value) + self._addrs[f'{addr.reverse_pointer}.'].append(record.fqdn) + + return desired + + def populate(self, zone, target=False, lenient=False): + self.log.debug('populate: zone=%s', zone.name) + before = len(zone.records) + + name = zone.name + for arpa, fqdns in self._addrs.items(): + if arpa.endswith(name): + record = Record.new( + zone, + zone.hostname_from_fqdn(arpa), + {'ttl': self.ttl, 'type': 'PTR', 'values': fqdns}, + ) + zone.add_record(record) + + self.log.info( + 'populate: found %s records', len(zone.records) - before + ) diff --git a/octodns/manager.py b/octodns/manager.py index 5677e6d..57037da 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -10,6 +10,7 @@ from sys import stdout import logging from . import __VERSION__ +from .auto_arpa import AutoArpa from .idna import IdnaDict, idna_decode, idna_encode from .provider.base import BaseProvider from .provider.plan import Plan @@ -99,7 +100,13 @@ class Manager(object): # TODO: all of this should get broken up, mainly so that it's not so huge # and each bit can be cleanly tested independently - def __init__(self, config_file, max_workers=None, include_meta=False): + def __init__( + self, + config_file, + max_workers=None, + include_meta=False, + enable_auto_arpa=False, + ): version = self._try_version('octodns', version=__VERSION__) self.log.info( '__init__: config_file=%s (octoDNS %s)', config_file, version @@ -119,8 +126,15 @@ class Manager(object): self.include_meta = self._config_include_meta( manager_config, include_meta ) + self.auto_arpa = self._config_auto_arpa( + manager_config, enable_auto_arpa + ) self.global_processors = manager_config.get('processors', []) + if self.auto_arpa: + # if enabled this need to run last on every zone so that it can get + # the final picture of any A/AAAA records + self.global_processors.append('auto-arpa') self.log.info('__init__: global_processors=%s', self.global_processors) providers_config = self.config['providers'] @@ -174,6 +188,17 @@ class Manager(object): self.log.info('_config_include_meta: include_meta=%s', include_meta) return include_meta + def _config_auto_arpa(self, manager_config, enable_auto_arpa=False): + enable_auto_arpa = enable_auto_arpa or manager_config.get( + 'enable_auto_arpa', False + ) + self.log.info( + '_config_auto_arpa: enable_auto_arpa=%s', enable_auto_arpa + ) + if enable_auto_arpa: + return AutoArpa() + return None + def _config_providers(self, providers_config): self.log.debug('_config_providers: configuring providers') providers = {} @@ -202,6 +227,9 @@ class Manager(object): 'Incorrect provider config for ' + provider_name ) + if self.auto_arpa: + providers['auto-arpa'] = self.auto_arpa + return providers def _config_processors(self, processors_config): @@ -229,6 +257,10 @@ class Manager(object): raise ManagerException( 'Incorrect processor config for ' + processor_name ) + + if self.auto_arpa: + processors['auto-arpa'] = self.auto_arpa + return processors def _config_plan_outputs(self, plan_outputs_config): @@ -476,6 +508,7 @@ class Manager(object): zones = IdnaDict({n: zones.get(n) for n in eligible_zones}) aliased_zones = {} + deferred_kwargs = [] futures = [] for zone_name, config in zones.items(): decoded_zone_name = idna_decode(zone_name) @@ -506,6 +539,10 @@ class Manager(object): f'Zone {decoded_zone_name} is missing sources' ) + # if auto-arpa is in the list (while it's still ids) this zone needs + # to be deferred until after everything else has run + deferred = 'auto-arpa' in sources + try: targets = config['targets'] except KeyError: @@ -572,16 +609,20 @@ class Manager(object): f'Zone {decoded_zone_name}, unknown ' f'target: {target}' ) - futures.append( - self._executor.submit( - self._populate_and_plan, - zone_name, - processors, - sources, - targets, - lenient=lenient, + kwargs = { + 'zone_name': zone_name, + 'processors': processors, + 'sources': sources, + 'targets': targets, + 'lenient': lenient, + } + if deferred: + self.log.debug('sync: deferring %s', zone_name) + deferred_kwargs.append(kwargs) + else: + futures.append( + self._executor.submit(self._populate_and_plan, **kwargs) ) - ) # Wait on all results and unpack/flatten the plans and store the # desired states in case we need them below @@ -621,6 +662,17 @@ class Manager(object): # as these are aliased zones plans += [p for f in futures for p in f.result()[0]] + if deferred_kwargs: + self.log.debug('sync: planning deferred zones') + # now we need to run any deferred planning + futures = [] + for kwargs in deferred_kwargs: + futures.append( + self._executor.submit(self._populate_and_plan, **kwargs) + ) + # wait for them to finish + plans += [p for f in futures for p in f.result()[0]] + # Best effort sort plans children first so that we create/update # children zones before parents which should allow us to more safely # extract things into sub-zones. Combining a child back into a parent diff --git a/tests/config/auto-arpa.yaml b/tests/config/auto-arpa.yaml new file mode 100644 index 0000000..9d74442 --- /dev/null +++ b/tests/config/auto-arpa.yaml @@ -0,0 +1,26 @@ +manager: + enable_auto_arpa: true + max_workers: 2 + +providers: + in: + class: octodns.provider.yaml.YamlProvider + directory: tests/config + supports_root_ns: False + dump: + class: octodns.provider.yaml.YamlProvider + directory: env/YAML_TMP_DIR + default_ttl: 999 + supports_root_ns: False + +zones: + 1.in-addr.arpa.: + sources: + - auto-arpa + targets: + - dump + unit.tests.: + sources: + - in + targets: + - dump diff --git a/tests/test_octodns_auto_arpa.py b/tests/test_octodns_auto_arpa.py new file mode 100644 index 0000000..61725da --- /dev/null +++ b/tests/test_octodns_auto_arpa.py @@ -0,0 +1,149 @@ +# +# +# + +from unittest import TestCase + +from octodns.auto_arpa import AutoArpa +from octodns.record import Record +from octodns.zone import Zone + + +class TestAutoArpa(TestCase): + def test_v4(self): + aa = AutoArpa() + + # a record it won't be interested in b/c of type + zone = Zone('unit.tests.', []) + record = Record.new( + zone, 'ns', {'type': 'NS', 'ttl': 1800, 'value': 'ns1.unit.tests.'} + ) + zone.add_record(record) + aa.process_source_zone(zone, []) + # nothing recorded + self.assertFalse(aa._addrs) + + # a record it will record + zone = Zone('unit.tests.', []) + record = Record.new( + zone, 'a', {'type': 'A', 'ttl': 1800, 'value': '10.0.0.1'} + ) + zone.add_record(record) + aa.process_source_zone(zone, []) + self.assertEqual( + {'1.0.0.10.in-addr.arpa.': ['a.unit.tests.']}, dict(aa._addrs) + ) + + # another record it will record + zone = Zone('unit.tests.', []) + record = Record.new( + zone, 'b', {'type': 'A', 'ttl': 1800, 'value': '10.0.42.1'} + ) + zone.add_record(record) + aa.process_source_zone(zone, []) + self.assertEqual( + { + '1.0.0.10.in-addr.arpa.': ['a.unit.tests.'], + '1.42.0.10.in-addr.arpa.': ['b.unit.tests.'], + }, + dict(aa._addrs), + ) + + # a second record pointed to the same IP + zone = Zone('unit.tests.', []) + record = Record.new( + zone, 'c', {'type': 'A', 'ttl': 1800, 'value': '10.0.42.1'} + ) + zone.add_record(record) + aa.process_source_zone(zone, []) + self.assertEqual( + { + '1.0.0.10.in-addr.arpa.': ['a.unit.tests.'], + '1.42.0.10.in-addr.arpa.': ['b.unit.tests.', 'c.unit.tests.'], + }, + dict(aa._addrs), + ) + + # subnet with just 1 record + zone = Zone('0.0.10.in-addr.arpa.', []) + aa.populate(zone) + self.assertEqual( + {'1.0.0.10.in-addr.arpa.': ['a.unit.tests.']}, + {r.fqdn: r.values for r in zone.records}, + ) + + # subnet with 2 records + zone = Zone('0.10.in-addr.arpa.', []) + aa.populate(zone) + self.assertEqual( + { + '1.0.0.10.in-addr.arpa.': ['a.unit.tests.'], + '1.42.0.10.in-addr.arpa.': ['b.unit.tests.', 'c.unit.tests.'], + }, + {r.fqdn: r.values for r in zone.records}, + ) + + def test_v6(self): + aa = AutoArpa() + + # a v6 record it will record + zone = Zone('unit.tests.', []) + record = Record.new( + zone, 'aaaa', {'type': 'AAAA', 'ttl': 1800, 'value': 'fc00::1'} + ) + zone.add_record(record) + aa.process_source_zone(zone, []) + self.assertEqual( + { + '1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.c.f.ip6.arpa.': [ + 'aaaa.unit.tests.' + ] + }, + dict(aa._addrs), + ) + + # another v6 record it will record + zone = Zone('unit.tests.', []) + record = Record.new( + zone, 'bbbb', {'type': 'AAAA', 'ttl': 1800, 'value': 'fc42::1'} + ) + zone.add_record(record) + aa.process_source_zone(zone, []) + self.assertEqual( + { + '1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.c.f.ip6.arpa.': [ + 'aaaa.unit.tests.' + ], + '1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.2.4.c.f.ip6.arpa.': [ + 'bbbb.unit.tests.' + ], + }, + dict(aa._addrs), + ) + + # subnet with just 1 record + zone = Zone('0.0.c.f.ip6.arpa.', []) + aa.populate(zone) + self.assertEqual( + { + '1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.c.f.ip6.arpa.': [ + 'aaaa.unit.tests.' + ] + }, + {r.fqdn: r.values for r in zone.records}, + ) + + # subnet with 2 records + zone = Zone('c.f.ip6.arpa.', []) + aa.populate(zone) + self.assertEqual( + { + '1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.c.f.ip6.arpa.': [ + 'aaaa.unit.tests.' + ], + '1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.2.4.c.f.ip6.arpa.': [ + 'bbbb.unit.tests.' + ], + }, + {r.fqdn: r.values for r in zone.records}, + ) diff --git a/tests/test_octodns_manager.py b/tests/test_octodns_manager.py index 1a36723..533a46a 100644 --- a/tests/test_octodns_manager.py +++ b/tests/test_octodns_manager.py @@ -176,6 +176,16 @@ class TestManager(TestCase): ).sync(dry_run=False, force=True) self.assertEqual(33, tc) + def test_auto_arpa(self): + with TemporaryDirectory() as tmpdir: + environ['YAML_TMP_DIR'] = tmpdir.dirname + environ['YAML_TMP_DIR2'] = tmpdir.dirname + + manager = Manager(get_config_filename('auto-arpa.yaml')) + tc = manager.sync(dry_run=False) + # we expect 22 records from unit.tests and 2 PTRs in the arpa zone + self.assertEqual(24, tc) + def test_idna_eligible_zones(self): # loading w/simple, but we'll be blowing it away and doing some manual # stuff