diff --git a/octodns/auto_arpa.py b/octodns/auto_arpa.py new file mode 100644 index 0000000..20f8a24 --- /dev/null +++ b/octodns/auto_arpa.py @@ -0,0 +1,51 @@ +# +# +# + +from collections import defaultdict +from ipaddress import ip_address +from logging import getLogger + +from .processor.base import BaseProcessor +from .record import Record +from .source.base import BaseSource + + +class AutoArpa(BaseProcessor, BaseSource): + SUPPORTS = set(('PTR',)) + SUPPORTS_GEO = False + + log = getLogger('AutoArpa') + + def __init__(self, ttl=3600): + super().__init__('auto-arpa') + self.ttl = ttl + + self._addrs = defaultdict(list) + + def process_source_zone(self, desired, sources): + for record in desired.records: + if record._type in ('A', 'AAAA'): + for value in record.values: + addr = ip_address(value) + self._addrs[f'{addr.reverse_pointer}.'].append(record.fqdn) + + return desired + + def populate(self, zone, target=False, lenient=False): + self.log.debug('populate: zone=%s', zone.name) + before = len(zone.records) + + name = zone.name + for arpa, fqdns in self._addrs.items(): + if arpa.endswith(name): + record = Record.new( + zone, + zone.hostname_from_fqdn(arpa), + {'ttl': self.ttl, 'type': 'PTR', 'values': fqdns}, + ) + zone.add_record(record) + + self.log.info( + 'populate: found %s records', len(zone.records) - before + ) diff --git a/octodns/manager.py b/octodns/manager.py index 5677e6d..b4b7c0f 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -10,6 +10,7 @@ from sys import stdout import logging from . import __VERSION__ +from .auto_arpa import AutoArpa from .idna import IdnaDict, idna_decode, idna_encode from .provider.base import BaseProvider from .provider.plan import Plan @@ -62,7 +63,7 @@ class _AggregateTarget(object): raise AttributeError(f'{klass} object has no attribute {name}') -class MakeThreadFuture(object): +class FakeThreadFuture(object): def __init__(self, func, args, kwargs): self.func = func self.args = args @@ -82,7 +83,7 @@ class MainThreadExecutor(object): ''' def submit(self, func, *args, **kwargs): - return MakeThreadFuture(func, args, kwargs) + return FakeThreadFuture(func, args, kwargs) class ManagerException(Exception): @@ -99,7 +100,9 @@ class Manager(object): # TODO: all of this should get broken up, mainly so that it's not so huge # and each bit can be cleanly tested independently - def __init__(self, config_file, max_workers=None, include_meta=False): + def __init__( + self, config_file, max_workers=None, include_meta=False, auto_arpa=False + ): version = self._try_version('octodns', version=__VERSION__) self.log.info( '__init__: config_file=%s (octoDNS %s)', config_file, version @@ -119,6 +122,7 @@ class Manager(object): self.include_meta = self._config_include_meta( manager_config, include_meta ) + self.auto_arpa = self._config_auto_arpa(manager_config, auto_arpa) self.global_processors = manager_config.get('processors', []) self.log.info('__init__: global_processors=%s', self.global_processors) @@ -174,6 +178,15 @@ class Manager(object): self.log.info('_config_include_meta: include_meta=%s', include_meta) return include_meta + def _config_auto_arpa(self, manager_config, auto_arpa=False): + auto_arpa = auto_arpa or manager_config.get('auto-arpa', False) + self.log.info('_config_auto_arpa: auto_arpa=%s', auto_arpa) + if auto_arpa: + if not isinstance(auto_arpa, dict): + auto_arpa = {} + return AutoArpa(**auto_arpa) + return None + def _config_providers(self, providers_config): self.log.debug('_config_providers: configuring providers') providers = {} @@ -202,6 +215,9 @@ class Manager(object): 'Incorrect provider config for ' + provider_name ) + if self.auto_arpa: + providers['auto-arpa'] = self.auto_arpa + return providers def _config_processors(self, processors_config): @@ -229,6 +245,10 @@ class Manager(object): raise ManagerException( 'Incorrect processor config for ' + processor_name ) + + if self.auto_arpa: + processors['auto-arpa'] = self.auto_arpa + return processors def _config_plan_outputs(self, plan_outputs_config): @@ -477,6 +497,7 @@ class Manager(object): aliased_zones = {} futures = [] + arpa_kwargs = [] for zone_name, config in zones.items(): decoded_zone_name = idna_decode(zone_name) self.log.info('sync: zone=%s', decoded_zone_name) @@ -537,6 +558,9 @@ class Manager(object): collected = [] for processor in self.global_processors + processors: collected.append(self.processors[processor]) + # always goes last + if self.auto_arpa: + collected.append(self.auto_arpa) processors = collected except KeyError: raise ManagerException( @@ -572,16 +596,25 @@ class Manager(object): f'Zone {decoded_zone_name}, unknown ' f'target: {target}' ) - futures.append( - self._executor.submit( - self._populate_and_plan, - zone_name, - processors, - sources, - targets, - lenient=lenient, + kwargs = { + 'zone_name': zone_name, + 'processors': processors, + 'sources': sources, + 'targets': targets, + 'lenient': lenient, + } + if self.auto_arpa and ( + zone_name.endswith('in-addr.arpa.') + or zone_name.endswith('ip6.arpa.') + ): + # auto arpa is enabled so we need to defer processing all arpa + # zones until after general ones have completed (async) so that + # they'll have access to all the recorded A/AAAA values + arpa_kwargs.append(kwargs) + else: + futures.append( + self._executor.submit(self._populate_and_plan, **kwargs) ) - ) # Wait on all results and unpack/flatten the plans and store the # desired states in case we need them below @@ -621,6 +654,24 @@ class Manager(object): # as these are aliased zones plans += [p for f in futures for p in f.result()[0]] + if self.auto_arpa and arpa_kwargs: + self.log.info( + 'sync: processing %d arpa reverse dns zones', len(arpa_kwargs) + ) + # all the general zones are done and we've recorded the A/AAAA + # records with the AutoPtr. We can no process ptr zones + futures = [] + for kwargs in arpa_kwargs: + futures.append( + self._executor.submit(self._populate_and_plan, **kwargs) + ) + + for future in futures: + ps, d = future.result() + desired[d.name] = d + for plan in ps: + plans.append(plan) + # Best effort sort plans children first so that we create/update # children zones before parents which should allow us to more safely # extract things into sub-zones. Combining a child back into a parent diff --git a/tests/config/simple.yaml b/tests/config/simple.yaml index 5040298..561139b 100644 --- a/tests/config/simple.yaml +++ b/tests/config/simple.yaml @@ -1,4 +1,5 @@ manager: + auto-arpa: true max_workers: 2 providers: in: @@ -44,3 +45,8 @@ zones: - in targets: - dump + 3.2.1.in-addr.arpa.: + sources: + - auto-arpa + targets: + - dump diff --git a/tests/test_octodns_manager.py b/tests/test_octodns_manager.py index 1a36723..e34f2a9 100644 --- a/tests/test_octodns_manager.py +++ b/tests/test_octodns_manager.py @@ -136,7 +136,7 @@ class TestManager(TestCase): environ['YAML_TMP_DIR'] = tmpdir.dirname environ['YAML_TMP_DIR2'] = tmpdir.dirname tc = Manager(get_config_filename('simple.yaml')).sync(dry_run=False) - self.assertEqual(28, tc) + self.assertEqual(30, tc) # try with just one of the zones tc = Manager(get_config_filename('simple.yaml')).sync( @@ -160,13 +160,13 @@ class TestManager(TestCase): tc = Manager(get_config_filename('simple.yaml')).sync( dry_run=False, force=True ) - self.assertEqual(28, tc) + self.assertEqual(30, tc) # Again with max_workers = 1 tc = Manager( get_config_filename('simple.yaml'), max_workers=1 ).sync(dry_run=False, force=True) - self.assertEqual(28, tc) + self.assertEqual(30, tc) # Include meta tc = Manager( @@ -174,7 +174,7 @@ class TestManager(TestCase): max_workers=1, include_meta=True, ).sync(dry_run=False, force=True) - self.assertEqual(33, tc) + self.assertEqual(36, tc) def test_idna_eligible_zones(self): # loading w/simple, but we'll be blowing it away and doing some manual @@ -186,9 +186,6 @@ class TestManager(TestCase): manager.config['zones'] = manager._config_zones( {'déjà.vu.': {}, 'deja.vu.': {}, idna_encode('こんにちは.jp.'): {}} ) - from pprint import pprint - - pprint(manager.config['zones']) # refer to them with utf-8 with self.assertRaises(ManagerException) as ctx: