POC of auto-arpa concept, think there's too many complications though

octodns/auto_arpa.py

@@ -0,0 +1,51 @@
+#
+#
+#
+
+from collections import defaultdict
+from ipaddress import ip_address
+from logging import getLogger
+
+from .processor.base import BaseProcessor
+from .record import Record
+from .source.base import BaseSource
+
+
+class AutoArpa(BaseProcessor, BaseSource):
+    SUPPORTS = set(('PTR',))
+    SUPPORTS_GEO = False
+
+    log = getLogger('AutoArpa')
+
+    def __init__(self, ttl=3600):
+        super().__init__('auto-arpa')
+        self.ttl = ttl
+
+        self._addrs = defaultdict(list)
+
+    def process_source_zone(self, desired, sources):
+        for record in desired.records:
+            if record._type in ('A', 'AAAA'):
+                for value in record.values:
+                    addr = ip_address(value)
+                    self._addrs[f'{addr.reverse_pointer}.'].append(record.fqdn)
+
+        return desired
+
+    def populate(self, zone, target=False, lenient=False):
+        self.log.debug('populate: zone=%s', zone.name)
+        before = len(zone.records)
+
+        name = zone.name
+        for arpa, fqdns in self._addrs.items():
+            if arpa.endswith(name):
+                record = Record.new(
+                    zone,
+                    zone.hostname_from_fqdn(arpa),
+                    {'ttl': self.ttl, 'type': 'PTR', 'values': fqdns},
+                )
+                zone.add_record(record)
+
+        self.log.info(
+            'populate:   found %s records', len(zone.records) - before
+        )

octodns/manager.py

@@ -10,6 +10,7 @@ from sys import stdout
 import logging
 
 from . import __VERSION__
+from .auto_arpa import AutoArpa
 from .idna import IdnaDict, idna_decode, idna_encode
 from .provider.base import BaseProvider
 from .provider.plan import Plan
@@ -99,7 +100,13 @@ class Manager(object):
 
     # TODO: all of this should get broken up, mainly so that it's not so huge
     # and each bit can be cleanly tested independently
-    def __init__(self, config_file, max_workers=None, include_meta=False):
+    def __init__(
+        self,
+        config_file,
+        max_workers=None,
+        include_meta=False,
+        enable_auto_arpa=False,
+    ):
         version = self._try_version('octodns', version=__VERSION__)
         self.log.info(
             '__init__: config_file=%s (octoDNS %s)', config_file, version
@@ -119,8 +126,15 @@ class Manager(object):
         self.include_meta = self._config_include_meta(
             manager_config, include_meta
         )
+        self.auto_arpa = self._config_auto_arpa(
+            manager_config, enable_auto_arpa
+        )
 
         self.global_processors = manager_config.get('processors', [])
+        if self.auto_arpa:
+            # if enabled this need to run last on every zone so that it can get
+            # the final picture of any A/AAAA records
+            self.global_processors.append('auto-arpa')
         self.log.info('__init__: global_processors=%s', self.global_processors)
 
         providers_config = self.config['providers']
@@ -174,6 +188,17 @@ class Manager(object):
         self.log.info('_config_include_meta: include_meta=%s', include_meta)
         return include_meta
 
+    def _config_auto_arpa(self, manager_config, enable_auto_arpa=False):
+        enable_auto_arpa = enable_auto_arpa or manager_config.get(
+            'enable_auto_arpa', False
+        )
+        self.log.info(
+            '_config_auto_arpa: enable_auto_arpa=%s', enable_auto_arpa
+        )
+        if enable_auto_arpa:
+            return AutoArpa()
+        return None
+
     def _config_providers(self, providers_config):
         self.log.debug('_config_providers: configuring providers')
         providers = {}
@@ -202,6 +227,9 @@ class Manager(object):
                     'Incorrect provider config for ' + provider_name
                 )
 
+        if self.auto_arpa:
+            providers['auto-arpa'] = self.auto_arpa
+
         return providers
 
     def _config_processors(self, processors_config):
@@ -229,6 +257,10 @@ class Manager(object):
                 raise ManagerException(
                     'Incorrect processor config for ' + processor_name
                 )
+
+        if self.auto_arpa:
+            processors['auto-arpa'] = self.auto_arpa
+
         return processors
 
     def _config_plan_outputs(self, plan_outputs_config):
@@ -476,6 +508,7 @@ class Manager(object):
             zones = IdnaDict({n: zones.get(n) for n in eligible_zones})
 
         aliased_zones = {}
+        deferred_kwargs = []
         futures = []
         for zone_name, config in zones.items():
             decoded_zone_name = idna_decode(zone_name)
@@ -506,6 +539,10 @@ class Manager(object):
                     f'Zone {decoded_zone_name} is missing sources'
                 )
 
+            # if auto-arpa is in the list (while it's still ids) this zone needs
+            # to be deferred until after everything else has run
+            deferred = 'auto-arpa' in sources
+
             try:
                 targets = config['targets']
             except KeyError:
@@ -572,16 +609,20 @@ class Manager(object):
                     f'Zone {decoded_zone_name}, unknown ' f'target: {target}'
                 )
 
-            futures.append(
-                self._executor.submit(
-                    self._populate_and_plan,
-                    zone_name,
-                    processors,
-                    sources,
-                    targets,
-                    lenient=lenient,
+            kwargs = {
+                'zone_name': zone_name,
+                'processors': processors,
+                'sources': sources,
+                'targets': targets,
+                'lenient': lenient,
+            }
+            if deferred:
+                self.log.debug('sync:   deferring %s', zone_name)
+                deferred_kwargs.append(kwargs)
+            else:
+                futures.append(
+                    self._executor.submit(self._populate_and_plan, **kwargs)
                 )
-            )
 
         # Wait on all results and unpack/flatten the plans and store the
         # desired states in case we need them below
@@ -621,6 +662,17 @@ class Manager(object):
         # as these are aliased zones
         plans += [p for f in futures for p in f.result()[0]]
 
+        if deferred_kwargs:
+            self.log.debug('sync: planning deferred zones')
+            # now we need to run any deferred planning
+            futures = []
+            for kwargs in deferred_kwargs:
+                futures.append(
+                    self._executor.submit(self._populate_and_plan, **kwargs)
+                )
+            # wait for them to finish
+            plans += [p for f in futures for p in f.result()[0]]
+
         # Best effort sort plans children first so that we create/update
         # children zones before parents which should allow us to more safely
         # extract things into sub-zones. Combining a child back into a parent

tests/config/auto-arpa.yaml

@@ -0,0 +1,26 @@
+manager:
+  enable_auto_arpa: true
+  max_workers: 2
+
+providers:
+  in:
+    class: octodns.provider.yaml.YamlProvider
+    directory: tests/config
+    supports_root_ns: False
+  dump:
+    class: octodns.provider.yaml.YamlProvider
+    directory: env/YAML_TMP_DIR
+    default_ttl: 999
+    supports_root_ns: False
+
+zones:
+  1.in-addr.arpa.:
+    sources:
+    - auto-arpa
+    targets:
+    - dump
+  unit.tests.:
+    sources:
+    - in
+    targets:
+    - dump

tests/test_octodns_auto_arpa.py

@@ -0,0 +1,149 @@
+#
+#
+#
+
+from unittest import TestCase
+
+from octodns.auto_arpa import AutoArpa
+from octodns.record import Record
+from octodns.zone import Zone
+
+
+class TestAutoArpa(TestCase):
+    def test_v4(self):
+        aa = AutoArpa()
+
+        # a record it won't be interested in b/c of type
+        zone = Zone('unit.tests.', [])
+        record = Record.new(
+            zone, 'ns', {'type': 'NS', 'ttl': 1800, 'value': 'ns1.unit.tests.'}
+        )
+        zone.add_record(record)
+        aa.process_source_zone(zone, [])
+        # nothing recorded
+        self.assertFalse(aa._addrs)
+
+        # a record it will record
+        zone = Zone('unit.tests.', [])
+        record = Record.new(
+            zone, 'a', {'type': 'A', 'ttl': 1800, 'value': '10.0.0.1'}
+        )
+        zone.add_record(record)
+        aa.process_source_zone(zone, [])
+        self.assertEqual(
+            {'1.0.0.10.in-addr.arpa.': ['a.unit.tests.']}, dict(aa._addrs)
+        )
+
+        # another record it will record
+        zone = Zone('unit.tests.', [])
+        record = Record.new(
+            zone, 'b', {'type': 'A', 'ttl': 1800, 'value': '10.0.42.1'}
+        )
+        zone.add_record(record)
+        aa.process_source_zone(zone, [])
+        self.assertEqual(
+            {
+                '1.0.0.10.in-addr.arpa.': ['a.unit.tests.'],
+                '1.42.0.10.in-addr.arpa.': ['b.unit.tests.'],
+            },
+            dict(aa._addrs),
+        )
+
+        # a second record pointed to the same IP
+        zone = Zone('unit.tests.', [])
+        record = Record.new(
+            zone, 'c', {'type': 'A', 'ttl': 1800, 'value': '10.0.42.1'}
+        )
+        zone.add_record(record)
+        aa.process_source_zone(zone, [])
+        self.assertEqual(
+            {
+                '1.0.0.10.in-addr.arpa.': ['a.unit.tests.'],
+                '1.42.0.10.in-addr.arpa.': ['b.unit.tests.', 'c.unit.tests.'],
+            },
+            dict(aa._addrs),
+        )
+
+        # subnet with just 1 record
+        zone = Zone('0.0.10.in-addr.arpa.', [])
+        aa.populate(zone)
+        self.assertEqual(
+            {'1.0.0.10.in-addr.arpa.': ['a.unit.tests.']},
+            {r.fqdn: r.values for r in zone.records},
+        )
+
+        # subnet with 2 records
+        zone = Zone('0.10.in-addr.arpa.', [])
+        aa.populate(zone)
+        self.assertEqual(
+            {
+                '1.0.0.10.in-addr.arpa.': ['a.unit.tests.'],
+                '1.42.0.10.in-addr.arpa.': ['b.unit.tests.', 'c.unit.tests.'],
+            },
+            {r.fqdn: r.values for r in zone.records},
+        )
+
+    def test_v6(self):
+        aa = AutoArpa()
+
+        # a v6 record it will record
+        zone = Zone('unit.tests.', [])
+        record = Record.new(
+            zone, 'aaaa', {'type': 'AAAA', 'ttl': 1800, 'value': 'fc00::1'}
+        )
+        zone.add_record(record)
+        aa.process_source_zone(zone, [])
+        self.assertEqual(
+            {
+                '1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.c.f.ip6.arpa.': [
+                    'aaaa.unit.tests.'
+                ]
+            },
+            dict(aa._addrs),
+        )
+
+        # another v6 record it will record
+        zone = Zone('unit.tests.', [])
+        record = Record.new(
+            zone, 'bbbb', {'type': 'AAAA', 'ttl': 1800, 'value': 'fc42::1'}
+        )
+        zone.add_record(record)
+        aa.process_source_zone(zone, [])
+        self.assertEqual(
+            {
+                '1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.c.f.ip6.arpa.': [
+                    'aaaa.unit.tests.'
+                ],
+                '1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.2.4.c.f.ip6.arpa.': [
+                    'bbbb.unit.tests.'
+                ],
+            },
+            dict(aa._addrs),
+        )
+
+        # subnet with just 1 record
+        zone = Zone('0.0.c.f.ip6.arpa.', [])
+        aa.populate(zone)
+        self.assertEqual(
+            {
+                '1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.c.f.ip6.arpa.': [
+                    'aaaa.unit.tests.'
+                ]
+            },
+            {r.fqdn: r.values for r in zone.records},
+        )
+
+        # subnet with 2 records
+        zone = Zone('c.f.ip6.arpa.', [])
+        aa.populate(zone)
+        self.assertEqual(
+            {
+                '1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.c.f.ip6.arpa.': [
+                    'aaaa.unit.tests.'
+                ],
+                '1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.2.4.c.f.ip6.arpa.': [
+                    'bbbb.unit.tests.'
+                ],
+            },
+            {r.fqdn: r.values for r in zone.records},
+        )

tests/test_octodns_manager.py

@@ -176,6 +176,16 @@ class TestManager(TestCase):
             ).sync(dry_run=False, force=True)
             self.assertEqual(33, tc)
 
+    def test_auto_arpa(self):
+        with TemporaryDirectory() as tmpdir:
+            environ['YAML_TMP_DIR'] = tmpdir.dirname
+            environ['YAML_TMP_DIR2'] = tmpdir.dirname
+
+            manager = Manager(get_config_filename('auto-arpa.yaml'))
+            tc = manager.sync(dry_run=False)
+            # we expect 22 records from unit.tests and 2 PTRs in the arpa zone
+            self.assertEqual(24, tc)
+
     def test_idna_eligible_zones(self):
         # loading w/simple, but we'll be blowing it away and doing some manual
         # stuff