Revert "WIP/expiriments with auto-arpa"

This reverts commit 8b82159ee0.

octodns/auto_arpa.py

@@ -1,51 +0,0 @@
-#
-#
-#
-
-from collections import defaultdict
-from ipaddress import ip_address
-from logging import getLogger
-
-from .processor.base import BaseProcessor
-from .record import Record
-from .source.base import BaseSource
-
-
-class AutoArpa(BaseProcessor, BaseSource):
-    SUPPORTS = set(('PTR',))
-    SUPPORTS_GEO = False
-
-    log = getLogger('AutoArpa')
-
-    def __init__(self, ttl=3600):
-        super().__init__('auto-arpa')
-        self.ttl = ttl
-
-        self._addrs = defaultdict(list)
-
-    def process_source_zone(self, desired, sources):
-        for record in desired.records:
-            if record._type in ('A', 'AAAA'):
-                for value in record.values:
-                    addr = ip_address(value)
-                    self._addrs[f'{addr.reverse_pointer}.'].append(record.fqdn)
-
-        return desired
-
-    def populate(self, zone, target=False, lenient=False):
-        self.log.debug('populate: zone=%s', zone.name)
-        before = len(zone.records)
-
-        name = zone.name
-        for arpa, fqdns in self._addrs.items():
-            if arpa.endswith(name):
-                record = Record.new(
-                    zone,
-                    zone.hostname_from_fqdn(arpa),
-                    {'ttl': self.ttl, 'type': 'PTR', 'values': fqdns},
-                )
-                zone.add_record(record)
-
-        self.log.info(
-            'populate:   found %s records', len(zone.records) - before
-        )

octodns/manager.py

@@ -10,7 +10,6 @@ from sys import stdout
 import logging
 
 from . import __VERSION__
-from .auto_arpa import AutoArpa
 from .idna import IdnaDict, idna_decode, idna_encode
 from .provider.base import BaseProvider
 from .provider.plan import Plan
@@ -63,7 +62,7 @@ class _AggregateTarget(object):
         raise AttributeError(f'{klass} object has no attribute {name}')
 
 
-class FakeThreadFuture(object):
+class MakeThreadFuture(object):
     def __init__(self, func, args, kwargs):
         self.func = func
         self.args = args
@@ -83,7 +82,7 @@ class MainThreadExecutor(object):
     '''
 
     def submit(self, func, *args, **kwargs):
-        return FakeThreadFuture(func, args, kwargs)
+        return MakeThreadFuture(func, args, kwargs)
 
 
 class ManagerException(Exception):
@@ -100,9 +99,7 @@ class Manager(object):
 
     # TODO: all of this should get broken up, mainly so that it's not so huge
     # and each bit can be cleanly tested independently
-    def __init__(
-        self, config_file, max_workers=None, include_meta=False, auto_arpa=False
-    ):
+    def __init__(self, config_file, max_workers=None, include_meta=False):
         version = self._try_version('octodns', version=__VERSION__)
         self.log.info(
             '__init__: config_file=%s (octoDNS %s)', config_file, version
@@ -122,7 +119,6 @@ class Manager(object):
         self.include_meta = self._config_include_meta(
             manager_config, include_meta
         )
-        self.auto_arpa = self._config_auto_arpa(manager_config, auto_arpa)
 
         self.global_processors = manager_config.get('processors', [])
         self.log.info('__init__: global_processors=%s', self.global_processors)
@@ -178,15 +174,6 @@ class Manager(object):
         self.log.info('_config_include_meta: include_meta=%s', include_meta)
         return include_meta
 
-    def _config_auto_arpa(self, manager_config, auto_arpa=False):
-        auto_arpa = auto_arpa or manager_config.get('auto-arpa', False)
-        self.log.info('_config_auto_arpa: auto_arpa=%s', auto_arpa)
-        if auto_arpa:
-            if not isinstance(auto_arpa, dict):
-                auto_arpa = {}
-            return AutoArpa(**auto_arpa)
-        return None
-
     def _config_providers(self, providers_config):
         self.log.debug('_config_providers: configuring providers')
         providers = {}
@@ -215,9 +202,6 @@ class Manager(object):
                     'Incorrect provider config for ' + provider_name
                 )
 
-        if self.auto_arpa:
-            providers['auto-arpa'] = self.auto_arpa
-
         return providers
 
     def _config_processors(self, processors_config):
@@ -245,10 +229,6 @@ class Manager(object):
                 raise ManagerException(
                     'Incorrect processor config for ' + processor_name
                 )
-
-        if self.auto_arpa:
-            processors['auto-arpa'] = self.auto_arpa
-
         return processors
 
     def _config_plan_outputs(self, plan_outputs_config):
@@ -497,7 +477,6 @@ class Manager(object):
 
         aliased_zones = {}
         futures = []
-        arpa_kwargs = []
         for zone_name, config in zones.items():
             decoded_zone_name = idna_decode(zone_name)
             self.log.info('sync:   zone=%s', decoded_zone_name)
@@ -558,9 +537,6 @@ class Manager(object):
                 collected = []
                 for processor in self.global_processors + processors:
                     collected.append(self.processors[processor])
-                # always goes last
-                if self.auto_arpa:
-                    collected.append(self.auto_arpa)
                 processors = collected
             except KeyError:
                 raise ManagerException(
@@ -596,25 +572,16 @@ class Manager(object):
                     f'Zone {decoded_zone_name}, unknown ' f'target: {target}'
                 )
 
-            kwargs = {
-                'zone_name': zone_name,
-                'processors': processors,
-                'sources': sources,
-                'targets': targets,
-                'lenient': lenient,
-            }
-            if self.auto_arpa and (
-                zone_name.endswith('in-addr.arpa.')
-                or zone_name.endswith('ip6.arpa.')
-            ):
-                # auto arpa is enabled so we need to defer processing all arpa
-                # zones until after general ones have completed (async) so that
-                # they'll have access to all the recorded A/AAAA values
-                arpa_kwargs.append(kwargs)
-            else:
-                futures.append(
-                    self._executor.submit(self._populate_and_plan, **kwargs)
+            futures.append(
+                self._executor.submit(
+                    self._populate_and_plan,
+                    zone_name,
+                    processors,
+                    sources,
+                    targets,
+                    lenient=lenient,
                 )
+            )
 
         # Wait on all results and unpack/flatten the plans and store the
         # desired states in case we need them below
@@ -654,24 +621,6 @@ class Manager(object):
         # as these are aliased zones
         plans += [p for f in futures for p in f.result()[0]]
 
-        if self.auto_arpa and arpa_kwargs:
-            self.log.info(
-                'sync: processing %d arpa reverse dns zones', len(arpa_kwargs)
-            )
-            # all the general zones are done and we've recorded the A/AAAA
-            # records with the AutoPtr. We can no process ptr zones
-            futures = []
-            for kwargs in arpa_kwargs:
-                futures.append(
-                    self._executor.submit(self._populate_and_plan, **kwargs)
-                )
-
-            for future in futures:
-                ps, d = future.result()
-                desired[d.name] = d
-                for plan in ps:
-                    plans.append(plan)
-
         # Best effort sort plans children first so that we create/update
         # children zones before parents which should allow us to more safely
         # extract things into sub-zones. Combining a child back into a parent

tests/config/simple.yaml

@@ -1,5 +1,4 @@
 manager:
-  auto-arpa: true
   max_workers: 2
 providers:
   in:
@@ -45,8 +44,3 @@ zones:
     - in
     targets:
     - dump
-  3.2.1.in-addr.arpa.:
-    sources:
-      - auto-arpa
-    targets:
-    - dump

tests/test_octodns_manager.py

@@ -136,7 +136,7 @@ class TestManager(TestCase):
             environ['YAML_TMP_DIR'] = tmpdir.dirname
             environ['YAML_TMP_DIR2'] = tmpdir.dirname
             tc = Manager(get_config_filename('simple.yaml')).sync(dry_run=False)
-            self.assertEqual(30, tc)
+            self.assertEqual(28, tc)
 
             # try with just one of the zones
             tc = Manager(get_config_filename('simple.yaml')).sync(
@@ -160,13 +160,13 @@ class TestManager(TestCase):
             tc = Manager(get_config_filename('simple.yaml')).sync(
                 dry_run=False, force=True
             )
-            self.assertEqual(30, tc)
+            self.assertEqual(28, tc)
 
             # Again with max_workers = 1
             tc = Manager(
                 get_config_filename('simple.yaml'), max_workers=1
             ).sync(dry_run=False, force=True)
-            self.assertEqual(30, tc)
+            self.assertEqual(28, tc)
 
             # Include meta
             tc = Manager(
@@ -174,7 +174,7 @@ class TestManager(TestCase):
                 max_workers=1,
                 include_meta=True,
             ).sync(dry_run=False, force=True)
-            self.assertEqual(36, tc)
+            self.assertEqual(33, tc)
 
     def test_idna_eligible_zones(self):
         # loading w/simple, but we'll be blowing it away and doing some manual
@@ -186,6 +186,9 @@ class TestManager(TestCase):
         manager.config['zones'] = manager._config_zones(
             {'déjà.vu.': {}, 'deja.vu.': {}, idna_encode('こんにちは.jp.'): {}}
         )
+        from pprint import pprint
+
+        pprint(manager.config['zones'])
 
         # refer to them with utf-8
         with self.assertRaises(ManagerException) as ctx: