POC of processors concept that can hook in to modify zones

octodns/manager.py

@@ -121,6 +121,25 @@ class Manager(object):
                 raise ManagerException('Incorrect provider config for {}'
                                        .format(provider_name))
 
+        self.processors = {}
+        for processor_name, processor_config in \
+                self.config.get('processors', {}).items():
+            try:
+                _class = processor_config.pop('class')
+            except KeyError:
+                self.log.exception('Invalid processor class')
+                raise ManagerException('Processor {} is missing class'
+                                       .format(processor_name))
+            _class = self._get_named_class('processor', _class)
+            kwargs = self._build_kwargs(processor_config)
+            try:
+                self.processors[processor_name] = _class(processor_name,
+                                                         **kwargs)
+            except TypeError:
+                self.log.exception('Invalid processor config')
+                raise ManagerException('Incorrect processor config for {}'
+                                       .format(processor_name))
+
         zone_tree = {}
         # sort by reversed strings so that parent zones always come first
         for name in sorted(self.config['zones'].keys(), key=lambda s: s[::-1]):
@@ -222,8 +241,8 @@ class Manager(object):
         self.log.debug('configured_sub_zones: subs=%s', sub_zone_names)
         return set(sub_zone_names)
 
-    def _populate_and_plan(self, zone_name, sources, targets, desired=None,
-                           lenient=False):
+    def _populate_and_plan(self, zone_name, processors, sources, targets,
+                           desired=None, lenient=False):
 
         self.log.debug('sync:   populating, zone=%s, lenient=%s',
                        zone_name, lenient)
@@ -248,6 +267,10 @@ class Manager(object):
                                   'param', source.__class__.__name__)
                     source.populate(zone)
 
+        self.log.debug('sync:   processing, zone=%s', zone_name)
+        for processor in processors:
+            zone = processor.process(zone)
+
         self.log.debug('sync:   planning, zone=%s', zone_name)
         plans = []
 
@@ -259,7 +282,9 @@ class Manager(object):
                     'value': 'provider={}'.format(target.id)
                 })
                 zone.add_record(meta, replace=True)
-            plan = target.plan(zone)
+            # TODO: if someone has overrriden plan already this will be a
+            # breaking change so we probably need to try both ways
+            plan = target.plan(zone, processors=processors)
             if plan:
                 plans.append((target, plan))
 
@@ -315,6 +340,8 @@ class Manager(object):
                 raise ManagerException('Zone {} is missing targets'
                                        .format(zone_name))
 
+            processors = config.get('processors', [])
+
             if (eligible_sources and not
                     [s for s in sources if s in eligible_sources]):
                 self.log.info('sync:   no eligible sources, skipping')
@@ -332,6 +359,15 @@ class Manager(object):
 
             self.log.info('sync:   sources=%s -> targets=%s', sources, targets)
 
+            try:
+                collected = []
+                for processor in processors:
+                    collected.append(self.processors[processor])
+                processors = collected
+            except KeyError:
+                raise ManagerException('Zone {}, unknown processor: {}'
+                                       .format(zone_name, processor))
+
             try:
                 # rather than using a list comprehension, we break this loop
                 # out so that the `except` block below can reference the
@@ -358,8 +394,9 @@ class Manager(object):
                                        .format(zone_name, target))
 
             futures.append(self._executor.submit(self._populate_and_plan,
-                                                 zone_name, sources,
-                                                 targets, lenient=lenient))
+                                                 zone_name, processors,
+                                                 sources, targets,
+                                                 lenient=lenient))
 
         # Wait on all results and unpack/flatten the plans and store the
         # desired states in case we need them below
@@ -378,6 +415,7 @@ class Manager(object):
             futures.append(self._executor.submit(
                 self._populate_and_plan,
                 zone_name,
+                processors,
                 [],
                 [self.providers[t] for t in source_config['targets']],
                 desired=desired[zone_source],
@@ -518,6 +556,9 @@ class Manager(object):
                 if isinstance(source, YamlProvider):
                     source.populate(zone)
 
+            # TODO: validate
+            # processors = config.get('processors', [])
+
     def get_zone(self, zone_name):
         if not zone_name[-1] == '.':
             raise ManagerException('Invalid zone name {}, missing ending dot'

octodns/processors/__init__.py

@@ -0,0 +1,17 @@
+#
+#
+#
+
+from __future__ import absolute_import, division, print_function, \
+    unicode_literals
+
+from ..zone import Zone
+
+
+class BaseProcessor(object):
+
+    def __init__(self, name):
+        self.name = name
+
+    def _create_zone(self, zone):
+        return Zone(zone.name, sub_zones=zone.sub_zones)

octodns/processors/filters.py

@@ -0,0 +1,38 @@
+#
+#
+#
+
+from __future__ import absolute_import, division, print_function, \
+    unicode_literals
+
+from . import BaseProcessor
+
+
+class TypeAllowlistFilter(BaseProcessor):
+
+    def __init__(self, name, allowlist):
+        super(TypeAllowlistFilter, self).__init__(name)
+        self.allowlist = allowlist
+
+    def process(self, zone):
+        ret = self._create_zone(zone)
+        for record in zone.records:
+            if record._type in self.allowlist:
+                ret.add_record(record)
+
+        return ret
+
+
+class TypeRejectlistFilter(BaseProcessor):
+
+    def __init__(self, name, rejectlist):
+        super(TypeRejectlistFilter, self).__init__(name)
+        self.rejectlist = rejectlist
+
+    def process(self, zone):
+        ret = self._create_zone(zone)
+        for record in zone.records:
+            if record._type not in self.rejectlist:
+                ret.add_record(record)
+
+        return ret

octodns/provider/base.py

@@ -44,7 +44,7 @@ class BaseProvider(BaseSource):
         '''
         return []
 
-    def plan(self, desired):
+    def plan(self, desired, processors=[]):
         self.log.info('plan: desired=%s', desired.name)
 
         existing = Zone(desired.name, desired.sub_zones)
@@ -55,6 +55,9 @@ class BaseProvider(BaseSource):
             self.log.warn('Provider %s used in target mode did not return '
                           'exists', self.id)
 
+        for processor in processors:
+            existing = processor.process(existing)
+
         # compute the changes at the zone/record level
         changes = existing.changes(desired, self)
 

tests/test_octodns_manager.py

@@ -352,7 +352,7 @@ class TestManager(TestCase):
                     pass
 
             # This should be ok, we'll fall back to not passing it
-            manager._populate_and_plan('unit.tests.', [NoLenient()], [])
+            manager._populate_and_plan('unit.tests.', [], [NoLenient()], [])
 
             class NoZone(SimpleProvider):