Merge pull request #1309 from octodns/dump-processor-support

Add processor support to octodns-dump

.changelog/fc2ce1a8cc5f4f1981b5b1911a28c2f8.md

@@ -0,0 +1,4 @@
+---
+type: minor
+---
+Add processor support to octodns-dump

octodns/manager.py

@@ -587,6 +587,27 @@ class Manager(object):
 
         return sources
 
+    def _get_processors(self, decoded_zone_name, config):
+        # Build list of processor names
+        processors = (
+            self.global_processors
+            + (config.get('processors') or [])
+            + self.global_post_processors
+        )
+
+        # Translate processor names to processor objects
+        try:
+            collected = []
+            for processor in processors:
+                collected.append(self.processors[processor])
+            processors = collected
+        except KeyError:
+            raise ManagerException(
+                f'Zone {decoded_zone_name}, unknown processor: {processor}'
+            )
+
+        return processors
+
     def _preprocess_zones(self, zones, eligible_sources=None, sources=None):
         '''
         This may modify the passed in zone object, it should be ignored after
@@ -757,12 +778,8 @@ class Manager(object):
                     f'Zone {decoded_zone_name} is missing targets'
                 )
 
-            processors = (
-                self.global_processors
-                + (config.get('processors') or [])
-                + self.global_post_processors
-            )
-            self.log.info('sync:     processors=%s', processors)
+            processors = self._get_processors(decoded_zone_name, config)
+            self.log.info('sync:     processors=%s', [p.id for p in processors])
 
             if not sources:
                 self.log.info('sync:   no eligible sources, skipping')
@@ -780,17 +797,6 @@ class Manager(object):
 
             self.log.info('sync:     targets=%s', targets)
 
-            try:
-                collected = []
-                for processor in processors:
-                    collected.append(self.processors[processor])
-                processors = collected
-            except KeyError:
-                raise ManagerException(
-                    f'Zone {decoded_zone_name}, unknown '
-                    f'processor: {processor}'
-                )
-
             try:
                 trgs = []
                 for target in targets:
@@ -1017,17 +1023,32 @@ class Manager(object):
         zones = self._preprocess_zones(zones, sources=sources)
 
         if '*' in zone:
-            # we want to do everything, just need the names though
-            zones = zones.keys()
+            # we want to do everything
+            zones = zones.items()
         else:
             # we want to do a specific zone
-            zones = [zone]
+            try:
+                zones = [(zone, zones[zone])]
+            except KeyError:
+                raise ManagerException(
+                    f'Requested zone "{zone}" not found in config'
+                )
+
+        for zone_name, config in zones:
+            decoded_zone_name = idna_decode(zone_name)
+            self.log.info('dump:   zone=%s', decoded_zone_name)
 
-        for zone in zones:
-            zone = self.get_zone(zone)
+            processors = self._get_processors(decoded_zone_name, config)
+            self.log.info('dump:     processors=%s', [p.id for p in processors])
+
+            zone = self.get_zone(zone_name)
             for source in sources:
                 source.populate(zone, lenient=lenient)
 
+            # Apply processors
+            for processor in processors:
+                zone = processor.process_source_zone(zone, sources=sources)
+
             plan = target.plan(zone)
             if plan is None:
                 plan = Plan(zone, zone, [], False)

tests/config/dump-processors.yaml

@@ -0,0 +1,35 @@
+providers:
+  config:
+    class: octodns.provider.yaml.YamlProvider
+    directory: tests/config
+    strict_supports: False
+  dump:
+    class: octodns.provider.yaml.YamlProvider
+    directory: env/YAML_TMP_DIR
+    supports_root_ns: False
+    strict_supports: False
+
+processors:
+  only-a-records:
+    class: octodns.processor.filter.TypeAllowlistFilter
+    allowlist:
+      - A
+  counter:
+    class: helpers.CountingProcessor
+
+zones:
+  unit.tests.:
+    sources:
+      - config
+    processors:
+      - only-a-records
+    targets:
+      - dump
+
+  bad.unit.tests.:
+    sources:
+      - config
+    processors:
+      - unknown-processor
+    targets:
+      - dump

tests/test_octodns_manager.py

@@ -27,6 +27,7 @@ from octodns.manager import (
     _AggregateTarget,
 )
 from octodns.processor.base import BaseProcessor
+from octodns.provider.yaml import YamlProvider
 from octodns.record import Create, Delete, Record, Update
 from octodns.secret.environ import EnvironSecretsException
 from octodns.yaml import safe_load
@@ -584,6 +585,38 @@ class TestManager(TestCase):
                     sources=['in'],
                 )
 
+    def test_dump_processors(self):
+        with TemporaryDirectory() as tmpdir:
+            environ['YAML_TMP_DIR'] = tmpdir.dirname
+            manager = Manager(get_config_filename('dump-processors.yaml'))
+
+            # Dump with processor that filters to only A records
+            manager.dump(
+                zone='unit.tests.',
+                output_dir=tmpdir.dirname,
+                sources=['config'],
+            )
+
+            # Read the dumped file and verify only A records are present
+            dumped = YamlProvider('dumped', tmpdir.dirname)
+            zone = Zone('unit.tests.', [])
+            dumped.populate(zone)
+
+            # Should only have A records, not AAAA, CNAME, etc.
+            record_types = {r._type for r in zone.records}
+            self.assertIn('A', record_types)
+            self.assertNotIn('AAAA', record_types)
+            self.assertNotIn('CNAME', record_types)
+
+            # Test unknown processor error
+            with self.assertRaises(ManagerException) as ctx:
+                manager.dump(
+                    zone='bad.unit.tests.',
+                    output_dir=tmpdir.dirname,
+                    sources=['config'],
+                )
+            self.assertIn('unknown processor', str(ctx.exception))
+
     def test_validate_configs(self):
         Manager(get_config_filename('simple-validate.yaml')).validate_configs()