Merge pull request #934 from octodns/global-processors

Global processors support

CHANGELOG.md

@@ -11,6 +11,8 @@
   decoded form. Both forms should be accepted in command line arguments.
   Providers may need to be updated to display the decoded form in their logs,
   until then they'd display the IDNA version.
+* Support for configuring global processors that apply to all zones with
+  `manager.processors`
 
 #### Stuff
 

octodns/manager.py

@@ -127,6 +127,9 @@ class Manager(object):
             manager_config, include_meta
         )
 
+        self.global_processors = manager_config.get('processors', [])
+        self.log.info('__init__: global_processors=%s', self.global_processors)
+
         providers_config = self.config['providers']
         self.providers = self._config_providers(providers_config)
 
@@ -539,7 +542,7 @@ class Manager(object):
 
             try:
                 collected = []
-                for processor in processors:
+                for processor in self.global_processors + processors:
                     collected.append(self.processors[processor])
                 processors = collected
             except KeyError:

tests/config/processors.yaml

@@ -1,3 +1,7 @@
+manager:
+  processors:
+    - global-counter
+
 providers:
   config:
     # This helps us get coverage when printing out provider versions
@@ -19,6 +23,8 @@ processors:
   test:
     # This helps us get coverage when printing out processor versions
     class: helpers.TestBaseProcessor
+  global-counter:
+    class: helpers.CountingProcessor
 
 zones:
   unit.tests.:

tests/helpers.py

@@ -131,3 +131,13 @@ class TestYamlProvider(YamlProvider):
 
 class TestBaseProcessor(BaseProcessor):
     pass
+
+
+class CountingProcessor(BaseProcessor):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.count = 0
+
+    def process_source_zone(self, zone, *args, **kwargs):
+        self.count += len(zone.records)
+        return zone

tests/test_octodns_manager.py

@@ -643,9 +643,16 @@ class TestManager(TestCase):
     def test_processor_config(self):
         # Smoke test loading a valid config
         manager = Manager(get_config_filename('processors.yaml'))
-        self.assertEqual(['noop', 'test'], list(manager.processors.keys()))
+        self.assertEqual(
+            ['noop', 'test', 'global-counter'], list(manager.processors.keys())
+        )
+        # make sure we got the global processor and that it's count is 0 now
+        self.assertEqual(['global-counter'], manager.global_processors)
+        self.assertEqual(0, manager.processors['global-counter'].count)
         # This zone specifies a valid processor
         manager.sync(['unit.tests.'])
+        # make sure the global processor ran and counted some records
+        self.assertTrue(manager.processors['global-counter'].count >= 25)
 
         with self.assertRaises(ManagerException) as ctx:
             # This zone specifies a non-existent processor