Add glob and regex support to dynamic zone config

.changelog/503eda9ec20e470cafb7904500dd22cb.md

@@ -0,0 +1,4 @@
+---
+type: minor
+---
+Add glob and regex support to dynamic zone config

octodns/manager.py

@@ -4,6 +4,7 @@
 
 from collections import deque
 from concurrent.futures import ThreadPoolExecutor
+from fnmatch import filter as fnmatch_filter
 from hashlib import sha256
 from importlib import import_module
 from importlib.metadata import PackageNotFoundError
@@ -593,13 +594,11 @@ class Manager(object):
         instead.
         '''
 
-        # sorting longest first with the assumption that'll longer wildcards or
-        # regexes will be more specific, but mostly it's just to make the
-        # behavior consistent
-        for name, config in sorted(
-            zones.items(), key=lambda d: len(d[0]), reverse=True
-        ):
-            if name[0] != '*' and name[-1] != '$':
+        source_zones = {}
+
+        # list since we'll be modifying zones in the loop
+        for name, config in list(zones.items()):
+            if name[0] != '*':
                 # this isn't a dynamic zone config, move along
                 continue
 
@@ -607,33 +606,51 @@ class Manager(object):
             found_sources = sources or self._get_sources(
                 name, config, eligible_sources
             )
-            self.log.info('sync:   dynamic zone=%s, sources=%s', name, sources)
-            sourced_zones = set()
+            self.log.info(
+                '_preprocess_zones: dynamic zone=%s, sources=%s',
+                name,
+                (s.id for s in found_sources),
+            )
+            candidates = set()
             for source in found_sources:
-                if not hasattr(source, 'list_zones'):
-                    raise ManagerException(
-                        f'dynamic zone={name} includes a source, {source.id}, that does not support `list_zones`'
+                if source.id not in source_zones:
+                    if not hasattr(source, 'list_zones'):
+                        raise ManagerException(
+                            f'dynamic zone={name} includes a source, {source.id}, that does not support `list_zones`'
+                        )
+                    # get this source's zones
+                    listed_zones = set(source.list_zones())
+                    # cache them
+                    source_zones[source.id] = listed_zones
+                    self.log.debug(
+                        '_preprocess_zones: source=%s, list_zones=%s',
+                        source.id,
+                        listed_zones,
                     )
-                sourced_zones |= set(source.list_zones())
-
-            self.log.debug('_preprocess_zones: sourced_zones=%s', sourced_zones)
-
-            if name[-1] == '$':
-                # it's an end-anchored regex
-                re = re_compile(name)
-                # filter the zones we sourced with it
-                sourced_zones = set(z for z in sourced_zones if re.match(z))
-            # old-style wildcards are implcit catch-alls so they don't need
-            # filtering
-
-            # we do want to remove any explicitly configured zones or those
-            # that matched a previous wildcard/regex
-            sourced_zones -= set(zones.keys())
+                # add this source's zones to the candidates
+                candidates |= source_zones[source.id]
+
+            self.log.debug('_preprocess_zones: candidates=%s', candidates)
+
+            # remove any zones that are already configured, either explicitly or
+            # from a previous dyanmic config
+            candidates -= set(zones.keys())
+
+            if glob := config.pop('glob', None):
+                self.log.debug('_preprocess_zones: glob=%s', glob)
+                candidates = set(fnmatch_filter(candidates, glob))
+            elif regex := config.pop('regex', None):
+                self.log.debug('_preprocess_zones: regex=%s', regex)
+                regex = re_compile(regex)
+                self.log.debug('_preprocess_zones: compiled=%s', regex)
+                candidates = set(z for z in candidates if regex.search(z))
+            else:
+                # old-style wildcard that uses everything
+                self.log.debug('_preprocess_zones: old semantics, catch all')
 
-            self.log.debug('_preprocess_zones: filtered=%s', sourced_zones)
+            self.log.debug('_preprocess_zones: matches=%s', candidates)
 
-            for match in sourced_zones:
-                self.log.info('sync:     adding dynamic zone=%s', match)
+            for match in candidates:
                 zones[match] = config
 
             # remove the dynamic config element so we don't try and populate it

tests/test_octodns_manager.py

@@ -1341,7 +1341,8 @@ class TestManager(TestCase):
             self.assertIsNone(zone_with_defaults.update_pcent_threshold)
             self.assertIsNone(zone_with_defaults.delete_pcent_threshold)
 
-    def test_preprocess_zones(self):
+    def test_preprocess_zones_original(self):
+        # these will be unused
         environ['YAML_TMP_DIR'] = '/tmp'
         environ['YAML_TMP_DIR2'] = '/tmp'
         manager = Manager(get_config_filename('simple.yaml'))
@@ -1405,27 +1406,158 @@ class TestManager(TestCase):
         )
         mock_source.list_zones.assert_called_once()
 
-        # multiple wildcards, this didn't make sense previously as the 2nd one
-        # would just win
-        mock_source.reset_mock()
+    def test_preprocess_zones_multiple_single_source(self):
+        # these will be unused
+        environ['YAML_TMP_DIR'] = '/tmp'
+        environ['YAML_TMP_DIR2'] = '/tmp'
+        manager = Manager(get_config_filename('simple.yaml'))
+
+        manager._get_sources = MagicMock()
+        mock_source = MagicMock()
+        mock_source.id = 'mm'
+        manager._get_sources = MagicMock()
+        manager._get_sources.return_value = [mock_source]
+
         config_a = {'foo': 42}
         config_b = {'bar': 43}
-        zones = {r'.*\.a\.com\.$': config_a, r'.*\.b\.com\.$': config_b}
-        mock_source.list_zones.return_value = [
-            'one.a.com.',
-            'two.a.com.',
-            'three.b.com.',
+        zones = {'*.a.com.': config_a, '*.b.com.': config_b}
+        mock_source.list_zones.side_effect = [
+            ['one.a.com.', 'two.a.com.', 'one.b.com.', 'two.b.com.']
         ]
-        got = manager._preprocess_zones(zones, sources=[mock_source])
+        got = manager._preprocess_zones(zones, sources=[])
+        # each zone will have it's sources looked up
+        self.assertEqual(2, manager._get_sources.call_count)
+        # but there's only one source so it's zones will be cached
+        self.assertEqual(1, mock_source.list_zones.call_count)
+        # everything will have been matched by the first old style wildcard and
+        # thus have its config, nothing will have b's
+        self.assertEqual(
+            {
+                'one.a.com.': config_a,
+                'two.a.com.': config_a,
+                'one.b.com.': config_a,
+                'two.b.com.': config_a,
+            },
+            got,
+        )
+
+    def test_preprocess_zones_multiple_seperate_sources(self):
+        # these will be unused
+        environ['YAML_TMP_DIR'] = '/tmp'
+        environ['YAML_TMP_DIR2'] = '/tmp'
+        manager = Manager(get_config_filename('simple.yaml'))
+
+        manager._get_sources = MagicMock()
+        mock_source_a = MagicMock()
+        mock_source_a.id = 'mm_a'
+        mock_source_b = MagicMock()
+        mock_source_b.id = 'mm_b'
+        manager._get_sources = MagicMock()
+        manager._get_sources.side_effect = [[mock_source_a], [mock_source_b]]
+
+        config_a = {'foo': 42}
+        config_b = {'bar': 43}
+        zones = {'*.a.com.': config_a, '*.b.com.': config_b}
+        mock_source_a.list_zones.side_effect = [['one.a.com.', 'two.a.com.']]
+        mock_source_b.list_zones.side_effect = [['one.b.com.', 'two.b.com.']]
+        got = manager._preprocess_zones(zones, sources=[])
+        # each zone will have it's sources looked up
+        self.assertEqual(2, manager._get_sources.call_count)
+        # so each mock will be called once
+        self.assertEqual(1, mock_source_a.list_zones.call_count)
+        self.assertEqual(1, mock_source_b.list_zones.call_count)
+        # the souces from each source will be matched with the coresponding config
+        self.assertEqual(
+            {
+                'one.a.com.': config_a,
+                'two.a.com.': config_a,
+                'one.b.com.': config_b,
+                'two.b.com.': config_b,
+            },
+            got,
+        )
+
+    def test_preprocess_zones_glob(self):
+        # these will be unused
+        environ['YAML_TMP_DIR'] = '/tmp'
+        environ['YAML_TMP_DIR2'] = '/tmp'
+        manager = Manager(get_config_filename('simple.yaml'))
+
+        manager._get_sources = MagicMock()
+        mock_source = MagicMock()
+        mock_source.id = 'mm'
+        manager._get_sources = MagicMock()
+        manager._get_sources.return_value = [mock_source]
+
+        # match things with .a.
+        config_a = {'foo': 42, 'glob': r'*.a.com.'}
+        # match things with .b.
+        config_b = {'bar': 43, 'glob': r'*.b.com.'}
+        zones = {'*.a.com.': config_a, '*.b.com.': config_b}
+        mock_source.list_zones.side_effect = [
+            [
+                'one.a.com.',
+                'two.a.com.',
+                'one.b.com.',
+                'two.b.com.',
+                'ignored.com.',
+            ]
+        ]
+        got = manager._preprocess_zones(zones, sources=[])
+        self.assertEqual(2, manager._get_sources.call_count)
+        self.assertEqual(1, mock_source.list_zones.call_count)
+        # a will glob match .a.com., b will .b.com., ignored.com. won't match
+        # anything
+        self.assertEqual(
+            {
+                'one.a.com.': config_a,
+                'two.a.com.': config_a,
+                'one.b.com.': config_b,
+                'two.b.com.': config_b,
+            },
+            got,
+        )
+
+    def test_preprocess_zones_regex(self):
+        # these will be unused
+        environ['YAML_TMP_DIR'] = '/tmp'
+        environ['YAML_TMP_DIR2'] = '/tmp'
+        manager = Manager(get_config_filename('simple.yaml'))
+
+        manager._get_sources = MagicMock()
+        mock_source = MagicMock()
+        mock_source.id = 'mm'
+        manager._get_sources = MagicMock()
+        manager._get_sources.return_value = [mock_source]
+
+        # match things with .a.
+        config_a = {'foo': 42, 'regex': r'\.a\.'}
+        # match things with .b.
+        config_b = {'bar': 43, 'regex': r'\.b\.'}
+        zones = {'*.a.com.': config_a, '*.b.com.': config_b}
+        mock_source.list_zones.side_effect = [
+            [
+                'one.a.com.',
+                'two.a.com.',
+                'one.b.com.',
+                'two.b.com.',
+                'ignored.com.',
+            ]
+        ]
+        got = manager._preprocess_zones(zones, sources=[])
+        self.assertEqual(2, manager._get_sources.call_count)
+        self.assertEqual(1, mock_source.list_zones.call_count)
+        # a will regex match .a.com., b will .b.com., ignored.com. won't match
+        # anything
         self.assertEqual(
             {
                 'one.a.com.': config_a,
                 'two.a.com.': config_a,
-                'three.b.com.': config_b,
+                'one.b.com.': config_b,
+                'two.b.com.': config_b,
             },
             got,
         )
-        self.assertEqual(2, mock_source.list_zones.call_count)
 
 
 class TestMainThreadExecutor(TestCase):