diff --git a/.changelog/503eda9ec20e470cafb7904500dd22cb.md b/.changelog/503eda9ec20e470cafb7904500dd22cb.md new file mode 100644 index 0000000..6f7da1d --- /dev/null +++ b/.changelog/503eda9ec20e470cafb7904500dd22cb.md @@ -0,0 +1,4 @@ +--- +type: minor +--- +Add glob and regex support to dynamic zone config \ No newline at end of file diff --git a/octodns/manager.py b/octodns/manager.py index c49c716..4478c71 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -4,6 +4,7 @@ from collections import deque from concurrent.futures import ThreadPoolExecutor +from fnmatch import filter as fnmatch_filter from hashlib import sha256 from importlib import import_module from importlib.metadata import PackageNotFoundError @@ -593,13 +594,11 @@ class Manager(object): instead. ''' - # sorting longest first with the assumption that'll longer wildcards or - # regexes will be more specific, but mostly it's just to make the - # behavior consistent - for name, config in sorted( - zones.items(), key=lambda d: len(d[0]), reverse=True - ): - if name[0] != '*' and name[-1] != '$': + source_zones = {} + + # list since we'll be modifying zones in the loop + for name, config in list(zones.items()): + if name[0] != '*': # this isn't a dynamic zone config, move along continue @@ -607,33 +606,51 @@ class Manager(object): found_sources = sources or self._get_sources( name, config, eligible_sources ) - self.log.info('sync: dynamic zone=%s, sources=%s', name, sources) - sourced_zones = set() + self.log.info( + '_preprocess_zones: dynamic zone=%s, sources=%s', + name, + (s.id for s in found_sources), + ) + candidates = set() for source in found_sources: - if not hasattr(source, 'list_zones'): - raise ManagerException( - f'dynamic zone={name} includes a source, {source.id}, that does not support `list_zones`' + if source.id not in source_zones: + if not hasattr(source, 'list_zones'): + raise ManagerException( + f'dynamic zone={name} includes a source, {source.id}, that does not support `list_zones`' + ) + # get this source's zones + listed_zones = set(source.list_zones()) + # cache them + source_zones[source.id] = listed_zones + self.log.debug( + '_preprocess_zones: source=%s, list_zones=%s', + source.id, + listed_zones, ) - sourced_zones |= set(source.list_zones()) - - self.log.debug('_preprocess_zones: sourced_zones=%s', sourced_zones) - - if name[-1] == '$': - # it's an end-anchored regex - re = re_compile(name) - # filter the zones we sourced with it - sourced_zones = set(z for z in sourced_zones if re.match(z)) - # old-style wildcards are implcit catch-alls so they don't need - # filtering - - # we do want to remove any explicitly configured zones or those - # that matched a previous wildcard/regex - sourced_zones -= set(zones.keys()) + # add this source's zones to the candidates + candidates |= source_zones[source.id] + + self.log.debug('_preprocess_zones: candidates=%s', candidates) + + # remove any zones that are already configured, either explicitly or + # from a previous dyanmic config + candidates -= set(zones.keys()) + + if glob := config.pop('glob', None): + self.log.debug('_preprocess_zones: glob=%s', glob) + candidates = set(fnmatch_filter(candidates, glob)) + elif regex := config.pop('regex', None): + self.log.debug('_preprocess_zones: regex=%s', regex) + regex = re_compile(regex) + self.log.debug('_preprocess_zones: compiled=%s', regex) + candidates = set(z for z in candidates if regex.search(z)) + else: + # old-style wildcard that uses everything + self.log.debug('_preprocess_zones: old semantics, catch all') - self.log.debug('_preprocess_zones: filtered=%s', sourced_zones) + self.log.debug('_preprocess_zones: matches=%s', candidates) - for match in sourced_zones: - self.log.info('sync: adding dynamic zone=%s', match) + for match in candidates: zones[match] = config # remove the dynamic config element so we don't try and populate it diff --git a/tests/test_octodns_manager.py b/tests/test_octodns_manager.py index 549658d..5ee22a3 100644 --- a/tests/test_octodns_manager.py +++ b/tests/test_octodns_manager.py @@ -1341,7 +1341,8 @@ class TestManager(TestCase): self.assertIsNone(zone_with_defaults.update_pcent_threshold) self.assertIsNone(zone_with_defaults.delete_pcent_threshold) - def test_preprocess_zones(self): + def test_preprocess_zones_original(self): + # these will be unused environ['YAML_TMP_DIR'] = '/tmp' environ['YAML_TMP_DIR2'] = '/tmp' manager = Manager(get_config_filename('simple.yaml')) @@ -1405,27 +1406,158 @@ class TestManager(TestCase): ) mock_source.list_zones.assert_called_once() - # multiple wildcards, this didn't make sense previously as the 2nd one - # would just win - mock_source.reset_mock() + def test_preprocess_zones_multiple_single_source(self): + # these will be unused + environ['YAML_TMP_DIR'] = '/tmp' + environ['YAML_TMP_DIR2'] = '/tmp' + manager = Manager(get_config_filename('simple.yaml')) + + manager._get_sources = MagicMock() + mock_source = MagicMock() + mock_source.id = 'mm' + manager._get_sources = MagicMock() + manager._get_sources.return_value = [mock_source] + config_a = {'foo': 42} config_b = {'bar': 43} - zones = {r'.*\.a\.com\.$': config_a, r'.*\.b\.com\.$': config_b} - mock_source.list_zones.return_value = [ - 'one.a.com.', - 'two.a.com.', - 'three.b.com.', + zones = {'*.a.com.': config_a, '*.b.com.': config_b} + mock_source.list_zones.side_effect = [ + ['one.a.com.', 'two.a.com.', 'one.b.com.', 'two.b.com.'] ] - got = manager._preprocess_zones(zones, sources=[mock_source]) + got = manager._preprocess_zones(zones, sources=[]) + # each zone will have it's sources looked up + self.assertEqual(2, manager._get_sources.call_count) + # but there's only one source so it's zones will be cached + self.assertEqual(1, mock_source.list_zones.call_count) + # everything will have been matched by the first old style wildcard and + # thus have its config, nothing will have b's + self.assertEqual( + { + 'one.a.com.': config_a, + 'two.a.com.': config_a, + 'one.b.com.': config_a, + 'two.b.com.': config_a, + }, + got, + ) + + def test_preprocess_zones_multiple_seperate_sources(self): + # these will be unused + environ['YAML_TMP_DIR'] = '/tmp' + environ['YAML_TMP_DIR2'] = '/tmp' + manager = Manager(get_config_filename('simple.yaml')) + + manager._get_sources = MagicMock() + mock_source_a = MagicMock() + mock_source_a.id = 'mm_a' + mock_source_b = MagicMock() + mock_source_b.id = 'mm_b' + manager._get_sources = MagicMock() + manager._get_sources.side_effect = [[mock_source_a], [mock_source_b]] + + config_a = {'foo': 42} + config_b = {'bar': 43} + zones = {'*.a.com.': config_a, '*.b.com.': config_b} + mock_source_a.list_zones.side_effect = [['one.a.com.', 'two.a.com.']] + mock_source_b.list_zones.side_effect = [['one.b.com.', 'two.b.com.']] + got = manager._preprocess_zones(zones, sources=[]) + # each zone will have it's sources looked up + self.assertEqual(2, manager._get_sources.call_count) + # so each mock will be called once + self.assertEqual(1, mock_source_a.list_zones.call_count) + self.assertEqual(1, mock_source_b.list_zones.call_count) + # the souces from each source will be matched with the coresponding config + self.assertEqual( + { + 'one.a.com.': config_a, + 'two.a.com.': config_a, + 'one.b.com.': config_b, + 'two.b.com.': config_b, + }, + got, + ) + + def test_preprocess_zones_glob(self): + # these will be unused + environ['YAML_TMP_DIR'] = '/tmp' + environ['YAML_TMP_DIR2'] = '/tmp' + manager = Manager(get_config_filename('simple.yaml')) + + manager._get_sources = MagicMock() + mock_source = MagicMock() + mock_source.id = 'mm' + manager._get_sources = MagicMock() + manager._get_sources.return_value = [mock_source] + + # match things with .a. + config_a = {'foo': 42, 'glob': r'*.a.com.'} + # match things with .b. + config_b = {'bar': 43, 'glob': r'*.b.com.'} + zones = {'*.a.com.': config_a, '*.b.com.': config_b} + mock_source.list_zones.side_effect = [ + [ + 'one.a.com.', + 'two.a.com.', + 'one.b.com.', + 'two.b.com.', + 'ignored.com.', + ] + ] + got = manager._preprocess_zones(zones, sources=[]) + self.assertEqual(2, manager._get_sources.call_count) + self.assertEqual(1, mock_source.list_zones.call_count) + # a will glob match .a.com., b will .b.com., ignored.com. won't match + # anything + self.assertEqual( + { + 'one.a.com.': config_a, + 'two.a.com.': config_a, + 'one.b.com.': config_b, + 'two.b.com.': config_b, + }, + got, + ) + + def test_preprocess_zones_regex(self): + # these will be unused + environ['YAML_TMP_DIR'] = '/tmp' + environ['YAML_TMP_DIR2'] = '/tmp' + manager = Manager(get_config_filename('simple.yaml')) + + manager._get_sources = MagicMock() + mock_source = MagicMock() + mock_source.id = 'mm' + manager._get_sources = MagicMock() + manager._get_sources.return_value = [mock_source] + + # match things with .a. + config_a = {'foo': 42, 'regex': r'\.a\.'} + # match things with .b. + config_b = {'bar': 43, 'regex': r'\.b\.'} + zones = {'*.a.com.': config_a, '*.b.com.': config_b} + mock_source.list_zones.side_effect = [ + [ + 'one.a.com.', + 'two.a.com.', + 'one.b.com.', + 'two.b.com.', + 'ignored.com.', + ] + ] + got = manager._preprocess_zones(zones, sources=[]) + self.assertEqual(2, manager._get_sources.call_count) + self.assertEqual(1, mock_source.list_zones.call_count) + # a will regex match .a.com., b will .b.com., ignored.com. won't match + # anything self.assertEqual( { 'one.a.com.': config_a, 'two.a.com.': config_a, - 'three.b.com.': config_b, + 'one.b.com.': config_b, + 'two.b.com.': config_b, }, got, ) - self.assertEqual(2, mock_source.list_zones.call_count) class TestMainThreadExecutor(TestCase):