From 689043cd3d0cbce25fed26e578d2e713eb3190db Mon Sep 17 00:00:00 2001 From: Christian Funkhouser Date: Mon, 8 Apr 2019 17:07:45 -0400 Subject: [PATCH] Merge SplitYamlProvider and YamlProvider tests Signed-off-by: Christian Funkhouser --- octodns/provider/yaml.py | 31 +++-- tests/test_octodns_provider_yaml.py | 199 +++++++++++++++++++++++++++- 2 files changed, 212 insertions(+), 18 deletions(-) diff --git a/octodns/provider/yaml.py b/octodns/provider/yaml.py index 99fc2e0..9e0fb24 100644 --- a/octodns/provider/yaml.py +++ b/octodns/provider/yaml.py @@ -116,20 +116,6 @@ class YamlProvider(BaseProvider): safe_dump(dict(data), fh) -# Any record name added to this set will be included in the catch-all file, -# instead of a file matching the record name. -_CATCHALL_RECORD_NAMES = ('*', '') - - -def list_all_yaml_files(directory): - yaml_files = set() - for f in listdir(directory): - filename = join(directory, '{}'.format(f)) - if f.endswith('.yaml') and isfile(filename): - yaml_files.add(filename) - return list(yaml_files) - - class SplitYamlProvider(YamlProvider): ''' Core provider for records configured in multiple YAML files on disk. @@ -153,6 +139,19 @@ class SplitYamlProvider(YamlProvider): enforce_order: True ''' + # Any record name added to this set will be included in the catch-all file, + # instead of a file matching the record name. + CATCHALL_RECORD_NAMES = ('*', '') + + @classmethod + def list_all_yaml_files(_, directory): + yaml_files = set() + for f in listdir(directory): + filename = join(directory, '{}'.format(f)) + if f.endswith('.yaml') and isfile(filename): + yaml_files.add(filename) + return list(yaml_files) + def __init__(self, id, directory, *args, **kwargs): super(SplitYamlProvider, self).__init__(id, directory, *args, **kwargs) self.log = logging.getLogger('SplitYamlProvider[{}]'.format(id)) @@ -170,7 +169,7 @@ class SplitYamlProvider(YamlProvider): return False before = len(zone.records) - yaml_filenames = list_all_yaml_files(self._zone_directory(zone)) + yaml_filenames = self.list_all_yaml_files(self._zone_directory(zone)) self.log.info('populate: found %s YAML files', len(yaml_filenames)) for yaml_filename in yaml_filenames: self._populate_from_file(yaml_filename, zone, lenient) @@ -186,7 +185,7 @@ class SplitYamlProvider(YamlProvider): catchall = dict() for record, config in data.items(): - if record in _CATCHALL_RECORD_NAMES: + if record in self.CATCHALL_RECORD_NAMES: catchall[record] = config continue filename = join(zone_dir, '{}.yaml'.format(record)) diff --git a/tests/test_octodns_provider_yaml.py b/tests/test_octodns_provider_yaml.py index 74261de..5ff97ba 100644 --- a/tests/test_octodns_provider_yaml.py +++ b/tests/test_octodns_provider_yaml.py @@ -5,13 +5,15 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -from os.path import dirname, isfile, join +from os import makedirs +from os.path import basename, dirname, isdir, isfile, join from unittest import TestCase from yaml import safe_load from yaml.constructor import ConstructorError from octodns.record import Create -from octodns.provider.yaml import YamlProvider +from octodns.provider.base import Plan +from octodns.provider.yaml import SplitYamlProvider, YamlProvider from octodns.zone import SubzoneRecordException, Zone from helpers import TemporaryDirectory @@ -176,3 +178,196 @@ class TestYamlProvider(TestCase): source.populate(zone) self.assertEquals('Record www.sub.unit.tests. is under a managed ' 'subzone', ctx.exception.message) + + +class TestSplitYamlProvider(TestCase): + + def test_list_all_yaml_files(self): + yaml_files = ('foo.yaml', '1.yaml', '$unit.tests.yaml') + all_files = ('something', 'else', '1', '$$', '-f') + yaml_files + all_dirs = ('dir1', 'dir2/sub', 'tricky.yaml') + + with TemporaryDirectory() as td: + directory = join(td.dirname) + + # Create some files, some of them with a .yaml extension, all of + # them empty. + for emptyfile in all_files: + open(join(directory, emptyfile), 'w').close() + # Do the same for some fake directories + for emptydir in all_dirs: + makedirs(join(directory, emptydir)) + + # This isn't great, but given the variable nature of the temp dir + # names, it's necessary. + got = (basename(f) + for f in SplitYamlProvider.list_all_yaml_files(directory)) + self.assertItemsEqual(yaml_files, got) + + def test_zone_directory(self): + source = SplitYamlProvider( + 'test', join(dirname(__file__), 'config/split')) + + zone = Zone('unit.tests.', []) + + self.assertEqual( + join(dirname(__file__), 'config/split/unit.tests.'), + source._zone_directory(zone)) + + def test_apply_handles_existing_zone_directory(self): + with TemporaryDirectory() as td: + provider = SplitYamlProvider('test', join(td.dirname, 'config')) + makedirs(join(td.dirname, 'config', 'does.exist.')) + + zone = Zone('does.exist.', []) + self.assertTrue(isdir(provider._zone_directory(zone))) + provider.apply(Plan(None, zone, [], True)) + self.assertTrue(isdir(provider._zone_directory(zone))) + + def test_provider(self): + source = SplitYamlProvider( + 'test', join(dirname(__file__), 'config/split')) + + zone = Zone('unit.tests.', []) + dynamic_zone = Zone('dynamic.tests.', []) + + # With target we don't add anything + source.populate(zone, target=source) + self.assertEquals(0, len(zone.records)) + + # without it we see everything + source.populate(zone) + self.assertEquals(18, len(zone.records)) + + source.populate(dynamic_zone) + self.assertEquals(5, len(dynamic_zone.records)) + + with TemporaryDirectory() as td: + # Add some subdirs to make sure that it can create them + directory = join(td.dirname, 'sub', 'dir') + zone_dir = join(directory, 'unit.tests.') + dynamic_zone_dir = join(directory, 'dynamic.tests.') + target = SplitYamlProvider('test', directory) + + # We add everything + plan = target.plan(zone) + self.assertEquals(15, len(filter(lambda c: isinstance(c, Create), + plan.changes))) + self.assertFalse(isdir(zone_dir)) + + # Now actually do it + self.assertEquals(15, target.apply(plan)) + + # Dynamic plan + plan = target.plan(dynamic_zone) + self.assertEquals(5, len(filter(lambda c: isinstance(c, Create), + plan.changes))) + self.assertFalse(isdir(dynamic_zone_dir)) + # Apply it + self.assertEquals(5, target.apply(plan)) + self.assertTrue(isdir(dynamic_zone_dir)) + + # There should be no changes after the round trip + reloaded = Zone('unit.tests.', []) + target.populate(reloaded) + self.assertDictEqual( + {'included': ['test']}, + filter( + lambda x: x.name == 'included', reloaded.records + )[0]._octodns) + + self.assertFalse(zone.changes(reloaded, target=source)) + + # A 2nd sync should still create everything + plan = target.plan(zone) + self.assertEquals(15, len(filter(lambda c: isinstance(c, Create), + plan.changes))) + + yaml_file = join(zone_dir, '$unit.tests.yaml') + self.assertTrue(isfile(yaml_file)) + with open(yaml_file) as fh: + data = safe_load(fh.read()) + roots = sorted(data.pop(''), key=lambda r: r['type']) + self.assertTrue('values' in roots[0]) # A + self.assertTrue('geo' in roots[0]) # geo made the trip + self.assertTrue('value' in roots[1]) # CAA + self.assertTrue('values' in roots[2]) # SSHFP + + # These records are stored as plural "values." Check each file to + # ensure correctness. + for record_name in ('_srv._tcp', 'mx', 'naptr', 'sub', 'txt'): + yaml_file = join(zone_dir, '{}.yaml'.format(record_name)) + self.assertTrue(isfile(yaml_file)) + with open(yaml_file) as fh: + data = safe_load(fh.read()) + self.assertTrue('values' in data.pop(record_name)) + + # These are stored as singular "value." Again, check each file. + for record_name in ('aaaa', 'cname', 'included', 'ptr', 'spf', + 'www.sub', 'www'): + yaml_file = join(zone_dir, '{}.yaml'.format(record_name)) + self.assertTrue(isfile(yaml_file)) + with open(yaml_file) as fh: + data = safe_load(fh.read()) + self.assertTrue('value' in data.pop(record_name)) + + # Again with the plural, this time checking dynamic.tests. + for record_name in ('a', 'aaaa', 'real-ish-a'): + yaml_file = join( + dynamic_zone_dir, '{}.yaml'.format(record_name)) + self.assertTrue(isfile(yaml_file)) + with open(yaml_file) as fh: + data = safe_load(fh.read()) + dyna = data.pop(record_name) + self.assertTrue('values' in dyna) + self.assertTrue('dynamic' in dyna) + + # Singular again. + for record_name in ('cname', 'simple-weighted'): + yaml_file = join( + dynamic_zone_dir, '{}.yaml'.format(record_name)) + self.assertTrue(isfile(yaml_file)) + with open(yaml_file) as fh: + data = safe_load(fh.read()) + dyna = data.pop(record_name) + self.assertTrue('value' in dyna) + self.assertTrue('dynamic' in dyna) + + def test_empty(self): + source = SplitYamlProvider( + 'test', join(dirname(__file__), 'config/split')) + + zone = Zone('empty.', []) + + # without it we see everything + source.populate(zone) + self.assertEquals(0, len(zone.records)) + + def test_unsorted(self): + source = SplitYamlProvider( + 'test', join(dirname(__file__), 'config/split')) + + zone = Zone('unordered.', []) + + with self.assertRaises(ConstructorError): + source.populate(zone) + + zone = Zone('unordered.', []) + + source = SplitYamlProvider( + 'test', join(dirname(__file__), 'config/split'), + enforce_order=False) + # no exception + source.populate(zone) + self.assertEqual(2, len(zone.records)) + + def test_subzone_handling(self): + source = SplitYamlProvider( + 'test', join(dirname(__file__), 'config/split')) + + # If we add `sub` as a sub-zone we'll reject `www.sub` + zone = Zone('unit.tests.', ['sub']) + with self.assertRaises(SubzoneRecordException) as ctx: + source.populate(zone) + self.assertEquals('Record www.sub.unit.tests. is under a managed ' + 'subzone', ctx.exception.message)