Merge SplitYamlProvider and YamlProvider tests

Signed-off-by: Christian Funkhouser <cfunkhouser@heroku.com>

octodns/provider/yaml.py

@@ -116,20 +116,6 @@ class YamlProvider(BaseProvider):
             safe_dump(dict(data), fh)
 
 
-# Any record name added to this set will be included in the catch-all file,
-# instead of a file matching the record name.
-_CATCHALL_RECORD_NAMES = ('*', '')
-
-
-def list_all_yaml_files(directory):
-    yaml_files = set()
-    for f in listdir(directory):
-        filename = join(directory, '{}'.format(f))
-        if f.endswith('.yaml') and isfile(filename):
-            yaml_files.add(filename)
-    return list(yaml_files)
-
-
 class SplitYamlProvider(YamlProvider):
     '''
     Core provider for records configured in multiple YAML files on disk.
@@ -153,6 +139,19 @@ class SplitYamlProvider(YamlProvider):
         enforce_order: True
     '''
 
+    # Any record name added to this set will be included in the catch-all file,
+    # instead of a file matching the record name.
+    CATCHALL_RECORD_NAMES = ('*', '')
+
+    @classmethod
+    def list_all_yaml_files(_, directory):
+        yaml_files = set()
+        for f in listdir(directory):
+            filename = join(directory, '{}'.format(f))
+            if f.endswith('.yaml') and isfile(filename):
+                yaml_files.add(filename)
+        return list(yaml_files)
+
     def __init__(self, id, directory, *args, **kwargs):
         super(SplitYamlProvider, self).__init__(id, directory, *args, **kwargs)
         self.log = logging.getLogger('SplitYamlProvider[{}]'.format(id))
@@ -170,7 +169,7 @@ class SplitYamlProvider(YamlProvider):
             return False
 
         before = len(zone.records)
-        yaml_filenames = list_all_yaml_files(self._zone_directory(zone))
+        yaml_filenames = self.list_all_yaml_files(self._zone_directory(zone))
         self.log.info('populate:   found %s YAML files', len(yaml_filenames))
         for yaml_filename in yaml_filenames:
             self._populate_from_file(yaml_filename, zone, lenient)
@@ -186,7 +185,7 @@ class SplitYamlProvider(YamlProvider):
 
         catchall = dict()
         for record, config in data.items():
-            if record in _CATCHALL_RECORD_NAMES:
+            if record in self.CATCHALL_RECORD_NAMES:
                 catchall[record] = config
                 continue
             filename = join(zone_dir, '{}.yaml'.format(record))

tests/test_octodns_provider_yaml.py

@@ -5,13 +5,15 @@
 from __future__ import absolute_import, division, print_function, \
     unicode_literals
 
-from os.path import dirname, isfile, join
+from os import makedirs
+from os.path import basename, dirname, isdir, isfile, join
 from unittest import TestCase
 from yaml import safe_load
 from yaml.constructor import ConstructorError
 
 from octodns.record import Create
-from octodns.provider.yaml import YamlProvider
+from octodns.provider.base import Plan
+from octodns.provider.yaml import SplitYamlProvider, YamlProvider
 from octodns.zone import SubzoneRecordException, Zone
 
 from helpers import TemporaryDirectory
@@ -176,3 +178,196 @@ class TestYamlProvider(TestCase):
             source.populate(zone)
         self.assertEquals('Record www.sub.unit.tests. is under a managed '
                           'subzone', ctx.exception.message)
+
+
+class TestSplitYamlProvider(TestCase):
+
+    def test_list_all_yaml_files(self):
+        yaml_files = ('foo.yaml', '1.yaml', '$unit.tests.yaml')
+        all_files = ('something', 'else', '1', '$$', '-f') + yaml_files
+        all_dirs = ('dir1', 'dir2/sub', 'tricky.yaml')
+
+        with TemporaryDirectory() as td:
+            directory = join(td.dirname)
+
+            # Create some files, some of them with a .yaml extension, all of
+            # them empty.
+            for emptyfile in all_files:
+                open(join(directory, emptyfile), 'w').close()
+            # Do the same for some fake directories
+            for emptydir in all_dirs:
+                makedirs(join(directory, emptydir))
+
+            # This isn't great, but given the variable nature of the temp dir
+            # names, it's necessary.
+            got = (basename(f)
+                   for f in SplitYamlProvider.list_all_yaml_files(directory))
+            self.assertItemsEqual(yaml_files, got)
+
+    def test_zone_directory(self):
+        source = SplitYamlProvider(
+            'test', join(dirname(__file__), 'config/split'))
+
+        zone = Zone('unit.tests.', [])
+
+        self.assertEqual(
+            join(dirname(__file__), 'config/split/unit.tests.'),
+            source._zone_directory(zone))
+
+    def test_apply_handles_existing_zone_directory(self):
+        with TemporaryDirectory() as td:
+            provider = SplitYamlProvider('test', join(td.dirname, 'config'))
+            makedirs(join(td.dirname, 'config', 'does.exist.'))
+
+            zone = Zone('does.exist.', [])
+            self.assertTrue(isdir(provider._zone_directory(zone)))
+            provider.apply(Plan(None, zone, [], True))
+            self.assertTrue(isdir(provider._zone_directory(zone)))
+
+    def test_provider(self):
+        source = SplitYamlProvider(
+            'test', join(dirname(__file__), 'config/split'))
+
+        zone = Zone('unit.tests.', [])
+        dynamic_zone = Zone('dynamic.tests.', [])
+
+        # With target we don't add anything
+        source.populate(zone, target=source)
+        self.assertEquals(0, len(zone.records))
+
+        # without it we see everything
+        source.populate(zone)
+        self.assertEquals(18, len(zone.records))
+
+        source.populate(dynamic_zone)
+        self.assertEquals(5, len(dynamic_zone.records))
+
+        with TemporaryDirectory() as td:
+            # Add some subdirs to make sure that it can create them
+            directory = join(td.dirname, 'sub', 'dir')
+            zone_dir = join(directory, 'unit.tests.')
+            dynamic_zone_dir = join(directory, 'dynamic.tests.')
+            target = SplitYamlProvider('test', directory)
+
+            # We add everything
+            plan = target.plan(zone)
+            self.assertEquals(15, len(filter(lambda c: isinstance(c, Create),
+                                             plan.changes)))
+            self.assertFalse(isdir(zone_dir))
+
+            # Now actually do it
+            self.assertEquals(15, target.apply(plan))
+
+            # Dynamic plan
+            plan = target.plan(dynamic_zone)
+            self.assertEquals(5, len(filter(lambda c: isinstance(c, Create),
+                                            plan.changes)))
+            self.assertFalse(isdir(dynamic_zone_dir))
+            # Apply it
+            self.assertEquals(5, target.apply(plan))
+            self.assertTrue(isdir(dynamic_zone_dir))
+
+            # There should be no changes after the round trip
+            reloaded = Zone('unit.tests.', [])
+            target.populate(reloaded)
+            self.assertDictEqual(
+                {'included': ['test']},
+                filter(
+                    lambda x: x.name == 'included', reloaded.records
+                )[0]._octodns)
+
+            self.assertFalse(zone.changes(reloaded, target=source))
+
+            # A 2nd sync should still create everything
+            plan = target.plan(zone)
+            self.assertEquals(15, len(filter(lambda c: isinstance(c, Create),
+                                             plan.changes)))
+
+            yaml_file = join(zone_dir, '$unit.tests.yaml')
+            self.assertTrue(isfile(yaml_file))
+            with open(yaml_file) as fh:
+                data = safe_load(fh.read())
+                roots = sorted(data.pop(''), key=lambda r: r['type'])
+                self.assertTrue('values' in roots[0])  # A
+                self.assertTrue('geo' in roots[0])  # geo made the trip
+                self.assertTrue('value' in roots[1])   # CAA
+                self.assertTrue('values' in roots[2])  # SSHFP
+
+            # These records are stored as plural "values." Check each file to
+            # ensure correctness.
+            for record_name in ('_srv._tcp', 'mx', 'naptr', 'sub', 'txt'):
+                yaml_file = join(zone_dir, '{}.yaml'.format(record_name))
+                self.assertTrue(isfile(yaml_file))
+                with open(yaml_file) as fh:
+                    data = safe_load(fh.read())
+                    self.assertTrue('values' in data.pop(record_name))
+
+            # These are stored as singular "value." Again, check each file.
+            for record_name in ('aaaa', 'cname', 'included', 'ptr', 'spf',
+                                'www.sub', 'www'):
+                yaml_file = join(zone_dir, '{}.yaml'.format(record_name))
+                self.assertTrue(isfile(yaml_file))
+                with open(yaml_file) as fh:
+                    data = safe_load(fh.read())
+                    self.assertTrue('value' in data.pop(record_name))
+
+            # Again with the plural, this time checking dynamic.tests.
+            for record_name in ('a', 'aaaa', 'real-ish-a'):
+                yaml_file = join(
+                    dynamic_zone_dir, '{}.yaml'.format(record_name))
+                self.assertTrue(isfile(yaml_file))
+                with open(yaml_file) as fh:
+                    data = safe_load(fh.read())
+                    dyna = data.pop(record_name)
+                    self.assertTrue('values' in dyna)
+                    self.assertTrue('dynamic' in dyna)
+
+            # Singular again.
+            for record_name in ('cname', 'simple-weighted'):
+                yaml_file = join(
+                    dynamic_zone_dir, '{}.yaml'.format(record_name))
+                self.assertTrue(isfile(yaml_file))
+                with open(yaml_file) as fh:
+                    data = safe_load(fh.read())
+                    dyna = data.pop(record_name)
+                    self.assertTrue('value' in dyna)
+                    self.assertTrue('dynamic' in dyna)
+
+    def test_empty(self):
+        source = SplitYamlProvider(
+            'test', join(dirname(__file__), 'config/split'))
+
+        zone = Zone('empty.', [])
+
+        # without it we see everything
+        source.populate(zone)
+        self.assertEquals(0, len(zone.records))
+
+    def test_unsorted(self):
+        source = SplitYamlProvider(
+            'test', join(dirname(__file__), 'config/split'))
+
+        zone = Zone('unordered.', [])
+
+        with self.assertRaises(ConstructorError):
+            source.populate(zone)
+
+        zone = Zone('unordered.', [])
+
+        source = SplitYamlProvider(
+            'test', join(dirname(__file__), 'config/split'),
+            enforce_order=False)
+        # no exception
+        source.populate(zone)
+        self.assertEqual(2, len(zone.records))
+
+    def test_subzone_handling(self):
+        source = SplitYamlProvider(
+            'test', join(dirname(__file__), 'config/split'))
+
+        # If we add `sub` as a sub-zone we'll reject `www.sub`
+        zone = Zone('unit.tests.', ['sub'])
+        with self.assertRaises(SubzoneRecordException) as ctx:
+            source.populate(zone)
+        self.assertEquals('Record www.sub.unit.tests. is under a managed '
+                          'subzone', ctx.exception.message)