Add array syntax support to !include tag for merging multiple files

.changelog/05f9b507087a4140932dd6a554c9d61d.md

@@ -0,0 +1,4 @@
+---
+type: minor
+---
+Add array syntax support to !include tag for merging multiple files

octodns/yaml.py

@@ -17,11 +17,15 @@ _natsort_key = staticmethod(natsort_keygen())
 
 
 class ContextLoader(SafeLoader):
+
+    def _context(self, node):
+        start_mark = node.start_mark
+        return f'{start_mark.name}, line {start_mark.line+1}, column {start_mark.column+1}'
+
     def _pairs(self, node):
         self.flatten_mapping(node)
         pairs = self.construct_pairs(node)
-        start_mark = node.start_mark
-        context = f'{start_mark.name}, line {start_mark.line+1}, column {start_mark.column+1}'
+        context = self._context(node)
         return ContextDict(pairs, context=context), pairs, context
 
     def _construct(self, node):
@@ -31,10 +35,54 @@ class ContextLoader(SafeLoader):
         mark = self.get_mark()
         directory = dirname(mark.name)
 
-        filename = join(directory, self.construct_scalar(node))
-
-        with open(filename, 'r') as fh:
-            return safe_load(fh, self.__class__)
+        def load_file(filename):
+            filename = join(directory, filename)
+            with open(filename, 'r') as fh:
+                return safe_load(fh, self.__class__)
+
+        if not isinstance(node.value, list):
+            # single filename, just load and return whatever is in it
+            scalar = node.value
+            return load_file(scalar)
+
+        scalars = node.value
+        data = [load_file(s.value) for s in scalars]
+
+        if not data:
+            return None
+        elif isinstance(data[0], list):
+            # we're working with lists
+            ret = data[0]
+            for i, d in enumerate(data[1:]):
+                if not isinstance(d, list):
+                    context = self._context(node)
+                    raise ConstructorError(
+                        None,
+                        None,
+                        f'!include first element contained a list, element {i+1} contained a {d.__class__.__name__} at {context}',
+                    )
+                ret.extend(d)
+            return ret
+        elif isinstance(data[0], dict):
+            # assume we're working with dict
+            ret = data[0]
+            for i, d in enumerate(data[1:]):
+                if not isinstance(d, dict):
+                    context = self._context(node)
+                    raise ConstructorError(
+                        None,
+                        None,
+                        f'!include first element contained a dict, element {i+1} contained a {d.__class__.__name__} at {context}',
+                    )
+                ret.update(d)
+            return ret
+
+        context = self._context(node)
+        raise ConstructorError(
+            None,
+            None,
+            f'!include first element contained an unsupported type, {data[0].__class__.__name__} at {context}',
+        )
 
 
 ContextLoader.add_constructor('!include', ContextLoader.include)

tests/config/include/dict_too.yaml

@@ -0,0 +1,3 @@
+---
+foo: bar
+z: 43

tests/config/include/include-array-with-dict.yaml

@@ -0,0 +1,4 @@
+---
+data: !include
+  - array.yaml
+  - dict.yaml

tests/config/include/include-array-with-non-existant.yaml

@@ -0,0 +1,4 @@
+---
+data: !include
+  - array.yaml
+  - does-not-exist.yaml

tests/config/include/include-array-with-unsupported.yaml

@@ -0,0 +1,3 @@
+---
+data: !include
+  - subdir/value.yaml

tests/config/include/include-dict-with-array.yaml

@@ -0,0 +1,4 @@
+---
+data: !include
+  - dict.yaml
+  - array.yaml

tests/config/include/main.yaml

@@ -4,5 +4,12 @@ included-dict: !include dict.yaml
 included-empty: !include empty.yaml
 included-nested: !include nested.yaml
 included-subdir: !include subdir/value.yaml
+included-array-of-arrays: !include
+  - array.yaml
+  - array.yaml
+included-array-of-dicts: !include
+  - dict.yaml
+  - dict_too.yaml
+included-empty-array: !include []
 key: value
 name: main

tests/test_octodns_yaml.py

@@ -65,7 +65,7 @@ class TestYaml(TestCase):
 
     def test_include(self):
         with open('tests/config/include/main.yaml') as fh:
-            data = safe_load(fh)
+            data = safe_load(fh, enforce_order=False)
         self.assertEqual(
             {
                 'included-array': [14, 15, 16, 72],
@@ -73,6 +73,9 @@ class TestYaml(TestCase):
                 'included-empty': None,
                 'included-nested': 'Hello World!',
                 'included-subdir': 'Hello World!',
+                'included-array-of-arrays': [14, 15, 16, 72, 14, 15, 16, 72],
+                'included-array-of-dicts': {'foo': 'bar', 'k': 'v', 'z': 43},
+                'included-empty-array': None,
                 'key': 'value',
                 'name': 'main',
             },
@@ -87,6 +90,42 @@ class TestYaml(TestCase):
                 str(ctx.exception),
             )
 
+        with open(
+            'tests/config/include/include-array-with-non-existant.yaml'
+        ) as fh:
+            with self.assertRaises(FileNotFoundError) as ctx:
+                data = safe_load(fh)
+            self.assertEqual(
+                "[Errno 2] No such file or directory: 'tests/config/include/does-not-exist.yaml'",
+                str(ctx.exception),
+            )
+
+        with open('tests/config/include/include-array-with-dict.yaml') as fh:
+            with self.assertRaises(ConstructorError) as ctx:
+                data = safe_load(fh)
+            self.assertEqual(
+                "!include first element contained a list, element 1 contained a ContextDict at tests/config/include/include-array-with-dict.yaml, line 2, column 7",
+                str(ctx.exception),
+            )
+
+        with open('tests/config/include/include-dict-with-array.yaml') as fh:
+            with self.assertRaises(ConstructorError) as ctx:
+                data = safe_load(fh)
+            self.assertEqual(
+                "!include first element contained a dict, element 1 contained a list at tests/config/include/include-dict-with-array.yaml, line 2, column 7",
+                str(ctx.exception),
+            )
+
+        with open(
+            'tests/config/include/include-array-with-unsupported.yaml'
+        ) as fh:
+            with self.assertRaises(ConstructorError) as ctx:
+                data = safe_load(fh)
+            self.assertEqual(
+                "!include first element contained an unsupported type, str at tests/config/include/include-array-with-unsupported.yaml, line 2, column 7",
+                str(ctx.exception),
+            )
+
     def test_order_mode(self):
         data = {'*.1.2': 'a', '*.10.1': 'c', '*.11.2': 'd', '*.2.2': 'b'}
         natural = '''---