From 80979194a5e1493ff15c6cee9d6dc8d3c06c963a Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Tue, 14 Oct 2025 17:23:38 -0700 Subject: [PATCH 1/5] Add array syntax support to !include tag for merging multiple files --- .../05f9b507087a4140932dd6a554c9d61d.md | 4 ++ octodns/yaml.py | 60 +++++++++++++++++-- tests/config/include/dict_too.yaml | 3 + .../include/include-array-with-dict.yaml | 4 ++ .../include-array-with-non-existant.yaml | 4 ++ .../include-array-with-unsupported.yaml | 3 + .../include/include-dict-with-array.yaml | 4 ++ tests/config/include/main.yaml | 7 +++ tests/test_octodns_yaml.py | 41 ++++++++++++- 9 files changed, 123 insertions(+), 7 deletions(-) create mode 100644 .changelog/05f9b507087a4140932dd6a554c9d61d.md create mode 100644 tests/config/include/dict_too.yaml create mode 100644 tests/config/include/include-array-with-dict.yaml create mode 100644 tests/config/include/include-array-with-non-existant.yaml create mode 100644 tests/config/include/include-array-with-unsupported.yaml create mode 100644 tests/config/include/include-dict-with-array.yaml diff --git a/.changelog/05f9b507087a4140932dd6a554c9d61d.md b/.changelog/05f9b507087a4140932dd6a554c9d61d.md new file mode 100644 index 0000000..dbbf71d --- /dev/null +++ b/.changelog/05f9b507087a4140932dd6a554c9d61d.md @@ -0,0 +1,4 @@ +--- +type: minor +--- +Add array syntax support to !include tag for merging multiple files \ No newline at end of file diff --git a/octodns/yaml.py b/octodns/yaml.py index 71e7d05..77c1053 100644 --- a/octodns/yaml.py +++ b/octodns/yaml.py @@ -17,11 +17,15 @@ _natsort_key = staticmethod(natsort_keygen()) class ContextLoader(SafeLoader): + + def _context(self, node): + start_mark = node.start_mark + return f'{start_mark.name}, line {start_mark.line+1}, column {start_mark.column+1}' + def _pairs(self, node): self.flatten_mapping(node) pairs = self.construct_pairs(node) - start_mark = node.start_mark - context = f'{start_mark.name}, line {start_mark.line+1}, column {start_mark.column+1}' + context = self._context(node) return ContextDict(pairs, context=context), pairs, context def _construct(self, node): @@ -31,10 +35,54 @@ class ContextLoader(SafeLoader): mark = self.get_mark() directory = dirname(mark.name) - filename = join(directory, self.construct_scalar(node)) - - with open(filename, 'r') as fh: - return safe_load(fh, self.__class__) + def load_file(filename): + filename = join(directory, filename) + with open(filename, 'r') as fh: + return safe_load(fh, self.__class__) + + if not isinstance(node.value, list): + # single filename, just load and return whatever is in it + scalar = node.value + return load_file(scalar) + + scalars = node.value + data = [load_file(s.value) for s in scalars] + + if not data: + return None + elif isinstance(data[0], list): + # we're working with lists + ret = data[0] + for i, d in enumerate(data[1:]): + if not isinstance(d, list): + context = self._context(node) + raise ConstructorError( + None, + None, + f'!include first element contained a list, element {i+1} contained a {d.__class__.__name__} at {context}', + ) + ret.extend(d) + return ret + elif isinstance(data[0], dict): + # assume we're working with dict + ret = data[0] + for i, d in enumerate(data[1:]): + if not isinstance(d, dict): + context = self._context(node) + raise ConstructorError( + None, + None, + f'!include first element contained a dict, element {i+1} contained a {d.__class__.__name__} at {context}', + ) + ret.update(d) + return ret + + context = self._context(node) + raise ConstructorError( + None, + None, + f'!include first element contained an unsupported type, {data[0].__class__.__name__} at {context}', + ) ContextLoader.add_constructor('!include', ContextLoader.include) diff --git a/tests/config/include/dict_too.yaml b/tests/config/include/dict_too.yaml new file mode 100644 index 0000000..ea4423a --- /dev/null +++ b/tests/config/include/dict_too.yaml @@ -0,0 +1,3 @@ +--- +foo: bar +z: 43 diff --git a/tests/config/include/include-array-with-dict.yaml b/tests/config/include/include-array-with-dict.yaml new file mode 100644 index 0000000..5cfb5c3 --- /dev/null +++ b/tests/config/include/include-array-with-dict.yaml @@ -0,0 +1,4 @@ +--- +data: !include + - array.yaml + - dict.yaml diff --git a/tests/config/include/include-array-with-non-existant.yaml b/tests/config/include/include-array-with-non-existant.yaml new file mode 100644 index 0000000..d633100 --- /dev/null +++ b/tests/config/include/include-array-with-non-existant.yaml @@ -0,0 +1,4 @@ +--- +data: !include + - array.yaml + - does-not-exist.yaml diff --git a/tests/config/include/include-array-with-unsupported.yaml b/tests/config/include/include-array-with-unsupported.yaml new file mode 100644 index 0000000..3a3ac2f --- /dev/null +++ b/tests/config/include/include-array-with-unsupported.yaml @@ -0,0 +1,3 @@ +--- +data: !include + - subdir/value.yaml diff --git a/tests/config/include/include-dict-with-array.yaml b/tests/config/include/include-dict-with-array.yaml new file mode 100644 index 0000000..80604cd --- /dev/null +++ b/tests/config/include/include-dict-with-array.yaml @@ -0,0 +1,4 @@ +--- +data: !include + - dict.yaml + - array.yaml diff --git a/tests/config/include/main.yaml b/tests/config/include/main.yaml index 11f6c6c..897b4d6 100644 --- a/tests/config/include/main.yaml +++ b/tests/config/include/main.yaml @@ -4,5 +4,12 @@ included-dict: !include dict.yaml included-empty: !include empty.yaml included-nested: !include nested.yaml included-subdir: !include subdir/value.yaml +included-array-of-arrays: !include + - array.yaml + - array.yaml +included-array-of-dicts: !include + - dict.yaml + - dict_too.yaml +included-empty-array: !include [] key: value name: main diff --git a/tests/test_octodns_yaml.py b/tests/test_octodns_yaml.py index a05d28d..c91ff72 100644 --- a/tests/test_octodns_yaml.py +++ b/tests/test_octodns_yaml.py @@ -65,7 +65,7 @@ class TestYaml(TestCase): def test_include(self): with open('tests/config/include/main.yaml') as fh: - data = safe_load(fh) + data = safe_load(fh, enforce_order=False) self.assertEqual( { 'included-array': [14, 15, 16, 72], @@ -73,6 +73,9 @@ class TestYaml(TestCase): 'included-empty': None, 'included-nested': 'Hello World!', 'included-subdir': 'Hello World!', + 'included-array-of-arrays': [14, 15, 16, 72, 14, 15, 16, 72], + 'included-array-of-dicts': {'foo': 'bar', 'k': 'v', 'z': 43}, + 'included-empty-array': None, 'key': 'value', 'name': 'main', }, @@ -87,6 +90,42 @@ class TestYaml(TestCase): str(ctx.exception), ) + with open( + 'tests/config/include/include-array-with-non-existant.yaml' + ) as fh: + with self.assertRaises(FileNotFoundError) as ctx: + data = safe_load(fh) + self.assertEqual( + "[Errno 2] No such file or directory: 'tests/config/include/does-not-exist.yaml'", + str(ctx.exception), + ) + + with open('tests/config/include/include-array-with-dict.yaml') as fh: + with self.assertRaises(ConstructorError) as ctx: + data = safe_load(fh) + self.assertEqual( + "!include first element contained a list, element 1 contained a ContextDict at tests/config/include/include-array-with-dict.yaml, line 2, column 7", + str(ctx.exception), + ) + + with open('tests/config/include/include-dict-with-array.yaml') as fh: + with self.assertRaises(ConstructorError) as ctx: + data = safe_load(fh) + self.assertEqual( + "!include first element contained a dict, element 1 contained a list at tests/config/include/include-dict-with-array.yaml, line 2, column 7", + str(ctx.exception), + ) + + with open( + 'tests/config/include/include-array-with-unsupported.yaml' + ) as fh: + with self.assertRaises(ConstructorError) as ctx: + data = safe_load(fh) + self.assertEqual( + "!include first element contained an unsupported type, str at tests/config/include/include-array-with-unsupported.yaml, line 2, column 7", + str(ctx.exception), + ) + def test_order_mode(self): data = {'*.1.2': 'a', '*.10.1': 'c', '*.11.2': 'd', '*.2.2': 'b'} natural = '''--- From 8f58e5c474322153326f9e990b16fc28678a2751 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Tue, 14 Oct 2025 19:03:53 -0700 Subject: [PATCH 2/5] pass at documenting the !include directive --- docs/include_directive.rst | 290 +++++++++++++++++++++++++++++++++++++ docs/index.rst | 1 + 2 files changed, 291 insertions(+) create mode 100644 docs/include_directive.rst diff --git a/docs/include_directive.rst b/docs/include_directive.rst new file mode 100644 index 0000000..79465b9 --- /dev/null +++ b/docs/include_directive.rst @@ -0,0 +1,290 @@ +YAML !include Directive +======================= + +The ``!include`` directive is a powerful feature in octoDNS that allows you to +reuse YAML content across multiple configuration files and zone files. This +helps reduce duplication and makes your DNS configuration more maintainable. + +Overview +-------- + +The ``!include`` directive can be used anywhere in your YAML files to include +content from other files. Files are resolved relative to the directory +containing the file with the ``!include`` directive. + +Basic Usage +----------- + +Single File Include +................... + +The simplest form includes the entire contents of a single file:: + + --- + # main.yaml + common-config: !include common.yaml + +If ``common.yaml`` contains:: + + --- + key: value + setting: 42 + +Then ``common-config`` will be set to ``{'key': 'value', 'setting': 42}``. + +The included file can contain any valid YAML type: dictionaries, lists, strings, +numbers, or even ``null`` values. + +Array Syntax +............ + +The ``!include`` directive also supports an array syntax to merge multiple files +together. This is useful for composing configurations from multiple sources. + +Merging Dictionaries +~~~~~~~~~~~~~~~~~~~~~ + +When including multiple files that contain dictionaries, the dictionaries are +merged together. Later files override keys from earlier files:: + + --- + # main.yaml + merged-config: !include + - base-config.yaml + - overrides.yaml + +If ``base-config.yaml`` contains:: + + --- + timeout: 30 + retries: 3 + debug: false + +And ``overrides.yaml`` contains:: + + --- + timeout: 60 + debug: true + added: hi + +Then ``merged-config`` will be:: + + { + 'timeout': 60, # overridden + 'retries': 3, # from base + 'debug': true, # overridden + 'added': 'hi' # added by overrides + } + +Merging Arrays +~~~~~~~~~~~~~~ + +When including multiple files that contain arrays, the arrays are concatenated +together:: + + --- + # main.yaml + all-values: !include + - values-1.yaml + - values-2.yaml + +If ``values-1.yaml`` contains:: + + --- + - item1 + - item2 + +And ``values-2.yaml`` contains:: + + --- + - item3 + - item4 + +Then ``all-values`` will be ``['item1', 'item2', 'item3', 'item4']``. + +Empty Arrays +~~~~~~~~~~~~ + +An empty array can be used with ``!include``, which results in ``null``:: + + --- + empty-value: !include [] + +This sets ``empty-value`` to ``null``. + +Use Cases +--------- + +Configuration Files +................... + +The ``!include`` directive is useful in octoDNS configuration files for sharing +common provider settings, processor configurations, or zone settings. + +Shared Provider Configuration:: + + --- + # production.yaml + providers: + base-config: !include providers/common.yaml + + route53: + class: octodns_route53.Route53Provider + access_key_id: env/AWS_ACCESS_KEY_ID + secret_access_key: env/AWS_SECRET_ACCESS_KEY + # Include common retry/timeout settings + settings: !include providers/aws-settings.yaml + +Shared Zone Configuration:: + + --- + # config.yaml + zones: + example.com.: &standard-setup !include zones/standard-setup.yaml + example.net.: *standard-setup + example.org.: *standard-setup + +Zone Files +.......... + +The ``!include`` directive is particularly powerful in zone files for reducing +duplication of common record configurations. + +Shared APEX Records +~~~~~~~~~~~~~~~~~~~ + +When you have multiple zones with shared APEX records but differing records +otherwise, you can share the APEX configuration:: + + --- + # example.com.yaml + '': !include common/apex.yaml + api: + type: A + value: 1.2.3.4 + web: + type: A + value: 1.2.3.5 + +Where ``common/apex.yaml`` might contain:: + + --- + - type: A + value: 1.2.3.4 + - type: MX + values: + - exchange: mail1.example.com. + preference: 10 + - exchange: mail2.example.com. + preference: 20 + - type: NS + values: + - 6.2.3.4. + - 7.2.3.4. + - type: TXT + values: + - some-domain-claiming-value=gimme + - v=spf1 -all + +Common Record Values +~~~~~~~~~~~~~~~~~~~~ + +You can merge multiple files to build up complex record sets:: + + --- + # zone.yaml + '': + type: TXT + values: !include + - txt-records/verification.yaml + - txt-records/spf.yaml + - txt-records/dmarc.yaml + +This combines TXT records from multiple files into a single record set. + +Subdirectories +.............. + +Files in subdirectories can be included using relative paths:: + + --- + # main.yaml + nested-config: !include subdir/nested.yaml + deeper: !include subdir/another/deep.yaml + parent: !include ../sibling/config.yaml + +Type Requirements +----------------- + +When using the array syntax to include multiple files, all files must contain +compatible types: + +* All files must contain **dictionaries**, or +* All files must contain **arrays** + +If the first file contains a dictionary and a subsequent file contains an array +(or vice versa), octoDNS will raise a ``ConstructorError`` with a clear message +indicating which file and position caused the type mismatch. + +Simple scalar values (strings, numbers, booleans) are not supported with the +array syntax. Use single file includes for scalar values. + +Examples +-------- + +Example 1: Shared Provider Settings +.................................... + +Create reusable provider configurations:: + + # providers/retry-settings.yaml + --- + max_retries: 5 + retry_delay: 2 + timeout: 30 + + # production.yaml + --- + providers: + dacloud: + class: octodns_route53.DaCloudProvider + access_key_id: env/DC_ACCESS_KEY_ID + secret_access_key: env/DC_SECRET_ACCESS_KEY + network: !include providers/retry-settings.yaml + +Example 2: Composing TXT Records +................................. + +Build TXT records from multiple sources:: + + # txt-records/spf.yaml + --- + - "v=spf1 include:_spf.google.com ~all" + + # txt-records/verification.yaml + --- + - "google-site-verification=abc123" + - "ms-domain-verification=xyz789" + + # example.com.yaml + --- + '': + type: TXT + values: !include + - txt-records/spf.yaml + - txt-records/verification.yaml + +Best Practices +-------------- + +1. **Organize shared files**: Create a dedicated directory structure for shared + configurations (e.g., ``shared/``, ``common/``) + +2. **Use descriptive filenames**: Name included files clearly to indicate their + purpose (e.g., ``spf-record.yaml``, ``geo-routing-rules.yaml``) + +3. **Keep includes shallow**: Avoid deeply nested includes as they can make + the configuration harder to understand and debug + +4. **Document shared files**: Add comments in shared files explaining their + purpose and where they're used \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst index 983a5ca..7d9f173 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -28,6 +28,7 @@ Documentation getting-started.rst records.md configuration.rst + include_directive.rst dynamic_zone_config.rst dynamic_records.rst auto_arpa.rst From 6654d13ba20e052da334be62c27e38555b79f739 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Wed, 15 Oct 2025 09:23:13 -0700 Subject: [PATCH 3/5] Add a test to ensure !include files use the same loader (settings) --- octodns/yaml.py | 2 +- tests/config/include/dict_too.yaml | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/octodns/yaml.py b/octodns/yaml.py index 77c1053..e40c1c1 100644 --- a/octodns/yaml.py +++ b/octodns/yaml.py @@ -38,7 +38,7 @@ class ContextLoader(SafeLoader): def load_file(filename): filename = join(directory, filename) with open(filename, 'r') as fh: - return safe_load(fh, self.__class__) + return load(fh, self.__class__) if not isinstance(node.value, list): # single filename, just load and return whatever is in it diff --git a/tests/config/include/dict_too.yaml b/tests/config/include/dict_too.yaml index ea4423a..918bdb0 100644 --- a/tests/config/include/dict_too.yaml +++ b/tests/config/include/dict_too.yaml @@ -1,3 +1,5 @@ --- -foo: bar z: 43 +# keys are intentionally out of order here to ensure !include files are loaded +# with the same laoder class and thus "settings" as their parent +foo: bar From 45e372acb12d0641e6b66e6f2fb4f3d712765e75 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 20 Oct 2025 13:10:27 -0700 Subject: [PATCH 4/5] Rework to adding support for merge include, <<: !include --- .../05f9b507087a4140932dd6a554c9d61d.md | 3 +- docs/include_directive.rst | 152 ++++------------ octodns/yaml.py | 162 +++++++++++------- tests/config/include/dict.yaml | 1 + tests/config/include/main.yaml | 7 - tests/config/include/merge.yaml | 6 + tests/test_octodns_yaml.py | 60 +++---- 7 files changed, 161 insertions(+), 230 deletions(-) create mode 100644 tests/config/include/merge.yaml diff --git a/.changelog/05f9b507087a4140932dd6a554c9d61d.md b/.changelog/05f9b507087a4140932dd6a554c9d61d.md index dbbf71d..21c6446 100644 --- a/.changelog/05f9b507087a4140932dd6a554c9d61d.md +++ b/.changelog/05f9b507087a4140932dd6a554c9d61d.md @@ -1,4 +1,5 @@ --- type: minor --- -Add array syntax support to !include tag for merging multiple files \ No newline at end of file + +Add merge syntax support to !include tag, `<<: !include file.yaml` diff --git a/docs/include_directive.rst b/docs/include_directive.rst index 79465b9..7e37e75 100644 --- a/docs/include_directive.rst +++ b/docs/include_directive.rst @@ -35,23 +35,24 @@ Then ``common-config`` will be set to ``{'key': 'value', 'setting': 42}``. The included file can contain any valid YAML type: dictionaries, lists, strings, numbers, or even ``null`` values. -Array Syntax +Merge Syntax ............ -The ``!include`` directive also supports an array syntax to merge multiple files -together. This is useful for composing configurations from multiple sources. +The ``!include`` directive can also be used with the merge operator, ``<<:``. +This is useful for composing configurations from multiple sources. Merging Dictionaries -~~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~ When including multiple files that contain dictionaries, the dictionaries are merged together. Later files override keys from earlier files:: --- # main.yaml - merged-config: !include - - base-config.yaml - - overrides.yaml + merged-config: + <<: !include base-config.yaml + <<: !include overrides.yaml + key: value If ``base-config.yaml`` contains:: @@ -73,45 +74,10 @@ Then ``merged-config`` will be:: 'timeout': 60, # overridden 'retries': 3, # from base 'debug': true, # overridden - 'added': 'hi' # added by overrides + 'added': 'hi', # added by overrides + 'key': 'value', # from main } -Merging Arrays -~~~~~~~~~~~~~~ - -When including multiple files that contain arrays, the arrays are concatenated -together:: - - --- - # main.yaml - all-values: !include - - values-1.yaml - - values-2.yaml - -If ``values-1.yaml`` contains:: - - --- - - item1 - - item2 - -And ``values-2.yaml`` contains:: - - --- - - item3 - - item4 - -Then ``all-values`` will be ``['item1', 'item2', 'item3', 'item4']``. - -Empty Arrays -~~~~~~~~~~~~ - -An empty array can be used with ``!include``, which results in ``null``:: - - --- - empty-value: !include [] - -This sets ``empty-value`` to ``null``. - Use Cases --------- @@ -126,7 +92,8 @@ Shared Provider Configuration:: --- # production.yaml providers: - base-config: !include providers/common.yaml + # contents will be merged with what's defined here + <<: !include providers/common.yaml route53: class: octodns_route53.Route53Provider @@ -135,15 +102,34 @@ Shared Provider Configuration:: # Include common retry/timeout settings settings: !include providers/aws-settings.yaml + --- + # providers/common.yaml + config: + class: octodns.providers.yaml.YamlProvider + directory: ./config/ + + internal: + class: octodns_powerdns.PdnsProvider + ... + Shared Zone Configuration:: --- # config.yaml zones: + # contents will become the value for example.com. example.com.: &standard-setup !include zones/standard-setup.yaml example.net.: *standard-setup example.org.: *standard-setup + --- + # zones/standard-ssetup.yaml + sources: + - config + targets: + - internal + - route53 + Zone Files .......... @@ -153,7 +139,7 @@ duplication of common record configurations. Shared APEX Records ~~~~~~~~~~~~~~~~~~~ -When you have multiple zones with shared APEX records but differing records +When you have multiple zones with shared APEX records but differing records otherwise, you can share the APEX configuration:: --- @@ -166,9 +152,9 @@ otherwise, you can share the APEX configuration:: type: A value: 1.2.3.5 -Where ``common/apex.yaml`` might contain:: --- + # common/apex.yaml - type: A value: 1.2.3.4 - type: MX @@ -186,22 +172,6 @@ Where ``common/apex.yaml`` might contain:: - some-domain-claiming-value=gimme - v=spf1 -all -Common Record Values -~~~~~~~~~~~~~~~~~~~~ - -You can merge multiple files to build up complex record sets:: - - --- - # zone.yaml - '': - type: TXT - values: !include - - txt-records/verification.yaml - - txt-records/spf.yaml - - txt-records/dmarc.yaml - -This combines TXT records from multiple files into a single record set. - Subdirectories .............. @@ -216,64 +186,12 @@ Files in subdirectories can be included using relative paths:: Type Requirements ----------------- -When using the array syntax to include multiple files, all files must contain -compatible types: - -* All files must contain **dictionaries**, or -* All files must contain **arrays** - -If the first file contains a dictionary and a subsequent file contains an array -(or vice versa), octoDNS will raise a ``ConstructorError`` with a clear message -indicating which file and position caused the type mismatch. - -Simple scalar values (strings, numbers, booleans) are not supported with the -array syntax. Use single file includes for scalar values. +Any valid YAML datatype can be used in the basic **!include** stile. -Examples --------- - -Example 1: Shared Provider Settings -.................................... - -Create reusable provider configurations:: - - # providers/retry-settings.yaml - --- - max_retries: 5 - retry_delay: 2 - timeout: 30 +When using the merge syntax all files must contain **dictionaries**. - # production.yaml - --- - providers: - dacloud: - class: octodns_route53.DaCloudProvider - access_key_id: env/DC_ACCESS_KEY_ID - secret_access_key: env/DC_SECRET_ACCESS_KEY network: !include providers/retry-settings.yaml -Example 2: Composing TXT Records -................................. - -Build TXT records from multiple sources:: - - # txt-records/spf.yaml - --- - - "v=spf1 include:_spf.google.com ~all" - - # txt-records/verification.yaml - --- - - "google-site-verification=abc123" - - "ms-domain-verification=xyz789" - - # example.com.yaml - --- - '': - type: TXT - values: !include - - txt-records/spf.yaml - - txt-records/verification.yaml - Best Practices -------------- diff --git a/octodns/yaml.py b/octodns/yaml.py index e40c1c1..7b7ed3e 100644 --- a/octodns/yaml.py +++ b/octodns/yaml.py @@ -5,7 +5,7 @@ from os.path import dirname, join from natsort import natsort_keygen -from yaml import SafeDumper, SafeLoader, dump, load +from yaml import SafeDumper, SafeLoader, compose, dump, load from yaml.constructor import ConstructorError from yaml.representer import SafeRepresenter @@ -18,91 +18,123 @@ _natsort_key = staticmethod(natsort_keygen()) class ContextLoader(SafeLoader): - def _context(self, node): - start_mark = node.start_mark - return f'{start_mark.name}, line {start_mark.line+1}, column {start_mark.column+1}' + def construct_include(self, node): + mark = self.get_mark() + directory = dirname(mark.name) - def _pairs(self, node): - self.flatten_mapping(node) - pairs = self.construct_pairs(node) - context = self._context(node) - return ContextDict(pairs, context=context), pairs, context + filename = join(directory, self.construct_scalar(node)) - def _construct(self, node): - return self._pairs(node)[0] + with open(filename, 'r') as fh: + return load(fh, self.__class__) - def include(self, node): + def flatten_include(self, node): mark = self.get_mark() directory = dirname(mark.name) - def load_file(filename): - filename = join(directory, filename) - with open(filename, 'r') as fh: - return load(fh, self.__class__) - - if not isinstance(node.value, list): - # single filename, just load and return whatever is in it - scalar = node.value - return load_file(scalar) - - scalars = node.value - data = [load_file(s.value) for s in scalars] - - if not data: - return None - elif isinstance(data[0], list): - # we're working with lists - ret = data[0] - for i, d in enumerate(data[1:]): - if not isinstance(d, list): - context = self._context(node) - raise ConstructorError( - None, - None, - f'!include first element contained a list, element {i+1} contained a {d.__class__.__name__} at {context}', - ) - ret.extend(d) - return ret - elif isinstance(data[0], dict): - # assume we're working with dict - ret = data[0] - for i, d in enumerate(data[1:]): - if not isinstance(d, dict): - context = self._context(node) + filename = join(directory, self.construct_scalar(node)) + + with open(filename, 'r') as fh: + yield compose(fh, self.__class__).value + + def construct_mapping(self, node, deep=False): + ''' + Calls our parent and wraps the resulting dict with a ContextDict + ''' + start_mark = node.start_mark + context = f'{start_mark.name}, line {start_mark.line+1}, column {start_mark.column+1}' + return ContextDict( + super().construct_mapping(node, deep), context=context + ) + + # the following 4 methods are ported out of + # https://github.com/yaml/pyyaml/pull/894 an intended to be used until we + # can (hopefully) require a version of pyyaml with that PR merged. + + @classmethod + def add_flattener(cls, tag, flattener): + if not 'yaml_flatteners' in cls.__dict__: + cls.yaml_flatteners = {} + cls.yaml_flatteners[tag] = flattener + + # this overwrites/ignores the built-in version of the method + def flatten_mapping(self, node): # pragma: no cover + merge = [] + for key_node, value_node in node.value: + if key_node.tag == 'tag:yaml.org,2002:merge': + flattener = self.yaml_flatteners.get(value_node.tag) + if flattener: + for value in flattener(self, value_node): + merge.extend(value) + else: raise ConstructorError( - None, - None, - f'!include first element contained a dict, element {i+1} contained a {d.__class__.__name__} at {context}', + "while constructing a mapping", + node.start_mark, + "expected a mapping or list of mappings for merging, but found %s" + % value_node.id, + value_node.start_mark, ) - ret.update(d) - return ret - - context = self._context(node) - raise ConstructorError( - None, - None, - f'!include first element contained an unsupported type, {data[0].__class__.__name__} at {context}', - ) + elif key_node.tag == 'tag:yaml.org,2002:value': + key_node.tag = 'tag:yaml.org,2002:str' + merge.append((key_node, value_node)) + else: + merge.append((key_node, value_node)) + + node.value = merge + + def flatten_yaml_seq(self, node): # pragma: no cover + submerge = [] + for subnode in node.value: + # we need to flatten each item in the seq, most likely they'll be mappings, + # but we need to allow for custom flatteners as well. + flattener = self.yaml_flatteners.get(subnode.tag) + if flattener: + for value in flattener(self, subnode): + submerge.append(value) + else: + raise ConstructorError( + "while constructing a mapping", + node.start_mark, + "expected a mapping for merging, but found %s" % subnode.id, + subnode.start_mark, + ) + submerge.reverse() + for value in submerge: + yield value + def flatten_yaml_map(self, node): # pragma: no cover + self.flatten_mapping(node) + yield node.value + + +# These 2 add's are also ported out of the PR +ContextLoader.add_flattener( + 'tag:yaml.org,2002:seq', ContextLoader.flatten_yaml_seq +) +ContextLoader.add_flattener( + 'tag:yaml.org,2002:map', ContextLoader.flatten_yaml_map +) -ContextLoader.add_constructor('!include', ContextLoader.include) ContextLoader.add_constructor( - ContextLoader.DEFAULT_MAPPING_TAG, ContextLoader._construct + ContextLoader.DEFAULT_MAPPING_TAG, ContextLoader.construct_mapping ) +ContextLoader.add_constructor('!include', ContextLoader.construct_include) +ContextLoader.add_flattener('!include', ContextLoader.flatten_include) # Found http://stackoverflow.com/a/21912744 which guided me on how to hook in # here class SortEnforcingLoader(ContextLoader): - def _construct(self, node): - ret, pairs, context = self._pairs(node) + def construct_mapping(self, node, deep=False): + ret = super().construct_mapping(node, deep) - keys = [d[0] for d in pairs] + keys = list(ret.keys()) keys_sorted = sorted(keys, key=self.KEYGEN) for key in keys: expected = keys_sorted.pop(0) if key != expected: + start_mark = node.start_mark + context = f'{start_mark.name}, line {start_mark.line+1}, column {start_mark.column+1}' raise ConstructorError( None, None, @@ -119,7 +151,7 @@ class NaturalSortEnforcingLoader(SortEnforcingLoader): NaturalSortEnforcingLoader.add_constructor( SortEnforcingLoader.DEFAULT_MAPPING_TAG, - NaturalSortEnforcingLoader._construct, + NaturalSortEnforcingLoader.construct_mapping, ) @@ -129,7 +161,7 @@ class SimpleSortEnforcingLoader(SortEnforcingLoader): SimpleSortEnforcingLoader.add_constructor( SortEnforcingLoader.DEFAULT_MAPPING_TAG, - SimpleSortEnforcingLoader._construct, + SimpleSortEnforcingLoader.construct_mapping, ) diff --git a/tests/config/include/dict.yaml b/tests/config/include/dict.yaml index da2e22f..2390cce 100644 --- a/tests/config/include/dict.yaml +++ b/tests/config/include/dict.yaml @@ -1,3 +1,4 @@ --- k: v +m: o z: 42 diff --git a/tests/config/include/main.yaml b/tests/config/include/main.yaml index 897b4d6..11f6c6c 100644 --- a/tests/config/include/main.yaml +++ b/tests/config/include/main.yaml @@ -4,12 +4,5 @@ included-dict: !include dict.yaml included-empty: !include empty.yaml included-nested: !include nested.yaml included-subdir: !include subdir/value.yaml -included-array-of-arrays: !include - - array.yaml - - array.yaml -included-array-of-dicts: !include - - dict.yaml - - dict_too.yaml -included-empty-array: !include [] key: value name: main diff --git a/tests/config/include/merge.yaml b/tests/config/include/merge.yaml new file mode 100644 index 0000000..393ca33 --- /dev/null +++ b/tests/config/include/merge.yaml @@ -0,0 +1,6 @@ +--- +parent: + k: overwritten + <<: !include dict.yaml + child: added + z: overwrote diff --git a/tests/test_octodns_yaml.py b/tests/test_octodns_yaml.py index c91ff72..7a8b51c 100644 --- a/tests/test_octodns_yaml.py +++ b/tests/test_octodns_yaml.py @@ -65,17 +65,14 @@ class TestYaml(TestCase): def test_include(self): with open('tests/config/include/main.yaml') as fh: - data = safe_load(fh, enforce_order=False) + data = safe_load(fh) self.assertEqual( { 'included-array': [14, 15, 16, 72], - 'included-dict': {'k': 'v', 'z': 42}, + 'included-dict': {'k': 'v', 'm': 'o', 'z': 42}, 'included-empty': None, 'included-nested': 'Hello World!', 'included-subdir': 'Hello World!', - 'included-array-of-arrays': [14, 15, 16, 72, 14, 15, 16, 72], - 'included-array-of-dicts': {'foo': 'bar', 'k': 'v', 'z': 43}, - 'included-empty-array': None, 'key': 'value', 'name': 'main', }, @@ -90,41 +87,24 @@ class TestYaml(TestCase): str(ctx.exception), ) - with open( - 'tests/config/include/include-array-with-non-existant.yaml' - ) as fh: - with self.assertRaises(FileNotFoundError) as ctx: - data = safe_load(fh) - self.assertEqual( - "[Errno 2] No such file or directory: 'tests/config/include/does-not-exist.yaml'", - str(ctx.exception), - ) - - with open('tests/config/include/include-array-with-dict.yaml') as fh: - with self.assertRaises(ConstructorError) as ctx: - data = safe_load(fh) - self.assertEqual( - "!include first element contained a list, element 1 contained a ContextDict at tests/config/include/include-array-with-dict.yaml, line 2, column 7", - str(ctx.exception), - ) - - with open('tests/config/include/include-dict-with-array.yaml') as fh: - with self.assertRaises(ConstructorError) as ctx: - data = safe_load(fh) - self.assertEqual( - "!include first element contained a dict, element 1 contained a list at tests/config/include/include-dict-with-array.yaml, line 2, column 7", - str(ctx.exception), - ) - - with open( - 'tests/config/include/include-array-with-unsupported.yaml' - ) as fh: - with self.assertRaises(ConstructorError) as ctx: - data = safe_load(fh) - self.assertEqual( - "!include first element contained an unsupported type, str at tests/config/include/include-array-with-unsupported.yaml, line 2, column 7", - str(ctx.exception), - ) + def test_include_merge(self): + with open('tests/config/include/merge.yaml') as fh: + data = safe_load(fh, enforce_order=False) + self.assertEqual( + { + 'parent': { + # overwritten by include + 'k': 'v', + # added by include + 'm': 'o', + # explicitly in parent + 'child': 'added', + # overrode by inlucd + 'z': 'overwrote', + } + }, + data, + ) def test_order_mode(self): data = {'*.1.2': 'a', '*.10.1': 'c', '*.11.2': 'd', '*.2.2': 'b'} From 0d52dc54dfe6c4aaf969b37713239d8419fc041c Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 10 Nov 2025 10:03:03 -0800 Subject: [PATCH 5/5] Bump pragma no cover for yaml ports --- script/coverage | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/coverage b/script/coverage index 3b5307c..674373e 100755 --- a/script/coverage +++ b/script/coverage @@ -10,8 +10,8 @@ SOURCE_DIR="octodns/" # Don't allow disabling coverage PRAGMA_OUTPUT=$(grep -r -I --line-number "# pragma: \+no.*cover" "$SOURCE_DIR" || echo) PRAGMA_COUNT=$(echo "$PRAGMA_OUTPUT" | grep -c . || true) -PRAGMA_ALLOWED=2 -if [ "$PRAGMA_COUNT" -gt "$PRAGMA_ALLOWED" ]; then +PRAGMA_ALLOWED=5 +if [ "$PRAGMA_COUNT" -ne "$PRAGMA_ALLOWED" ]; then echo "Found $PRAGMA_COUNT instances of 'pragma: no cover' (no more than $PRAGMA_ALLOWED allowed):" echo "$PRAGMA_OUTPUT" echo "Code coverage should not be disabled, except for version handling blocks"