From 135f826b7ee580b607bf1c258405d7c2b79fb820 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 27 Apr 2019 15:08:09 -0700 Subject: [PATCH 1/3] Add OverridingYamlProvider and tests --- octodns/provider/yaml.py | 60 ++++++++++++++++++++++-- tests/config/override/dynamic.tests.yaml | 13 +++++ tests/test_octodns_provider_yaml.py | 37 ++++++++++++++- 3 files changed, 105 insertions(+), 5 deletions(-) create mode 100644 tests/config/override/dynamic.tests.yaml diff --git a/octodns/provider/yaml.py b/octodns/provider/yaml.py index 966e96e..aa04528 100644 --- a/octodns/provider/yaml.py +++ b/octodns/provider/yaml.py @@ -47,7 +47,7 @@ class YamlProvider(BaseProvider): self.default_ttl = default_ttl self.enforce_order = enforce_order - def _populate_from_file(self, filename, zone, lenient): + def _populate_from_file(self, filename, zone, lenient, replace=False): with open(filename, 'r') as fh: yaml_data = safe_load(fh, enforce_order=self.enforce_order) if yaml_data: @@ -59,9 +59,10 @@ class YamlProvider(BaseProvider): d['ttl'] = self.default_ttl record = Record.new(zone, name, d, source=self, lenient=lenient) - zone.add_record(record, lenient=lenient) - self.log.debug( - '_populate_from_file: successfully loaded "%s"', filename) + zone.add_record(record, lenient=lenient, + replace=replace) + self.log.debug('_populate_from_file: successfully loaded "%s"', + filename) def populate(self, zone, target=False, lenient=False): self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, @@ -211,3 +212,54 @@ class SplitYamlProvider(YamlProvider): self.log.debug('_apply: writing catchall filename=%s', filename) with open(filename, 'w') as fh: safe_dump(catchall, fh) + + +class OverridingYamlProvider(YamlProvider): + ''' + Provider that builds on YamlProvider to allow overriding specific records. + + Works identically to YamlProvider with the additional behavior of loading + data from a second zonefile in override_directory if it exists. Records in + this second file will override (replace) those previously seen in the + primary. Records that do not exist in the primary will just be added. There + is currently no mechinism to remove records from the primary zone. + + config: + class: octodns.provider.yaml.OverridingYamlProvider + # The location of yaml config files (required) + directory: ./config + # The location of overriding yaml config files (required) + override_directory: ./config + # The ttl to use for records when not specified in the data + # (optional, default 3600) + default_ttl: 3600 + # Whether or not to enforce sorting order on the yaml config + # (optional, default True) + enforce_order: True + ''' + + def __init__(self, id, directory, override_directory, *args, **kwargs): + super(OverridingYamlProvider, self).__init__(id, directory, *args, + **kwargs) + self.override_directory = override_directory + + def populate(self, zone, target=False, lenient=False): + self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, + target, lenient) + + if target: + # When acting as a target we ignore any existing records so that we + # create a completely new copy + return False + + before = len(zone.records) + filename = join(self.directory, '{}yaml'.format(zone.name)) + self._populate_from_file(filename, zone, lenient) + + filename = join(self.override_directory, '{}yaml'.format(zone.name)) + if isfile(filename): + self._populate_from_file(filename, zone, lenient, replace=True) + + self.log.info('populate: found %s records, exists=False', + len(zone.records) - before) + return False diff --git a/tests/config/override/dynamic.tests.yaml b/tests/config/override/dynamic.tests.yaml new file mode 100644 index 0000000..d79e092 --- /dev/null +++ b/tests/config/override/dynamic.tests.yaml @@ -0,0 +1,13 @@ +--- +# Replace 'a' with a generic record +a: + type: A + values: + - 4.4.4.4 + - 5.5.5.5 +# Add another record +added: + type: A + values: + - 6.6.6.6 + - 7.7.7.7 diff --git a/tests/test_octodns_provider_yaml.py b/tests/test_octodns_provider_yaml.py index d5d5e37..123f9b2 100644 --- a/tests/test_octodns_provider_yaml.py +++ b/tests/test_octodns_provider_yaml.py @@ -14,7 +14,7 @@ from yaml.constructor import ConstructorError from octodns.record import Create from octodns.provider.base import Plan from octodns.provider.yaml import _list_all_yaml_files, \ - SplitYamlProvider, YamlProvider + OverridingYamlProvider, SplitYamlProvider, YamlProvider from octodns.zone import SubzoneRecordException, Zone from helpers import TemporaryDirectory @@ -372,3 +372,38 @@ class TestSplitYamlProvider(TestCase): source.populate(zone) self.assertEquals('Record www.sub.unit.tests. is under a managed ' 'subzone', ctx.exception.message) + + +class TestOverridingYamlProvider(TestCase): + + def test_provider(self): + config = join(dirname(__file__), 'config') + override_config = join(dirname(__file__), 'config', 'override') + source = OverridingYamlProvider('test', config, override_config) + + zone = Zone('unit.tests.', []) + dynamic_zone = Zone('dynamic.tests.', []) + + # With target we don't add anything (same as base) + source.populate(zone, target=source) + self.assertEquals(0, len(zone.records)) + + # without it we see everything + source.populate(zone) + self.assertEquals(18, len(zone.records)) + + # Load the dynamic records + source.populate(dynamic_zone) + + got = {r.name: r for r in dynamic_zone.records} + # We see both the base and override files, 1 extra record + self.assertEquals(6, len(got)) + + # 'a' was replaced with a generic record + self.assertEquals({ + 'ttl': 3600, + 'values': ['4.4.4.4', '5.5.5.5'] + }, got['a'].data) + + # And we have a new override + self.assertTrue('added' in got) From a078ec9d3134535c56d437edd2098d5f996385b3 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 6 Jan 2020 14:16:47 -0800 Subject: [PATCH 2/3] Move to populate_should_replace rather then OverridingYamlProvider --- octodns/provider/yaml.py | 136 ++++++++++++++++------------ tests/test_octodns_provider_yaml.py | 42 ++++----- 2 files changed, 99 insertions(+), 79 deletions(-) diff --git a/octodns/provider/yaml.py b/octodns/provider/yaml.py index aa04528..a010084 100644 --- a/octodns/provider/yaml.py +++ b/octodns/provider/yaml.py @@ -28,7 +28,78 @@ class YamlProvider(BaseProvider): default_ttl: 3600 # Whether or not to enforce sorting order on the yaml config # (optional, default True) - enforce_order: True + enforce_order: true + # Whether duplicate records should replace rather than error + # (optiona, default False) + populate_should_replace: false + + Overriding values can be accomplished using multiple yaml providers in the + `sources` list where subsequent providers have `populate_should_replace` + set to `true`. An example use of this would be a zone that you want to push + to external DNS providers and internally, but you want to modify some of + the records in the internal version. + + config/octodns.com.yaml + --- + other: + type: A + values: + - 192.30.252.115 + - 192.30.252.116 + www: + type: A + values: + - 192.30.252.113 + - 192.30.252.114 + + + internal/octodns.com.yaml + --- + 'www': + type: A + values: + - 10.0.0.12 + - 10.0.0.13 + + external.yaml + --- + providers: + config: + class: octodns.provider.yaml.YamlProvider + directory: ./config + + zones: + + octodns.com.: + sources: + - config + targets: + - route53 + + internal.yaml + --- + providers: + config: + class: octodns.provider.yaml.YamlProvider + directory: ./config + + internal: + class: octodns.provider.yaml.YamlProvider + directory: ./internal + + zones: + + octodns.com.: + sources: + - config + - internal + targets: + - pdns + + You can then sync our records eternally with `--config-file=external.yaml` + and internally (with the custom overrides) with + `--config-file=internal.yaml` + ''' SUPPORTS_GEO = True SUPPORTS_DYNAMIC = True @@ -36,18 +107,20 @@ class YamlProvider(BaseProvider): 'PTR', 'SSHFP', 'SPF', 'SRV', 'TXT')) def __init__(self, id, directory, default_ttl=3600, enforce_order=True, - *args, **kwargs): + populate_should_replace=False, *args, **kwargs): self.log = logging.getLogger('{}[{}]'.format( self.__class__.__name__, id)) self.log.debug('__init__: id=%s, directory=%s, default_ttl=%d, ' - 'enforce_order=%d', id, directory, default_ttl, - enforce_order) + 'enforce_order=%d, populate_should_replace=%d', + id, directory, default_ttl, enforce_order, + populate_should_replace) super(YamlProvider, self).__init__(id, *args, **kwargs) self.directory = directory self.default_ttl = default_ttl self.enforce_order = enforce_order + self.populate_should_replace = populate_should_replace - def _populate_from_file(self, filename, zone, lenient, replace=False): + def _populate_from_file(self, filename, zone, lenient): with open(filename, 'r') as fh: yaml_data = safe_load(fh, enforce_order=self.enforce_order) if yaml_data: @@ -60,7 +133,7 @@ class YamlProvider(BaseProvider): record = Record.new(zone, name, d, source=self, lenient=lenient) zone.add_record(record, lenient=lenient, - replace=replace) + replace=self.populate_should_replace) self.log.debug('_populate_from_file: successfully loaded "%s"', filename) @@ -212,54 +285,3 @@ class SplitYamlProvider(YamlProvider): self.log.debug('_apply: writing catchall filename=%s', filename) with open(filename, 'w') as fh: safe_dump(catchall, fh) - - -class OverridingYamlProvider(YamlProvider): - ''' - Provider that builds on YamlProvider to allow overriding specific records. - - Works identically to YamlProvider with the additional behavior of loading - data from a second zonefile in override_directory if it exists. Records in - this second file will override (replace) those previously seen in the - primary. Records that do not exist in the primary will just be added. There - is currently no mechinism to remove records from the primary zone. - - config: - class: octodns.provider.yaml.OverridingYamlProvider - # The location of yaml config files (required) - directory: ./config - # The location of overriding yaml config files (required) - override_directory: ./config - # The ttl to use for records when not specified in the data - # (optional, default 3600) - default_ttl: 3600 - # Whether or not to enforce sorting order on the yaml config - # (optional, default True) - enforce_order: True - ''' - - def __init__(self, id, directory, override_directory, *args, **kwargs): - super(OverridingYamlProvider, self).__init__(id, directory, *args, - **kwargs) - self.override_directory = override_directory - - def populate(self, zone, target=False, lenient=False): - self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, - target, lenient) - - if target: - # When acting as a target we ignore any existing records so that we - # create a completely new copy - return False - - before = len(zone.records) - filename = join(self.directory, '{}yaml'.format(zone.name)) - self._populate_from_file(filename, zone, lenient) - - filename = join(self.override_directory, '{}yaml'.format(zone.name)) - if isfile(filename): - self._populate_from_file(filename, zone, lenient, replace=True) - - self.log.info('populate: found %s records, exists=False', - len(zone.records) - before) - return False diff --git a/tests/test_octodns_provider_yaml.py b/tests/test_octodns_provider_yaml.py index 0efcee9..f858c05 100644 --- a/tests/test_octodns_provider_yaml.py +++ b/tests/test_octodns_provider_yaml.py @@ -15,7 +15,7 @@ from yaml.constructor import ConstructorError from octodns.record import Create from octodns.provider.base import Plan from octodns.provider.yaml import _list_all_yaml_files, \ - OverridingYamlProvider, SplitYamlProvider, YamlProvider + SplitYamlProvider, YamlProvider from octodns.zone import SubzoneRecordException, Zone from helpers import TemporaryDirectory @@ -377,31 +377,29 @@ class TestOverridingYamlProvider(TestCase): def test_provider(self): config = join(dirname(__file__), 'config') override_config = join(dirname(__file__), 'config', 'override') - source = OverridingYamlProvider('test', config, override_config) - - zone = Zone('unit.tests.', []) - dynamic_zone = Zone('dynamic.tests.', []) - - # With target we don't add anything (same as base) - source.populate(zone, target=source) - self.assertEquals(0, len(zone.records)) - - # without it we see everything - source.populate(zone) - self.assertEquals(18, len(zone.records)) - - # Load the dynamic records - source.populate(dynamic_zone) - - got = {r.name: r for r in dynamic_zone.records} - # We see both the base and override files, 1 extra record + base = YamlProvider('base', config, populate_should_replace=False) + override = YamlProvider('test', override_config, + populate_should_replace=True) + + zone = Zone('dynamic.tests.', []) + + # Load the base, should see the 5 records + base.populate(zone) + got = {r.name: r for r in zone.records} + self.assertEquals(5, len(got)) + # We get the "dynamic" A from the bae config + self.assertTrue('dynamic' in got['a'].data) + # No added + self.assertFalse('added' in got) + + # Load the overrides, should replace one and add 1 + override.populate(zone) + got = {r.name: r for r in zone.records} self.assertEquals(6, len(got)) - # 'a' was replaced with a generic record self.assertEquals({ 'ttl': 3600, 'values': ['4.4.4.4', '5.5.5.5'] }, got['a'].data) - - # And we have a new override + # And we have the new one self.assertTrue('added' in got) From f1cc392bc41bddd24f480a1178ccd50ab15019b5 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Thu, 9 Jan 2020 07:41:30 -0800 Subject: [PATCH 3/3] Include populate_should_replace in yaml example. --- octodns/provider/yaml.py | 1 + 1 file changed, 1 insertion(+) diff --git a/octodns/provider/yaml.py b/octodns/provider/yaml.py index a010084..10add5a 100644 --- a/octodns/provider/yaml.py +++ b/octodns/provider/yaml.py @@ -86,6 +86,7 @@ class YamlProvider(BaseProvider): internal: class: octodns.provider.yaml.YamlProvider directory: ./internal + populate_should_replace: true zones: