From e4d6084b4c85da0e97b9f88d146583aac074794d Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Thu, 3 Dec 2020 17:50:56 -0800 Subject: [PATCH 01/92] POC of processors concept that can hook in to modify zones --- octodns/manager.py | 51 ++++++++++++++++++++++++++++++---- octodns/processors/__init__.py | 17 ++++++++++++ octodns/processors/filters.py | 38 +++++++++++++++++++++++++ octodns/provider/base.py | 5 +++- tests/test_octodns_manager.py | 2 +- 5 files changed, 106 insertions(+), 7 deletions(-) create mode 100644 octodns/processors/__init__.py create mode 100644 octodns/processors/filters.py diff --git a/octodns/manager.py b/octodns/manager.py index fc05810..9116742 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -121,6 +121,25 @@ class Manager(object): raise ManagerException('Incorrect provider config for {}' .format(provider_name)) + self.processors = {} + for processor_name, processor_config in \ + self.config.get('processors', {}).items(): + try: + _class = processor_config.pop('class') + except KeyError: + self.log.exception('Invalid processor class') + raise ManagerException('Processor {} is missing class' + .format(processor_name)) + _class = self._get_named_class('processor', _class) + kwargs = self._build_kwargs(processor_config) + try: + self.processors[processor_name] = _class(processor_name, + **kwargs) + except TypeError: + self.log.exception('Invalid processor config') + raise ManagerException('Incorrect processor config for {}' + .format(processor_name)) + zone_tree = {} # sort by reversed strings so that parent zones always come first for name in sorted(self.config['zones'].keys(), key=lambda s: s[::-1]): @@ -222,8 +241,8 @@ class Manager(object): self.log.debug('configured_sub_zones: subs=%s', sub_zone_names) return set(sub_zone_names) - def _populate_and_plan(self, zone_name, sources, targets, desired=None, - lenient=False): + def _populate_and_plan(self, zone_name, processors, sources, targets, + desired=None, lenient=False): self.log.debug('sync: populating, zone=%s, lenient=%s', zone_name, lenient) @@ -248,6 +267,10 @@ class Manager(object): 'param', source.__class__.__name__) source.populate(zone) + self.log.debug('sync: processing, zone=%s', zone_name) + for processor in processors: + zone = processor.process(zone) + self.log.debug('sync: planning, zone=%s', zone_name) plans = [] @@ -259,7 +282,9 @@ class Manager(object): 'value': 'provider={}'.format(target.id) }) zone.add_record(meta, replace=True) - plan = target.plan(zone) + # TODO: if someone has overrriden plan already this will be a + # breaking change so we probably need to try both ways + plan = target.plan(zone, processors=processors) if plan: plans.append((target, plan)) @@ -315,6 +340,8 @@ class Manager(object): raise ManagerException('Zone {} is missing targets' .format(zone_name)) + processors = config.get('processors', []) + if (eligible_sources and not [s for s in sources if s in eligible_sources]): self.log.info('sync: no eligible sources, skipping') @@ -332,6 +359,15 @@ class Manager(object): self.log.info('sync: sources=%s -> targets=%s', sources, targets) + try: + collected = [] + for processor in processors: + collected.append(self.processors[processor]) + processors = collected + except KeyError: + raise ManagerException('Zone {}, unknown processor: {}' + .format(zone_name, processor)) + try: # rather than using a list comprehension, we break this loop # out so that the `except` block below can reference the @@ -358,8 +394,9 @@ class Manager(object): .format(zone_name, target)) futures.append(self._executor.submit(self._populate_and_plan, - zone_name, sources, - targets, lenient=lenient)) + zone_name, processors, + sources, targets, + lenient=lenient)) # Wait on all results and unpack/flatten the plans and store the # desired states in case we need them below @@ -378,6 +415,7 @@ class Manager(object): futures.append(self._executor.submit( self._populate_and_plan, zone_name, + processors, [], [self.providers[t] for t in source_config['targets']], desired=desired[zone_source], @@ -518,6 +556,9 @@ class Manager(object): if isinstance(source, YamlProvider): source.populate(zone) + # TODO: validate + # processors = config.get('processors', []) + def get_zone(self, zone_name): if not zone_name[-1] == '.': raise ManagerException('Invalid zone name {}, missing ending dot' diff --git a/octodns/processors/__init__.py b/octodns/processors/__init__.py new file mode 100644 index 0000000..9431e6a --- /dev/null +++ b/octodns/processors/__init__.py @@ -0,0 +1,17 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from ..zone import Zone + + +class BaseProcessor(object): + + def __init__(self, name): + self.name = name + + def _create_zone(self, zone): + return Zone(zone.name, sub_zones=zone.sub_zones) diff --git a/octodns/processors/filters.py b/octodns/processors/filters.py new file mode 100644 index 0000000..d483ab7 --- /dev/null +++ b/octodns/processors/filters.py @@ -0,0 +1,38 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from . import BaseProcessor + + +class TypeAllowlistFilter(BaseProcessor): + + def __init__(self, name, allowlist): + super(TypeAllowlistFilter, self).__init__(name) + self.allowlist = allowlist + + def process(self, zone): + ret = self._create_zone(zone) + for record in zone.records: + if record._type in self.allowlist: + ret.add_record(record) + + return ret + + +class TypeRejectlistFilter(BaseProcessor): + + def __init__(self, name, rejectlist): + super(TypeRejectlistFilter, self).__init__(name) + self.rejectlist = rejectlist + + def process(self, zone): + ret = self._create_zone(zone) + for record in zone.records: + if record._type not in self.rejectlist: + ret.add_record(record) + + return ret diff --git a/octodns/provider/base.py b/octodns/provider/base.py index ae87844..2a4ab11 100644 --- a/octodns/provider/base.py +++ b/octodns/provider/base.py @@ -44,7 +44,7 @@ class BaseProvider(BaseSource): ''' return [] - def plan(self, desired): + def plan(self, desired, processors=[]): self.log.info('plan: desired=%s', desired.name) existing = Zone(desired.name, desired.sub_zones) @@ -55,6 +55,9 @@ class BaseProvider(BaseSource): self.log.warn('Provider %s used in target mode did not return ' 'exists', self.id) + for processor in processors: + existing = processor.process(existing) + # compute the changes at the zone/record level changes = existing.changes(desired, self) diff --git a/tests/test_octodns_manager.py b/tests/test_octodns_manager.py index dc047e8..3a09809 100644 --- a/tests/test_octodns_manager.py +++ b/tests/test_octodns_manager.py @@ -352,7 +352,7 @@ class TestManager(TestCase): pass # This should be ok, we'll fall back to not passing it - manager._populate_and_plan('unit.tests.', [NoLenient()], []) + manager._populate_and_plan('unit.tests.', [], [NoLenient()], []) class NoZone(SimpleProvider): From 95d9ffc221917e592ec169fb05124f2bd29a2ed3 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Thu, 3 Dec 2020 18:12:16 -0800 Subject: [PATCH 02/92] Tell the processor when it's being called in a target context --- octodns/processors/filters.py | 4 ++-- octodns/provider/base.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/octodns/processors/filters.py b/octodns/processors/filters.py index d483ab7..09613d3 100644 --- a/octodns/processors/filters.py +++ b/octodns/processors/filters.py @@ -14,7 +14,7 @@ class TypeAllowlistFilter(BaseProcessor): super(TypeAllowlistFilter, self).__init__(name) self.allowlist = allowlist - def process(self, zone): + def process(self, zone, target=False): ret = self._create_zone(zone) for record in zone.records: if record._type in self.allowlist: @@ -29,7 +29,7 @@ class TypeRejectlistFilter(BaseProcessor): super(TypeRejectlistFilter, self).__init__(name) self.rejectlist = rejectlist - def process(self, zone): + def process(self, zone, target=False): ret = self._create_zone(zone) for record in zone.records: if record._type not in self.rejectlist: diff --git a/octodns/provider/base.py b/octodns/provider/base.py index 2a4ab11..137b664 100644 --- a/octodns/provider/base.py +++ b/octodns/provider/base.py @@ -56,7 +56,7 @@ class BaseProvider(BaseSource): 'exists', self.id) for processor in processors: - existing = processor.process(existing) + existing = processor.process(existing, target=True) # compute the changes at the zone/record level changes = existing.changes(desired, self) From f9a4239af5de677624509cda88864f759f694d79 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Thu, 3 Dec 2020 18:12:43 -0800 Subject: [PATCH 03/92] Marking side of ownership processor --- octodns/processors/ownership.py | 40 +++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 octodns/processors/ownership.py diff --git a/octodns/processors/ownership.py b/octodns/processors/ownership.py new file mode 100644 index 0000000..c132b2e --- /dev/null +++ b/octodns/processors/ownership.py @@ -0,0 +1,40 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from ..record import Record + +from . import BaseProcessor + + +class OwnershipProcessor(BaseProcessor): + + def __init__(self, name, txt_name='_owner'): + super(OwnershipProcessor, self).__init__(name) + self.txt_name = txt_name + + def add_ownerships(self, zone): + ret = self._create_zone(zone) + for record in zone.records: + ret.add_record(record) + name = '{}.{}.{}'.format(self.txt_name, record._type, record.name), + txt = Record.new(zone, name, { + 'type': 'TXT', + 'ttl': 60, + 'value': 'octodns', + }) + ret.add_record(txt) + + return ret + + def remove_unowned(self, zone): + ret = self._create_zone(zone) + return ret + + def process(self, zone, target=False): + if target: + return self.remove_unowned(zone) + return self.add_ownerships(zone) From 261abeb133ca90e2782f64a4fdb6d2fe3956a093 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sun, 6 Dec 2020 13:02:56 -0800 Subject: [PATCH 04/92] Sketch at process: source, target, plan setup, with ownership --- octodns/manager.py | 7 ++- octodns/processors/__init__.py | 13 ++++ octodns/processors/filters.py | 10 +++- octodns/processors/ownership.py | 103 +++++++++++++++++++++++++++----- octodns/provider/base.py | 2 +- 5 files changed, 115 insertions(+), 20 deletions(-) diff --git a/octodns/manager.py b/octodns/manager.py index 9116742..aebd0d5 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -255,7 +255,6 @@ class Manager(object): for _, records in desired._records.items(): for record in records: zone.add_record(record.copy(zone=zone), lenient=lenient) - else: for source in sources: try: @@ -267,9 +266,8 @@ class Manager(object): 'param', source.__class__.__name__) source.populate(zone) - self.log.debug('sync: processing, zone=%s', zone_name) for processor in processors: - zone = processor.process(zone) + zone = processor.process_source_zone(zone, sources=sources) self.log.debug('sync: planning, zone=%s', zone_name) plans = [] @@ -285,6 +283,9 @@ class Manager(object): # TODO: if someone has overrriden plan already this will be a # breaking change so we probably need to try both ways plan = target.plan(zone, processors=processors) + for processor in processors: + plan = processor.process_plan(plan, sources=sources, + target=target) if plan: plans.append((target, plan)) diff --git a/octodns/processors/__init__.py b/octodns/processors/__init__.py index 9431e6a..6b999e2 100644 --- a/octodns/processors/__init__.py +++ b/octodns/processors/__init__.py @@ -15,3 +15,16 @@ class BaseProcessor(object): def _create_zone(self, zone): return Zone(zone.name, sub_zones=zone.sub_zones) + + def process_source_zone(self, zone, sources): + # sources may be empty, as will be the case for aliased zones + return zone + + def process_target_zone(self, zone, target): + return zone + + def process_plan(self, plan, sources, target): + # plan may be None if no changes were detected up until now, the + # process may still create a plan. + # sources may be empty, as will be the case for aliased zones + return plan diff --git a/octodns/processors/filters.py b/octodns/processors/filters.py index 09613d3..413964f 100644 --- a/octodns/processors/filters.py +++ b/octodns/processors/filters.py @@ -14,7 +14,7 @@ class TypeAllowlistFilter(BaseProcessor): super(TypeAllowlistFilter, self).__init__(name) self.allowlist = allowlist - def process(self, zone, target=False): + def _process(self, zone, *args, **kwargs): ret = self._create_zone(zone) for record in zone.records: if record._type in self.allowlist: @@ -22,6 +22,9 @@ class TypeAllowlistFilter(BaseProcessor): return ret + process_source_zone = _process + process_target_zone = _process + class TypeRejectlistFilter(BaseProcessor): @@ -29,10 +32,13 @@ class TypeRejectlistFilter(BaseProcessor): super(TypeRejectlistFilter, self).__init__(name) self.rejectlist = rejectlist - def process(self, zone, target=False): + def _process(self, zone, *args, **kwargs): ret = self._create_zone(zone) for record in zone.records: if record._type not in self.rejectlist: ret.add_record(record) return ret + + process_source_zone = _process + process_target_zone = _process diff --git a/octodns/processors/ownership.py b/octodns/processors/ownership.py index c132b2e..27a0385 100644 --- a/octodns/processors/ownership.py +++ b/octodns/processors/ownership.py @@ -5,36 +5,111 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals +from collections import defaultdict +from pprint import pprint + +from ..provider.plan import Plan from ..record import Record from . import BaseProcessor +# Mark anything octoDNS is managing that way it can know it's safe to modify or +# delete. We'll take ownership of existing records that we're told to manage +# and thus "own" them going forward. class OwnershipProcessor(BaseProcessor): - def __init__(self, name, txt_name='_owner'): + def __init__(self, name, txt_name='_owner', txt_value='*octodns*'): super(OwnershipProcessor, self).__init__(name) self.txt_name = txt_name + self.txt_value = txt_value + self._txt_values = [txt_value] - def add_ownerships(self, zone): + def process_source_zone(self, zone, *args, **kwargs): ret = self._create_zone(zone) for record in zone.records: + # Always copy over the source records ret.add_record(record) - name = '{}.{}.{}'.format(self.txt_name, record._type, record.name), + # Then create and add an ownership TXT for each of them + record_name = record.name.replace('*', '_wildcard') + if record.name: + name = '{}.{}.{}'.format(self.txt_name, record._type, + record_name) + else: + name = '{}.{}'.format(self.txt_name, record._type) txt = Record.new(zone, name, { - 'type': 'TXT', - 'ttl': 60, - 'value': 'octodns', - }) + 'type': 'TXT', + 'ttl': 60, + 'value': self.txt_value, + }) ret.add_record(txt) return ret - def remove_unowned(self, zone): - ret = self._create_zone(zone) - return ret + def _is_ownership(self, record): + return record._type == 'TXT' and \ + record.name.startswith(self.txt_name) \ + and record.values == self._txt_values + + def process_plan(self, plan, *args, **kwargs): + if not plan: + # If we don't have any change there's nothing to do + return plan + + # First find all the ownership info + owned = defaultdict(dict) + # We need to look for ownership in both the desired and existing + # states, many things will show up in both, but that's fine. + for record in list(plan.existing.records) + list(plan.desired.records): + if self._is_ownership(record): + pieces = record.name.split('.', 2) + if len(pieces) > 2: + _, _type, name = pieces + else: + _type = pieces[1] + name = '' + name = name.replace('_wildcard', '*') + owned[name][_type.upper()] = True + + pprint(dict(owned)) + + # Cases: + # - Configured in source + # - We'll fully CRU/manage it adding ownership TXT, + # thanks to process_source_zone, if needed + # - Not in source + # - Has an ownership TXT - delete it & the ownership TXT + # - Does not have an ownership TXT - don't delete it + # - Special records like octodns-meta + # - Should be left alone and should not have ownerthis TXTs + + pprint(plan.changes) + + filtered_changes = [] + for change in plan.changes: + record = change.record + + pprint([change, + not self._is_ownership(record), + record._type not in owned[record.name], + record.name != 'octodns-meta']) + + if not self._is_ownership(record) and \ + record._type not in owned[record.name] and \ + record.name != 'octodns-meta': + # It's not an ownership TXT, it's not owned, and it's not + # special we're going to ignore it + continue + + # We own this record or owned it up until now so whatever the + # change is we should do + filtered_changes.append(change) + + pprint(filtered_changes) + + if plan.changes != filtered_changes: + return Plan(plan.existing, plan.desired, filtered_changes, + plan.exists, plan.update_pcent_threshold, + plan.delete_pcent_threshold) - def process(self, zone, target=False): - if target: - return self.remove_unowned(zone) - return self.add_ownerships(zone) + return plan diff --git a/octodns/provider/base.py b/octodns/provider/base.py index 137b664..b28dd6e 100644 --- a/octodns/provider/base.py +++ b/octodns/provider/base.py @@ -56,7 +56,7 @@ class BaseProvider(BaseSource): 'exists', self.id) for processor in processors: - existing = processor.process(existing, target=True) + existing = processor.process_target_zone(existing, target=self) # compute the changes at the zone/record level changes = existing.changes(desired, self) From 040afe6fc1dcef9327ba0e68fd62d9cc8990ffb7 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Tue, 8 Dec 2020 08:13:36 -0800 Subject: [PATCH 05/92] Don't need to _wildcard empty names --- octodns/processors/ownership.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/processors/ownership.py b/octodns/processors/ownership.py index 27a0385..374e380 100644 --- a/octodns/processors/ownership.py +++ b/octodns/processors/ownership.py @@ -65,10 +65,10 @@ class OwnershipProcessor(BaseProcessor): pieces = record.name.split('.', 2) if len(pieces) > 2: _, _type, name = pieces + name = name.replace('_wildcard', '*') else: _type = pieces[1] name = '' - name = name.replace('_wildcard', '*') owned[name][_type.upper()] = True pprint(dict(owned)) From 1aefa1a496271b2fc1922d9ddd55cf06a3e29123 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 25 Mar 2021 21:56:29 +0000 Subject: [PATCH 06/92] Bump pyyaml from 5.3.1 to 5.4 Bumps [pyyaml](https://github.com/yaml/pyyaml) from 5.3.1 to 5.4. - [Release notes](https://github.com/yaml/pyyaml/releases) - [Changelog](https://github.com/yaml/pyyaml/blob/master/CHANGES) - [Commits](https://github.com/yaml/pyyaml/compare/5.3.1...5.4) Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8b9c052..933ac60 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -PyYaml==5.3.1 +PyYaml==5.4 azure-common==1.1.25 azure-mgmt-dns==3.0.0 boto3==1.15.9 From a99c71b3e566cb2619696c8a545028675127a6b5 Mon Sep 17 00:00:00 2001 From: Scott Little Date: Fri, 26 Mar 2021 19:48:38 +0100 Subject: [PATCH 07/92] Change the auth value to have a proper string not a binary rep --- octodns/provider/easydns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/easydns.py b/octodns/provider/easydns.py index 835fcb9..d7a75a4 100644 --- a/octodns/provider/easydns.py +++ b/octodns/provider/easydns.py @@ -59,7 +59,7 @@ class EasyDNSClient(object): self.base_path = self.SANDBOX if sandbox else self.LIVE sess = Session() sess.headers.update({'Authorization': 'Basic {}' - .format(self.auth_key)}) + .format(self.auth_key.decode('utf-8'))}) sess.headers.update({'accept': 'application/json'}) self._sess = sess From 374a5c62f25e93f155f8066717b5eeb71b2d1552 Mon Sep 17 00:00:00 2001 From: Maikel Poot Date: Wed, 31 Mar 2021 12:49:01 +0200 Subject: [PATCH 08/92] Enable NS support for non-root NS records --- octodns/provider/transip.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/transip.py b/octodns/provider/transip.py index 7458e36..ae2c5d5 100644 --- a/octodns/provider/transip.py +++ b/octodns/provider/transip.py @@ -50,7 +50,7 @@ class TransipProvider(BaseProvider): SUPPORTS_GEO = False SUPPORTS_DYNAMIC = False SUPPORTS = set( - ('A', 'AAAA', 'CNAME', 'MX', 'SRV', 'SPF', 'TXT', 'SSHFP', 'CAA')) + ('A', 'AAAA', 'CNAME', 'MX', 'SRV', 'SPF', 'TXT', 'SSHFP', 'CAA', 'NS')) # unsupported by OctoDNS: 'TLSA' MIN_TTL = 120 TIMEOUT = 15 From 87c31ca23d757a3de4606d830a9dc1e66fbe23f7 Mon Sep 17 00:00:00 2001 From: Maikel Poot Date: Wed, 31 Mar 2021 13:31:24 +0200 Subject: [PATCH 09/92] Update transip unittest --- tests/test_octodns_provider_transip.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/test_octodns_provider_transip.py b/tests/test_octodns_provider_transip.py index 84cfebc..234c95e 100644 --- a/tests/test_octodns_provider_transip.py +++ b/tests/test_octodns_provider_transip.py @@ -56,10 +56,11 @@ class MockDomainService(DomainService): _dns_entries.extend(entries_for(name, record)) - # NS is not supported as a DNS Entry, - # so it should cover the if statement + # Add a non-supported type + # so it triggers the "is supported" (transip.py:115) check and + # give 100% code coverage _dns_entries.append( - DnsEntry('@', '3600', 'NS', 'ns01.transip.nl.')) + DnsEntry('@', '3600', 'BOGUS', 'ns01.transip.nl.')) self.mockupEntries = _dns_entries @@ -222,7 +223,7 @@ N4OiVz1I3rbZGYa396lpxO6ku8yCglisL1yrSP6DdEUp66ntpKVd provider._client = MockDomainService('unittest', self.bogus_key) plan = provider.plan(_expected) - self.assertEqual(14, plan.change_counts['Create']) + self.assertEqual(15, plan.change_counts['Create']) self.assertEqual(0, plan.change_counts['Update']) self.assertEqual(0, plan.change_counts['Delete']) @@ -235,7 +236,7 @@ N4OiVz1I3rbZGYa396lpxO6ku8yCglisL1yrSP6DdEUp66ntpKVd provider = TransipProvider('test', 'unittest', self.bogus_key) provider._client = MockDomainService('unittest', self.bogus_key) plan = provider.plan(_expected) - self.assertEqual(14, len(plan.changes)) + self.assertEqual(15, len(plan.changes)) changes = provider.apply(plan) self.assertEqual(changes, len(plan.changes)) From 60e6f185cdbbf66544734bc2f6e21b65f4233f8c Mon Sep 17 00:00:00 2001 From: Maikel Poot Date: Wed, 31 Mar 2021 13:41:55 +0200 Subject: [PATCH 10/92] Update Documentation --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0a26da9..3186ed2 100644 --- a/README.md +++ b/README.md @@ -204,7 +204,7 @@ The above command pulled the existing data out of Route53 and placed the results | [Rackspace](/octodns/provider/rackspace.py) | | A, AAAA, ALIAS, CNAME, MX, NS, PTR, SPF, TXT | No | | | [Route53](/octodns/provider/route53.py) | boto3 | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | Both | CNAME health checks don't support a Host header | | [Selectel](/octodns/provider/selectel.py) | | A, AAAA, CNAME, MX, NS, SPF, SRV, TXT | No | | -| [Transip](/octodns/provider/transip.py) | transip | A, AAAA, CNAME, MX, SRV, SPF, TXT, SSHFP, CAA | No | | +| [Transip](/octodns/provider/transip.py) | transip | A, AAAA, CNAME, MX, NS, SRV, SPF, TXT, SSHFP, CAA | No | | | [UltraDns](/octodns/provider/ultra.py) | | A, AAAA, CAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | | | [AxfrSource](/octodns/source/axfr.py) | | A, AAAA, CAA, CNAME, LOC, MX, NS, PTR, SPF, SRV, TXT | No | read-only | | [ZoneFileSource](/octodns/source/axfr.py) | | A, AAAA, CAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | read-only | From 840cb03bf432e8c7cfba2df25aed45e838fedb19 Mon Sep 17 00:00:00 2001 From: Maikel Poot Date: Wed, 31 Mar 2021 13:44:37 +0200 Subject: [PATCH 11/92] Fix lint error --- octodns/provider/transip.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/octodns/provider/transip.py b/octodns/provider/transip.py index ae2c5d5..6ccbe22 100644 --- a/octodns/provider/transip.py +++ b/octodns/provider/transip.py @@ -49,8 +49,8 @@ class TransipProvider(BaseProvider): ''' SUPPORTS_GEO = False SUPPORTS_DYNAMIC = False - SUPPORTS = set( - ('A', 'AAAA', 'CNAME', 'MX', 'SRV', 'SPF', 'TXT', 'SSHFP', 'CAA', 'NS')) + SUPPORTS = set(('A', 'AAAA', 'CNAME', 'MX', 'NS', 'SRV', 'SPF', 'TXT', + 'SSHFP', 'CAA')) # unsupported by OctoDNS: 'TLSA' MIN_TTL = 120 TIMEOUT = 15 From 090dbe351554ca3e929f099fa9e127aceedccd79 Mon Sep 17 00:00:00 2001 From: Christian Funkhouser Date: Wed, 7 Apr 2021 16:00:05 -0400 Subject: [PATCH 12/92] sync accepts file handle for plan output --- octodns/manager.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/octodns/manager.py b/octodns/manager.py index 9ce10ff..f4f1491 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -9,6 +9,7 @@ from concurrent.futures import ThreadPoolExecutor from importlib import import_module from os import environ from six import text_type +from sys import stdout import logging from .provider.base import BaseProvider @@ -267,7 +268,7 @@ class Manager(object): return plans, zone def sync(self, eligible_zones=[], eligible_sources=[], eligible_targets=[], - dry_run=True, force=False): + dry_run=True, force=False, plan_output_fh=stdout): self.log.info('sync: eligible_zones=%s, eligible_targets=%s, ' 'dry_run=%s, force=%s', eligible_zones, eligible_targets, dry_run, force) @@ -276,7 +277,7 @@ class Manager(object): if eligible_zones: zones = [z for z in zones if z[0] in eligible_zones] - aliased_zones = {} + aliased_zones = {} futures = [] for zone_name, config in zones: self.log.info('sync: zone=%s', zone_name) @@ -402,7 +403,7 @@ class Manager(object): plans.sort(key=self._plan_keyer, reverse=True) for output in self.plan_outputs.values(): - output.run(plans=plans, log=self.log) + output.run(plans=plans, log=self.log, fh=plan_output_fh) if not force: self.log.debug('sync: checking safety') From 2075550f078942d642c873bd2ec62343eb1a0f16 Mon Sep 17 00:00:00 2001 From: Christian Funkhouser Date: Wed, 7 Apr 2021 18:21:34 -0400 Subject: [PATCH 13/92] Test that Manager passes fh to _PlanOutputs --- octodns/manager.py | 5 +++-- tests/config/plan-output-filehandle.yaml | 6 ++++++ tests/test_octodns_manager.py | 23 ++++++++++++++++++++++- 3 files changed, 31 insertions(+), 3 deletions(-) create mode 100644 tests/config/plan-output-filehandle.yaml diff --git a/octodns/manager.py b/octodns/manager.py index f4f1491..e94e41e 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -270,8 +270,9 @@ class Manager(object): def sync(self, eligible_zones=[], eligible_sources=[], eligible_targets=[], dry_run=True, force=False, plan_output_fh=stdout): self.log.info('sync: eligible_zones=%s, eligible_targets=%s, ' - 'dry_run=%s, force=%s', eligible_zones, eligible_targets, - dry_run, force) + 'dry_run=%s, force=%s, plan_output_fh=%s', + eligible_zones, eligible_targets, dry_run, force, + plan_output_fh) zones = self.config['zones'].items() if eligible_zones: diff --git a/tests/config/plan-output-filehandle.yaml b/tests/config/plan-output-filehandle.yaml new file mode 100644 index 0000000..9c9bb87 --- /dev/null +++ b/tests/config/plan-output-filehandle.yaml @@ -0,0 +1,6 @@ +manager: + plan_outputs: + "doesntexist": + class: octodns.provider.plan.DoesntExist +providers: {} +zones: {} diff --git a/tests/test_octodns_manager.py b/tests/test_octodns_manager.py index 442ed49..003d414 100644 --- a/tests/test_octodns_manager.py +++ b/tests/test_octodns_manager.py @@ -8,7 +8,6 @@ from __future__ import absolute_import, division, print_function, \ from os import environ from os.path import dirname, join from six import text_type -from unittest import TestCase from octodns.record import Record from octodns.manager import _AggregateTarget, MainThreadExecutor, Manager, \ @@ -16,6 +15,10 @@ from octodns.manager import _AggregateTarget, MainThreadExecutor, Manager, \ from octodns.yaml import safe_load from octodns.zone import Zone +from mock import MagicMock, patch +from unittest import TestCase + + from helpers import DynamicProvider, GeoProvider, NoSshFpProvider, \ SimpleProvider, TemporaryDirectory @@ -371,6 +374,24 @@ class TestManager(TestCase): with self.assertRaises(TypeError): manager._populate_and_plan('unit.tests.', [NoZone()], []) + @patch('octodns.manager.Manager._get_named_class') + def test_sync_passes_file_handle(self, mock): + plan_output_mock = MagicMock() + plan_output_class_mock = MagicMock() + plan_output_class_mock.return_value = plan_output_mock + mock.return_value = plan_output_class_mock + fh_mock = MagicMock() + + Manager(get_config_filename('plan-output-filehandle.yaml') + ).sync(plan_output_fh=fh_mock) + + # Since we only care about the fh kwarg, and different _PlanOutputs are + # are free to require arbitrary kwargs anyway, we concern ourselves + # with checking the value of fh only. + plan_output_mock.run.assert_called() + _, kwargs = plan_output_mock.run.call_args + self.assertEqual(fh_mock, kwargs.get('fh')) + class TestMainThreadExecutor(TestCase): From 55c194c2032fd65b1d61b0309d3d9c55f8e22489 Mon Sep 17 00:00:00 2001 From: Christian Funkhouser Date: Thu, 8 Apr 2021 09:40:04 -0400 Subject: [PATCH 14/92] Update tests/test_octodns_manager.py Co-authored-by: Ross McFarland --- tests/test_octodns_manager.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_octodns_manager.py b/tests/test_octodns_manager.py index 003d414..91ee374 100644 --- a/tests/test_octodns_manager.py +++ b/tests/test_octodns_manager.py @@ -18,7 +18,6 @@ from octodns.zone import Zone from mock import MagicMock, patch from unittest import TestCase - from helpers import DynamicProvider, GeoProvider, NoSshFpProvider, \ SimpleProvider, TemporaryDirectory From aa93e20f2e4f7fa597e61f11a5096a34fea32f20 Mon Sep 17 00:00:00 2001 From: Christian Funkhouser Date: Thu, 8 Apr 2021 11:03:30 -0400 Subject: [PATCH 15/92] Represent plan_output_fh less verbosely. Co-authored-by: Ross McFarland --- octodns/manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/manager.py b/octodns/manager.py index e94e41e..7326207 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -272,7 +272,7 @@ class Manager(object): self.log.info('sync: eligible_zones=%s, eligible_targets=%s, ' 'dry_run=%s, force=%s, plan_output_fh=%s', eligible_zones, eligible_targets, dry_run, force, - plan_output_fh) + getattr(plan_output_fh, 'name', plan_output_fh.__class__.__name__)) zones = self.config['zones'].items() if eligible_zones: From ada61f8d764cf4175f83376d989eaec3984e716a Mon Sep 17 00:00:00 2001 From: Christian Funkhouser Date: Thu, 8 Apr 2021 11:56:17 -0400 Subject: [PATCH 16/92] De-lint an aggressively-long log line --- octodns/manager.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/octodns/manager.py b/octodns/manager.py index 7326207..9b10196 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -269,10 +269,12 @@ class Manager(object): def sync(self, eligible_zones=[], eligible_sources=[], eligible_targets=[], dry_run=True, force=False, plan_output_fh=stdout): - self.log.info('sync: eligible_zones=%s, eligible_targets=%s, ' - 'dry_run=%s, force=%s, plan_output_fh=%s', - eligible_zones, eligible_targets, dry_run, force, - getattr(plan_output_fh, 'name', plan_output_fh.__class__.__name__)) + + self.log.info( + 'sync: eligible_zones=%s, eligible_targets=%s, dry_run=%s, ' + 'force=%s, plan_output_fh=%s', + eligible_zones, eligible_targets, dry_run, force, + getattr(plan_output_fh, 'name', plan_output_fh.__class__.__name__)) zones = self.config['zones'].items() if eligible_zones: From c71784b0e832850f1c535485994976d9d48b29da Mon Sep 17 00:00:00 2001 From: Ricard Bejarano Date: Mon, 12 Apr 2021 08:06:59 +0200 Subject: [PATCH 17/92] initial work on Hetzner provider - implemented HetznerClient API class - tested manually, lacking formal tests --- octodns/provider/hetzner.py | 91 +++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 octodns/provider/hetzner.py diff --git a/octodns/provider/hetzner.py b/octodns/provider/hetzner.py new file mode 100644 index 0000000..17dc046 --- /dev/null +++ b/octodns/provider/hetzner.py @@ -0,0 +1,91 @@ +# +# +# + +from requests import Session + + +class HetznerClientException(Exception): + pass + + +class HetznerClientNotFound(HetznerClientException): + + def __init__(self): + super(HetznerClientNotFound, self).__init__('Not Found') + + +class HetznerClient(object): + BASE_URL = 'https://dns.hetzner.com/api/v1' + + def __init__(self, token): + session = Session() + session.headers.update({'Auth-API-Token': token}) + self._session = session + + def _do(self, method, path, params=None, data=None): + url = self.BASE_URL + path + response = self._session.request(method, url, params=params, json=data) + if response.status_code == 404: + raise HetznerClientNotFound() + response.raise_for_status() + return response.json() + + def _do_with_pagination(self, method, path, key, params=None, data=None, + per_page=100): + pagination_params = {'page': 1, 'per_page': per_page} + if params is not None: + params = {**params, **pagination_params} + else: + params = pagination_params + + items = [] + while True: + response = self._do(method, path, params, data) + items += response[key] + if response['meta']['pagination']['page'] >= \ + response['meta']['pagination']['last_page']: + break + params['page'] += 1 + return items + + def zones_get(self, name=None, search_name=None): + params = {'name': name, 'search_name': search_name} + return self._do_with_pagination('GET', '/zones', 'zones', + params=params) + + def zone_get(self, zone_id): + return self._do('GET', '/zones/' + zone_id)['zone'] + + def zone_create(self, name, ttl=None): + data = {'name': name, 'ttl': ttl} + return self._do('POST', '/zones', data=data)['zone'] + + def zone_update(self, zone_id, name, ttl=None): + data = {'name': name, 'ttl': ttl} + return self._do('PUT', '/zones/' + zone_id, data=data)['zone'] + + def zone_delete(self, zone_id): + return self._do('DELETE', '/zones/' + zone_id) + + def zone_records_get(self, zone_id): + params = {'zone_id': zone_id} + # No need to handle pagination as it returns all records by default. + return self._do('GET', '/records', params=params)['records'] + + def zone_record_get(self, record_id): + return self._do('GET', '/records/' + record_id)['record'] + + def zone_record_create(self, zone_id, name, _type, value, ttl=None): + data = {'name': name, 'ttl': ttl, 'type': _type, 'value': value, + 'zone_id': zone_id} + return self._do('POST', '/records', data=data)['record'] + + def zone_record_update(self, zone_id, record_id, name, _type, value, + ttl=None): + data = {'name': name, 'ttl': ttl, 'type': _type, 'value': value, + 'zone_id': zone_id} + return self._do('PUT', '/records/' + record_id, data=data)['record'] + + def zone_record_delete(self, zone_id, record_id): + return self._do('DELETE', '/records/' + record_id) From c8e91c1e11b86106645ac67055b34eec4b6724f4 Mon Sep 17 00:00:00 2001 From: Ricard Bejarano Date: Mon, 12 Apr 2021 21:08:53 +0200 Subject: [PATCH 18/92] added HetznerClient docstring --- octodns/provider/hetzner.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/octodns/provider/hetzner.py b/octodns/provider/hetzner.py index 17dc046..1002e7d 100644 --- a/octodns/provider/hetzner.py +++ b/octodns/provider/hetzner.py @@ -16,6 +16,15 @@ class HetznerClientNotFound(HetznerClientException): class HetznerClient(object): + ''' + Hetzner DNS Public API v1 client class. + + Zone and Record resources are (almost) fully supported, even if unnecessary + to future-proof this client. Bulk Record create/update is not supported. + + No support for Primary Servers. + ''' + BASE_URL = 'https://dns.hetzner.com/api/v1' def __init__(self, token): From 8a9743b36ec56b81429b0bf9481c0f674a9834a9 Mon Sep 17 00:00:00 2001 From: Ricard Bejarano Date: Tue, 13 Apr 2021 08:08:48 +0200 Subject: [PATCH 19/92] added HetznerClient._replace_at to address the "@"/"" record name problem --- octodns/provider/hetzner.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/octodns/provider/hetzner.py b/octodns/provider/hetzner.py index 1002e7d..2e069d0 100644 --- a/octodns/provider/hetzner.py +++ b/octodns/provider/hetzner.py @@ -77,24 +77,34 @@ class HetznerClient(object): def zone_delete(self, zone_id): return self._do('DELETE', '/zones/' + zone_id) + def _replace_at(self, record): + record['name'] = '' if record['name'] == '@' else record['name'] + return record + def zone_records_get(self, zone_id): params = {'zone_id': zone_id} # No need to handle pagination as it returns all records by default. - return self._do('GET', '/records', params=params)['records'] + return [ + self._replace_at(record) + for record in self._do('GET', '/records', params=params)['records'] + ] def zone_record_get(self, record_id): - return self._do('GET', '/records/' + record_id)['record'] + record = self._do('GET', '/records/' + record_id)['record'] + return self._replace_at(record) def zone_record_create(self, zone_id, name, _type, value, ttl=None): - data = {'name': name, 'ttl': ttl, 'type': _type, 'value': value, + data = {'name': name or '@', 'ttl': ttl, 'type': _type, 'value': value, 'zone_id': zone_id} - return self._do('POST', '/records', data=data)['record'] + record = self._do('POST', '/records', data=data)['record'] + return self._replace_at(record) def zone_record_update(self, zone_id, record_id, name, _type, value, ttl=None): - data = {'name': name, 'ttl': ttl, 'type': _type, 'value': value, + data = {'name': name or '@', 'ttl': ttl, 'type': _type, 'value': value, 'zone_id': zone_id} - return self._do('PUT', '/records/' + record_id, data=data)['record'] + record = self._do('PUT', '/records/' + record_id, data=data)['record'] + return self._replace_at(record) def zone_record_delete(self, zone_id, record_id): return self._do('DELETE', '/records/' + record_id) From f507349ce506af5554de338a0eb56977b8632a60 Mon Sep 17 00:00:00 2001 From: Ricard Bejarano Date: Wed, 14 Apr 2021 07:57:23 +0200 Subject: [PATCH 20/92] implemented HetznerProvider --- octodns/provider/hetzner.py | 254 ++++++++++++++++++++++++++++++++++++ 1 file changed, 254 insertions(+) diff --git a/octodns/provider/hetzner.py b/octodns/provider/hetzner.py index 2e069d0..570cefe 100644 --- a/octodns/provider/hetzner.py +++ b/octodns/provider/hetzner.py @@ -2,8 +2,17 @@ # # +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +import logging +from collections import defaultdict + from requests import Session +from ..record import Record +from .base import BaseProvider + class HetznerClientException(Exception): pass @@ -108,3 +117,248 @@ class HetznerClient(object): def zone_record_delete(self, zone_id, record_id): return self._do('DELETE', '/records/' + record_id) + + +class HetznerProvider(BaseProvider): + ''' + Hetzner DNS provider using API v1 + + hetzner: + class: octodns.provider.hetzner.HetznerProvider + # Your Hetzner API token (required) + token: foo + ''' + SUPPORTS_GEO = False + SUPPORTS_DYNAMIC = False + SUPPORTS = set(('A', 'AAAA', 'CAA', 'CNAME', 'MX', 'NS', 'SRV', 'TXT')) + + def __init__(self, id, token, *args, **kwargs): + self.log = logging.getLogger('HetznerProvider[{}]'.format(id)) + self.log.debug('__init__: id=%s, token=***', id) + super(HetznerProvider, self).__init__(id, *args, **kwargs) + self._client = HetznerClient(token) + + self._zone_records = {} + + def _append_dot(self, value): + return value if value[-1] == '.' else '{}.'.format(value) + + def _data_for_multiple(self, _type, records): + values = [record['value'].replace(';', '\\;') for record in records] + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': values + } + + _data_for_A = _data_for_multiple + _data_for_AAAA = _data_for_multiple + + def _data_for_CAA(self, _type, records): + values = [] + for record in records: + value_without_spaces = record['value'].replace(' ', '') + flags = value_without_spaces[0] + tag = value_without_spaces[1:].split('"')[0] + value = record['value'].split('"')[1] + values.append({ + 'flags': int(flags), + 'tag': tag, + 'value': value, + }) + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': values + } + + def _data_for_CNAME(self, _type, records): + record = records[0] + return { + 'ttl': record['ttl'], + 'type': _type, + 'value': self._append_dot(record['value']) + } + + def _data_for_MX(self, _type, records): + values = [] + for record in records: + value_stripped_split = record['value'].strip().split(' ') + preference = value_stripped_split[0] + exchange = value_stripped_split[-1] + values.append({ + 'preference': int(preference), + 'exchange': self._append_dot(exchange) + }) + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': values + } + + def _data_for_NS(self, _type, records): + values = [] + for record in records: + values.append(self._append_dot(record['value'])) + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': values, + } + + def _data_for_SRV(self, _type, records): + values = [] + for record in records: + value_stripped = record['value'].strip() + priority = value_stripped.split(' ')[0] + weight = value_stripped[len(priority):].strip().split(' ')[0] + target = value_stripped.split(' ')[-1] + port = value_stripped[:-len(target)].strip().split(' ')[-1] + values.append({ + 'port': int(port), + 'priority': int(priority), + 'target': self._append_dot(target), + 'weight': int(weight) + }) + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': values + } + + _data_for_TXT = _data_for_multiple + + def zone_records(self, zone): + if zone.name not in self._zone_records: + try: + zone_id = self._client.zones_get(name=zone.name[:-1])[0]['id'] + self._zone_records[zone.name] = \ + self._client.zone_records_get(zone_id) + except HetznerClientNotFound: + return [] + + return self._zone_records[zone.name] + + def populate(self, zone, target=False, lenient=False): + self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, + target, lenient) + + values = defaultdict(lambda: defaultdict(list)) + for record in self.zone_records(zone): + _type = record['type'] + if _type not in self.SUPPORTS: + self.log.warning('populate: skipping unsupported %s record', + _type) + continue + values[record['name']][record['type']].append(record) + + before = len(zone.records) + for name, types in values.items(): + for _type, records in types.items(): + data_for = getattr(self, '_data_for_{}'.format(_type)) + record = Record.new(zone, name, data_for(_type, records), + source=self, lenient=lenient) + zone.add_record(record, lenient=lenient) + + exists = zone.name in self._zone_records + self.log.info('populate: found %s records, exists=%s', + len(zone.records) - before, exists) + return exists + + def _params_for_multiple(self, record): + for value in record.values: + yield { + 'value': value.replace('\\;', ';'), + 'name': record.name, + 'ttl': record.ttl, + 'type': record._type + } + + _params_for_A = _params_for_multiple + _params_for_AAAA = _params_for_multiple + + def _params_for_CAA(self, record): + for value in record.values: + data = '{} {} "{}"'.format(value.flags, value.tag, value.value) + yield { + 'value': data, + 'name': record.name, + 'ttl': record.ttl, + 'type': record._type + } + + def _params_for_single(self, record): + yield { + 'value': record.value, + 'name': record.name, + 'ttl': record.ttl, + 'type': record._type + } + + _params_for_CNAME = _params_for_single + + def _params_for_MX(self, record): + for value in record.values: + data = '{} {}'.format(value.preference, value.exchange) + yield { + 'value': data, + 'name': record.name, + 'ttl': record.ttl, + 'type': record._type + } + + _params_for_NS = _params_for_multiple + + def _params_for_SRV(self, record): + for value in record.values: + data = '{} {} {} {}'.format(value.priority, value.weight, + value.port, value.target) + yield { + 'value': data, + 'name': record.name, + 'ttl': record.ttl, + 'type': record._type + } + + _params_for_TXT = _params_for_multiple + + def _apply_Create(self, zone_id, change): + new = change.new + params_for = getattr(self, '_params_for_{}'.format(new._type)) + for params in params_for(new): + self._client.zone_record_create(zone_id, params['name'], + params['type'], params['value'], + params['ttl']) + + def _apply_Update(self, zone_id, change): + # It's way simpler to delete-then-recreate than to update + self._apply_Delete(zone_id, change) + self._apply_Create(zone_id, change) + + def _apply_Delete(self, zone_id, change): + existing = change.existing + zone = existing.zone + for record in self.zone_records(zone): + if existing.name == record['name'] and \ + existing._type == record['type']: + self._client.zone_record_delete(zone_id, record['id']) + + def _apply(self, plan): + desired = plan.desired + changes = plan.changes + self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name, + len(changes)) + + zone_name = desired.name[:-1] + try: + zone_id = self._client.zones_get(name=zone_name)[0]['id'] + except HetznerClientNotFound: + self.log.debug('_apply: no matching zone, creating domain') + zone_id = self._client.zone_create(zone_name)['id'] + + for change in changes: + class_name = change.__class__.__name__ + getattr(self, '_apply_{}'.format(class_name))(zone_id, change) + + # Clear out the cache if any + self._zone_records.pop(desired.name, None) From 8d1dd926ea6a8241928140e62258221f8636b782 Mon Sep 17 00:00:00 2001 From: Ricard Bejarano Date: Wed, 14 Apr 2021 07:59:00 +0200 Subject: [PATCH 21/92] added Hetzner provider to README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 0a26da9..42c91a0 100644 --- a/README.md +++ b/README.md @@ -197,6 +197,7 @@ The above command pulled the existing data out of Route53 and placed the results | [EnvVarSource](/octodns/source/envvar.py) | | TXT | No | read-only environment variable injection | | [GandiProvider](/octodns/provider/gandi.py) | | A, AAAA, ALIAS, CAA, CNAME, DNAME, MX, NS, PTR, SPF, SRV, SSHFP, TXT | No | | | [GoogleCloudProvider](/octodns/provider/googlecloud.py) | google-cloud-dns | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | No | | +| [HetznerProvider](/octodns/provider/hetzner.py) | | A, AAAA, CAA, CNAME, MX, NS, SRV, TXT | No | | | [MythicBeastsProvider](/octodns/provider/mythicbeasts.py) | Mythic Beasts | A, AAAA, ALIAS, CNAME, MX, NS, SRV, SSHFP, CAA, TXT | No | | | [Ns1Provider](/octodns/provider/ns1.py) | ns1-python | All | Yes | Missing `NA` geo target | | [OVH](/octodns/provider/ovh.py) | ovh | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, SSHFP, TXT, DKIM | No | | From 1a2cb50c6326b22a870694c51374bd2bb05556fa Mon Sep 17 00:00:00 2001 From: Ricard Bejarano Date: Wed, 21 Apr 2021 07:52:38 +0200 Subject: [PATCH 22/92] fixed potential KeyError when record ttl field is missing --- octodns/provider/hetzner.py | 38 ++++++++++++++++++++++++++----------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/octodns/provider/hetzner.py b/octodns/provider/hetzner.py index 570cefe..17b1141 100644 --- a/octodns/provider/hetzner.py +++ b/octodns/provider/hetzner.py @@ -5,10 +5,9 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -import logging from collections import defaultdict - from requests import Session +import logging from ..record import Record from .base import BaseProvider @@ -24,6 +23,12 @@ class HetznerClientNotFound(HetznerClientException): super(HetznerClientNotFound, self).__init__('Not Found') +class HetznerClientUnauthorized(HetznerClientException): + + def __init__(self): + super(HetznerClientUnauthorized, self).__init__('Unauthorized') + + class HetznerClient(object): ''' Hetzner DNS Public API v1 client class. @@ -44,6 +49,8 @@ class HetznerClient(object): def _do(self, method, path, params=None, data=None): url = self.BASE_URL + path response = self._session.request(method, url, params=params, json=data) + if response.status_code == 401: + raise HetznerClientUnauthorized() if response.status_code == 404: raise HetznerClientNotFound() response.raise_for_status() @@ -139,14 +146,22 @@ class HetznerProvider(BaseProvider): self._client = HetznerClient(token) self._zone_records = {} + self._zone_metadata = {} def _append_dot(self, value): - return value if value[-1] == '.' else '{}.'.format(value) + if value == '@' or value[-1] == '.': + return value + return '{}.'.format(value) + + def _record_ttl(self, record): + if 'ttl' in record: + return record['ttl'] + return self._zone_metadata[record['zone_id']]['ttl'] def _data_for_multiple(self, _type, records): values = [record['value'].replace(';', '\\;') for record in records] return { - 'ttl': records[0]['ttl'], + 'ttl': self._record_ttl(records[0]), 'type': _type, 'values': values } @@ -167,7 +182,7 @@ class HetznerProvider(BaseProvider): 'value': value, }) return { - 'ttl': records[0]['ttl'], + 'ttl': self._record_ttl(records[0]), 'type': _type, 'values': values } @@ -175,7 +190,7 @@ class HetznerProvider(BaseProvider): def _data_for_CNAME(self, _type, records): record = records[0] return { - 'ttl': record['ttl'], + 'ttl': self._record_ttl(record), 'type': _type, 'value': self._append_dot(record['value']) } @@ -191,7 +206,7 @@ class HetznerProvider(BaseProvider): 'exchange': self._append_dot(exchange) }) return { - 'ttl': records[0]['ttl'], + 'ttl': self._record_ttl(records[0]), 'type': _type, 'values': values } @@ -201,7 +216,7 @@ class HetznerProvider(BaseProvider): for record in records: values.append(self._append_dot(record['value'])) return { - 'ttl': records[0]['ttl'], + 'ttl': self._record_ttl(records[0]), 'type': _type, 'values': values, } @@ -221,7 +236,7 @@ class HetznerProvider(BaseProvider): 'weight': int(weight) }) return { - 'ttl': records[0]['ttl'], + 'ttl': self._record_ttl(records[0]), 'type': _type, 'values': values } @@ -231,9 +246,10 @@ class HetznerProvider(BaseProvider): def zone_records(self, zone): if zone.name not in self._zone_records: try: - zone_id = self._client.zones_get(name=zone.name[:-1])[0]['id'] + zone_metadata = self._client.zones_get(name=zone.name[:-1])[0] + self._zone_metadata[zone_metadata['id']] = zone_metadata self._zone_records[zone.name] = \ - self._client.zone_records_get(zone_id) + self._client.zone_records_get(zone_metadata['id']) except HetznerClientNotFound: return [] From ab436af92d42643eac3796ee58a8109be737ace9 Mon Sep 17 00:00:00 2001 From: Ricard Bejarano Date: Wed, 21 Apr 2021 07:55:18 +0200 Subject: [PATCH 23/92] added populate() tests --- tests/fixtures/hetzner-records.json | 223 +++++++++++++++++++++++++ tests/fixtures/hetzner-zones.json | 43 +++++ tests/test_octodns_provider_hetzner.py | 84 ++++++++++ 3 files changed, 350 insertions(+) create mode 100644 tests/fixtures/hetzner-records.json create mode 100644 tests/fixtures/hetzner-zones.json create mode 100644 tests/test_octodns_provider_hetzner.py diff --git a/tests/fixtures/hetzner-records.json b/tests/fixtures/hetzner-records.json new file mode 100644 index 0000000..bbafdcb --- /dev/null +++ b/tests/fixtures/hetzner-records.json @@ -0,0 +1,223 @@ +{ + "records": [ + { + "id": "SOA", + "type": "SOA", + "name": "@", + "value": "hydrogen.ns.hetzner.com. dns.hetzner.com. 1 86400 10800 3600000 3600", + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "NS:sub:0", + "type": "NS", + "name": "sub", + "value": "6.2.3.4", + "ttl": 3600, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "NS:sub:1", + "type": "NS", + "name": "sub", + "value": "7.2.3.4", + "ttl": 3600, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "SRV:_srv._tcp:0", + "type": "SRV", + "name": "_srv._tcp", + "value": "10 20 30 foo-1.unit.tests", + "ttl": 600, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "SRV:_srv._tcp:1", + "type": "SRV", + "name": "_srv._tcp", + "value": "12 20 30 foo-2.unit.tests", + "ttl": 600, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "TXT:txt:0", + "type": "TXT", + "name": "txt", + "value": "\"Bah bah black sheep\"", + "ttl": 600, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "TXT:txt:1", + "type": "TXT", + "name": "txt", + "value": "\"have you any wool.\"", + "ttl": 600, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "A:@:0", + "type": "A", + "name": "@", + "value": "1.2.3.4", + "ttl": 300, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "A:@:1", + "type": "A", + "name": "@", + "value": "1.2.3.5", + "ttl": 300, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "A:www:0", + "type": "A", + "name": "www", + "value": "2.2.3.6", + "ttl": 300, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "MX:mx:0", + "type": "MX", + "name": "mx", + "value": "10 smtp-4.unit.tests", + "ttl": 300, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "MX:mx:1", + "type": "MX", + "name": "mx", + "value": "20 smtp-2.unit.tests", + "ttl": 300, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "MX:mx:2", + "type": "MX", + "name": "mx", + "value": "30 smtp-3.unit.tests", + "ttl": 300, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "MX:mx:3", + "type": "MX", + "name": "mx", + "value": "40 smtp-1.unit.tests", + "ttl": 300, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "AAAA:aaaa:0", + "type": "AAAA", + "name": "aaaa", + "value": "2601:644:500:e210:62f8:1dff:feb8:947a", + "ttl": 600, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "CNAME:cname:0", + "type": "CNAME", + "name": "cname", + "value": "unit.tests", + "ttl": 300, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "A:www.sub:0", + "type": "A", + "name": "www.sub", + "value": "2.2.3.6", + "ttl": 300, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "TXT:txt:2", + "type": "TXT", + "name": "txt", + "value": "v=DKIM1;k=rsa;s=email;h=sha256;p=A/kinda+of/long/string+with+numb3rs", + "ttl": 600, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "CAA:@:0", + "type": "CAA", + "name": "@", + "value": "0 issue \"ca.unit.tests\"", + "ttl": 3600, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "CNAME:included:0", + "type": "CNAME", + "name": "included", + "value": "unit.tests", + "ttl": 3600, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "SRV:_imap._tcp:0", + "type": "SRV", + "name": "_imap._tcp", + "value": "0 0 0 .", + "ttl": 600, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "SRV:_pop3._tcp:0", + "type": "SRV", + "name": "_pop3._tcp", + "value": "0 0 0 .", + "ttl": 600, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + } + ] +} diff --git a/tests/fixtures/hetzner-zones.json b/tests/fixtures/hetzner-zones.json new file mode 100644 index 0000000..4d9b897 --- /dev/null +++ b/tests/fixtures/hetzner-zones.json @@ -0,0 +1,43 @@ +{ + "zones": [ + { + "id": "unit.tests", + "name": "unit.tests", + "ttl": 3600, + "registrar": "", + "legacy_dns_host": "", + "legacy_ns": [], + "ns": [], + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "verified": "", + "modified": "0000-00-00 00:00:00.000 +0000 UTC", + "project": "", + "owner": "", + "permission": "", + "zone_type": { + "id": "", + "name": "", + "description": "", + "prices": null + }, + "status": "verified", + "paused": false, + "is_secondary_dns": false, + "txt_verification": { + "name": "", + "token": "" + }, + "records_count": null + } + ], + "meta": { + "pagination": { + "page": 1, + "per_page": 100, + "previous_page": 1, + "next_page": 1, + "last_page": 1, + "total_entries": 1 + } + } +} diff --git a/tests/test_octodns_provider_hetzner.py b/tests/test_octodns_provider_hetzner.py new file mode 100644 index 0000000..bf619fb --- /dev/null +++ b/tests/test_octodns_provider_hetzner.py @@ -0,0 +1,84 @@ +# +# +# + + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from os.path import dirname, join +from requests import HTTPError +from requests_mock import ANY, mock as requests_mock +from six import text_type +from unittest import TestCase + +from octodns.provider.hetzner import HetznerProvider +from octodns.provider.yaml import YamlProvider +from octodns.zone import Zone + + +class TestdHetznerProvider(TestCase): + expected = Zone('unit.tests.', []) + source = YamlProvider('test', join(dirname(__file__), 'config')) + source.populate(expected) + + def test_populate(self): + provider = HetznerProvider('test', 'token') + + # Bad auth + with requests_mock() as mock: + mock.get(ANY, status_code=401, + text='{"message":"Invalid authentication credentials"}') + + with self.assertRaises(Exception) as ctx: + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals('Unauthorized', text_type(ctx.exception)) + + # General error + with requests_mock() as mock: + mock.get(ANY, status_code=502, text='Things caught fire') + + with self.assertRaises(HTTPError) as ctx: + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals(502, ctx.exception.response.status_code) + + # Non-existent zone doesn't populate anything + with requests_mock() as mock: + mock.get(ANY, status_code=404, + text='{"zone":{"id":"","name":"","ttl":0,"registrar":"",' + '"legacy_dns_host":"","legacy_ns":null,"ns":null,' + '"created":"","verified":"","modified":"","project":"",' + '"owner":"","permission":"","zone_type":{"id":"",' + '"name":"","description":"","prices":null},"status":"",' + '"paused":false,"is_secondary_dns":false,' + '"txt_verification":{"name":"","token":""},' + '"records_count":0},"error":{' + '"message":"zone not found","code":404}}') + + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals(set(), zone.records) + + # No diffs == no changes + with requests_mock() as mock: + base = 'https://dns.hetzner.com/api/v1' + with open('tests/fixtures/hetzner-zones.json') as fh: + mock.get('{}/zones'.format(base), text=fh.read()) + with open('tests/fixtures/hetzner-records.json') as fh: + mock.get('{}/records'.format(base), text=fh.read()) + + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals(13, len(zone.records)) + changes = self.expected.changes(zone, provider) + self.assertEquals(0, len(changes)) + + # 2nd populate makes no network calls/all from cache + again = Zone('unit.tests.', []) + provider.populate(again) + self.assertEquals(13, len(again.records)) + + # bust the cache + del provider._zone_records[zone.name] From 3451c79c6ac012a3dda29fdca36234a5bfe0966f Mon Sep 17 00:00:00 2001 From: Cadu Ribeiro Date: Wed, 21 Apr 2021 12:41:11 -0300 Subject: [PATCH 24/92] Change DNSimple's Provider to query zones instead domains A quick summary of a problem with have with DNSimple provider: Let's suppose that I have the following config: zones: 30.114.195.in-addr.arpa.: sources: - config targets: - dnsimple Even if a customer has this Reverse zone configured in DNSimple, this fails with: 400 Bad Request for url: https://api.sandbox.dnsimple.com/v2/x/domains because it is trying to create a domain because the zone wasn't found. octodns.provider.dnsimple.DnsimpleClientNotFound: Not found This happens because the GET /domains endpoint at DNSimple does not bring Reverse Zones. To make this work nice, we should use /zones/ instead making it return properly the reverse zones. --- octodns/provider/dnsimple.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/octodns/provider/dnsimple.py b/octodns/provider/dnsimple.py index 599eacb..a4b9350 100644 --- a/octodns/provider/dnsimple.py +++ b/octodns/provider/dnsimple.py @@ -51,8 +51,8 @@ class DnsimpleClient(object): resp.raise_for_status() return resp - def domain(self, name): - path = '/domains/{}'.format(name) + def zone(self, name): + path = '/zones/{}'.format(name) return self._request('GET', path).json() def domain_create(self, name): @@ -442,7 +442,7 @@ class DnsimpleProvider(BaseProvider): domain_name = desired.name[:-1] try: - self._client.domain(domain_name) + self._client.zone(domain_name) except DnsimpleClientNotFound: self.log.debug('_apply: no matching zone, creating domain') self._client.domain_create(domain_name) From e90aeb5d34fe0b1d423daffb2cdd2b831fbf1775 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Thu, 22 Apr 2021 18:45:14 -0700 Subject: [PATCH 25/92] pools used as fallbacks should count as seen --- octodns/record/__init__.py | 12 ++++++++---- tests/config/dynamic.tests.yaml | 23 +++++++++++++++++++++++ tests/test_octodns_provider_yaml.py | 14 +++++++++----- 3 files changed, 40 insertions(+), 9 deletions(-) diff --git a/octodns/record/__init__.py b/octodns/record/__init__.py index 8ee2eaa..8c8760a 100644 --- a/octodns/record/__init__.py +++ b/octodns/record/__init__.py @@ -531,6 +531,7 @@ class _DynamicMixin(object): pools_exist = set() pools_seen = set() + pools_seen_as_fallback = set() if not isinstance(pools, dict): reasons.append('pools must be a dict') elif not pools: @@ -573,9 +574,12 @@ class _DynamicMixin(object): 'value {}'.format(_id, value_num)) fallback = pool.get('fallback', None) - if fallback is not None and fallback not in pools: - reasons.append('undefined fallback "{}" for pool "{}"' - .format(fallback, _id)) + if fallback is not None: + if fallback in pools: + pools_seen_as_fallback.add(fallback) + else: + reasons.append('undefined fallback "{}" for pool "{}"' + .format(fallback, _id)) # Check for loops fallback = pools[_id].get('fallback', None) @@ -644,7 +648,7 @@ class _DynamicMixin(object): reasons.extend(GeoCodes.validate(geo, 'rule {} ' .format(rule_num))) - unused = pools_exist - pools_seen + unused = pools_exist - pools_seen - pools_seen_as_fallback if unused: unused = '", "'.join(sorted(unused)) reasons.append('unused pools: "{}"'.format(unused)) diff --git a/tests/config/dynamic.tests.yaml b/tests/config/dynamic.tests.yaml index 4bd97a7..d25f63a 100644 --- a/tests/config/dynamic.tests.yaml +++ b/tests/config/dynamic.tests.yaml @@ -109,6 +109,29 @@ cname: - pool: iad type: CNAME value: target.unit.tests. +pool-only-in-fallback: + dynamic: + pools: + one: + fallback: two + values: + - value: 1.1.1.1 + three: + values: + - value: 3.3.3.3 + two: + values: + - value: 2.2.2.2 + rules: + - geos: + - NA-US + pool: one + - geos: + - AS-SG + pool: three + ttl: 300 + type: A + values: [4.4.4.4] real-ish-a: dynamic: pools: diff --git a/tests/test_octodns_provider_yaml.py b/tests/test_octodns_provider_yaml.py index 2ce9b4a..872fcca 100644 --- a/tests/test_octodns_provider_yaml.py +++ b/tests/test_octodns_provider_yaml.py @@ -38,7 +38,7 @@ class TestYamlProvider(TestCase): self.assertEquals(22, len(zone.records)) source.populate(dynamic_zone) - self.assertEquals(5, len(dynamic_zone.records)) + self.assertEquals(6, len(dynamic_zone.records)) # Assumption here is that a clean round-trip means that everything # worked as expected, data that went in came back out and could be @@ -68,11 +68,11 @@ class TestYamlProvider(TestCase): # Dynamic plan plan = target.plan(dynamic_zone) - self.assertEquals(5, len([c for c in plan.changes + self.assertEquals(6, len([c for c in plan.changes if isinstance(c, Create)])) self.assertFalse(isfile(dynamic_yaml_file)) # Apply it - self.assertEquals(5, target.apply(plan)) + self.assertEquals(6, target.apply(plan)) self.assertTrue(isfile(dynamic_yaml_file)) # There should be no changes after the round trip @@ -148,6 +148,10 @@ class TestYamlProvider(TestCase): self.assertTrue('value' in dyna) # self.assertTrue('dynamic' in dyna) + dyna = data.pop('pool-only-in-fallback') + self.assertTrue('value' in dyna) + # self.assertTrue('dynamic' in dyna) + # make sure nothing is left self.assertEquals([], list(data.keys())) @@ -397,7 +401,7 @@ class TestOverridingYamlProvider(TestCase): # Load the base, should see the 5 records base.populate(zone) got = {r.name: r for r in zone.records} - self.assertEquals(5, len(got)) + self.assertEquals(6, len(got)) # We get the "dynamic" A from the base config self.assertTrue('dynamic' in got['a'].data) # No added @@ -406,7 +410,7 @@ class TestOverridingYamlProvider(TestCase): # Load the overrides, should replace one and add 1 override.populate(zone) got = {r.name: r for r in zone.records} - self.assertEquals(6, len(got)) + self.assertEquals(7, len(got)) # 'a' was replaced with a generic record self.assertEquals({ 'ttl': 3600, From b55a647e6e915ea5395711152ce3f2745a376062 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 26 Apr 2021 06:36:05 -0700 Subject: [PATCH 26/92] Remove redudant pools_seen.add(pool) call --- octodns/record/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/octodns/record/__init__.py b/octodns/record/__init__.py index 8c8760a..135baf8 100644 --- a/octodns/record/__init__.py +++ b/octodns/record/__init__.py @@ -628,7 +628,6 @@ class _DynamicMixin(object): if pool not in pools: reasons.append('rule {} undefined pool "{}"' .format(rule_num, pool)) - pools_seen.add(pool) elif pool in pools_seen and geos: reasons.append('rule {} invalid, target pool "{}" ' 'reused'.format(rule_num, pool)) From 078576520d67e7b6360f5f1ef1166ee6b3f0f2d2 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 26 Apr 2021 08:34:02 -0700 Subject: [PATCH 27/92] Rework NS1 pool handling to support fallback-only pools --- octodns/provider/ns1.py | 63 +++++++++++++++++++----------- tests/test_octodns_provider_ns1.py | 15 +++++-- 2 files changed, 51 insertions(+), 27 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 6cea185..7724fcb 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -406,7 +406,7 @@ class Ns1Provider(BaseProvider): for piece in note.split(' '): try: k, v = piece.split(':', 1) - data[k] = v + data[k] = v if v != '' else None except ValueError: pass return data @@ -479,31 +479,45 @@ class Ns1Provider(BaseProvider): # region. pools = defaultdict(lambda: {'fallback': None, 'values': []}) for answer in record['answers']: - # region (group name in the UI) is the pool name - pool_name = answer['region'] - # Get the actual pool name by removing the type - pool_name = self._parse_dynamic_pool_name(pool_name) - pool = pools[pool_name] - meta = answer['meta'] + notes = self._parse_notes(meta.get('note', '')) + value = text_type(answer['answer'][0]) - if meta['priority'] == 1: - # priority 1 means this answer is part of the pools own values - value_dict = { - 'value': value, - 'weight': int(meta.get('weight', 1)), - } - # If we have the original pool name and the catchall pool name - # in the answers, they point at the same pool. Add values only - # once - if value_dict not in pool['values']: - pool['values'].append(value_dict) + if notes.get('from', False) == '--default--': + # It's a final/default value, record it and move on + default.add(value) + continue + + # NS1 pool names can be found in notes > v0.9.11, in order to allow + # us to find fallback-only pools/values. Before that we used + # `region` (group name in the UI) and only paid attention to + # priority=1 (first level) + notes_pool_name = notes.get('pool', None) + if notes_pool_name is None: + # < v0.9.11 + if meta['priority'] != 1: + # Ignore all but priority 1 + continue + # And use region's pool name as the pool name + pool_name = self._parse_dynamic_pool_name(answer['region']) else: - # It's a fallback, we only care about it if it's a - # final/default - notes = self._parse_notes(meta.get('note', '')) - if notes.get('from', False) == '--default--': - default.add(value) + # > v0.9.11, use the notes-based name and consider all values + pool_name = notes_pool_name + + pool = pools[pool_name] + value_dict = { + 'value': value, + 'weight': int(meta.get('weight', 1)), + } + if value_dict not in pool['values']: + # If we haven't seen this value before add it to the pool + pool['values'].append(value_dict) + + # If there's a fallback recorded in the value for its pool go ahead + # and use it, another v0.9.11 thing + fallback = notes.get('fallback', None) + if fallback is not None: + pool['fallback'] = fallback # The regions objects map to rules, but it's a bit fuzzy since they're # tied to pools on the NS1 side, e.g. we can only have 1 rule per pool, @@ -978,12 +992,15 @@ class Ns1Provider(BaseProvider): seen.add(current_pool_name) pool = pools[current_pool_name] for answer in pool_answers[current_pool_name]: + fallback = pool.data['fallback'] answer = { 'answer': answer['answer'], 'meta': { 'priority': priority, 'note': self._encode_notes({ 'from': pool_label, + 'pool': current_pool_name, + 'fallback': fallback or '', }), 'up': { 'feed': answer['feed_id'], diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index 00b068b..b7270fb 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -1117,14 +1117,21 @@ class TestNs1ProviderDynamic(TestCase): # finally has a catchall. Those are examples of the two ways pools get # expanded. # - # lhr splits in two, with a region and country. + # lhr splits in two, with a region and country and includes a fallback + # + # All values now include their own `pool:` name # # well as both lhr georegion (for contients) and country. The first is # an example of a repeated target pool in a rule (only allowed when the # 2nd is a catchall.) - self.assertEquals(['from:--default--', 'from:iad__catchall', - 'from:iad__country', 'from:iad__georegion', - 'from:lhr__country', 'from:lhr__georegion'], + self.assertEquals(['fallback: from:iad__catchall pool:iad', + 'fallback: from:iad__country pool:iad', + 'fallback: from:iad__georegion pool:iad', + 'fallback: from:lhr__country pool:iad', + 'fallback: from:lhr__georegion pool:iad', + 'fallback:iad from:lhr__country pool:lhr', + 'fallback:iad from:lhr__georegion pool:lhr', + 'from:--default--'], sorted(notes.keys())) # All the iad's should match (after meta and region were removed) From fe74c462138a951577ce4693e8eafa5e32764557 Mon Sep 17 00:00:00 2001 From: Ricard Bejarano Date: Mon, 26 Apr 2021 19:02:44 +0200 Subject: [PATCH 28/92] made hetzner.HetznerProvider._do Mock-able for testing purposes making it a wrapper for _do_raw --- octodns/provider/hetzner.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/octodns/provider/hetzner.py b/octodns/provider/hetzner.py index 17b1141..78be756 100644 --- a/octodns/provider/hetzner.py +++ b/octodns/provider/hetzner.py @@ -46,7 +46,7 @@ class HetznerClient(object): session.headers.update({'Auth-API-Token': token}) self._session = session - def _do(self, method, path, params=None, data=None): + def _do_raw(self, method, path, params=None, data=None): url = self.BASE_URL + path response = self._session.request(method, url, params=params, json=data) if response.status_code == 401: @@ -54,7 +54,10 @@ class HetznerClient(object): if response.status_code == 404: raise HetznerClientNotFound() response.raise_for_status() - return response.json() + return response + + def _do(self, method, path, params=None, data=None): + return self._do_raw(method, path, params, data).json() def _do_with_pagination(self, method, path, key, params=None, data=None, per_page=100): From 612738b327b26b417a683ec5dd0c3c55291b6bd4 Mon Sep 17 00:00:00 2001 From: Ricard Bejarano Date: Mon, 26 Apr 2021 19:03:25 +0200 Subject: [PATCH 29/92] renamed TestdHetznerProvider -> TestHetznerProvider (missing "d") --- tests/test_octodns_provider_hetzner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_octodns_provider_hetzner.py b/tests/test_octodns_provider_hetzner.py index bf619fb..a8a67fb 100644 --- a/tests/test_octodns_provider_hetzner.py +++ b/tests/test_octodns_provider_hetzner.py @@ -17,7 +17,7 @@ from octodns.provider.yaml import YamlProvider from octodns.zone import Zone -class TestdHetznerProvider(TestCase): +class TestHetznerProvider(TestCase): expected = Zone('unit.tests.', []) source = YamlProvider('test', join(dirname(__file__), 'config')) source.populate(expected) From 3f3d500152baa55e9d88c27ced0e83101699472c Mon Sep 17 00:00:00 2001 From: Ricard Bejarano Date: Mon, 26 Apr 2021 19:06:29 +0200 Subject: [PATCH 30/92] fixed missing whitespace in plan() debug logging When debug logging is enabled, the plan() function logs lines like: Plan: DEBUG: __init__: Creates=13, Updates=0, Deletes=0Existing=0 Where a space between "Deletes=0" and "Existing=0" is missing. This adds it. --- octodns/provider/plan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/plan.py b/octodns/provider/plan.py index af6863a..69bd2b2 100644 --- a/octodns/provider/plan.py +++ b/octodns/provider/plan.py @@ -50,7 +50,7 @@ class Plan(object): except AttributeError: existing_n = 0 - self.log.debug('__init__: Creates=%d, Updates=%d, Deletes=%d' + self.log.debug('__init__: Creates=%d, Updates=%d, Deletes=%d ' 'Existing=%d', self.change_counts['Create'], self.change_counts['Update'], From fbd838990364dec250d3aa58e3ed1b441f853ef4 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 26 Apr 2021 17:10:22 -0700 Subject: [PATCH 31/92] Tests for new-style ns1 data_for_dynamic_A fallback only pools --- octodns/provider/ns1.py | 2 +- tests/test_octodns_provider_ns1.py | 109 +++++++++++++++++++++++++++++ 2 files changed, 110 insertions(+), 1 deletion(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 7724fcb..52fb3d2 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -542,7 +542,7 @@ class Ns1Provider(BaseProvider): rules[rule_order] = rule # The group notes field in the UI is a `note` on the region here, - # that's where we can find our pool's fallback. + # that's where we can find our pool's fallback in < v0.9.11 anyway if 'fallback' in notes: # set the fallback pool name pools[pool_name]['fallback'] = notes['fallback'] diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index b7270fb..db713eb 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -1478,6 +1478,115 @@ class TestNs1ProviderDynamic(TestCase): self.assertTrue( 'OC-{}'.format(c) in data4['dynamic']['rules'][0]['geos']) + # Test out fallback only pools and new-style notes + ns1_record = { + 'answers': [{ + 'answer': ['1.1.1.1'], + 'meta': { + 'priority': 1, + 'note': 'from:one__country pool:one fallback:two', + }, + 'region': 'one_country', + }, { + 'answer': ['2.2.2.2'], + 'meta': { + 'priority': 2, + 'note': 'from:one__country pool:two fallback:three', + }, + 'region': 'one_country', + }, { + 'answer': ['3.3.3.3'], + 'meta': { + 'priority': 3, + 'note': 'from:one__country pool:three fallback:', + }, + 'region': 'one_country', + }, { + 'answer': ['5.5.5.5'], + 'meta': { + 'priority': 4, + 'note': 'from:--default--', + }, + 'region': 'one_country', + }, { + 'answer': ['4.4.4.4'], + 'meta': { + 'priority': 1, + 'note': 'from:four__country pool:four fallback:', + }, + 'region': 'four_country', + }, { + 'answer': ['5.5.5.5'], + 'meta': { + 'priority': 2, + 'note': 'from:--default--', + }, + 'region': 'four_country', + }], + 'domain': 'unit.tests', + 'filters': filters, + 'regions': { + 'one__country': { + 'meta': { + 'note': 'rule-order:1 fallback:two', + 'country': ['CA'], + 'us_state': ['OR'], + }, + }, + 'four__country': { + 'meta': { + 'note': 'rule-order:2', + 'country': ['CA'], + 'us_state': ['OR'], + }, + }, + catchall_pool_name: { + 'meta': { + 'note': 'rule-order:3', + }, + } + }, + 'tier': 3, + 'ttl': 42, + } + data = provider._data_for_dynamic_A('A', ns1_record) + self.assertEquals({ + 'dynamic': { + 'pools': { + 'four': { + 'fallback': None, + 'values': [{'value': '4.4.4.4', 'weight': 1}] + }, + 'one': { + 'fallback': 'two', + 'values': [{'value': '1.1.1.1', 'weight': 1}] + }, + 'three': { + 'fallback': None, + 'values': [{'value': '3.3.3.3', 'weight': 1}] + }, + 'two': { + 'fallback': 'three', + 'values': [{'value': '2.2.2.2', 'weight': 1}] + }, + }, + 'rules': [{ + '_order': '1', + 'geos': ['NA-CA', 'NA-US-OR'], + 'pool': 'one' + }, { + '_order': '2', + 'geos': ['NA-CA', 'NA-US-OR'], + 'pool': 'four' + }, { + '_order': '3', 'pool': 'iad'} + ] + }, + 'ttl': 42, + 'type': 'A', + 'values': ['5.5.5.5'] + }, data) + @patch('ns1.rest.records.Records.retrieve') @patch('ns1.rest.zones.Zones.retrieve') @patch('octodns.provider.ns1.Ns1Provider._monitors_for') From 7832fb880910e50c7cb65d80b3a7659d9dac941e Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 26 Apr 2021 20:08:42 -0700 Subject: [PATCH 32/92] Better name for _create_zone --- octodns/processors/__init__.py | 2 +- octodns/processors/filters.py | 4 ++-- octodns/processors/ownership.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/octodns/processors/__init__.py b/octodns/processors/__init__.py index 6b999e2..327b1c2 100644 --- a/octodns/processors/__init__.py +++ b/octodns/processors/__init__.py @@ -13,7 +13,7 @@ class BaseProcessor(object): def __init__(self, name): self.name = name - def _create_zone(self, zone): + def _clone_zone(self, zone): return Zone(zone.name, sub_zones=zone.sub_zones) def process_source_zone(self, zone, sources): diff --git a/octodns/processors/filters.py b/octodns/processors/filters.py index 413964f..2c5f87f 100644 --- a/octodns/processors/filters.py +++ b/octodns/processors/filters.py @@ -15,7 +15,7 @@ class TypeAllowlistFilter(BaseProcessor): self.allowlist = allowlist def _process(self, zone, *args, **kwargs): - ret = self._create_zone(zone) + ret = self._clone_zone(zone) for record in zone.records: if record._type in self.allowlist: ret.add_record(record) @@ -33,7 +33,7 @@ class TypeRejectlistFilter(BaseProcessor): self.rejectlist = rejectlist def _process(self, zone, *args, **kwargs): - ret = self._create_zone(zone) + ret = self._clone_zone(zone) for record in zone.records: if record._type not in self.rejectlist: ret.add_record(record) diff --git a/octodns/processors/ownership.py b/octodns/processors/ownership.py index 374e380..67bbb9a 100644 --- a/octodns/processors/ownership.py +++ b/octodns/processors/ownership.py @@ -26,7 +26,7 @@ class OwnershipProcessor(BaseProcessor): self._txt_values = [txt_value] def process_source_zone(self, zone, *args, **kwargs): - ret = self._create_zone(zone) + ret = self._clone_zone(zone) for record in zone.records: # Always copy over the source records ret.add_record(record) From 0de9efd03268ae0d447997fe1960dc314880348d Mon Sep 17 00:00:00 2001 From: Ricard Bejarano Date: Tue, 27 Apr 2021 07:31:05 +0200 Subject: [PATCH 33/92] removed unused HetznerClient methods to fix imparial coverage --- octodns/provider/hetzner.py | 70 +++++++++---------------------------- 1 file changed, 17 insertions(+), 53 deletions(-) diff --git a/octodns/provider/hetzner.py b/octodns/provider/hetzner.py index 78be756..0be01ea 100644 --- a/octodns/provider/hetzner.py +++ b/octodns/provider/hetzner.py @@ -32,11 +32,6 @@ class HetznerClientUnauthorized(HetznerClientException): class HetznerClient(object): ''' Hetzner DNS Public API v1 client class. - - Zone and Record resources are (almost) fully supported, even if unnecessary - to future-proof this client. Bulk Record create/update is not supported. - - No support for Primary Servers. ''' BASE_URL = 'https://dns.hetzner.com/api/v1' @@ -46,8 +41,8 @@ class HetznerClient(object): session.headers.update({'Auth-API-Token': token}) self._session = session - def _do_raw(self, method, path, params=None, data=None): - url = self.BASE_URL + path + def _do(self, method, path, params=None, data=None): + url = '{}{}'.format(self.BASE_URL, path) response = self._session.request(method, url, params=params, json=data) if response.status_code == 401: raise HetznerClientUnauthorized() @@ -56,20 +51,15 @@ class HetznerClient(object): response.raise_for_status() return response - def _do(self, method, path, params=None, data=None): - return self._do_raw(method, path, params, data).json() - - def _do_with_pagination(self, method, path, key, params=None, data=None, - per_page=100): - pagination_params = {'page': 1, 'per_page': per_page} - if params is not None: - params = {**params, **pagination_params} - else: - params = pagination_params + def _do_json(self, method, path, params=None, data=None): + return self._do(method, path, params, data).json() + def _do_json_paginate(self, method, path, key, params=None, data=None, + per_page=100): items = [] + params = {**{'page': 1, 'per_page': per_page}, **params} while True: - response = self._do(method, path, params, data) + response = self._do_json(method, path, params, data) items += response[key] if response['meta']['pagination']['page'] >= \ response['meta']['pagination']['last_page']: @@ -79,54 +69,28 @@ class HetznerClient(object): def zones_get(self, name=None, search_name=None): params = {'name': name, 'search_name': search_name} - return self._do_with_pagination('GET', '/zones', 'zones', - params=params) - - def zone_get(self, zone_id): - return self._do('GET', '/zones/' + zone_id)['zone'] + return self._do_json_paginate('GET', '/zones', 'zones', params=params) def zone_create(self, name, ttl=None): data = {'name': name, 'ttl': ttl} - return self._do('POST', '/zones', data=data)['zone'] - - def zone_update(self, zone_id, name, ttl=None): - data = {'name': name, 'ttl': ttl} - return self._do('PUT', '/zones/' + zone_id, data=data)['zone'] - - def zone_delete(self, zone_id): - return self._do('DELETE', '/zones/' + zone_id) - - def _replace_at(self, record): - record['name'] = '' if record['name'] == '@' else record['name'] - return record + return self._do_json('POST', '/zones', data=data)['zone'] def zone_records_get(self, zone_id): params = {'zone_id': zone_id} # No need to handle pagination as it returns all records by default. - return [ - self._replace_at(record) - for record in self._do('GET', '/records', params=params)['records'] - ] - - def zone_record_get(self, record_id): - record = self._do('GET', '/records/' + record_id)['record'] - return self._replace_at(record) + records = self._do_json('GET', '/records', params=params)['records'] + for record in records: + if record['name'] == '@': + record['name'] = '' + return records def zone_record_create(self, zone_id, name, _type, value, ttl=None): data = {'name': name or '@', 'ttl': ttl, 'type': _type, 'value': value, 'zone_id': zone_id} - record = self._do('POST', '/records', data=data)['record'] - return self._replace_at(record) - - def zone_record_update(self, zone_id, record_id, name, _type, value, - ttl=None): - data = {'name': name or '@', 'ttl': ttl, 'type': _type, 'value': value, - 'zone_id': zone_id} - record = self._do('PUT', '/records/' + record_id, data=data)['record'] - return self._replace_at(record) + self._do('POST', '/records', data=data) def zone_record_delete(self, zone_id, record_id): - return self._do('DELETE', '/records/' + record_id) + self._do('DELETE', '/records/{}'.format(record_id)) class HetznerProvider(BaseProvider): From 192231109181dac8f61c1b41be156ba4c53cbac9 Mon Sep 17 00:00:00 2001 From: Ricard Bejarano Date: Tue, 27 Apr 2021 07:53:17 +0200 Subject: [PATCH 34/92] WIP added TestHetznerProvider.test_apply --- tests/test_octodns_provider_hetzner.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/tests/test_octodns_provider_hetzner.py b/tests/test_octodns_provider_hetzner.py index a8a67fb..0c18731 100644 --- a/tests/test_octodns_provider_hetzner.py +++ b/tests/test_octodns_provider_hetzner.py @@ -6,13 +6,15 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals +from mock import Mock from os.path import dirname, join from requests import HTTPError from requests_mock import ANY, mock as requests_mock from six import text_type from unittest import TestCase -from octodns.provider.hetzner import HetznerProvider +from octodns.provider.hetzner import HetznerClientNotFound, \ + HetznerProvider from octodns.provider.yaml import YamlProvider from octodns.zone import Zone @@ -82,3 +84,24 @@ class TestHetznerProvider(TestCase): # bust the cache del provider._zone_records[zone.name] + + def test_apply(self): + provider = HetznerProvider('test', 'token') + + resp = Mock() + resp.json = Mock() + provider._client._do = Mock(return_value=resp) + + # non-existent domain, create everything + resp.json.side_effect = [ + HetznerClientNotFound, # no zone in populate + HetznerClientNotFound, # no zone during apply + {"zone": {"id": "string"}} + ] + plan = provider.plan(self.expected) + + # No root NS, no ignored, no excluded, no unsupported + n = len(self.expected.records) - 9 + self.assertEquals(n, len(plan.changes)) + self.assertEquals(n, provider.apply(plan)) + self.assertFalse(plan.exists) From 716d06819611b917d0d6c81c5138eb4f5b5d4809 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Tue, 27 Apr 2021 06:44:09 -0700 Subject: [PATCH 35/92] Backwards compat for plan overrides, 100% manager coverage, singular processor module name --- octodns/manager.py | 14 +- octodns/processor/__init__.py | 6 + .../__init__.py => processor/base.py} | 0 octodns/{processors => processor}/filters.py | 0 .../{processors => processor}/ownership.py | 0 tests/config/processors-missing-class.yaml | 23 ++++ tests/config/processors-wants-config.yaml | 25 ++++ tests/config/processors.yaml | 33 +++++ tests/helpers.py | 30 ++++ tests/test_octodns_manager.py | 129 +++++++++++++++++- tests/test_octodns_provider_base.py | 67 ++++++++- 11 files changed, 314 insertions(+), 13 deletions(-) create mode 100644 octodns/processor/__init__.py rename octodns/{processors/__init__.py => processor/base.py} (100%) rename octodns/{processors => processor}/filters.py (100%) rename octodns/{processors => processor}/ownership.py (100%) create mode 100644 tests/config/processors-missing-class.yaml create mode 100644 tests/config/processors-wants-config.yaml create mode 100644 tests/config/processors.yaml diff --git a/octodns/manager.py b/octodns/manager.py index f2a2da6..c7173c6 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -263,7 +263,7 @@ class Manager(object): except TypeError as e: if "keyword argument 'lenient'" not in text_type(e): raise - self.log.warn(': provider %s does not accept lenient ' + self.log.warn('provider %s does not accept lenient ' 'param', source.__class__.__name__) source.populate(zone) @@ -281,9 +281,15 @@ class Manager(object): 'value': 'provider={}'.format(target.id) }) zone.add_record(meta, replace=True) - # TODO: if someone has overrriden plan already this will be a - # breaking change so we probably need to try both ways - plan = target.plan(zone, processors=processors) + try: + plan = target.plan(zone, processors=processors) + except TypeError as e: + if "keyword argument 'processors'" not in text_type(e): + raise + self.log.warn('provider.plan %s does not accept processors ' + 'param', target.__class__.__name__) + plan = target.plan(zone) + for processor in processors: plan = processor.process_plan(plan, sources=sources, target=target) diff --git a/octodns/processor/__init__.py b/octodns/processor/__init__.py new file mode 100644 index 0000000..14ccf18 --- /dev/null +++ b/octodns/processor/__init__.py @@ -0,0 +1,6 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals diff --git a/octodns/processors/__init__.py b/octodns/processor/base.py similarity index 100% rename from octodns/processors/__init__.py rename to octodns/processor/base.py diff --git a/octodns/processors/filters.py b/octodns/processor/filters.py similarity index 100% rename from octodns/processors/filters.py rename to octodns/processor/filters.py diff --git a/octodns/processors/ownership.py b/octodns/processor/ownership.py similarity index 100% rename from octodns/processors/ownership.py rename to octodns/processor/ownership.py diff --git a/tests/config/processors-missing-class.yaml b/tests/config/processors-missing-class.yaml new file mode 100644 index 0000000..4594307 --- /dev/null +++ b/tests/config/processors-missing-class.yaml @@ -0,0 +1,23 @@ +providers: + config: + class: octodns.provider.yaml.YamlProvider + directory: tests/config + dump: + class: octodns.provider.yaml.YamlProvider + directory: env/YAML_TMP_DIR + geo: + class: helpers.GeoProvider + nosshfp: + class: helpers.NoSshFpProvider + +processors: + no-class: {} + +zones: + unit.tests.: + processors: + - noop + sources: + - in + targets: + - dump diff --git a/tests/config/processors-wants-config.yaml b/tests/config/processors-wants-config.yaml new file mode 100644 index 0000000..53fc397 --- /dev/null +++ b/tests/config/processors-wants-config.yaml @@ -0,0 +1,25 @@ +providers: + config: + class: octodns.provider.yaml.YamlProvider + directory: tests/config + dump: + class: octodns.provider.yaml.YamlProvider + directory: env/YAML_TMP_DIR + geo: + class: helpers.GeoProvider + nosshfp: + class: helpers.NoSshFpProvider + +processors: + # valid class, but it wants a param and we're not passing it + wants-config: + class: helpers.WantsConfigProcessor + +zones: + unit.tests.: + processors: + - noop + sources: + - in + targets: + - dump diff --git a/tests/config/processors.yaml b/tests/config/processors.yaml new file mode 100644 index 0000000..097024b --- /dev/null +++ b/tests/config/processors.yaml @@ -0,0 +1,33 @@ +providers: + config: + class: octodns.provider.yaml.YamlProvider + directory: tests/config + dump: + class: octodns.provider.yaml.YamlProvider + directory: env/YAML_TMP_DIR + geo: + class: helpers.GeoProvider + nosshfp: + class: helpers.NoSshFpProvider + +processors: + # Just testing config so any processor will do + noop: + class: octodns.processor.base.BaseProcessor + +zones: + unit.tests.: + processors: + - noop + sources: + - config + targets: + - dump + + bad.unit.tests.: + processors: + - doesnt-exist + sources: + - in + targets: + - dump diff --git a/tests/helpers.py b/tests/helpers.py index ff7f7cc..eedfd8b 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -7,6 +7,10 @@ from __future__ import absolute_import, division, print_function, \ from shutil import rmtree from tempfile import mkdtemp +from logging import getLogger + +from octodns.processor.base import BaseProcessor +from octodns.provider.base import BaseProvider class SimpleSource(object): @@ -90,3 +94,29 @@ class TemporaryDirectory(object): rmtree(self.dirname) else: raise Exception(self.dirname) + + +class WantsConfigProcessor(BaseProcessor): + + def __init__(self, name, some_config): + super(WantsConfigProcessor, self).__init__(name) + + +class PlannableProvider(BaseProvider): + log = getLogger('PlannableProvider') + + SUPPORTS_GEO = False + SUPPORTS_DYNAMIC = False + SUPPORTS = set(('A',)) + + def __init__(self, *args, **kwargs): + super(PlannableProvider, self).__init__(*args, **kwargs) + + def populate(self, zone, source=False, target=False, lenient=False): + pass + + def supports(self, record): + return True + + def __repr__(self): + return self.__class__.__name__ diff --git a/tests/test_octodns_manager.py b/tests/test_octodns_manager.py index e4b5211..069bc0b 100644 --- a/tests/test_octodns_manager.py +++ b/tests/test_octodns_manager.py @@ -9,9 +9,10 @@ from os import environ from os.path import dirname, join from six import text_type -from octodns.record import Record from octodns.manager import _AggregateTarget, MainThreadExecutor, Manager, \ ManagerException +from octodns.processor.base import BaseProcessor +from octodns.record import Create, Delete, Record from octodns.yaml import safe_load from octodns.zone import Zone @@ -19,7 +20,7 @@ from mock import MagicMock, patch from unittest import TestCase from helpers import DynamicProvider, GeoProvider, NoSshFpProvider, \ - SimpleProvider, TemporaryDirectory + PlannableProvider, SimpleProvider, TemporaryDirectory config_dir = join(dirname(__file__), 'config') @@ -358,20 +359,48 @@ class TestManager(TestCase): class NoLenient(SimpleProvider): - def populate(self, zone, source=False): + def populate(self, zone): pass # This should be ok, we'll fall back to not passing it manager._populate_and_plan('unit.tests.', [], [NoLenient()], []) - class NoZone(SimpleProvider): + class OtherType(SimpleProvider): - def populate(self, lenient=False): + def populate(self, zone, lenient=False): + raise TypeError('something else') + + # This will blow up, we don't fallback for source + with self.assertRaises(TypeError) as ctx: + manager._populate_and_plan('unit.tests.', [], [OtherType()], + []) + self.assertEquals('something else', text_type(ctx.exception)) + + def test_plan_processors_fallback(self): + with TemporaryDirectory() as tmpdir: + environ['YAML_TMP_DIR'] = tmpdir.dirname + # Only allow a target that doesn't exist + manager = Manager(get_config_filename('simple.yaml')) + + class NoProcessors(SimpleProvider): + + def plan(self, zone): pass + # This should be ok, we'll fall back to not passing it + manager._populate_and_plan('unit.tests.', [], [], + [NoProcessors()]) + + class OtherType(SimpleProvider): + + def plan(self, zone, processors): + raise TypeError('something else') + # This will blow up, we don't fallback for source - with self.assertRaises(TypeError): - manager._populate_and_plan('unit.tests.', [NoZone()], []) + with self.assertRaises(TypeError) as ctx: + manager._populate_and_plan('unit.tests.', [], [], + [OtherType()]) + self.assertEquals('something else', text_type(ctx.exception)) @patch('octodns.manager.Manager._get_named_class') def test_sync_passes_file_handle(self, mock): @@ -391,6 +420,92 @@ class TestManager(TestCase): _, kwargs = plan_output_mock.run.call_args self.assertEqual(fh_mock, kwargs.get('fh')) + def test_processor_config(self): + # Smoke test loading a valid config + manager = Manager(get_config_filename('processors.yaml')) + self.assertEquals(['noop'], list(manager.processors.keys())) + # This zone specifies a valid processor + manager.sync(['unit.tests.']) + + with self.assertRaises(ManagerException) as ctx: + # This zone specifies a non-existant processor + manager.sync(['bad.unit.tests.']) + self.assertTrue('Zone bad.unit.tests., unknown processor: ' + 'doesnt-exist' in text_type(ctx.exception)) + + with self.assertRaises(ManagerException) as ctx: + Manager(get_config_filename('processors-missing-class.yaml')) + self.assertTrue('Processor no-class is missing class' in + text_type(ctx.exception)) + + with self.assertRaises(ManagerException) as ctx: + Manager(get_config_filename('processors-wants-config.yaml')) + self.assertTrue('Incorrect processor config for wants-config' in + text_type(ctx.exception)) + + def test_processors(self): + manager = Manager(get_config_filename('simple.yaml')) + + targets = [PlannableProvider('prov')] + + zone = Zone('unit.tests.', []) + record = Record.new(zone, 'a', { + 'ttl': 30, + 'type': 'A', + 'value': '1.2.3.4', + }) + + # muck with sources + class MockProcessor(BaseProcessor): + + def process_source_zone(self, zone, sources): + zone = self._clone_zone(zone) + zone.add_record(record) + return zone + + mock = MockProcessor('mock') + plans, zone = manager._populate_and_plan('unit.tests.', [mock], [], + targets) + # Our mock was called and added the record + self.assertEquals(record, list(zone.records)[0]) + # We got a create for the thing added to the expected state (source) + self.assertIsInstance(plans[0][1].changes[0], Create) + + # muck with targets + class MockProcessor(BaseProcessor): + + def process_target_zone(self, zone, target): + zone = self._clone_zone(zone) + zone.add_record(record) + return zone + + mock = MockProcessor('mock') + plans, zone = manager._populate_and_plan('unit.tests.', [mock], [], + targets) + # No record added since it's target this time + self.assertFalse(zone.records) + # We got a delete for the thing added to the existing state (target) + self.assertIsInstance(plans[0][1].changes[0], Delete) + + # muck with plans + class MockProcessor(BaseProcessor): + + def process_target_zone(self, zone, target): + zone = self._clone_zone(zone) + zone.add_record(record) + return zone + + def process_plan(self, plans, sources, target): + # get rid of the change + plans.changes.pop(0) + + mock = MockProcessor('mock') + plans, zone = manager._populate_and_plan('unit.tests.', [mock], [], + targets) + # We planned a delete again, but this time removed it from the plan, so + # no plans + self.assertFalse(plans) + class TestMainThreadExecutor(TestCase): diff --git a/tests/test_octodns_provider_base.py b/tests/test_octodns_provider_base.py index f33db0f..4dfce48 100644 --- a/tests/test_octodns_provider_base.py +++ b/tests/test_octodns_provider_base.py @@ -9,9 +9,10 @@ from logging import getLogger from six import text_type from unittest import TestCase -from octodns.record import Create, Delete, Record, Update +from octodns.processor.base import BaseProcessor from octodns.provider.base import BaseProvider from octodns.provider.plan import Plan, UnsafePlan +from octodns.record import Create, Delete, Record, Update from octodns.zone import Zone @@ -21,7 +22,7 @@ class HelperProvider(BaseProvider): SUPPORTS = set(('A',)) id = 'test' - def __init__(self, extra_changes, apply_disabled=False, + def __init__(self, extra_changes=[], apply_disabled=False, include_change_callback=None): self.__extra_changes = extra_changes self.apply_disabled = apply_disabled @@ -43,6 +44,29 @@ class HelperProvider(BaseProvider): pass +class TrickyProcessor(BaseProcessor): + + def __init__(self, name, add_during_process_target_zone): + super(TrickyProcessor, self).__init__(name) + self.add_during_process_target_zone = add_during_process_target_zone + self.reset() + + def reset(self): + self.existing = None + self.target = None + + def process_target_zone(self, existing, target): + self.existing = existing + self.target = target + + new = self._clone_zone(existing) + for record in existing.records: + new.add_record(record) + for record in self.add_during_process_target_zone: + new.add_record(record) + return new + + class TestBaseProvider(TestCase): def test_base_provider(self): @@ -138,6 +162,45 @@ class TestBaseProvider(TestCase): self.assertTrue(plan) self.assertEquals(1, len(plan.changes)) + def test_plan_with_processors(self): + zone = Zone('unit.tests.', []) + + record = Record.new(zone, 'a', { + 'ttl': 30, + 'type': 'A', + 'value': '1.2.3.4', + }) + provider = HelperProvider() + # Processor that adds a record to the zone, which planning will then + # delete since it won't know anything about it + tricky = TrickyProcessor('tricky', [record]) + plan = provider.plan(zone, processors=[tricky]) + self.assertTrue(plan) + self.assertEquals(1, len(plan.changes)) + self.assertIsInstance(plan.changes[0], Delete) + # Called processor stored its params + self.assertTrue(tricky.existing) + self.assertEquals(zone.name, tricky.existing.name) + + # Chain of processors happen one after the other + other = Record.new(zone, 'b', { + 'ttl': 30, + 'type': 'A', + 'value': '5.6.7.8', + }) + # Another processor will add its record, thus 2 deletes + another = TrickyProcessor('tricky', [other]) + plan = provider.plan(zone, processors=[tricky, another]) + self.assertTrue(plan) + self.assertEquals(2, len(plan.changes)) + self.assertIsInstance(plan.changes[0], Delete) + self.assertIsInstance(plan.changes[1], Delete) + # 2nd processor stored its params, and we'll see the record the + # first one added + self.assertTrue(another.existing) + self.assertEquals(zone.name, another.existing.name) + self.assertEquals(1, len(another.existing.records)) + def test_apply(self): ignored = Zone('unit.tests.', []) From f387a61561e3841d77f2040c6302b04b1d391248 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Tue, 27 Apr 2021 07:52:41 -0700 Subject: [PATCH 36/92] Allow/reject use sets and now have tests --- octodns/processor/{filters.py => filter.py} | 6 +- tests/test_octodns_processor_filter.py | 92 +++++++++++++++++++++ 2 files changed, 95 insertions(+), 3 deletions(-) rename octodns/processor/{filters.py => filter.py} (89%) create mode 100644 tests/test_octodns_processor_filter.py diff --git a/octodns/processor/filters.py b/octodns/processor/filter.py similarity index 89% rename from octodns/processor/filters.py rename to octodns/processor/filter.py index 2c5f87f..369e987 100644 --- a/octodns/processor/filters.py +++ b/octodns/processor/filter.py @@ -5,14 +5,14 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -from . import BaseProcessor +from .base import BaseProcessor class TypeAllowlistFilter(BaseProcessor): def __init__(self, name, allowlist): super(TypeAllowlistFilter, self).__init__(name) - self.allowlist = allowlist + self.allowlist = set(allowlist) def _process(self, zone, *args, **kwargs): ret = self._clone_zone(zone) @@ -30,7 +30,7 @@ class TypeRejectlistFilter(BaseProcessor): def __init__(self, name, rejectlist): super(TypeRejectlistFilter, self).__init__(name) - self.rejectlist = rejectlist + self.rejectlist = set(rejectlist) def _process(self, zone, *args, **kwargs): ret = self._clone_zone(zone) diff --git a/tests/test_octodns_processor_filter.py b/tests/test_octodns_processor_filter.py new file mode 100644 index 0000000..1febfbd --- /dev/null +++ b/tests/test_octodns_processor_filter.py @@ -0,0 +1,92 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from logging import getLogger +from six import StringIO, text_type +from unittest import TestCase + +from octodns.processor.filter import TypeAllowlistFilter, TypeRejectlistFilter +from octodns.record import Record +from octodns.zone import Zone + +zone = Zone('unit.tests.', []) +for record in [ + Record.new(zone, 'a', { + 'ttl': 30, + 'type': 'A', + 'value': '1.2.3.4', + }), + Record.new(zone, 'aaaa', { + 'ttl': 30, + 'type': 'AAAA', + 'value': '::1', + }), + Record.new(zone, 'txt', { + 'ttl': 30, + 'type': 'TXT', + 'value': 'Hello World!', + }), + Record.new(zone, 'a2', { + 'ttl': 30, + 'type': 'A', + 'value': '2.3.4.5', + }), + Record.new(zone, 'txt2', { + 'ttl': 30, + 'type': 'TXT', + 'value': 'That will do', + }), +]: + zone.add_record(record) + + +class TestTypeAllowListFilter(TestCase): + + def test_basics(self): + filter_a = TypeAllowlistFilter('only-a', set(('A'))) + + got = filter_a.process_source_zone(zone) + self.assertEquals(['a', 'a2'], sorted([r.name for r in got.records])) + + filter_aaaa = TypeAllowlistFilter('only-aaaa', ('AAAA',)) + got = filter_aaaa.process_source_zone(zone) + self.assertEquals(['aaaa'], sorted([r.name for r in got.records])) + + filter_txt = TypeAllowlistFilter('only-txt', ['TXT']) + got = filter_txt.process_source_zone(zone) + self.assertEquals(['txt', 'txt2'], + sorted([r.name for r in got.records])) + + filter_a_aaaa = TypeAllowlistFilter('only-aaaa', set(('A', 'AAAA'))) + got = filter_a_aaaa.process_source_zone(zone) + self.assertEquals(['a', 'a2', 'aaaa'], + sorted([r.name for r in got.records])) + + +class TestTypeRejectListFilter(TestCase): + + def test_basics(self): + filter_a = TypeRejectlistFilter('not-a', set(('A'))) + + got = filter_a.process_source_zone(zone) + self.assertEquals(['aaaa', 'txt', 'txt2'], + sorted([r.name for r in got.records])) + + filter_aaaa = TypeRejectlistFilter('not-aaaa', ('AAAA',)) + got = filter_aaaa.process_source_zone(zone) + self.assertEquals(['a', 'a2', 'txt', 'txt2'], + sorted([r.name for r in got.records])) + + filter_txt = TypeRejectlistFilter('not-txt', ['TXT']) + got = filter_txt.process_source_zone(zone) + self.assertEquals(['a', 'a2', 'aaaa'], + sorted([r.name for r in got.records])) + + filter_a_aaaa = TypeRejectlistFilter('not-a-aaaa', set(('A', 'AAAA'))) + got = filter_a_aaaa.process_source_zone(zone) + self.assertEquals(['txt', 'txt2'], + sorted([r.name for r in got.records])) From 540fb27263a3b16524c125340c580efd59ed61ec Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Tue, 27 Apr 2021 09:25:24 -0700 Subject: [PATCH 37/92] Clean up and test OwnershipProcessor --- octodns/processor/ownership.py | 14 +------------- tests/test_octodns_processor_filter.py | 10 ++++------ 2 files changed, 5 insertions(+), 19 deletions(-) diff --git a/octodns/processor/ownership.py b/octodns/processor/ownership.py index 67bbb9a..172f349 100644 --- a/octodns/processor/ownership.py +++ b/octodns/processor/ownership.py @@ -6,12 +6,11 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals from collections import defaultdict -from pprint import pprint from ..provider.plan import Plan from ..record import Record -from . import BaseProcessor +from .base import BaseProcessor # Mark anything octoDNS is managing that way it can know it's safe to modify or @@ -71,8 +70,6 @@ class OwnershipProcessor(BaseProcessor): name = '' owned[name][_type.upper()] = True - pprint(dict(owned)) - # Cases: # - Configured in source # - We'll fully CRU/manage it adding ownership TXT, @@ -83,17 +80,10 @@ class OwnershipProcessor(BaseProcessor): # - Special records like octodns-meta # - Should be left alone and should not have ownerthis TXTs - pprint(plan.changes) - filtered_changes = [] for change in plan.changes: record = change.record - pprint([change, - not self._is_ownership(record), - record._type not in owned[record.name], - record.name != 'octodns-meta']) - if not self._is_ownership(record) and \ record._type not in owned[record.name] and \ record.name != 'octodns-meta': @@ -105,8 +95,6 @@ class OwnershipProcessor(BaseProcessor): # change is we should do filtered_changes.append(change) - pprint(filtered_changes) - if plan.changes != filtered_changes: return Plan(plan.existing, plan.desired, filtered_changes, plan.exists, plan.update_pcent_threshold, diff --git a/tests/test_octodns_processor_filter.py b/tests/test_octodns_processor_filter.py index 1febfbd..f421676 100644 --- a/tests/test_octodns_processor_filter.py +++ b/tests/test_octodns_processor_filter.py @@ -5,8 +5,6 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -from logging import getLogger -from six import StringIO, text_type from unittest import TestCase from octodns.processor.filter import TypeAllowlistFilter, TypeRejectlistFilter @@ -57,12 +55,12 @@ class TestTypeAllowListFilter(TestCase): self.assertEquals(['aaaa'], sorted([r.name for r in got.records])) filter_txt = TypeAllowlistFilter('only-txt', ['TXT']) - got = filter_txt.process_source_zone(zone) + got = filter_txt.process_target_zone(zone) self.assertEquals(['txt', 'txt2'], sorted([r.name for r in got.records])) filter_a_aaaa = TypeAllowlistFilter('only-aaaa', set(('A', 'AAAA'))) - got = filter_a_aaaa.process_source_zone(zone) + got = filter_a_aaaa.process_target_zone(zone) self.assertEquals(['a', 'a2', 'aaaa'], sorted([r.name for r in got.records])) @@ -82,11 +80,11 @@ class TestTypeRejectListFilter(TestCase): sorted([r.name for r in got.records])) filter_txt = TypeRejectlistFilter('not-txt', ['TXT']) - got = filter_txt.process_source_zone(zone) + got = filter_txt.process_target_zone(zone) self.assertEquals(['a', 'a2', 'aaaa'], sorted([r.name for r in got.records])) filter_a_aaaa = TypeRejectlistFilter('not-a-aaaa', set(('A', 'AAAA'))) - got = filter_a_aaaa.process_source_zone(zone) + got = filter_a_aaaa.process_target_zone(zone) self.assertEquals(['txt', 'txt2'], sorted([r.name for r in got.records])) From f8659bec0451d7b61d4c6135d3a6a87610026fff Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Tue, 27 Apr 2021 09:30:34 -0700 Subject: [PATCH 38/92] Helps if you add the file --- tests/test_octodns_processor_ownership.py | 146 ++++++++++++++++++++++ 1 file changed, 146 insertions(+) create mode 100644 tests/test_octodns_processor_ownership.py diff --git a/tests/test_octodns_processor_ownership.py b/tests/test_octodns_processor_ownership.py new file mode 100644 index 0000000..959f4c2 --- /dev/null +++ b/tests/test_octodns_processor_ownership.py @@ -0,0 +1,146 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from unittest import TestCase + +from octodns.processor.ownership import OwnershipProcessor +from octodns.record import Delete, Record +from octodns.zone import Zone + +from helpers import PlannableProvider + + +zone = Zone('unit.tests.', []) +records = {} +for record in [ + Record.new(zone, '', { + 'ttl': 30, + 'type': 'A', + 'values': [ + '1.2.3.4', + '5.6.7.8', + ], + }), + Record.new(zone, 'the-a', { + 'ttl': 30, + 'type': 'A', + 'value': '1.2.3.4', + }), + Record.new(zone, 'the-aaaa', { + 'ttl': 30, + 'type': 'AAAA', + 'value': '::1', + }), + Record.new(zone, 'the-txt', { + 'ttl': 30, + 'type': 'TXT', + 'value': 'Hello World!', + }), + Record.new(zone, '*', { + 'ttl': 30, + 'type': 'A', + 'value': '4.3.2.1', + }), +]: + records[record.name] = record + zone.add_record(record) + + +class TestOwnershipProcessor(TestCase): + + def test_process_source_zone(self): + ownership = OwnershipProcessor('ownership') + + got = ownership.process_source_zone(zone) + self.assertEquals([ + '', + '*', + '_owner.a', + '_owner.a._wildcard', + '_owner.a.the-a', + '_owner.aaaa.the-aaaa', + '_owner.txt.the-txt', + 'the-a', + 'the-aaaa', + 'the-txt', + ], sorted([r.name for r in got.records])) + + found = False + for record in got.records: + if record.name.startswith(ownership.txt_name): + self.assertEquals([ownership.txt_value], record.values) + # test _is_ownership while we're in here + self.assertTrue(ownership._is_ownership(record)) + found = True + else: + self.assertFalse(ownership._is_ownership(record)) + self.assertTrue(found) + + def test_process_plan(self): + ownership = OwnershipProcessor('ownership') + provider = PlannableProvider('helper') + + # No plan, is a quick noop + self.assertFalse(ownership.process_plan(None)) + + # Nothing exists create both records and ownership + ownership_added = ownership.process_source_zone(zone) + plan = provider.plan(ownership_added) + self.assertTrue(plan) + # Double the number of records + self.assertEquals(len(records) * 2, len(plan.changes)) + # Now process the plan, shouldn't make any changes, we're creating + # everything + got = ownership.process_plan(plan) + self.assertTrue(got) + self.assertEquals(len(records) * 2, len(got.changes)) + + # Something extra exists and doesn't have ownership TXT, leave it + # alone, we don't own it. + extra_a = Record.new(zone, 'extra-a', { + 'ttl': 30, + 'type': 'A', + 'value': '4.4.4.4', + }) + plan.existing.add_record(extra_a) + # If we'd done a "real" plan we'd have a delete for the extra thing. + plan.changes.append(Delete(extra_a)) + # Process the plan, shouldn't make any changes since the extra bit is + # something we don't own + got = ownership.process_plan(plan) + self.assertTrue(got) + self.assertEquals(len(records) * 2, len(got.changes)) + + # Something extra exists and does have an ownership record so we will + # delete it... + copy = Zone('unit.tests.', []) + for record in records.values(): + if record.name != 'the-a': + copy.add_record(record) + # New ownership, without the `the-a` + ownership_added = ownership.process_source_zone(copy) + self.assertEquals(len(records) * 2 - 2, len(ownership_added.records)) + plan = provider.plan(ownership_added) + # Fake the extra existing by adding the record, its ownership, and the + # two delete changes. + the_a = records['the-a'] + plan.existing.add_record(the_a) + name = '{}.a.the-a'.format(ownership.txt_name) + the_a_ownership = Record.new(zone, name, { + 'ttl': 30, + 'type': 'TXT', + 'value': ownership.txt_value, + }) + plan.existing.add_record(the_a_ownership) + plan.changes.append(Delete(the_a)) + plan.changes.append(Delete(the_a_ownership)) + # Finally process the plan, should be a noop and we should get the same + # plan out, meaning the planned deletes were allowed to happen. + got = ownership.process_plan(plan) + self.assertTrue(got) + self.assertEquals(plan, got) + self.assertEquals(len(plan.changes), len(got.changes)) From a0c4e9ecd76af5f2a830b8cb430468b147e070f3 Mon Sep 17 00:00:00 2001 From: Ricard Bejarano Date: Tue, 27 Apr 2021 19:56:39 +0200 Subject: [PATCH 39/92] simplified HetznerClient by removing unused pagination handling --- octodns/provider/hetzner.py | 24 +++--------------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/octodns/provider/hetzner.py b/octodns/provider/hetzner.py index 0be01ea..d396e18 100644 --- a/octodns/provider/hetzner.py +++ b/octodns/provider/hetzner.py @@ -30,10 +30,6 @@ class HetznerClientUnauthorized(HetznerClientException): class HetznerClient(object): - ''' - Hetzner DNS Public API v1 client class. - ''' - BASE_URL = 'https://dns.hetzner.com/api/v1' def __init__(self, token): @@ -54,22 +50,9 @@ class HetznerClient(object): def _do_json(self, method, path, params=None, data=None): return self._do(method, path, params, data).json() - def _do_json_paginate(self, method, path, key, params=None, data=None, - per_page=100): - items = [] - params = {**{'page': 1, 'per_page': per_page}, **params} - while True: - response = self._do_json(method, path, params, data) - items += response[key] - if response['meta']['pagination']['page'] >= \ - response['meta']['pagination']['last_page']: - break - params['page'] += 1 - return items - - def zones_get(self, name=None, search_name=None): - params = {'name': name, 'search_name': search_name} - return self._do_json_paginate('GET', '/zones', 'zones', params=params) + def zone_get(self, name): + params = {'name': name} + return self._do_json('GET', '/zones', params)['zones'][0] def zone_create(self, name, ttl=None): data = {'name': name, 'ttl': ttl} @@ -77,7 +60,6 @@ class HetznerClient(object): def zone_records_get(self, zone_id): params = {'zone_id': zone_id} - # No need to handle pagination as it returns all records by default. records = self._do_json('GET', '/records', params=params)['records'] for record in records: if record['name'] == '@': From d4c6836d0bb8b0d8bd6b3fb58ba9abb2378f706a Mon Sep 17 00:00:00 2001 From: Ricard Bejarano Date: Tue, 27 Apr 2021 19:58:07 +0200 Subject: [PATCH 40/92] implemented HetznerProvider.zone_metadata as the equivalent to zone_records for zone metadata --- octodns/provider/hetzner.py | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/octodns/provider/hetzner.py b/octodns/provider/hetzner.py index d396e18..53914f5 100644 --- a/octodns/provider/hetzner.py +++ b/octodns/provider/hetzner.py @@ -96,16 +96,28 @@ class HetznerProvider(BaseProvider): self._zone_records = {} self._zone_metadata = {} + self._zone_name_to_id = {} def _append_dot(self, value): if value == '@' or value[-1] == '.': return value return '{}.'.format(value) + def zone_metadata(self, zone_id=None, zone_name=None): + if zone_name is not None: + if zone_name in self._zone_name_to_id: + zone_id = self._zone_name_to_id[zone_name] + else: + zone = self._client.zone_get(name=zone_name[:-1]) + zone_id = zone['id'] + self._zone_name_to_id[zone_name] = zone_id + self._zone_metadata[zone_id] = zone + + return self._zone_metadata[zone_id] + def _record_ttl(self, record): - if 'ttl' in record: - return record['ttl'] - return self._zone_metadata[record['zone_id']]['ttl'] + default_ttl = self.zone_metadata(zone_id=record['zone_id'])['ttl'] + return record['ttl'] if 'ttl' in record else default_ttl def _data_for_multiple(self, _type, records): values = [record['value'].replace(';', '\\;') for record in records] @@ -195,10 +207,9 @@ class HetznerProvider(BaseProvider): def zone_records(self, zone): if zone.name not in self._zone_records: try: - zone_metadata = self._client.zones_get(name=zone.name[:-1])[0] - self._zone_metadata[zone_metadata['id']] = zone_metadata + zone_id = self.zone_metadata(zone_name=zone.name)['id'] self._zone_records[zone.name] = \ - self._client.zone_records_get(zone_metadata['id']) + self._client.zone_records_get(zone_id) except HetznerClientNotFound: return [] @@ -314,12 +325,11 @@ class HetznerProvider(BaseProvider): self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name, len(changes)) - zone_name = desired.name[:-1] try: - zone_id = self._client.zones_get(name=zone_name)[0]['id'] + zone_id = self.zone_metadata(zone_name=desired.name)['id'] except HetznerClientNotFound: self.log.debug('_apply: no matching zone, creating domain') - zone_id = self._client.zone_create(zone_name)['id'] + zone_id = self._client.zone_create(desired.name[:-1])['id'] for change in changes: class_name = change.__class__.__name__ From 6bf24d867894c8df992b8491762a9217e563d7d5 Mon Sep 17 00:00:00 2001 From: Ricard Bejarano Date: Tue, 27 Apr 2021 19:59:00 +0200 Subject: [PATCH 41/92] implemented TestHetznerProvider.test_apply --- tests/test_octodns_provider_hetzner.py | 230 ++++++++++++++++++++++++- 1 file changed, 229 insertions(+), 1 deletion(-) diff --git a/tests/test_octodns_provider_hetzner.py b/tests/test_octodns_provider_hetzner.py index 0c18731..02da32e 100644 --- a/tests/test_octodns_provider_hetzner.py +++ b/tests/test_octodns_provider_hetzner.py @@ -6,13 +6,14 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -from mock import Mock +from mock import Mock, call from os.path import dirname, join from requests import HTTPError from requests_mock import ANY, mock as requests_mock from six import text_type from unittest import TestCase +from octodns.record import Record from octodns.provider.hetzner import HetznerClientNotFound, \ HetznerProvider from octodns.provider.yaml import YamlProvider @@ -105,3 +106,230 @@ class TestHetznerProvider(TestCase): self.assertEquals(n, len(plan.changes)) self.assertEquals(n, provider.apply(plan)) self.assertFalse(plan.exists) + + provider._client._do.assert_has_calls([ + # created the zone + call('POST', '/zones', None, { + 'name': 'unit.tests', + 'ttl': None, + }), + # created all the records with their expected data + call('POST', '/records', data={ + 'name': '@', + 'ttl': 300, + 'type': 'A', + 'value': '1.2.3.4', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': '@', + 'ttl': 300, + 'type': 'A', + 'value': '1.2.3.5', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': '@', + 'ttl': 3600, + 'type': 'CAA', + 'value': '0 issue "ca.unit.tests"', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': '_imap._tcp', + 'ttl': 600, + 'type': 'SRV', + 'value': '0 0 0 .', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': '_pop3._tcp', + 'ttl': 600, + 'type': 'SRV', + 'value': '0 0 0 .', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': '_srv._tcp', + 'ttl': 600, + 'type': 'SRV', + 'value': '10 20 30 foo-1.unit.tests.', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': '_srv._tcp', + 'ttl': 600, + 'type': 'SRV', + 'value': '12 20 30 foo-2.unit.tests.', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': 'aaaa', + 'ttl': 600, + 'type': 'AAAA', + 'value': '2601:644:500:e210:62f8:1dff:feb8:947a', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': 'cname', + 'ttl': 300, + 'type': 'CNAME', + 'value': 'unit.tests.', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': 'included', + 'ttl': 3600, + 'type': 'CNAME', + 'value': 'unit.tests.', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': 'mx', + 'ttl': 300, + 'type': 'MX', + 'value': '10 smtp-4.unit.tests.', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': 'mx', + 'ttl': 300, + 'type': 'MX', + 'value': '20 smtp-2.unit.tests.', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': 'mx', + 'ttl': 300, + 'type': 'MX', + 'value': '30 smtp-3.unit.tests.', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': 'mx', + 'ttl': 300, + 'type': 'MX', + 'value': '40 smtp-1.unit.tests.', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': 'sub', + 'ttl': 3600, + 'type': 'NS', + 'value': '6.2.3.4.', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': 'sub', + 'ttl': 3600, + 'type': 'NS', + 'value': '7.2.3.4.', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': 'txt', + 'ttl': 600, + 'type': 'TXT', + 'value': 'Bah bah black sheep', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': 'txt', + 'ttl': 600, + 'type': 'TXT', + 'value': 'have you any wool.', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': 'txt', + 'ttl': 600, + 'type': 'TXT', + 'value': 'v=DKIM1;k=rsa;s=email;h=sha256;' + 'p=A/kinda+of/long/string+with+numb3rs', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': 'www', + 'ttl': 300, + 'type': 'A', + 'value': '2.2.3.6', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': 'www.sub', + 'ttl': 300, + 'type': 'A', + 'value': '2.2.3.6', + 'zone_id': 'unit.tests', + }), + ]) + self.assertEquals(24, provider._client._do.call_count) + + provider._client._do.reset_mock() + + # delete 1 and update 1 + provider._client.zone_get = Mock(return_value={ + 'id': 'unit.tests', + 'name': 'unit.tests', + 'ttl': 3600, + }) + provider._client.zone_records_get = Mock(return_value=[ + { + 'type': 'A', + 'id': 'one', + 'created': '0000-00-00T00:00:00Z', + 'modified': '0000-00-00T00:00:00Z', + 'zone_id': 'unit.tests', + 'name': 'www', + 'value': '1.2.3.4', + 'ttl': 300, + }, + { + 'type': 'A', + 'id': 'two', + 'created': '0000-00-00T00:00:00Z', + 'modified': '0000-00-00T00:00:00Z', + 'zone_id': 'unit.tests', + 'name': 'www', + 'value': '2.2.3.4', + 'ttl': 300, + }, + { + 'type': 'A', + 'id': 'three', + 'created': '0000-00-00T00:00:00Z', + 'modified': '0000-00-00T00:00:00Z', + 'zone_id': 'unit.tests', + 'name': 'ttl', + 'value': '3.2.3.4', + 'ttl': 600, + }, + ]) + + # Domain exists, we don't care about return + resp.json.side_effect = ['{}'] + + wanted = Zone('unit.tests.', []) + wanted.add_record(Record.new(wanted, 'ttl', { + 'ttl': 300, + 'type': 'A', + 'value': '3.2.3.4', + })) + + plan = provider.plan(wanted) + self.assertTrue(plan.exists) + self.assertEquals(2, len(plan.changes)) + self.assertEquals(2, provider.apply(plan)) + # recreate for update, and delete for the 2 parts of the other + provider._client._do.assert_has_calls([ + call('POST', '/records', data={ + 'name': 'ttl', + 'ttl': 300, + 'type': 'A', + 'value': '3.2.3.4', + 'zone_id': 'unit.tests', + }), + call('DELETE', '/records/one'), + call('DELETE', '/records/two'), + call('DELETE', '/records/three'), + ], any_order=True) From b3c394a5e0056e5c0f4382e89e7854b4e02c02e4 Mon Sep 17 00:00:00 2001 From: Ricard Bejarano Date: Tue, 27 Apr 2021 19:59:26 +0200 Subject: [PATCH 42/92] minor correctness tweaks --- tests/test_octodns_provider_hetzner.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/test_octodns_provider_hetzner.py b/tests/test_octodns_provider_hetzner.py index 02da32e..4167944 100644 --- a/tests/test_octodns_provider_hetzner.py +++ b/tests/test_octodns_provider_hetzner.py @@ -66,7 +66,7 @@ class TestHetznerProvider(TestCase): # No diffs == no changes with requests_mock() as mock: - base = 'https://dns.hetzner.com/api/v1' + base = provider._client.BASE_URL with open('tests/fixtures/hetzner-zones.json') as fh: mock.get('{}/zones'.format(base), text=fh.read()) with open('tests/fixtures/hetzner-records.json') as fh: @@ -93,11 +93,17 @@ class TestHetznerProvider(TestCase): resp.json = Mock() provider._client._do = Mock(return_value=resp) + domain_after_creation = {'zone': { + 'id': 'unit.tests', + 'name': 'unit.tests', + 'ttl': 3600, + }} + # non-existent domain, create everything resp.json.side_effect = [ HetznerClientNotFound, # no zone in populate HetznerClientNotFound, # no zone during apply - {"zone": {"id": "string"}} + domain_after_creation, ] plan = provider.plan(self.expected) From a564270ef5cf8030e55e3a2434ad2e7b51c9c370 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Thu, 29 Apr 2021 09:12:35 -0700 Subject: [PATCH 43/92] Add a blurb on pip installing a sha --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index cd9b884..5eaca41 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,14 @@ $ pip install octodns $ mkdir config ``` +#### Installing a specific commit SHA + +If you'd like to install a version that has not yet been released in a repetable/safe manner you can do the following. In general octoDNS is fairly stable inbetween releases thanks to the plan and apply process, but care should be taken regardless. + +```shell +$ pip install -e git+https://git@github.com/github/octodns.git@#egg=octodns +``` + ### Config We start by creating a config file to tell OctoDNS about our providers and the zone(s) we want it to manage. Below we're setting up a `YamlProvider` to source records from our config files and both a `Route53Provider` and `DynProvider` to serve as the targets for those records. You can have any number of zones set up and any number of sources of data and targets for records for each. You can also have multiple config files, that make use of separate accounts and each manage a distinct set of zones. A good example of this this might be `./config/staging.yaml` & `./config/production.yaml`. We'll focus on a `config/production.yaml`. From ad37b997739fc76c370aba1983f156bd8e32c03a Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Thu, 29 Apr 2021 09:12:38 -0700 Subject: [PATCH 44/92] Add shell code block types --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 5eaca41..4ab3c6b 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ It is similar to [Netflix/denominator](https://github.com/Netflix/denominator). Running through the following commands will install the latest release of OctoDNS and set up a place for your config files to live. To determine if provider specific requirements are necessary see the [Supported providers table](#supported-providers) below. -``` +```shell $ mkdir dns $ cd dns $ virtualenv env @@ -121,7 +121,7 @@ Further information can be found in [Records Documentation](/docs/records.md). We're ready to do a dry-run with our new setup to see what changes it would make. Since we're pretending here we'll act like there are no existing records for `example.com.` in our accounts on either provider. -``` +```shell $ octodns-sync --config-file=./config/production.yaml ... ******************************************************************************** @@ -145,7 +145,7 @@ There will be other logging information presented on the screen, but successful Now it's time to tell OctoDNS to make things happen. We'll invoke it again with the same options and add a `--doit` on the end to tell it this time we actually want it to try and make the specified changes. -``` +```shell $ octodns-sync --config-file=./config/production.yaml --doit ... ``` @@ -176,7 +176,7 @@ If that goes smoothly, you again see the expected changes, and verify them with Very few situations will involve starting with a blank slate which is why there's tooling built in to pull existing data out of providers into a matching config file. -``` +```shell $ octodns-dump --config-file=config/production.yaml --output-dir=tmp/ example.com. route53 2017-03-15T13:33:34 INFO Manager __init__: config_file=tmp/production.yaml 2017-03-15T13:33:34 INFO Manager dump: zone=example.com., sources=('route53',) From 58c7f431e86ce441767ab1f5a7f6dee7837465e8 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 30 Apr 2021 14:43:23 -0700 Subject: [PATCH 45/92] v0.9.12 version bump and CHANGELOG update --- CHANGELOG.md | 25 ++++++++++++++++++++++++- octodns/__init__.py | 2 +- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fb68e25..1d16544 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,29 @@ +## v0.9.12 - 2021-04-30 - Enough time has passed + +#### Noteworthy changes + +* Formal Python 2.7 support removed, deps and tooling were becoming + unmaintainable +* octodns/octodns move, from github/octodns, more to come + +#### Stuff + +* ZoneFileSource supports specifying an extension & no files end in . to better + support Windows +* LOC record type support added +* Support for pre-release versions of PowerDNS +* PowerDNS delete before create which allows A <-> CNAME etc. +* Improved validation of fqdn's in ALIAS, CNAME, etc. +* Transip support for NS records +* Support for sending plan output to a file +* DNSimple uses zone api rather than domain to support non-registered stuff, + e.g. reverse zones. +* Support for fallback-only dynamic pools and related fixes to NS1 provider +* Initial Hetzner provider + ## v0.9.11 - 2020-11-05 - We still don't know edition -#### Noteworthy changtes +#### Noteworthy changes * ALIAS records only allowed at the root of zones - see `leient` in record docs for work-arounds if you really need them. diff --git a/octodns/__init__.py b/octodns/__init__.py index 3fcdaa1..1885d42 100644 --- a/octodns/__init__.py +++ b/octodns/__init__.py @@ -3,4 +3,4 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -__VERSION__ = '0.9.11' +__VERSION__ = '0.9.12' From 40569945d2bc0fb89735fd97e3cdbbd00b1379c1 Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Fri, 30 Apr 2021 16:53:37 -0700 Subject: [PATCH 46/92] Add support for dynamic CNAME records in NS1 --- octodns/provider/ns1.py | 24 +++++- tests/test_octodns_provider_ns1.py | 118 +++++++++++++++++++++++++++++ 2 files changed, 138 insertions(+), 4 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 6cea185..4da0d33 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -588,16 +588,22 @@ class Ns1Provider(BaseProvider): rules = list(rules.values()) rules.sort(key=lambda r: (r['_order'], r['pool'])) - return { + data = { 'dynamic': { 'pools': pools, 'rules': rules, }, 'ttl': record['ttl'], 'type': _type, - 'values': sorted(default), } + if _type == 'CNAME': + data['value'] = default[0] + else: + data['values'] = default + + return data + def _data_for_A(self, _type, record): if record.get('tier', 1) > 1: # Advanced record, see if it's first answer has a note @@ -646,6 +652,10 @@ class Ns1Provider(BaseProvider): } def _data_for_CNAME(self, _type, record): + if record.get('tier', 1) > 1: + # Advanced dynamic record + return self._data_for_dynamic_A(_type, record) + try: value = record['short_answers'][0] except IndexError: @@ -1114,10 +1124,14 @@ class Ns1Provider(BaseProvider): 'feed_id': feed_id, }) + if record._type == 'CNAME': + default_values = [record.value] + else: + default_values = record.values default_answers = [{ 'answer': [v], 'weight': 1, - } for v in record.values] + } for v in default_values] # Build our list of answers # The regions dictionary built above already has the required pool @@ -1171,8 +1185,10 @@ class Ns1Provider(BaseProvider): values = [(v.flags, v.tag, v.value) for v in record.values] return {'answers': values, 'ttl': record.ttl}, None - # TODO: dynamic CNAME support def _params_for_CNAME(self, record): + if getattr(record, 'dynamic', False): + return self._params_for_dynamic_A(record) + return {'answers': [record.value], 'ttl': record.ttl}, None _params_for_ALIAS = _params_for_CNAME diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index 00b068b..c16a573 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -1270,6 +1270,63 @@ class TestNs1ProviderDynamic(TestCase): params, _ = provider._params_for_geo_A(record) self.assertEquals([], params['filters']) + @patch('octodns.provider.ns1.Ns1Provider._monitor_sync') + @patch('octodns.provider.ns1.Ns1Provider._monitors_for') + def test_params_for_dynamic_CNAME(self, monitors_for_mock, + monitor_sync_mock): + provider = Ns1Provider('test', 'api-key') + + # pre-fill caches to avoid extranious calls (things we're testing + # elsewhere) + provider._client._datasource_id = 'foo' + provider._client._feeds_for_monitors = { + 'mon-id': 'feed-id', + } + + # provider._params_for_A() calls provider._monitors_for() and + # provider._monitor_sync(). Mock their return values so that we don't + # make NS1 API calls during tests + monitors_for_mock.reset_mock() + monitor_sync_mock.reset_mock() + monitors_for_mock.side_effect = [{ + 'iad.unit.tests.': 'mid-1', + }] + monitor_sync_mock.side_effect = [ + ('mid-1', 'fid-1'), + ] + + record = Record.new(self.zone, 'foo', { + 'dynamic': { + 'pools': { + 'iad': { + 'values': [{ + 'value': 'iad.unit.tests.', + }], + }, + }, + 'rules': [{ + 'pool': 'iad', + }], + }, + 'octodns': { + 'healthcheck': { + 'host': 'send.me', + 'path': '/_ping', + 'port': 80, + 'protocol': 'HTTP', + } + }, + 'ttl': 32, + 'type': 'CNAME', + 'value': 'value.unit.tests.', + 'meta': {}, + }) + ret, _ = provider._params_for_CNAME(record) + + # Check if the default value was correctly read and populated + # All other dynamic record test cases are covered by dynamic_A tests + self.assertEquals(ret['answers'][1]['answer'][0], 'value.unit.tests.') + def test_data_for_dynamic_A(self): provider = Ns1Provider('test', 'api-key') @@ -1471,6 +1528,67 @@ class TestNs1ProviderDynamic(TestCase): self.assertTrue( 'OC-{}'.format(c) in data4['dynamic']['rules'][0]['geos']) + def test_data_for_dynamic_CNAME(self): + provider = Ns1Provider('test', 'api-key') + + # Test out a small setup that just covers default value validation + # Everything else is same as dynamic A whose tests will cover all + # other options and test cases + # Not testing for geo/region specific cases + filters = provider._get_updated_filter_chain(False, False) + catchall_pool_name = 'iad__catchall' + ns1_record = { + 'answers': [{ + 'answer': ['2.3.4.5.unit.tests.'], + 'meta': { + 'priority': 1, + 'weight': 12, + 'note': 'from:{}'.format(catchall_pool_name), + }, + 'region': catchall_pool_name, + }, { + 'answer': ['1.2.3.4.unit.tests.'], + 'meta': { + 'priority': 2, + 'note': 'from:--default--', + }, + 'region': catchall_pool_name, + }], + 'domain': 'foo.unit.tests', + 'filters': filters, + 'regions': { + catchall_pool_name: { + 'meta': { + 'note': 'rule-order:1', + }, + } + }, + 'tier': 3, + 'ttl': 42, + 'type': 'CNAME', + } + data = provider._data_for_CNAME('CNAME', ns1_record) + self.assertEquals({ + 'dynamic': { + 'pools': { + 'iad': { + 'fallback': None, + 'values': [{ + 'value': '2.3.4.5.unit.tests.', + 'weight': 12, + }], + }, + }, + 'rules': [{ + '_order': '1', + 'pool': 'iad', + }], + }, + 'ttl': 42, + 'type': 'CNAME', + 'value': '1.2.3.4.unit.tests.', + }, data) + @patch('ns1.rest.records.Records.retrieve') @patch('ns1.rest.zones.Zones.retrieve') @patch('octodns.provider.ns1.Ns1Provider._monitors_for') From 15eb23eeb672f743cb21724eddb2fa394c931aea Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Fri, 30 Apr 2021 20:26:11 -0700 Subject: [PATCH 47/92] Trim trailing dot from CNAME answers for NS1 monitors --- octodns/provider/ns1.py | 11 +++- tests/test_octodns_provider_ns1.py | 87 +++++++++++++++++++----------- 2 files changed, 64 insertions(+), 34 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 4da0d33..6ade3b9 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -832,6 +832,10 @@ class Ns1Provider(BaseProvider): # This monitor does not belong to this record config = monitor['config'] value = config['host'] + if record._type == 'CNAME': + # Append a trailing dot for CNAME records so that + # lookup by a CNAME answer works + value = value + '.' monitors[value] = monitor return monitors @@ -882,6 +886,10 @@ class Ns1Provider(BaseProvider): host = record.fqdn[:-1] _type = record._type + if _type == 'CNAME': + # NS1 does not accept a host value with a trailing dot + value = value[:-1] + ret = { 'active': True, 'config': { @@ -1266,8 +1274,7 @@ class Ns1Provider(BaseProvider): extra.append(Update(record, record)) continue - for have in self._monitors_for(record).values(): - value = have['config']['host'] + for value, have in self._monitors_for(record).items(): expected = self._monitor_gen(record, value) # TODO: find values which have missing monitors if not self._monitor_is_match(expected, have): diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index c16a573..a5c41a6 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -578,6 +578,34 @@ class TestNs1ProviderDynamic(TestCase): 'meta': {}, }) + def cname_record(self): + return Record.new(self.zone, 'foo', { + 'dynamic': { + 'pools': { + 'iad': { + 'values': [{ + 'value': 'iad.unit.tests.', + }], + }, + }, + 'rules': [{ + 'pool': 'iad', + }], + }, + 'octodns': { + 'healthcheck': { + 'host': 'send.me', + 'path': '/_ping', + 'port': 80, + 'protocol': 'HTTP', + } + }, + 'ttl': 33, + 'type': 'CNAME', + 'value': 'value.unit.tests.', + 'meta': {}, + }) + def test_notes(self): provider = Ns1Provider('test', 'api-key') @@ -609,6 +637,12 @@ class TestNs1ProviderDynamic(TestCase): }, 'notes': 'host:unit.tests type:A', } + monitor_five = { + 'config': { + 'host': 'iad.unit.tests', + }, + 'notes': 'host:foo.unit.tests type:CNAME', + } provider._client._monitors_cache = { 'one': monitor_one, 'two': { @@ -624,6 +658,7 @@ class TestNs1ProviderDynamic(TestCase): 'notes': 'host:other.unit.tests type:A', }, 'four': monitor_four, + 'five': monitor_five, } # Would match, but won't get there b/c it's not dynamic @@ -641,6 +676,11 @@ class TestNs1ProviderDynamic(TestCase): '2.3.4.5': monitor_four, }, provider._monitors_for(self.record())) + # Check match for CNAME values + self.assertEquals({ + 'iad.unit.tests.': monitor_five, + }, provider._monitors_for(self.cname_record())) + def test_uuid(self): # Just a smoke test/for coverage provider = Ns1Provider('test', 'api-key') @@ -728,6 +768,14 @@ class TestNs1ProviderDynamic(TestCase): # No http response expected self.assertFalse('rules' in monitor) + def test_monitor_gen_CNAME(self): + provider = Ns1Provider('test', 'api-key') + + value = 'iad.unit.tests.' + record = self.cname_record() + monitor = provider._monitor_gen(record, value) + self.assertEquals(value[:-1], monitor['config']['host']) + def test_monitor_is_match(self): provider = Ns1Provider('test', 'api-key') @@ -1295,32 +1343,7 @@ class TestNs1ProviderDynamic(TestCase): ('mid-1', 'fid-1'), ] - record = Record.new(self.zone, 'foo', { - 'dynamic': { - 'pools': { - 'iad': { - 'values': [{ - 'value': 'iad.unit.tests.', - }], - }, - }, - 'rules': [{ - 'pool': 'iad', - }], - }, - 'octodns': { - 'healthcheck': { - 'host': 'send.me', - 'path': '/_ping', - 'port': 80, - 'protocol': 'HTTP', - } - }, - 'ttl': 32, - 'type': 'CNAME', - 'value': 'value.unit.tests.', - 'meta': {}, - }) + record = self.cname_record() ret, _ = provider._params_for_CNAME(record) # Check if the default value was correctly read and populated @@ -1539,7 +1562,7 @@ class TestNs1ProviderDynamic(TestCase): catchall_pool_name = 'iad__catchall' ns1_record = { 'answers': [{ - 'answer': ['2.3.4.5.unit.tests.'], + 'answer': ['iad.unit.tests.'], 'meta': { 'priority': 1, 'weight': 12, @@ -1547,7 +1570,7 @@ class TestNs1ProviderDynamic(TestCase): }, 'region': catchall_pool_name, }, { - 'answer': ['1.2.3.4.unit.tests.'], + 'answer': ['value.unit.tests.'], 'meta': { 'priority': 2, 'note': 'from:--default--', @@ -1564,7 +1587,7 @@ class TestNs1ProviderDynamic(TestCase): } }, 'tier': 3, - 'ttl': 42, + 'ttl': 43, 'type': 'CNAME', } data = provider._data_for_CNAME('CNAME', ns1_record) @@ -1574,7 +1597,7 @@ class TestNs1ProviderDynamic(TestCase): 'iad': { 'fallback': None, 'values': [{ - 'value': '2.3.4.5.unit.tests.', + 'value': 'iad.unit.tests.', 'weight': 12, }], }, @@ -1584,9 +1607,9 @@ class TestNs1ProviderDynamic(TestCase): 'pool': 'iad', }], }, - 'ttl': 42, + 'ttl': 43, 'type': 'CNAME', - 'value': '1.2.3.4.unit.tests.', + 'value': 'value.unit.tests.', }, data) @patch('ns1.rest.records.Records.retrieve') From 3de5cd27404f5fae35feb14e7e4f01addace2528 Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Fri, 30 Apr 2021 20:38:08 -0700 Subject: [PATCH 48/92] More future proof index lookup --- tests/test_octodns_provider_ns1.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index a5c41a6..c14173b 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -1348,7 +1348,7 @@ class TestNs1ProviderDynamic(TestCase): # Check if the default value was correctly read and populated # All other dynamic record test cases are covered by dynamic_A tests - self.assertEquals(ret['answers'][1]['answer'][0], 'value.unit.tests.') + self.assertEquals(ret['answers'][-1]['answer'][0], 'value.unit.tests.') def test_data_for_dynamic_A(self): provider = Ns1Provider('test', 'api-key') From 6d7cab43e881247201ea77cd5ef281fb925674a6 Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Mon, 3 May 2021 10:59:12 -0700 Subject: [PATCH 49/92] Rename data/params for dynamic methods --- octodns/provider/ns1.py | 14 +++++++------- tests/test_octodns_provider_ns1.py | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 6ade3b9..38f9da3 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -464,10 +464,10 @@ class Ns1Provider(BaseProvider): pass return pool_name - def _data_for_dynamic_A(self, _type, record): + def _data_for_dynamic(self, _type, record): # First make sure we have the expected filters config if not self._valid_filter_config(record['filters'], record['domain']): - self.log.error('_data_for_dynamic_A: %s %s has unsupported ' + self.log.error('_data_for_dynamic: %s %s has unsupported ' 'filters', record['domain'], _type) raise Ns1Exception('Unrecognized advanced record') @@ -613,7 +613,7 @@ class Ns1Provider(BaseProvider): first_answer_note = '' # If that note includes a `from` (pool name) it's a dynamic record if 'from:' in first_answer_note: - return self._data_for_dynamic_A(_type, record) + return self._data_for_dynamic(_type, record) # If not it's an old geo record return self._data_for_geo_A(_type, record) @@ -654,7 +654,7 @@ class Ns1Provider(BaseProvider): def _data_for_CNAME(self, _type, record): if record.get('tier', 1) > 1: # Advanced dynamic record - return self._data_for_dynamic_A(_type, record) + return self._data_for_dynamic(_type, record) try: value = record['short_answers'][0] @@ -1031,7 +1031,7 @@ class Ns1Provider(BaseProvider): } answers.append(answer) - def _params_for_dynamic_A(self, record): + def _params_for_dynamic(self, record): pools = record.dynamic.pools # Convert rules to regions @@ -1168,7 +1168,7 @@ class Ns1Provider(BaseProvider): def _params_for_A(self, record): if getattr(record, 'dynamic', False): - return self._params_for_dynamic_A(record) + return self._params_for_dynamic(record) elif hasattr(record, 'geo'): return self._params_for_geo_A(record) @@ -1195,7 +1195,7 @@ class Ns1Provider(BaseProvider): def _params_for_CNAME(self, record): if getattr(record, 'dynamic', False): - return self._params_for_dynamic_A(record) + return self._params_for_dynamic(record) return {'answers': [record.value], 'ttl': record.ttl}, None diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index c14173b..4d6f02a 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -1290,7 +1290,7 @@ class TestNs1ProviderDynamic(TestCase): ('mid-2', 'fid-2'), ('mid-3', 'fid-3'), ] - # This indirectly calls into _params_for_dynamic_A and tests the + # This indirectly calls into _params_for_dynamic and tests the # handling to get there record = self.record() ret, _ = provider._params_for_A(record) @@ -1350,7 +1350,7 @@ class TestNs1ProviderDynamic(TestCase): # All other dynamic record test cases are covered by dynamic_A tests self.assertEquals(ret['answers'][-1]['answer'][0], 'value.unit.tests.') - def test_data_for_dynamic_A(self): + def test_data_for_dynamic(self): provider = Ns1Provider('test', 'api-key') # Unexpected filters throws an error @@ -1359,7 +1359,7 @@ class TestNs1ProviderDynamic(TestCase): 'filters': [], } with self.assertRaises(Ns1Exception) as ctx: - provider._data_for_dynamic_A('A', ns1_record) + provider._data_for_dynamic('A', ns1_record) self.assertEquals('Unrecognized advanced record', text_type(ctx.exception)) @@ -1371,7 +1371,7 @@ class TestNs1ProviderDynamic(TestCase): 'regions': {}, 'ttl': 42, } - data = provider._data_for_dynamic_A('A', ns1_record) + data = provider._data_for_dynamic('A', ns1_record) self.assertEquals({ 'dynamic': { 'pools': {}, @@ -1476,7 +1476,7 @@ class TestNs1ProviderDynamic(TestCase): 'tier': 3, 'ttl': 42, } - data = provider._data_for_dynamic_A('A', ns1_record) + data = provider._data_for_dynamic('A', ns1_record) self.assertEquals({ 'dynamic': { 'pools': { @@ -1520,7 +1520,7 @@ class TestNs1ProviderDynamic(TestCase): }, data) # Same answer if we go through _data_for_A which out sources the job to - # _data_for_dynamic_A + # _data_for_dynamic data2 = provider._data_for_A('A', ns1_record) self.assertEquals(data, data2) @@ -1531,7 +1531,7 @@ class TestNs1ProviderDynamic(TestCase): ns1_record['regions'][old_style_catchall_pool_name] = \ ns1_record['regions'][catchall_pool_name] del ns1_record['regions'][catchall_pool_name] - data3 = provider._data_for_dynamic_A('A', ns1_record) + data3 = provider._data_for_dynamic('A', ns1_record) self.assertEquals(data, data2) # Oceania test cases From ebfb9355b136dfce46db0905bd6c4e714d6563b7 Mon Sep 17 00:00:00 2001 From: omar Date: Mon, 10 May 2021 19:32:38 -0700 Subject: [PATCH 50/92] Update the AzureProvider to support azure-mgmt-dns 8.0.0 and azure-identity. --- README.md | 2 +- octodns/provider/azuredns.py | 22 +++++++++++----------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 4ab3c6b..0a04e27 100644 --- a/README.md +++ b/README.md @@ -192,7 +192,7 @@ The above command pulled the existing data out of Route53 and placed the results | Provider | Requirements | Record Support | Dynamic | Notes | |--|--|--|--|--| -| [AzureProvider](/octodns/provider/azuredns.py) | azure-mgmt-dns | A, AAAA, CAA, CNAME, MX, NS, PTR, SRV, TXT | No | | +| [AzureProvider](/octodns/provider/azuredns.py) | azure-identity, azure-mgmt-dns | A, AAAA, CAA, CNAME, MX, NS, PTR, SRV, TXT | No | | | [Akamai](/octodns/provider/edgedns.py) | edgegrid-python | A, AAAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, SSHFP, TXT | No | | | [CloudflareProvider](/octodns/provider/cloudflare.py) | | A, AAAA, ALIAS, CAA, CNAME, LOC, MX, NS, PTR, SPF, SRV, TXT | No | CAA tags restricted | | [ConstellixProvider](/octodns/provider/constellix.py) | | A, AAAA, ALIAS (ANAME), CAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | CAA tags restricted | diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 2fca5af..282077f 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -5,7 +5,7 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -from azure.common.credentials import ServicePrincipalCredentials +from azure.identity import ClientSecretCredential from azure.mgmt.dns import DnsManagementClient from azure.mgmt.dns.models import ARecord, AaaaRecord, CaaRecord, \ @@ -71,10 +71,10 @@ class _AzureRecord(object): '''Constructor for _AzureRecord. Notes on Azure records: An Azure record set has the form - RecordSet(name=<...>, type=<...>, arecords=[...], aaaa_records, ..) + RecordSet(name=<...>, type=<...>, a_records=[...], aaaa_records, ..) When constructing an azure record as done in self._apply_Create, the argument parameters for an A record would be - parameters={'ttl': , 'arecords': [ARecord(),]}. + parameters={'ttl': , 'a_records': [ARecord(),]}. As another example for CNAME record: parameters={'ttl': , 'cname_record': CnameRecord()}. @@ -263,7 +263,7 @@ def _parse_azure_type(string): def _check_for_alias(azrecord): - if (azrecord.target_resource.id and not azrecord.arecords and not + if (azrecord.target_resource.id and not azrecord.a_records and not azrecord.cname_record): return True return False @@ -343,14 +343,14 @@ class AzureProvider(BaseProvider): @property def _dns_client(self): if self.__dns_client is None: - credentials = ServicePrincipalCredentials( - self._dns_client_client_id, - secret=self._dns_client_key, - tenant=self._dns_client_directory_id + credential = ClientSecretCredential( + client_id=self._dns_client_client_id, + client_secret=self._dns_client_key, + tenant_id=self._dns_client_directory_id ) self.__dns_client = DnsManagementClient( - credentials, - self._dns_client_subscription_id + credential=credential, + subscription_id=self._dns_client_subscription_id ) return self.__dns_client @@ -452,7 +452,7 @@ class AzureProvider(BaseProvider): return exists def _data_for_A(self, azrecord): - return {'values': [ar.ipv4_address for ar in azrecord.arecords]} + return {'values': [ar.ipv4_address for ar in azrecord.a_records]} def _data_for_AAAA(self, azrecord): return {'values': [ar.ipv6_address for ar in azrecord.aaaa_records]} From 93de918e01b0c97a89ba8ff74f926534c966863b Mon Sep 17 00:00:00 2001 From: omar Date: Mon, 10 May 2021 20:38:30 -0700 Subject: [PATCH 51/92] Fix lint, requirements.txt, and all the tests but one. --- octodns/provider/azuredns.py | 3 ++- requirements.txt | 5 +++-- tests/test_octodns_provider_azuredns.py | 28 ++++++++++++------------- 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 282077f..7a1883f 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -71,7 +71,8 @@ class _AzureRecord(object): '''Constructor for _AzureRecord. Notes on Azure records: An Azure record set has the form - RecordSet(name=<...>, type=<...>, a_records=[...], aaaa_records, ..) + RecordSet(name=<...>, type=<...>, a_records=[...], + aaaa_records=[...], ...) When constructing an azure record as done in self._apply_Create, the argument parameters for an A record would be parameters={'ttl': , 'a_records': [ARecord(),]}. diff --git a/requirements.txt b/requirements.txt index 933ac60..54bd2a3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ PyYaml==5.4 -azure-common==1.1.25 -azure-mgmt-dns==3.0.0 +azure-common==1.1.27 +azure-identity==1.5.0 +azure-mgmt-dns==8.0.0 boto3==1.15.9 botocore==1.18.9 dnspython==1.16.0 diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index 9523b51..d9b2b5a 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -152,8 +152,8 @@ _base0.zone_name = 'unit.tests' _base0.relative_record_set_name = '@' _base0.record_type = 'A' _base0.params['ttl'] = 0 -_base0.params['arecords'] = [ARecord(ipv4_address='1.2.3.4'), - ARecord(ipv4_address='10.10.10.10')] +_base0.params['a_records'] = [ARecord(ipv4_address='1.2.3.4'), + ARecord(ipv4_address='10.10.10.10')] azure_records.append(_base0) _base1 = _AzureRecord('TestAzure', octo_records[1]) @@ -161,8 +161,8 @@ _base1.zone_name = 'unit.tests' _base1.relative_record_set_name = 'a' _base1.record_type = 'A' _base1.params['ttl'] = 1 -_base1.params['arecords'] = [ARecord(ipv4_address='1.2.3.4'), - ARecord(ipv4_address='1.1.1.1')] +_base1.params['a_records'] = [ARecord(ipv4_address='1.2.3.4'), + ARecord(ipv4_address='1.1.1.1')] azure_records.append(_base1) _base2 = _AzureRecord('TestAzure', octo_records[2]) @@ -170,7 +170,7 @@ _base2.zone_name = 'unit.tests' _base2.relative_record_set_name = 'aa' _base2.record_type = 'A' _base2.params['ttl'] = 9001 -_base2.params['arecords'] = ARecord(ipv4_address='1.2.4.3') +_base2.params['a_records'] = ARecord(ipv4_address='1.2.4.3') azure_records.append(_base2) _base3 = _AzureRecord('TestAzure', octo_records[3]) @@ -178,7 +178,7 @@ _base3.zone_name = 'unit.tests' _base3.relative_record_set_name = 'aaa' _base3.record_type = 'A' _base3.params['ttl'] = 2 -_base3.params['arecords'] = ARecord(ipv4_address='1.1.1.3') +_base3.params['a_records'] = ARecord(ipv4_address='1.1.1.3') azure_records.append(_base3) _base4 = _AzureRecord('TestAzure', octo_records[4]) @@ -366,7 +366,7 @@ class Test_CheckAzureAlias(TestCase): alias_record = type('C', (object,), {}) alias_record.target_resource = type('C', (object,), {}) alias_record.target_resource.id = "/subscriptions/x/resourceGroups/y/z" - alias_record.arecords = None + alias_record.a_records = None alias_record.cname_record = None self.assertEquals(_check_for_alias(alias_record), True) @@ -377,7 +377,7 @@ class TestAzureDnsProvider(TestCase): return self._get_provider('mock_spc', 'mock_dns_client') @patch('octodns.provider.azuredns.DnsManagementClient') - @patch('octodns.provider.azuredns.ServicePrincipalCredentials') + @patch('octodns.provider.azuredns.ClientSecretCredential') def _get_provider(self, mock_spc, mock_dns_client): '''Returns a mock AzureProvider object to use in testing. @@ -399,12 +399,12 @@ class TestAzureDnsProvider(TestCase): provider = self._get_provider() rs = [] - recordSet = RecordSet(arecords=[ARecord(ipv4_address='1.1.1.1')]) + recordSet = RecordSet(a_records=[ARecord(ipv4_address='1.1.1.1')]) recordSet.name, recordSet.ttl, recordSet.type = 'a1', 0, 'A' recordSet.target_resource = SubResource() rs.append(recordSet) - recordSet = RecordSet(arecords=[ARecord(ipv4_address='1.1.1.1'), - ARecord(ipv4_address='2.2.2.2')]) + recordSet = RecordSet(a_records=[ARecord(ipv4_address='1.1.1.1'), + ARecord(ipv4_address='2.2.2.2')]) recordSet.name, recordSet.ttl, recordSet.type = 'a2', 1, 'A' recordSet.target_resource = SubResource() rs.append(recordSet) @@ -575,11 +575,11 @@ class TestAzureDnsProvider(TestCase): provider = self._get_provider() rs = [] - recordSet = RecordSet(arecords=[ARecord(ipv4_address='1.1.1.1')]) + recordSet = RecordSet(a_records=[ARecord(ipv4_address='1.1.1.1')]) recordSet.name, recordSet.ttl, recordSet.type = 'a1', 0, 'A' rs.append(recordSet) - recordSet = RecordSet(arecords=[ARecord(ipv4_address='1.1.1.1'), - ARecord(ipv4_address='2.2.2.2')]) + recordSet = RecordSet(a_records=[ARecord(ipv4_address='1.1.1.1'), + ARecord(ipv4_address='2.2.2.2')]) recordSet.name, recordSet.ttl, recordSet.type = 'a2', 1, 'A' rs.append(recordSet) From 758c7fab61209084dc96cc845b08fa154835943c Mon Sep 17 00:00:00 2001 From: omar Date: Mon, 10 May 2021 20:49:28 -0700 Subject: [PATCH 52/92] Fix the last test. --- octodns/provider/azuredns.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 7a1883f..83cfeb0 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -102,8 +102,7 @@ class _AzureRecord(object): return # Refer to function docstring for key_name and class_name. - format_u_s = '' if record._type == 'A' else '_' - key_name = '{}{}records'.format(self.record_type, format_u_s).lower() + key_name = '{}_records'.format(self.record_type).lower() if record._type == 'CNAME': key_name = key_name[:len(key_name) - 1] azure_class = self.TYPE_MAP[self.record_type] From d619025040dc1e8087290e70554f9ccc9c028b78 Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Tue, 11 May 2021 15:39:00 -0700 Subject: [PATCH 53/92] Support dynamic records in Azure DNS --- README.md | 2 +- octodns/provider/azuredns.py | 632 +++++++++++++++-- requirements.txt | 1 + tests/test_octodns_provider_azuredns.py | 887 +++++++++++++++++++++++- 4 files changed, 1440 insertions(+), 82 deletions(-) diff --git a/README.md b/README.md index 0a04e27..8d59cd0 100644 --- a/README.md +++ b/README.md @@ -192,7 +192,7 @@ The above command pulled the existing data out of Route53 and placed the results | Provider | Requirements | Record Support | Dynamic | Notes | |--|--|--|--|--| -| [AzureProvider](/octodns/provider/azuredns.py) | azure-identity, azure-mgmt-dns | A, AAAA, CAA, CNAME, MX, NS, PTR, SRV, TXT | No | | +| [AzureProvider](/octodns/provider/azuredns.py) | azure-identity, azure-mgmt-dns, azure-mgmt-trafficmanager | A, AAAA, CAA, CNAME, MX, NS, PTR, SRV, TXT | Yes (CNAMEs only) | | | [Akamai](/octodns/provider/edgedns.py) | edgegrid-python | A, AAAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, SSHFP, TXT | No | | | [CloudflareProvider](/octodns/provider/cloudflare.py) | | A, AAAA, ALIAS, CAA, CNAME, LOC, MX, NS, PTR, SPF, SRV, TXT | No | CAA tags restricted | | [ConstellixProvider](/octodns/provider/constellix.py) | | A, AAAA, ALIAS (ANAME), CAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | CAA tags restricted | diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 83cfeb0..4d3dcc5 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -5,18 +5,28 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals +from collections import defaultdict + from azure.identity import ClientSecretCredential +from azure.common.credentials import ServicePrincipalCredentials from azure.mgmt.dns import DnsManagementClient +from azure.mgmt.trafficmanager import TrafficManagerManagementClient from azure.mgmt.dns.models import ARecord, AaaaRecord, CaaRecord, \ CnameRecord, MxRecord, SrvRecord, NsRecord, PtrRecord, TxtRecord, Zone +from azure.mgmt.trafficmanager.models import Profile, DnsConfig, \ + MonitorConfig, Endpoint, MonitorConfigCustomHeadersItem import logging from functools import reduce -from ..record import Record +from ..record import Record, Update, GeoCodes from .base import BaseProvider +class AzureException(Exception): + pass + + def escape_semicolon(s): assert s return s.replace(';', '\\;') @@ -67,7 +77,8 @@ class _AzureRecord(object): 'TXT': TxtRecord } - def __init__(self, resource_group, record, delete=False): + def __init__(self, resource_group, record, delete=False, + traffic_manager=None): '''Constructor for _AzureRecord. Notes on Azure records: An Azure record set has the form @@ -94,9 +105,11 @@ class _AzureRecord(object): self.log = logging.getLogger('AzureRecord') self.resource_group = resource_group - self.zone_name = record.zone.name[:len(record.zone.name) - 1] + self.zone_name = record.zone.name[:-1] self.relative_record_set_name = record.name or '@' self.record_type = record._type + self._record = record + self.traffic_manager = traffic_manager if delete: return @@ -104,11 +117,11 @@ class _AzureRecord(object): # Refer to function docstring for key_name and class_name. key_name = '{}_records'.format(self.record_type).lower() if record._type == 'CNAME': - key_name = key_name[:len(key_name) - 1] + key_name = key_name[:-1] azure_class = self.TYPE_MAP[self.record_type] - self.params = getattr(self, '_params_for_{}'.format(record._type)) - self.params = self.params(record.data, key_name, azure_class) + params_for = getattr(self, '_params_for_{}'.format(record._type)) + self.params = params_for(record.data, key_name, azure_class) self.params['ttl'] = record.ttl def _params_for_A(self, data, key_name, azure_class): @@ -139,6 +152,9 @@ class _AzureRecord(object): return {key_name: params} def _params_for_CNAME(self, data, key_name, azure_class): + if self._record.dynamic and self.traffic_manager: + return {'target_resource': self.traffic_manager} + return {key_name: azure_class(cname=data['value'])} def _params_for_MX(self, data, key_name, azure_class): @@ -227,25 +243,6 @@ class _AzureRecord(object): (parse_dict(self.params) == parse_dict(b.params)) & \ (self.relative_record_set_name == b.relative_record_set_name) - def __str__(self): - '''String representation of an _AzureRecord. - :type return: str - ''' - string = 'Zone: {}; '.format(self.zone_name) - string += 'Name: {}; '.format(self.relative_record_set_name) - string += 'Type: {}; '.format(self.record_type) - if not hasattr(self, 'params'): - return string - string += 'Ttl: {}; '.format(self.params['ttl']) - for char in self.params: - if char != 'ttl': - try: - for rec in self.params[char]: - string += 'Record: {}; '.format(rec.__dict__) - except: - string += 'Record: {}; '.format(self.params[char].__dict__) - return string - def _check_endswith_dot(string): return string if string.endswith('.') else string + '.' @@ -259,14 +256,88 @@ def _parse_azure_type(string): :type return: str ''' - return string.split('/')[len(string.split('/')) - 1] - + return string.split('/')[-1] + + +def _traffic_manager_suffix(record): + return record.fqdn[:-1].replace('.', '-') + + +def _get_monitor(record): + monitor = MonitorConfig( + protocol=record.healthcheck_protocol, + port=record.healthcheck_port, + path=record.healthcheck_path, + ) + host = record.healthcheck_host + if host: + monitor.custom_headers = [MonitorConfigCustomHeadersItem( + name='Host', value=host + )] + return monitor + + +def _profile_is_match(have, desired): + if have is None or desired is None: + return False + + # compare basic attributes + if have.name != desired.name or \ + have.traffic_routing_method != desired.traffic_routing_method or \ + have.dns_config.ttl != desired.dns_config.ttl or \ + len(have.endpoints) != len(desired.endpoints): + return False + + # compare monitoring configuration + monitor_have = have.monitor_config + monitor_desired = desired.monitor_config + if monitor_have.protocol != monitor_desired.protocol or \ + monitor_have.port != monitor_desired.port or \ + monitor_have.path != monitor_desired.path or \ + monitor_have.custom_headers != monitor_desired.custom_headers: + return False + + # compare endpoints + method = have.traffic_routing_method + if method == 'Priority': + have_endpoints = sorted(have.endpoints, key=lambda e: e.priority) + desired_endpoints = sorted(desired.endpoints, + key=lambda e: e.priority) + elif method == 'Weighted': + have_endpoints = sorted(have.endpoints, key=lambda e: e.target) + desired_endpoints = sorted(desired.endpoints, key=lambda e: e.target) + else: + have_endpoints = have.endpoints + desired_endpoints = desired.endpoints + endpoints = zip(have_endpoints, desired_endpoints) + for have_endpoint, desired_endpoint in endpoints: + if have_endpoint.name != desired_endpoint.name or \ + have_endpoint.type != desired_endpoint.type: + return False + target_type = have_endpoint.type.split('/')[-1] + if target_type == 'externalEndpoints': + # compare value, weight, priority + if have_endpoint.target != desired_endpoint.target: + return False + if method == 'Weighted' and \ + have_endpoint.weight != desired_endpoint.weight: + return False + elif target_type == 'nestedEndpoints': + # compare targets + if have_endpoint.target_resource_id != \ + desired_endpoint.target_resource_id: + return False + # compare geos + if method == 'Geographic': + have_geos = sorted(have_endpoint.geo_mapping) + desired_geos = sorted(desired_endpoint.geo_mapping) + if have_geos != desired_geos: + return False + else: + # unexpected, give up + return False -def _check_for_alias(azrecord): - if (azrecord.target_resource.id and not azrecord.a_records and not - azrecord.cname_record): - return True - return False + return True class AzureProvider(BaseProvider): @@ -318,7 +389,7 @@ class AzureProvider(BaseProvider): possible to also hard-code into the config file: eg, resource_group. ''' SUPPORTS_GEO = False - SUPPORTS_DYNAMIC = False + SUPPORTS_DYNAMIC = True SUPPORTS = set(('A', 'AAAA', 'CAA', 'CNAME', 'MX', 'NS', 'PTR', 'SRV', 'TXT')) @@ -336,24 +407,45 @@ class AzureProvider(BaseProvider): self._dns_client_directory_id = directory_id self._dns_client_subscription_id = sub_id self.__dns_client = None + self.__tm_client = None self._resource_group = resource_group self._azure_zones = set() + self._traffic_managers = dict() @property def _dns_client(self): if self.__dns_client is None: - credential = ClientSecretCredential( - client_id=self._dns_client_client_id, - client_secret=self._dns_client_key, - tenant_id=self._dns_client_directory_id - ) + # Azure's logger spits out a lot of debug messages at 'INFO' + # level, override it by re-assigning `info` method to `debug` + # (ugly hack until I find a better way) + logger_name = 'azure.core.pipeline.policies.http_logging_policy' + logger = logging.getLogger(logger_name) + logger.info = logger.debug self.__dns_client = DnsManagementClient( - credential=credential, - subscription_id=self._dns_client_subscription_id + credential=ClientSecretCredential( + client_id=self._dns_client_client_id, + client_secret=self._dns_client_key, + tenant_id=self._dns_client_directory_id, + logger=logger, + ), + subscription_id=self._dns_client_subscription_id, ) return self.__dns_client + @property + def _tm_client(self): + if self.__tm_client is None: + self.__tm_client = TrafficManagerManagementClient( + ServicePrincipalCredentials( + self._dns_client_client_id, + secret=self._dns_client_key, + tenant=self._dns_client_directory_id, + ), + self._dns_client_subscription_id, + ) + return self.__tm_client + def _populate_zones(self): self.log.debug('azure_zones: loading') list_zones = self._dns_client.zones.list_by_resource_group @@ -388,6 +480,42 @@ class AzureProvider(BaseProvider): # Else return nothing (aka false) return + def _populate_traffic_managers(self): + self.log.debug('traffic managers: loading') + list_profiles = self._tm_client.profiles.list_by_resource_group + for profile in list_profiles(self._resource_group): + self._traffic_managers[profile.id] = profile + # link nested profiles in advance for convenience + for _, profile in self._traffic_managers.items(): + self._populate_nested_profiles(profile) + + def _populate_nested_profiles(self, profile): + for ep in profile.endpoints: + target_id = ep.target_resource_id + if target_id and target_id in self._traffic_managers: + target = self._traffic_managers[target_id] + ep.target_resource = self._populate_nested_profiles(target) + return profile + + def _get_tm_profile_by_id(self, resource_id): + if not self._traffic_managers: + self._populate_traffic_managers() + return self._traffic_managers.get(resource_id) + + def _profile_name_to_id(self, name): + return '/subscriptions/' + self._dns_client_subscription_id + \ + '/resourceGroups/' + self._resource_group + \ + '/providers/Microsoft.Network/trafficManagerProfiles/' + \ + name + + def _get_tm_profile_by_name(self, name): + profile_id = self._profile_name_to_id(name) + return self._get_tm_profile_by_id(profile_id) + + def _get_tm_for_dynamic_record(self, record): + name = _traffic_manager_suffix(record) + return self._get_tm_profile_by_name(name) + def populate(self, zone, target=False, lenient=False): '''Required function of manager.py to collect records from zone. @@ -417,40 +545,35 @@ class AzureProvider(BaseProvider): exists = False before = len(zone.records) - zone_name = zone.name[:len(zone.name) - 1] + zone_name = zone.name[:-1] self._populate_zones() - self._check_zone(zone_name) - _records = [] records = self._dns_client.record_sets.list_by_dns_zone if self._check_zone(zone_name): exists = True for azrecord in records(self._resource_group, zone_name): - if _parse_azure_type(azrecord.type) in self.SUPPORTS: - _records.append(azrecord) - for azrecord in _records: - record_name = azrecord.name if azrecord.name != '@' else '' typ = _parse_azure_type(azrecord.type) + if typ not in self.SUPPORTS: + continue - if typ in ['A', 'CNAME']: - if _check_for_alias(azrecord): - self.log.debug( - 'Skipping - ALIAS. zone=%s record=%s, type=%s', - zone_name, record_name, typ) # pragma: no cover - continue # pragma: no cover - - data = getattr(self, '_data_for_{}'.format(typ)) - data = data(azrecord) - data['type'] = typ - data['ttl'] = azrecord.ttl - record = Record.new(zone, record_name, data, source=self) - + record = self._populate_record(zone, azrecord, lenient) zone.add_record(record, lenient=lenient) self.log.info('populate: found %s records, exists=%s', len(zone.records) - before, exists) return exists + def _populate_record(self, zone, azrecord, lenient=False): + record_name = azrecord.name if azrecord.name != '@' else '' + typ = _parse_azure_type(azrecord.type) + + data_for = getattr(self, '_data_for_{}'.format(typ)) + data = data_for(azrecord) + data['type'] = typ + data['ttl'] = azrecord.ttl + return Record.new(zone, record_name, data, source=self, + lenient=lenient) + def _data_for_A(self, azrecord): return {'values': [ar.ipv4_address for ar in azrecord.a_records]} @@ -470,6 +593,9 @@ class AzureProvider(BaseProvider): :type return: dict ''' + if azrecord.cname_record is None and azrecord.target_resource.id: + return self._data_for_dynamic(azrecord) + return {'value': _check_endswith_dot(azrecord.cname_record.cname)} def _data_for_MX(self, azrecord): @@ -495,6 +621,322 @@ class AzureProvider(BaseProvider): ar.value)) for ar in azrecord.txt_records]} + def _data_for_dynamic(self, azrecord): + default = set() + pools = defaultdict(lambda: {'fallback': None, 'values': []}) + rules = [] + + # top level geo profile + geo_profile = self._get_tm_profile_by_id(azrecord.target_resource.id) + for geo_ep in geo_profile.endpoints: + rule = {} + + # resolve list of regions + geo_map = list(geo_ep.geo_mapping) + if geo_map != ['WORLD']: + if 'GEO-ME' in geo_map: + # Azure treats Middle East as a separate group, but + # its part of Asia in octoDNS, so we need to remove GEO-ME + # if GEO-AS is also in the list + # Throw exception otherwise, it should not happen if the + # profile was generated by octoDNS + if 'GEO-AS' not in geo_map: + msg = '_data_for_dynamic: Profile={}: '.format( + geo_profile.name) + msg += 'Middle East (GEO-ME) is not supported by ' + \ + 'octoDNS. It needs to be either paired ' + \ + 'with Asia (GEO-AS) or expanded into ' + \ + 'individual list of countries.' + raise AzureException(msg) + geo_map.remove('GEO-ME') + geos = rule.setdefault('geos', []) + for code in geo_map: + if code.startswith('GEO-'): + geos.append(code[len('GEO-'):]) + elif '-' in code: + country, province = code.split('-', 1) + country = GeoCodes.country_to_code(country) + geos.append('{}-{}'.format(country, province)) + else: + geos.append(GeoCodes.country_to_code(code)) + + # second level priority profile + pool = None + rule_endpoints = geo_ep.target_resource.endpoints + rule_endpoints = sorted(rule_endpoints, key=lambda e: e.priority) + for rule_ep in rule_endpoints: + pool_name = rule_ep.name + + # third (and last) level weighted RR profile + # these should be leaf node profiles with no further nesting + pool_profile = rule_ep.target_resource + + # last/default pool + if pool_name == '--default--': + for pool_ep in pool_profile.endpoints: + default.add(pool_ep.target) + # this should be the last one, so let's break here + break + + # set first priority endpoint as the rule's primary pool + if 'pool' not in rule: + rule['pool'] = pool_name + + if pool: + # set current pool as fallback of the previous pool + pool['fallback'] = pool_name + + pool = pools[pool_name] + for pool_ep in pool_profile.endpoints: + val = pool_ep.target + value_dict = { + 'value': _check_endswith_dot(val), + 'weight': pool_ep.weight, + } + if value_dict not in pool['values']: + pool['values'].append(value_dict) + + if 'pool' not in rule or not default: + # this will happen if the priority profile does not have + # enough endpoints + msg = 'Expected at least 2 endpoints in {}, got {}'.format( + geo_ep.target_resource.name, len(rule_endpoints) + ) + raise AzureException(msg) + rules.append(rule) + + # Order and convert to a list + default = sorted(default) + + data = { + 'dynamic': { + 'pools': pools, + 'rules': rules, + }, + 'value': _check_endswith_dot(default[0]), + } + + return data + + def _extra_changes(self, existing, desired, changes): + changed = set() + + # Abort if there are non-CNAME dynamic records + for change in changes: + record = change.record + changed.add(record) + typ = record._type + dynamic = getattr(record, 'dynamic', False) + if dynamic and typ != 'CNAME': + msg = '{}: Dynamic records in Azure must be of type CNAME' + msg = msg.format(record.fqdn) + raise AzureException(msg) + + log = self.log.info + extra = [] + for record in desired.records: + if not getattr(record, 'dynamic', False): + # Already changed, or not dynamic, no need to check it + continue + + # let's walk through and show what will be changed even if + # the record is already be in list of changes + added = (record in changed) + + active = set() + profiles = self._generate_traffic_managers(record) + for profile in profiles: + name = profile.name + active.add(name) + existing_profile = self._get_tm_profile_by_name(name) + if not _profile_is_match(existing_profile, profile): + log('_extra_changes: Profile name=%s will be synced', + name) + if not added: + extra.append(Update(record, record)) + added = True + + existing_profiles = self._find_traffic_managers(record) + for name in existing_profiles - active: + log('_extra_changes: Profile name=%s will be destroyed', name) + if not added: + extra.append(Update(record, record)) + added = True + + return extra + + def _generate_tm_profile(self, name, routing, endpoints, record): + # set appropriate endpoint types + endpoint_type_prefix = 'Microsoft.Network/trafficManagerProfiles/' + for ep in endpoints: + if ep.target_resource_id: + ep.type = endpoint_type_prefix + 'nestedEndpoints' + elif ep.target: + ep.type = endpoint_type_prefix + 'externalEndpoints' + else: + msg = ('_generate_tm_profile: Invalid endpoint {} ' + + 'in profile {}, needs to have either target or ' + + 'target_resource_id').format(ep.name, name) + raise AzureException(msg) + + # build and return + return Profile( + id=self._profile_name_to_id(name), + name=name, + traffic_routing_method=routing, + dns_config=DnsConfig( + relative_name=name, + ttl=record.ttl, + ), + monitor_config=_get_monitor(record), + endpoints=endpoints, + location='global', + ) + + def _generate_traffic_managers(self, record): + traffic_managers = [] + pools = record.dynamic.pools + + tm_suffix = _traffic_manager_suffix(record) + profile = self._generate_tm_profile + + # construct the default pool that will be used at the end of + # all rules + target = record.value[:-1] + default_endpoints = [Endpoint( + name=target, + target=target, + weight=1, + )] + default_profile_name = 'default--{}'.format(tm_suffix) + default_profile = profile(default_profile_name, 'Weighted', + default_endpoints, record) + traffic_managers.append(default_profile) + + geo_endpoints = [] + + for rule in record.dynamic.rules: + pool_name = rule.data['pool'] + rule_endpoints = [] + priority = 1 + + while pool_name: + # iterate until we reach end of fallback chain + pool = pools[pool_name].data + profile_name = 'pool-{}--{}'.format(pool_name, tm_suffix) + endpoints = [] + for val in pool['values']: + target = val['value'] + # strip trailing dot from CNAME value + target = target[:-1] + endpoints.append(Endpoint( + name=target, + target=target, + weight=val.get('weight', 1), + )) + pool_profile = profile(profile_name, 'Weighted', endpoints, + record) + traffic_managers.append(pool_profile) + + # append pool to endpoint list of fallback rule profile + rule_endpoints.append(Endpoint( + name=pool_name, + target_resource_id=pool_profile.id, + priority=priority, + )) + + priority += 1 + pool_name = pool.get('fallback') + + # append default profile to the end + rule_endpoints.append(Endpoint( + name='--default--', + target_resource_id=default_profile.id, + priority=priority, + )) + # create rule profile with fallback chain + rule_profile_name = 'rule-{}--{}'.format(rule.data['pool'], + tm_suffix) + rule_profile = profile(rule_profile_name, 'Priority', + rule_endpoints, record) + traffic_managers.append(rule_profile) + + # append rule profile to top-level geo profile + rule_geos = rule.data.get('geos', []) + geos = [] + if len(rule_geos) > 0: + for geo in rule_geos: + if '-' in geo: + geos.append(geo.split('-', 1)[-1]) + else: + geos.append('GEO-{}'.format(geo)) + if geo == 'AS': + # Middle East is part of Asia in octoDNS, but + # Azure treats it as a separate "group", so let's + # add it in the list of geo mappings. We will drop + # it when we later parse the list of regions. + geos.append('GEO-ME') + else: + geos.append('WORLD') + geo_endpoints.append(Endpoint( + name='rule-{}'.format(rule.data['pool']), + target_resource_id=rule_profile.id, + geo_mapping=geos, + )) + + geo_profile = profile(tm_suffix, 'Geographic', geo_endpoints, record) + traffic_managers.append(geo_profile) + + return traffic_managers + + def _sync_traffic_managers(self, record): + desired_profiles = self._generate_traffic_managers(record) + seen = set() + + tm_sync = self._tm_client.profiles.create_or_update + populate = self._populate_nested_profiles + + for desired in desired_profiles: + name = desired.name + if name in seen: + continue + + existing = self._get_tm_profile_by_name(name) + if not _profile_is_match(existing, desired): + self.log.info( + '_sync_traffic_managers: Syncing profile=%s', name) + profile = tm_sync(self._resource_group, name, desired) + self._traffic_managers[profile.id] = populate(profile) + else: + self.log.debug( + '_sync_traffic_managers: Skipping profile=%s: up to date', + name) + seen.add(name) + + return seen + + def _find_traffic_managers(self, record): + tm_suffix = _traffic_manager_suffix(record) + + profiles = set() + for profile_id in self._traffic_managers: + # match existing profiles with record's suffix + name = profile_id.split('/')[-1] + if name == tm_suffix or \ + name.endswith('--{}'.format(tm_suffix)): + profiles.add(name) + + return profiles + + def _traffic_managers_gc(self, record, active_profiles): + existing_profiles = self._find_traffic_managers(record) + + # delete unused profiles + for profile_name in existing_profiles - active_profiles: + self.log.info('_traffic_managers_gc: Deleting profile=%s', + profile_name) + self._tm_client.profiles.delete(self._resource_group, profile_name) + def _apply_Create(self, change): '''A record from change must be created. @@ -503,7 +945,15 @@ class AzureProvider(BaseProvider): :type return: void ''' - ar = _AzureRecord(self._resource_group, change.new) + record = change.new + + dynamic = getattr(record, 'dynamic', False) + if dynamic: + self._sync_traffic_managers(record) + + profile = self._get_tm_for_dynamic_record(record) + ar = _AzureRecord(self._resource_group, record, + traffic_manager=profile) create = self._dns_client.record_sets.create_or_update create(resource_group_name=ar.resource_group, @@ -512,17 +962,71 @@ class AzureProvider(BaseProvider): record_type=ar.record_type, parameters=ar.params) - self.log.debug('* Success Create/Update: {}'.format(ar)) + self.log.debug('* Success Create: {}'.format(record)) - _apply_Update = _apply_Create + def _apply_Update(self, change): + '''A record from change must be created. + + :param change: a change object + :type change: octodns.record.Change + + :type return: void + ''' + existing = change.existing + new = change.new + existing_is_dynamic = getattr(existing, 'dynamic', False) + new_is_dynamic = getattr(new, 'dynamic', False) + + update_record = True + + if new_is_dynamic: + active = self._sync_traffic_managers(new) + # only TTL is configured in record, everything else goes inside + # traffic managers, so no need to update if TTL is unchanged + # and existing record is already aliased to its traffic manager + if existing.ttl == new.ttl and existing_is_dynamic: + update_record = False + + if update_record: + profile = self._get_tm_for_dynamic_record(new) + ar = _AzureRecord(self._resource_group, new, + traffic_manager=profile) + update = self._dns_client.record_sets.create_or_update + + update(resource_group_name=ar.resource_group, + zone_name=ar.zone_name, + relative_record_set_name=ar.relative_record_set_name, + record_type=ar.record_type, + parameters=ar.params) + + if new_is_dynamic: + # let's cleanup unused traffic managers + self._traffic_managers_gc(new, active) + elif existing_is_dynamic: + # cleanup traffic managers when a dynamic record gets + # changed to a simple record + self._traffic_managers_gc(existing, set()) + + self.log.debug('* Success Update: {}'.format(new)) def _apply_Delete(self, change): - ar = _AzureRecord(self._resource_group, change.existing, delete=True) + '''A record from change must be deleted. + + :param change: a change object + :type change: octodns.record.Change + + :type return: void + ''' + record = change.record + ar = _AzureRecord(self._resource_group, record, delete=True) delete = self._dns_client.record_sets.delete delete(self._resource_group, ar.zone_name, ar.relative_record_set_name, ar.record_type) + if getattr(record, 'dynamic', False): + self._traffic_managers_gc(record, set()) + self.log.debug('* Success Delete: {}'.format(ar)) def _apply(self, plan): diff --git a/requirements.txt b/requirements.txt index 54bd2a3..143ba67 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,7 @@ PyYaml==5.4 azure-common==1.1.27 azure-identity==1.5.0 azure-mgmt-dns==8.0.0 +azure-mgmt-trafficmanager==0.51.0 boto3==1.15.9 botocore==1.18.9 dnspython==1.16.0 diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index d9b2b5a..5655719 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -5,19 +5,23 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -from octodns.record import Create, Delete, Record +from octodns.record import Create, Update, Delete, Record from octodns.provider.azuredns import _AzureRecord, AzureProvider, \ - _check_endswith_dot, _parse_azure_type, _check_for_alias + _check_endswith_dot, _parse_azure_type, _traffic_manager_suffix, \ + _get_monitor, _profile_is_match, AzureException from octodns.zone import Zone from octodns.provider.base import Plan from azure.mgmt.dns.models import ARecord, AaaaRecord, CaaRecord, \ CnameRecord, MxRecord, SrvRecord, NsRecord, PtrRecord, TxtRecord, \ RecordSet, SoaRecord, SubResource, Zone as AzureZone +from azure.mgmt.trafficmanager.models import Profile, DnsConfig, \ + MonitorConfig, Endpoint, MonitorConfigCustomHeadersItem from msrestazure.azure_exceptions import CloudError +from six import text_type from unittest import TestCase -from mock import Mock, patch +from mock import Mock, patch, call zone = Zone(name='unit.tests.', sub_zones=[]) @@ -343,6 +347,43 @@ class Test_AzureRecord(TestCase): assert(azure_records[i]._equals(octo)) +class Test_DynamicAzureRecord(TestCase): + def test_azure_record(self): + tm_profile = Profile() + data = { + 'ttl': 60, + 'type': 'CNAME', + 'value': 'default.unit.tests.', + 'dynamic': { + 'pools': { + 'one': { + 'values': [ + {'value': 'one.unit.tests.', 'weight': 1} + ], + 'fallback': 'two', + }, + 'two': { + 'values': [ + {'value': 'two.unit.tests.', 'weight': 1} + ], + }, + }, + 'rules': [ + {'geos': ['AF'], 'pool': 'one'}, + {'pool': 'two'}, + ], + } + } + octo_record = Record.new(zone, 'foo', data) + azure_record = _AzureRecord('TestAzure', octo_record, + traffic_manager=tm_profile) + self.assertEqual(azure_record.zone_name, zone.name[:-1]) + self.assertEqual(azure_record.relative_record_set_name, 'foo') + self.assertEqual(azure_record.record_type, 'CNAME') + self.assertEqual(azure_record.params['ttl'], 60) + self.assertEqual(azure_record.params['target_resource'], tm_profile) + + class Test_ParseAzureType(TestCase): def test_parse_azure_type(self): for expected, test in [['A', 'Microsoft.Network/dnszones/A'], @@ -361,40 +402,369 @@ class Test_CheckEndswithDot(TestCase): self.assertEquals(expected, _check_endswith_dot(test)) -class Test_CheckAzureAlias(TestCase): - def test_check_for_alias(self): - alias_record = type('C', (object,), {}) - alias_record.target_resource = type('C', (object,), {}) - alias_record.target_resource.id = "/subscriptions/x/resourceGroups/y/z" - alias_record.a_records = None - alias_record.cname_record = None +class Test_TrafficManagerSuffix(TestCase): + def test_traffic_manager_suffix(self): + test = Record.new(zone, 'foo', data={ + 'ttl': 60, 'type': 'CNAME', 'value': 'default.unit.tests.', + }) + self.assertEqual(_traffic_manager_suffix(test), 'foo-unit-tests') + + +class Test_GetMonitor(TestCase): + def test_get_monitor(self): + record = Record.new(zone, 'foo', data={ + 'type': 'CNAME', 'ttl': 60, 'value': 'default.unit.tests.', + 'octodns': { + 'healthcheck': { + 'path': '/_ping', + 'port': 4443, + 'protocol': 'HTTPS', + } + }, + }) + + monitor = _get_monitor(record) + self.assertEqual(monitor.protocol, 'HTTPS') + self.assertEqual(monitor.port, 4443) + self.assertEqual(monitor.path, '/_ping') + headers = monitor.custom_headers + self.assertIsInstance(headers, list) + self.assertEquals(len(headers), 1) + headers = headers[0] + self.assertEqual(headers.name, 'Host') + self.assertEqual(headers.value, record.healthcheck_host) + + # test TCP monitor + record._octodns['healthcheck']['protocol'] = 'TCP' + monitor = _get_monitor(record) + self.assertEqual(monitor.protocol, 'TCP') + self.assertIsNone(monitor.custom_headers) + + +class Test_ProfileIsMatch(TestCase): + def test_profile_is_match(self): + is_match = _profile_is_match + + self.assertFalse(is_match(None, Profile())) + + # Profile object builder with default property values that can be + # overridden for testing below + def profile( + name = 'foo-unit-tests', + ttl = 60, + method = 'Geographic', + monitor_proto = 'HTTPS', + monitor_port = 4443, + monitor_path = '/_ping', + endpoints = 1, + endpoint_name = 'name', + endpoint_type = 'profile/nestedEndpoints', + target = 'target.unit.tests', + target_id = 'resource/id', + geos = ['GEO-AF'], + weight = 1, + priority = 1, + ): + dns = DnsConfig(ttl=ttl) + return Profile( + name=name, traffic_routing_method=method, dns_config=dns, + monitor_config=MonitorConfig( + protocol=monitor_proto, + port=monitor_port, + path=monitor_path, + ), + endpoints=[Endpoint( + name=endpoint_name, + type=endpoint_type, + target=target, + target_resource_id=target_id, + geo_mapping=geos, + weight=weight, + priority=priority, + )] + [Endpoint()] * (endpoints - 1), + ) + + self.assertTrue(is_match(profile(), profile())) + + self.assertFalse(is_match(profile(), profile(name='two'))) + self.assertFalse(is_match(profile(), profile(endpoints=2))) + self.assertFalse(is_match(profile(), profile(monitor_proto='HTTP'))) + self.assertFalse(is_match(profile(), profile(endpoint_name='a'))) + self.assertFalse(is_match(profile(), profile(endpoint_type='b'))) + self.assertFalse( + is_match(profile(endpoint_type='b'), profile(endpoint_type='b')) + ) + self.assertFalse(is_match(profile(), profile(target_id='rsrc/id2'))) + self.assertFalse(is_match(profile(), profile(geos=['IN']))) + + def wprofile(**kwargs): + kwargs['method'] = 'Weighted' + kwargs['endpoint_type'] = 'profile/externalEndpoints' + return profile(**kwargs) - self.assertEquals(_check_for_alias(alias_record), True) + self.assertFalse(is_match(wprofile(), wprofile(target='bar.unit'))) + self.assertFalse(is_match(wprofile(), wprofile(weight=3))) class TestAzureDnsProvider(TestCase): def _provider(self): return self._get_provider('mock_spc', 'mock_dns_client') + @patch('octodns.provider.azuredns.TrafficManagerManagementClient') @patch('octodns.provider.azuredns.DnsManagementClient') @patch('octodns.provider.azuredns.ClientSecretCredential') - def _get_provider(self, mock_spc, mock_dns_client): + @patch('octodns.provider.azuredns.ServicePrincipalCredentials') + def _get_provider(self, mock_spc, mock_css, mock_dns_client, + mock_tm_client): '''Returns a mock AzureProvider object to use in testing. :param mock_spc: placeholder :type mock_spc: str :param mock_dns_client: placeholder :type mock_dns_client: str + :param mock_tm_client: placeholder + :type mock_tm_client: str :type return: AzureProvider ''' provider = AzureProvider('mock_id', 'mock_client', 'mock_key', 'mock_directory', 'mock_sub', 'mock_rg' ) + # Fetch the client to force it to load the creds provider._dns_client + + # set critical functions to return properly + tm_list = provider._tm_client.profiles.list_by_resource_group + tm_list.return_value = [] + tm_sync = provider._tm_client.profiles.create_or_update + + def side_effect(rg, name, profile): + return profile + + tm_sync.side_effect = side_effect + return provider + def _get_dynamic_record(self, zone): + return Record.new(zone, 'foo', data={ + 'type': 'CNAME', + 'ttl': 60, + 'value': 'default.unit.tests.', + 'dynamic': { + 'pools': { + 'one': { + 'values': [ + {'value': 'one.unit.tests.', 'weight': 11}, + ], + 'fallback': 'two', + }, + 'two': { + 'values': [ + {'value': 'two1.unit.tests.', 'weight': 3}, + {'value': 'two2.unit.tests.', 'weight': 4}, + ], + 'fallback': 'three', + }, + 'three': { + 'values': [ + {'value': 'three.unit.tests.', 'weight': 13}, + ], + }, + }, + 'rules': [ + {'geos': ['AF', 'EU-DE', 'NA-US-CA'], 'pool': 'one'}, + {'pool': 'three'}, + ], + }, + 'octodns': { + 'healthcheck': { + 'path': '/_ping', + 'port': 4443, + 'protocol': 'HTTPS', + } + }, + }) + + def _get_tm_profiles(self, provider): + sub = provider._dns_client_subscription_id + rg = provider._resource_group + base_id = '/subscriptions/' + sub + \ + '/resourceGroups/' + rg + \ + '/providers/Microsoft.Network/trafficManagerProfiles/' + suffix = 'foo-unit-tests' + id_format = base_id + '{}--' + suffix + name_format = '{}--' + suffix + + dns = DnsConfig(ttl=60) + header = MonitorConfigCustomHeadersItem(name='Host', + value='foo.unit.tests') + monitor = MonitorConfig(protocol='HTTPS', port=4443, path='/_ping', + custom_headers=[header]) + external = 'Microsoft.Network/trafficManagerProfiles/externalEndpoints' + nested = 'Microsoft.Network/trafficManagerProfiles/nestedEndpoints' + + return [ + Profile( + id=id_format.format('default'), + name=name_format.format('default'), + traffic_routing_method='Weighted', + dns_config=dns, + monitor_config=monitor, + endpoints=[ + Endpoint( + name='default.unit.tests', + type=external, + target='default.unit.tests', + weight=1, + ), + ], + ), + Profile( + id=id_format.format('pool-one'), + name=name_format.format('pool-one'), + traffic_routing_method='Weighted', + dns_config=dns, + monitor_config=monitor, + endpoints=[ + Endpoint( + name='one.unit.tests', + type=external, + target='one.unit.tests', + weight=11, + ), + ], + ), + Profile( + id=id_format.format('pool-two'), + name=name_format.format('pool-two'), + traffic_routing_method='Weighted', + dns_config=dns, + monitor_config=monitor, + endpoints=[ + Endpoint( + name='two1.unit.tests', + type=external, + target='two1.unit.tests', + weight=3, + ), + Endpoint( + name='two2.unit.tests', + type=external, + target='two2.unit.tests', + weight=4, + ), + ], + ), + Profile( + id=id_format.format('pool-three'), + name=name_format.format('pool-three'), + traffic_routing_method='Weighted', + dns_config=dns, + monitor_config=monitor, + endpoints=[ + Endpoint( + name='three.unit.tests', + type=external, + target='three.unit.tests', + weight=13, + ), + ], + ), + Profile( + id=id_format.format('rule-one'), + name=name_format.format('rule-one'), + traffic_routing_method='Priority', + dns_config=dns, + monitor_config=monitor, + endpoints=[ + Endpoint( + name='one', + type=nested, + target_resource_id=id_format.format('pool-one'), + priority=1, + ), + Endpoint( + name='two', + type=nested, + target_resource_id=id_format.format('pool-two'), + priority=2, + ), + Endpoint( + name='three', + type=nested, + target_resource_id=id_format.format('pool-three'), + priority=3, + ), + Endpoint( + name='--default--', + type=nested, + target_resource_id=id_format.format('default'), + priority=4, + ), + ], + ), + Profile( + id=id_format.format('rule-three'), + name=name_format.format('rule-three'), + traffic_routing_method='Priority', + dns_config=dns, + monitor_config=monitor, + endpoints=[ + Endpoint( + name='three', + type=nested, + target_resource_id=id_format.format('pool-three'), + priority=1, + ), + Endpoint( + name='--default--', + type=nested, + target_resource_id=id_format.format('default'), + priority=2, + ), + ], + ), + Profile( + id=base_id + suffix, + name=suffix, + traffic_routing_method='Geographic', + dns_config=dns, + monitor_config=monitor, + endpoints=[ + Endpoint( + geo_mapping=['GEO-AF', 'DE', 'US-CA'], + name='rule-one', + type=nested, + target_resource_id=id_format.format('rule-one'), + ), + Endpoint( + geo_mapping=['WORLD'], + name='rule-three', + type=nested, + target_resource_id=id_format.format('rule-three'), + ), + ], + ), + ] + + def _get_dynamic_package(self): + '''Convenience function to setup a sample dynamic record. + ''' + provider = self._get_provider() + + # setup traffic manager profiles + tm_list = provider._tm_client.profiles.list_by_resource_group + tm_list.return_value = self._get_tm_profiles(provider) + + # setup zone with dynamic record + zone = Zone(name='unit.tests.', sub_zones=[]) + record = self._get_dynamic_record(zone) + zone.add_record(record) + + # return everything + return provider, zone, record + def test_populate_records(self): provider = self._get_provider() @@ -510,6 +880,121 @@ class TestAzureDnsProvider(TestCase): self.assertEquals(len(zone.records), 17) self.assertTrue(exists) + def test_populate_dynamic(self): + # Middle east without Asia raises exception + provider, zone, record = self._get_dynamic_package() + tm_suffix = _traffic_manager_suffix(record) + tm_id = provider._profile_name_to_id + tm_list = provider._tm_client.profiles.list_by_resource_group + rule_name = 'rule-one--{}'.format(tm_suffix) + nested = 'Microsoft.Network/trafficManagerProfiles/nestedEndpoints' + tm_list.return_value = [ + Profile( + id=tm_id(tm_suffix), + name=tm_suffix, + traffic_routing_method='Geographic', + endpoints=[ + Endpoint( + geo_mapping=['GEO-ME'], + ), + ], + ), + ] + azrecord = RecordSet( + ttl=60, + target_resource=SubResource(id=tm_id(tm_suffix)), + ) + azrecord.name = record.name or '@' + azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) + with self.assertRaises(AzureException) as ctx: + provider._populate_record(zone, azrecord) + self.assertTrue(text_type(ctx).startswith( + 'Middle East (GEO-ME) is not supported' + )) + + # empty priority profile raises exception + provider, zone, record = self._get_dynamic_package() + tm_list = provider._tm_client.profiles.list_by_resource_group + rule_name = 'rule-one--{}'.format(tm_suffix) + nested = 'Microsoft.Network/trafficManagerProfiles/nestedEndpoints' + tm_list.return_value = [ + Profile( + id=tm_id(rule_name), + name=rule_name, + traffic_routing_method='Priority', + endpoints=[], + ), + Profile( + id=tm_id(tm_suffix), + name=tm_suffix, + traffic_routing_method='Geographic', + endpoints=[ + Endpoint( + geo_mapping=['WORLD'], + name='rule-one', + type=nested, + target_resource_id=tm_id(rule_name), + ), + ], + ), + ] + with self.assertRaises(AzureException) as ctx: + provider._populate_record(zone, azrecord) + self.assertTrue(text_type(ctx).startswith( + 'Expected at least 2 endpoints' + )) + + # valid set of profiles produce expected dynamic record + provider, zone, record = self._get_dynamic_package() + root_profile_id = provider._profile_name_to_id( + _traffic_manager_suffix(record) + ) + azrecord = RecordSet( + ttl=60, + target_resource=SubResource(id=root_profile_id), + ) + azrecord.name = record.name or '@' + azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) + + record = provider._populate_record(zone, azrecord) + self.assertEqual(record.name, 'foo') + self.assertEqual(record.ttl, 60) + self.assertEqual(record.value, 'default.unit.tests.') + self.assertEqual(record.dynamic._data(), { + 'pools': { + 'one': { + 'values': [ + {'value': 'one.unit.tests.', 'weight': 11}, + ], + 'fallback': 'two', + }, + 'two': { + 'values': [ + {'value': 'two1.unit.tests.', 'weight': 3}, + {'value': 'two2.unit.tests.', 'weight': 4}, + ], + 'fallback': 'three', + }, + 'three': { + 'values': [ + {'value': 'three.unit.tests.', 'weight': 13}, + ], + 'fallback': None, + }, + }, + 'rules': [ + {'geos': ['AF', 'EU-DE', 'NA-US-CA'], 'pool': 'one'}, + {'pool': 'three'}, + ], + }) + + # valid profiles with Middle East test case + geo_profile = provider._get_tm_for_dynamic_record(record) + geo_profile.endpoints[0].geo_mapping.extend(['GEO-ME', 'GEO-AS']) + record = provider._populate_record(zone, azrecord) + self.assertIn('AS', record.dynamic.rules[0].data['geos']) + self.assertNotIn('ME', record.dynamic.rules[0].data['geos']) + def test_populate_zone(self): provider = self._get_provider() @@ -541,20 +1026,388 @@ class TestAzureDnsProvider(TestCase): None ) + def test_extra_changes(self): + provider, existing, record = self._get_dynamic_package() + + # test simple records produce no extra changes + desired = Zone(name=existing.name, sub_zones=[]) + desired.add_record(Record.new(desired, 'simple', data={ + 'type': record._type, + 'ttl': record.ttl, + 'value': record.value, + })) + extra = provider._extra_changes(desired, desired, []) + self.assertEqual(len(extra), 0) + + # test an unchanged dynamic record produces no extra changes + desired.add_record(record) + extra = provider._extra_changes(existing, desired, []) + self.assertEqual(len(extra), 0) + + # test unused TM produces the extra change for clean up + sample_profile = self._get_tm_profiles(provider)[0] + tm_id = provider._profile_name_to_id + root_profile_name = _traffic_manager_suffix(record) + extra_profile = Profile( + id=tm_id('random--{}'.format(root_profile_name)), + name='random--{}'.format(root_profile_name), + traffic_routing_method='Weighted', + dns_config=sample_profile.dns_config, + monitor_config=sample_profile.monitor_config, + endpoints=sample_profile.endpoints, + ) + tm_list = provider._tm_client.profiles.list_by_resource_group + tm_list.return_value.append(extra_profile) + provider._populate_traffic_managers() + extra = provider._extra_changes(existing, desired, []) + self.assertEqual(len(extra), 1) + extra = extra[0] + self.assertIsInstance(extra, Update) + self.assertEqual(extra.new, record) + desired._remove_record(record) + tm_list.return_value.pop() + + # test new dynamic record does not produce an extra change for it + new_dynamic = Record.new(desired, record.name + '2', data={ + 'type': record._type, + 'ttl': record.ttl, + 'value': record.value, + 'dynamic': record.dynamic._data(), + 'octodns': record._octodns, + }) + # test change in healthcheck by using a different port number + update_dynamic = Record.new(desired, record.name, data={ + 'type': record._type, + 'ttl': record.ttl, + 'value': record.value, + 'dynamic': record.dynamic._data(), + 'octodns': { + 'healthcheck': { + 'path': '/_ping', + 'port': 443, + 'protocol': 'HTTPS', + }, + }, + }) + desired.add_record(new_dynamic) + desired.add_record(update_dynamic) + changes = [Create(new_dynamic)] + extra = provider._extra_changes(existing, desired, changes) + # implicitly asserts that new_dynamic was not added to extra changes + # as it was already in the `changes` list + self.assertEqual(len(extra), 1) + extra = extra[0] + self.assertIsInstance(extra, Update) + self.assertEqual(extra.new, update_dynamic) + + # test non-CNAME dynamic record throws exception + a_dynamic = Record.new(desired, record.name + '3', data={ + 'type': 'A', + 'ttl': record.ttl, + 'values': ['1.1.1.1'], + 'dynamic': { + 'pools': { + 'one': {'values': [{'value': '2.2.2.2'}]}, + }, + 'rules': [ + {'pool': 'one'}, + ], + }, + }) + desired.add_record(a_dynamic) + changes.append(Create(a_dynamic)) + with self.assertRaises(AzureException): + provider._extra_changes(existing, desired, changes) + + def test_generate_tm_profile(self): + provider, zone, record = self._get_dynamic_package() + profile_gen = provider._generate_tm_profile + + name = 'foobar' + routing = 'Priority' + endpoints = [ + Endpoint(target='one.unit.tests'), + Endpoint(target_resource_id='/s/1/rg/foo/tm/foobar2'), + Endpoint(name='invalid'), + ] + + # invalid endpoint raises exception + with self.assertRaises(AzureException): + profile_gen(name, routing, endpoints, record) + + # regular test + endpoints.pop() + profile = profile_gen(name, routing, endpoints, record) + + # implicitly tests _profile_name_to_id + sub = provider._dns_client_subscription_id + rg = provider._resource_group + expected_id = '/subscriptions/' + sub + \ + '/resourceGroups/' + rg + \ + '/providers/Microsoft.Network/trafficManagerProfiles/' + name + self.assertEqual(profile.id, expected_id) + self.assertEqual(profile.name, name) + self.assertEqual(profile.traffic_routing_method, routing) + self.assertEqual(profile.dns_config.ttl, record.ttl) + self.assertEqual(len(profile.endpoints), len(endpoints)) + + self.assertEqual( + profile.endpoints[0].type, + 'Microsoft.Network/trafficManagerProfiles/externalEndpoints' + ) + self.assertEqual( + profile.endpoints[1].type, + 'Microsoft.Network/trafficManagerProfiles/nestedEndpoints' + ) + + def test_generate_traffic_managers(self): + provider, zone, record = self._get_dynamic_package() + profiles = provider._generate_traffic_managers(record) + deduped = [] + seen = set() + for profile in profiles: + if profile.name not in seen: + deduped.append(profile) + seen.add(profile.name) + + # check that every profile is a match with what we expect + expected_profiles = self._get_tm_profiles(provider) + self.assertEqual(len(expected_profiles), len(deduped)) + for have, expected in zip(deduped, expected_profiles): + self.assertTrue(_profile_is_match(have, expected)) + + # check Asia/Middle East test case + record.dynamic._data()['rules'][0]['geos'].append('AS') + profiles = provider._generate_traffic_managers(record) + geo_profile_name = _traffic_manager_suffix(record) + geo_profile = next( + profile + for profile in profiles + if profile.name == geo_profile_name + ) + self.assertIn('GEO-ME', geo_profile.endpoints[0].geo_mapping) + self.assertIn('GEO-AS', geo_profile.endpoints[0].geo_mapping) + + def test_sync_traffic_managers(self): + provider, zone, record = self._get_dynamic_package() + provider._populate_traffic_managers() + + tm_sync = provider._tm_client.profiles.create_or_update + + suffix = 'foo-unit-tests' + expected_seen = { + suffix, 'default--{}'.format(suffix), + 'rule-one--{}'.format(suffix), 'rule-three--{}'.format(suffix), + 'pool-one--{}'.format(suffix), 'pool-two--{}'.format(suffix), + 'pool-three--{}'.format(suffix), + } + + # test no change + seen = provider._sync_traffic_managers(record) + self.assertEqual(seen, expected_seen) + tm_sync.assert_not_called() + + # test that changing weight causes update API call + dynamic = record.dynamic._data() + dynamic['pools']['one']['values'][0]['weight'] = 14 + data = { + 'type': 'CNAME', + 'ttl': record.ttl, + 'value': record.value, + 'dynamic': dynamic, + 'octodns': record._octodns, + } + new_record = Record.new(zone, record.name, data) + tm_sync.reset_mock() + seen2 = provider._sync_traffic_managers(new_record) + self.assertEqual(seen2, expected_seen) + tm_sync.assert_called_once() + + # test that new profile was successfully inserted in cache + new_profile = provider._get_tm_profile_by_name( + 'pool-one--{}'.format(suffix) + ) + self.assertEqual(new_profile.endpoints[0].weight, 14) + + def test_find_traffic_managers(self): + provider, zone, record = self._get_dynamic_package() + + # insert a non-matching profile + sample_profile = self._get_tm_profiles(provider)[0] + # dummy record for generating suffix + record2 = Record.new(zone, record.name + '2', data={ + 'type': record._type, + 'ttl': record.ttl, + 'value': record.value, + }) + suffix2 = _traffic_manager_suffix(record2) + tm_id = provider._profile_name_to_id + extra_profile = Profile( + id=tm_id('random--{}'.format(suffix2)), + name='random--{}'.format(suffix2), + traffic_routing_method='Weighted', + dns_config=sample_profile.dns_config, + monitor_config=sample_profile.monitor_config, + endpoints=sample_profile.endpoints, + ) + tm_list = provider._tm_client.profiles.list_by_resource_group + tm_list.return_value.append(extra_profile) + provider._populate_traffic_managers() + + # implicitly asserts that non-matching profile is not included + suffix = _traffic_manager_suffix(record) + self.assertEqual(provider._find_traffic_managers(record), { + suffix, 'default--{}'.format(suffix), + 'rule-one--{}'.format(suffix), 'rule-three--{}'.format(suffix), + 'pool-one--{}'.format(suffix), 'pool-two--{}'.format(suffix), + 'pool-three--{}'.format(suffix), + }) + + def test_traffic_manager_gc(self): + provider, zone, record = self._get_dynamic_package() + provider._populate_traffic_managers() + + profiles = provider._find_traffic_managers(record) + profile_delete_mock = provider._tm_client.profiles.delete + + provider._traffic_managers_gc(record, profiles) + profile_delete_mock.assert_not_called() + + profile_delete_mock.reset_mock() + remove = list(profiles)[3] + profiles.discard(remove) + + provider._traffic_managers_gc(record, profiles) + profile_delete_mock.assert_has_calls( + [call(provider._resource_group, remove)] + ) + def test_apply(self): provider = self._get_provider() - changes = [] - deletes = [] - for i in octo_records: - changes.append(Create(i)) - deletes.append(Delete(i)) + half = int(len(octo_records) / 2) + changes = [Create(r) for r in octo_records[:half]] + \ + [Update(r, r) for r in octo_records[half:]] + deletes = [Delete(r) for r in octo_records] self.assertEquals(19, provider.apply(Plan(None, zone, changes, True))) self.assertEquals(19, provider.apply(Plan(zone, zone, deletes, True))) + def test_apply_create_dynamic(self): + provider = self._get_provider() + + tm_list = provider._tm_client.profiles.list_by_resource_group + tm_list.return_value = [] + + tm_sync = provider._tm_client.profiles.create_or_update + + zone = Zone(name='unit.tests.', sub_zones=[]) + record = self._get_dynamic_record(zone) + + profiles = self._get_tm_profiles(provider) + + provider._apply_Create(Create(record)) + # create was called as many times as number of profiles required for + # the dynamic record + self.assertEqual(tm_sync.call_count, len(profiles)) + + create = provider._dns_client.record_sets.create_or_update + create.assert_called_once() + + def test_apply_update_dynamic(self): + # existing is simple, new is dynamic + provider = self._get_provider() + tm_list = provider._tm_client.profiles.list_by_resource_group + tm_list.return_value = [] + profiles = self._get_tm_profiles(provider) + dynamic_record = self._get_dynamic_record(zone) + simple_record = Record.new(zone, dynamic_record.name, data={ + 'type': 'CNAME', + 'ttl': 3600, + 'value': 'cname.unit.tests.', + }) + change = Update(simple_record, dynamic_record) + provider._apply_Update(change) + tm_sync, dns_update, tm_delete = ( + provider._tm_client.profiles.create_or_update, + provider._dns_client.record_sets.create_or_update, + provider._tm_client.profiles.delete + ) + self.assertEqual(tm_sync.call_count, len(profiles)) + dns_update.assert_called_once() + tm_delete.assert_not_called() + + # existing is dynamic, new is simple + provider, existing, dynamic_record = self._get_dynamic_package() + profiles = self._get_tm_profiles(provider) + change = Update(dynamic_record, simple_record) + provider._apply_Update(change) + tm_sync, dns_update, tm_delete = ( + provider._tm_client.profiles.create_or_update, + provider._dns_client.record_sets.create_or_update, + provider._tm_client.profiles.delete + ) + tm_sync.assert_not_called() + dns_update.assert_called_once() + self.assertEqual(tm_delete.call_count, len(profiles)) + + # both are dynamic, healthcheck port is changed + provider, existing, dynamic_record = self._get_dynamic_package() + profiles = self._get_tm_profiles(provider) + dynamic_record2 = self._get_dynamic_record(existing) + dynamic_record2._octodns['healthcheck']['port'] += 1 + change = Update(dynamic_record, dynamic_record2) + provider._apply_Update(change) + tm_sync, dns_update, tm_delete = ( + provider._tm_client.profiles.create_or_update, + provider._dns_client.record_sets.create_or_update, + provider._tm_client.profiles.delete + ) + self.assertEqual(tm_sync.call_count, len(profiles)) + dns_update.assert_not_called() + tm_delete.assert_not_called() + + # both are dynamic, extra profile should be deleted + provider, existing, dynamic_record = self._get_dynamic_package() + sample_profile = self._get_tm_profiles(provider)[0] + tm_id = provider._profile_name_to_id + root_profile_name = _traffic_manager_suffix(dynamic_record) + extra_profile = Profile( + id=tm_id('random--{}'.format(root_profile_name)), + name='random--{}'.format(root_profile_name), + traffic_routing_method='Weighted', + dns_config=sample_profile.dns_config, + monitor_config=sample_profile.monitor_config, + endpoints=sample_profile.endpoints, + ) + tm_list = provider._tm_client.profiles.list_by_resource_group + tm_list.return_value.append(extra_profile) + change = Update(dynamic_record, dynamic_record) + provider._apply_Update(change) + tm_sync, dns_update, tm_delete = ( + provider._tm_client.profiles.create_or_update, + provider._dns_client.record_sets.create_or_update, + provider._tm_client.profiles.delete + ) + tm_sync.assert_not_called() + dns_update.assert_not_called() + tm_delete.assert_called_once() + + def test_apply_delete_dynamic(self): + provider, existing, record = self._get_dynamic_package() + provider._populate_traffic_managers() + profiles = self._get_tm_profiles(provider) + change = Delete(record) + provider._apply_Delete(change) + dns_delete, tm_delete = ( + provider._dns_client.record_sets.delete, + provider._tm_client.profiles.delete + ) + dns_delete.assert_called_once() + self.assertEqual(tm_delete.call_count, len(profiles)) + def test_create_zone(self): provider = self._get_provider() From 72b0438b7fb120e1155bf9934ddde34e9f28e267 Mon Sep 17 00:00:00 2001 From: Steven Hollingsworth Date: Wed, 12 May 2021 09:40:06 -0700 Subject: [PATCH 54/92] Updated doc to reflect config file location for zone wide lenient flag --- docs/records.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/records.md b/docs/records.md index e39a85d..c6b2a77 100644 --- a/docs/records.md +++ b/docs/records.md @@ -114,8 +114,7 @@ If you'd like to enable lenience for a whole zone you can do so with the followi ```yaml non-compliant-zone.com.: - octodns: - lenient: true + lenient: true sources: - route53 targets: From 9b5c8be01ece38e740fb7ee77aefec783812c852 Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Wed, 12 May 2021 10:02:04 -0700 Subject: [PATCH 55/92] optimize by not creating traffic manager for single-value pools If single-value pools have a weight defined, it will be lost by this optimization. Next time octodns-sync is run, it will show an update for setting the weight on remote. To overcome this, this commit includes a change to Record object that ignores the weight in single-value pools. --- octodns/provider/azuredns.py | 82 +++++++++--------- octodns/record/__init__.py | 4 + tests/test_octodns_provider_azuredns.py | 105 +++++++----------------- tests/test_octodns_record.py | 1 + 4 files changed, 78 insertions(+), 114 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 4d3dcc5..a792b9b 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -667,14 +667,9 @@ class AzureProvider(BaseProvider): for rule_ep in rule_endpoints: pool_name = rule_ep.name - # third (and last) level weighted RR profile - # these should be leaf node profiles with no further nesting - pool_profile = rule_ep.target_resource - # last/default pool if pool_name == '--default--': - for pool_ep in pool_profile.endpoints: - default.add(pool_ep.target) + default.add(rule_ep.target) # this should be the last one, so let's break here break @@ -687,11 +682,20 @@ class AzureProvider(BaseProvider): pool['fallback'] = pool_name pool = pools[pool_name] - for pool_ep in pool_profile.endpoints: + endpoints = [] + # these should be leaf node entries with no further nesting + if rule_ep.target_resource_id: + # third (and last) level weighted RR profile + endpoints = rule_ep.target_resource.endpoints + else: + # single-value pool + endpoints = [rule_ep] + + for pool_ep in endpoints: val = pool_ep.target value_dict = { 'value': _check_endswith_dot(val), - 'weight': pool_ep.weight, + 'weight': pool_ep.weight or 1, } if value_dict not in pool['values']: pool['values'].append(value_dict) @@ -703,6 +707,7 @@ class AzureProvider(BaseProvider): geo_ep.target_resource.name, len(rule_endpoints) ) raise AzureException(msg) + rules.append(rule) # Order and convert to a list @@ -800,19 +805,6 @@ class AzureProvider(BaseProvider): tm_suffix = _traffic_manager_suffix(record) profile = self._generate_tm_profile - # construct the default pool that will be used at the end of - # all rules - target = record.value[:-1] - default_endpoints = [Endpoint( - name=target, - target=target, - weight=1, - )] - default_profile_name = 'default--{}'.format(tm_suffix) - default_profile = profile(default_profile_name, 'Weighted', - default_endpoints, record) - traffic_managers.append(default_profile) - geo_endpoints = [] for rule in record.dynamic.rules: @@ -824,26 +816,36 @@ class AzureProvider(BaseProvider): # iterate until we reach end of fallback chain pool = pools[pool_name].data profile_name = 'pool-{}--{}'.format(pool_name, tm_suffix) - endpoints = [] - for val in pool['values']: - target = val['value'] - # strip trailing dot from CNAME value - target = target[:-1] - endpoints.append(Endpoint( - name=target, + if len(pool['values']) > 1: + # create Weighted profile for multi-value pool + endpoints = [] + for val in pool['values']: + target = val['value'] + # strip trailing dot from CNAME value + target = target[:-1] + endpoints.append(Endpoint( + name=target, + target=target, + weight=val.get('weight', 1), + )) + pool_profile = profile(profile_name, 'Weighted', endpoints, + record) + traffic_managers.append(pool_profile) + + # append pool to endpoint list of fallback rule profile + rule_endpoints.append(Endpoint( + name=pool_name, + target_resource_id=pool_profile.id, + priority=priority, + )) + else: + # add single-value pool as an external endpoint + target = pool['values'][0]['value'][:-1] + rule_endpoints.append(Endpoint( + name=pool_name, target=target, - weight=val.get('weight', 1), + priority=priority, )) - pool_profile = profile(profile_name, 'Weighted', endpoints, - record) - traffic_managers.append(pool_profile) - - # append pool to endpoint list of fallback rule profile - rule_endpoints.append(Endpoint( - name=pool_name, - target_resource_id=pool_profile.id, - priority=priority, - )) priority += 1 pool_name = pool.get('fallback') @@ -851,7 +853,7 @@ class AzureProvider(BaseProvider): # append default profile to the end rule_endpoints.append(Endpoint( name='--default--', - target_resource_id=default_profile.id, + target=record.value[:-1], priority=priority, )) # create rule profile with fallback chain diff --git a/octodns/record/__init__.py b/octodns/record/__init__.py index 135baf8..cc503d6 100644 --- a/octodns/record/__init__.py +++ b/octodns/record/__init__.py @@ -429,6 +429,10 @@ class _DynamicPool(object): ] values.sort(key=lambda d: d['value']) + # normalize weight of a single-value pool + if len(values) == 1: + values[0]['weight'] = 1 + fallback = data.get('fallback', None) self.data = { 'fallback': fallback if fallback != 'default' else None, diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index 5655719..fe867fe 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -574,7 +574,7 @@ class TestAzureDnsProvider(TestCase): }, 'rules': [ {'geos': ['AF', 'EU-DE', 'NA-US-CA'], 'pool': 'one'}, - {'pool': 'three'}, + {'pool': 'two'}, ], }, 'octodns': { @@ -605,36 +605,6 @@ class TestAzureDnsProvider(TestCase): nested = 'Microsoft.Network/trafficManagerProfiles/nestedEndpoints' return [ - Profile( - id=id_format.format('default'), - name=name_format.format('default'), - traffic_routing_method='Weighted', - dns_config=dns, - monitor_config=monitor, - endpoints=[ - Endpoint( - name='default.unit.tests', - type=external, - target='default.unit.tests', - weight=1, - ), - ], - ), - Profile( - id=id_format.format('pool-one'), - name=name_format.format('pool-one'), - traffic_routing_method='Weighted', - dns_config=dns, - monitor_config=monitor, - endpoints=[ - Endpoint( - name='one.unit.tests', - type=external, - target='one.unit.tests', - weight=11, - ), - ], - ), Profile( id=id_format.format('pool-two'), name=name_format.format('pool-two'), @@ -656,21 +626,6 @@ class TestAzureDnsProvider(TestCase): ), ], ), - Profile( - id=id_format.format('pool-three'), - name=name_format.format('pool-three'), - traffic_routing_method='Weighted', - dns_config=dns, - monitor_config=monitor, - endpoints=[ - Endpoint( - name='three.unit.tests', - type=external, - target='three.unit.tests', - weight=13, - ), - ], - ), Profile( id=id_format.format('rule-one'), name=name_format.format('rule-one'), @@ -680,8 +635,8 @@ class TestAzureDnsProvider(TestCase): endpoints=[ Endpoint( name='one', - type=nested, - target_resource_id=id_format.format('pool-one'), + type=external, + target='one.unit.tests', priority=1, ), Endpoint( @@ -692,37 +647,43 @@ class TestAzureDnsProvider(TestCase): ), Endpoint( name='three', - type=nested, - target_resource_id=id_format.format('pool-three'), + type=external, + target='three.unit.tests', priority=3, ), Endpoint( name='--default--', - type=nested, - target_resource_id=id_format.format('default'), + type=external, + target='default.unit.tests', priority=4, ), ], ), Profile( - id=id_format.format('rule-three'), - name=name_format.format('rule-three'), + id=id_format.format('rule-two'), + name=name_format.format('rule-two'), traffic_routing_method='Priority', dns_config=dns, monitor_config=monitor, endpoints=[ Endpoint( - name='three', + name='two', type=nested, - target_resource_id=id_format.format('pool-three'), + target_resource_id=id_format.format('pool-two'), priority=1, ), Endpoint( - name='--default--', - type=nested, - target_resource_id=id_format.format('default'), + name='three', + type=external, + target='three.unit.tests', priority=2, ), + Endpoint( + name='--default--', + type=external, + target='default.unit.tests', + priority=3, + ), ], ), Profile( @@ -740,9 +701,9 @@ class TestAzureDnsProvider(TestCase): ), Endpoint( geo_mapping=['WORLD'], - name='rule-three', + name='rule-two', type=nested, - target_resource_id=id_format.format('rule-three'), + target_resource_id=id_format.format('rule-two'), ), ], ), @@ -964,7 +925,7 @@ class TestAzureDnsProvider(TestCase): 'pools': { 'one': { 'values': [ - {'value': 'one.unit.tests.', 'weight': 11}, + {'value': 'one.unit.tests.', 'weight': 1}, ], 'fallback': 'two', }, @@ -977,14 +938,14 @@ class TestAzureDnsProvider(TestCase): }, 'three': { 'values': [ - {'value': 'three.unit.tests.', 'weight': 13}, + {'value': 'three.unit.tests.', 'weight': 1}, ], 'fallback': None, }, }, 'rules': [ {'geos': ['AF', 'EU-DE', 'NA-US-CA'], 'pool': 'one'}, - {'pool': 'three'}, + {'pool': 'two'}, ], }) @@ -1196,10 +1157,8 @@ class TestAzureDnsProvider(TestCase): suffix = 'foo-unit-tests' expected_seen = { - suffix, 'default--{}'.format(suffix), - 'rule-one--{}'.format(suffix), 'rule-three--{}'.format(suffix), - 'pool-one--{}'.format(suffix), 'pool-two--{}'.format(suffix), - 'pool-three--{}'.format(suffix), + suffix, 'pool-two--{}'.format(suffix), + 'rule-one--{}'.format(suffix), 'rule-two--{}'.format(suffix), } # test no change @@ -1209,7 +1168,7 @@ class TestAzureDnsProvider(TestCase): # test that changing weight causes update API call dynamic = record.dynamic._data() - dynamic['pools']['one']['values'][0]['weight'] = 14 + dynamic['pools']['two']['values'][0]['weight'] = 14 data = { 'type': 'CNAME', 'ttl': record.ttl, @@ -1225,7 +1184,7 @@ class TestAzureDnsProvider(TestCase): # test that new profile was successfully inserted in cache new_profile = provider._get_tm_profile_by_name( - 'pool-one--{}'.format(suffix) + 'pool-two--{}'.format(suffix) ) self.assertEqual(new_profile.endpoints[0].weight, 14) @@ -1257,10 +1216,8 @@ class TestAzureDnsProvider(TestCase): # implicitly asserts that non-matching profile is not included suffix = _traffic_manager_suffix(record) self.assertEqual(provider._find_traffic_managers(record), { - suffix, 'default--{}'.format(suffix), - 'rule-one--{}'.format(suffix), 'rule-three--{}'.format(suffix), - 'pool-one--{}'.format(suffix), 'pool-two--{}'.format(suffix), - 'pool-three--{}'.format(suffix), + suffix, 'pool-two--{}'.format(suffix), + 'rule-one--{}'.format(suffix), 'rule-two--{}'.format(suffix), }) def test_traffic_manager_gc(self): diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index ce40b9b..3bb5d24 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -3013,6 +3013,7 @@ class TestDynamicRecords(TestCase): 'pools': { 'one': { 'values': [{ + 'weight': 10, 'value': '3.3.3.3', }], }, From 5df2077ed02124487b7a69be024ddb2a0e6a9386 Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Fri, 14 May 2021 23:21:26 -0700 Subject: [PATCH 56/92] clarify the limitations and caveats of azure dynamic records --- README.md | 2 +- octodns/provider/azuredns.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 8d59cd0..2ce9445 100644 --- a/README.md +++ b/README.md @@ -192,7 +192,7 @@ The above command pulled the existing data out of Route53 and placed the results | Provider | Requirements | Record Support | Dynamic | Notes | |--|--|--|--|--| -| [AzureProvider](/octodns/provider/azuredns.py) | azure-identity, azure-mgmt-dns, azure-mgmt-trafficmanager | A, AAAA, CAA, CNAME, MX, NS, PTR, SRV, TXT | Yes (CNAMEs only) | | +| [AzureProvider](/octodns/provider/azuredns.py) | azure-identity, azure-mgmt-dns, azure-mgmt-trafficmanager | A, AAAA, CAA, CNAME, MX, NS, PTR, SRV, TXT | Alpha (CNAMEs only) | | | [Akamai](/octodns/provider/edgedns.py) | edgegrid-python | A, AAAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, SSHFP, TXT | No | | | [CloudflareProvider](/octodns/provider/cloudflare.py) | | A, AAAA, ALIAS, CAA, CNAME, LOC, MX, NS, PTR, SPF, SRV, TXT | No | CAA tags restricted | | [ConstellixProvider](/octodns/provider/constellix.py) | | A, AAAA, ALIAS (ANAME), CAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | CAA tags restricted | diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index a792b9b..8474014 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -387,6 +387,9 @@ class AzureProvider(BaseProvider): The first four variables above can be hidden in environment variables and octoDNS will automatically search for them in the shell. It is possible to also hard-code into the config file: eg, resource_group. + + Please read https://github.com/octodns/octodns/pull/706 for an overview + of how dynamic records are designed and caveats of using them. ''' SUPPORTS_GEO = False SUPPORTS_DYNAMIC = True From 1b5bf75c585a3a79c72ee655cbb383b8b962df72 Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Mon, 17 May 2021 13:16:00 -0700 Subject: [PATCH 57/92] drop method names from exceptions --- octodns/provider/azuredns.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 8474014..5c2d23c 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -644,8 +644,8 @@ class AzureProvider(BaseProvider): # Throw exception otherwise, it should not happen if the # profile was generated by octoDNS if 'GEO-AS' not in geo_map: - msg = '_data_for_dynamic: Profile={}: '.format( - geo_profile.name) + msg = 'Profile={} for record {}: '.format( + geo_profile.name, azrecord.fqdn) msg += 'Middle East (GEO-ME) is not supported by ' + \ 'octoDNS. It needs to be either paired ' + \ 'with Asia (GEO-AS) or expanded into ' + \ @@ -782,9 +782,9 @@ class AzureProvider(BaseProvider): elif ep.target: ep.type = endpoint_type_prefix + 'externalEndpoints' else: - msg = ('_generate_tm_profile: Invalid endpoint {} ' + - 'in profile {}, needs to have either target or ' + - 'target_resource_id').format(ep.name, name) + msg = ('Invalid endpoint {} in profile {}, needs to have' + + 'either target or target_resource_id').format( + ep.name, name) raise AzureException(msg) # build and return @@ -1032,7 +1032,7 @@ class AzureProvider(BaseProvider): if getattr(record, 'dynamic', False): self._traffic_managers_gc(record, set()) - self.log.debug('* Success Delete: {}'.format(ar)) + self.log.debug('* Success Delete: {}'.format(record)) def _apply(self, plan): '''Required function of manager.py to actually apply a record change. From cca288faa6463e2abfd572612ff1209e8d1053c5 Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Mon, 17 May 2021 13:19:43 -0700 Subject: [PATCH 58/92] log warning when non-1 weight is set for single-value pools --- octodns/record/__init__.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/octodns/record/__init__.py b/octodns/record/__init__.py index cc503d6..45cea51 100644 --- a/octodns/record/__init__.py +++ b/octodns/record/__init__.py @@ -417,6 +417,7 @@ class _ValueMixin(object): class _DynamicPool(object): + log = getLogger('_DynamicPool') def __init__(self, _id, data): self._id = _id @@ -431,7 +432,12 @@ class _DynamicPool(object): # normalize weight of a single-value pool if len(values) == 1: - values[0]['weight'] = 1 + weight = data['values'][0].get('weight', 1) + if weight != 1: + self.log.warn( + 'Using weight=1 instead of %s for single-value pool %s', + weight, _id) + values[0]['weight'] = 1 fallback = data.get('fallback', None) self.data = { From e1d262a3013942e2a614622ce6d819deab3909c9 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 17 May 2021 17:06:40 -0700 Subject: [PATCH 59/92] Add a validation requiring single value weight=1 --- octodns/record/__init__.py | 4 ++++ tests/test_octodns_record.py | 29 ++++++++++++++++++++++++++++- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/octodns/record/__init__.py b/octodns/record/__init__.py index 135baf8..f960998 100644 --- a/octodns/record/__init__.py +++ b/octodns/record/__init__.py @@ -573,6 +573,10 @@ class _DynamicMixin(object): reasons.append('missing value in pool "{}" ' 'value {}'.format(_id, value_num)) + if len(values) == 1 and values[0].get('weight', 1) != 1: + reasons.append('pool "{}" has single value with ' + 'weight!=1'.format(_id)) + fallback = pool.get('fallback', None) if fallback is not None: if fallback in pools: diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index ce40b9b..5a4704a 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -3412,7 +3412,7 @@ class TestDynamicRecords(TestCase): self.assertEquals(['pool "one" is missing values'], ctx.exception.reasons) - # pool valu not a dict + # pool value not a dict a_data = { 'dynamic': { 'pools': { @@ -3596,6 +3596,33 @@ class TestDynamicRecords(TestCase): self.assertEquals(['invalid weight "foo" in pool "three" value 2'], ctx.exception.reasons) + # single value with weight!=1 + a_data = { + 'dynamic': { + 'pools': { + 'one': { + 'values': [{ + 'weight': 12, + 'value': '6.6.6.6', + }], + }, + }, + 'rules': [{ + 'pool': 'one', + }], + }, + 'ttl': 60, + 'type': 'A', + 'values': [ + '1.1.1.1', + '2.2.2.2', + ], + } + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, 'bad', a_data) + self.assertEquals(['pool "one" has single value with weight!=1'], + ctx.exception.reasons) + # invalid fallback a_data = { 'dynamic': { From 753a337ecc604d837f018ac7f29313fd2e083b36 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 17 May 2021 17:49:31 -0700 Subject: [PATCH 60/92] Fix weights on new Azure dynamic record tests --- tests/test_octodns_provider_azuredns.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index fe867fe..609fc5c 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -555,7 +555,7 @@ class TestAzureDnsProvider(TestCase): 'pools': { 'one': { 'values': [ - {'value': 'one.unit.tests.', 'weight': 11}, + {'value': 'one.unit.tests.', 'weight': 1}, ], 'fallback': 'two', }, @@ -568,7 +568,7 @@ class TestAzureDnsProvider(TestCase): }, 'three': { 'values': [ - {'value': 'three.unit.tests.', 'weight': 13}, + {'value': 'three.unit.tests.', 'weight': 1}, ], }, }, From b02d5d0a2de8ff61271458da9d570283e325aff3 Mon Sep 17 00:00:00 2001 From: Tom Kaminski Date: Mon, 17 May 2021 19:29:10 -0500 Subject: [PATCH 61/92] Do not trigger change for health checks on cname dynamic records --- octodns/provider/route53.py | 11 +- tests/test_octodns_provider_route53.py | 157 +++++++++++++++++++++++++ 2 files changed, 167 insertions(+), 1 deletion(-) diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index 0d5bab9..a3b3a57 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -1253,7 +1253,16 @@ class Route53Provider(BaseProvider): return self._gen_mods('DELETE', existing_records, existing_rrsets) def _extra_changes_update_needed(self, record, rrset): - healthcheck_host = record.healthcheck_host + value = rrset['ResourceRecords'][0]['Value'] + + try: + ip_address(text_type(value)) + # We're working with an IP + healthcheck_host = record.healthcheck_host + except (AddressValueError, ValueError): + # This isn't an IP, host is the value + healthcheck_host = value + healthcheck_path = record.healthcheck_path healthcheck_protocol = record.healthcheck_protocol healthcheck_port = record.healthcheck_port diff --git a/tests/test_octodns_provider_route53.py b/tests/test_octodns_provider_route53.py index a2b61e7..1e7210d 100644 --- a/tests/test_octodns_provider_route53.py +++ b/tests/test_octodns_provider_route53.py @@ -1962,6 +1962,163 @@ class TestRoute53Provider(TestCase): self.assertEquals(1, len(extra)) stubber.assert_no_pending_responses() + def test_extra_change_dynamic_has_health_check_cname(self): + provider, stubber = self._get_stubbed_provider() + + list_hosted_zones_resp = { + 'HostedZones': [{ + 'Name': 'unit.tests.', + 'Id': 'z42', + 'CallerReference': 'abc', + }], + 'Marker': 'm', + 'IsTruncated': False, + 'MaxItems': '100', + } + stubber.add_response('list_hosted_zones', list_hosted_zones_resp, {}) + + # record with geo and no health check returns change + desired = Zone('unit.tests.', []) + record = Record.new(desired, 'cname', { + 'ttl': 30, + 'type': 'CNAME', + 'value': 'cname.unit.tests.', + 'dynamic': { + 'pools': { + 'one': { + 'values': [{ + 'value': 'one.cname.unit.tests.', + }], + }, + }, + 'rules': [{ + 'pool': 'one', + }], + }, + }) + desired.add_record(record) + list_resource_record_sets_resp = { + 'ResourceRecordSets': [{ + # Not dynamic value and other name + 'Name': 'unit.tests.', + 'Type': 'CNAME', + 'GeoLocation': { + 'CountryCode': '*', + }, + 'ResourceRecords': [{ + 'Value': 'cname.unit.tests.', + }], + 'TTL': 61, + # All the non-matches have a different Id so we'll fail if they + # match + 'HealthCheckId': '33', + }, { + # Not dynamic value, matching name, other type + 'Name': 'cname.unit.tests.', + 'Type': 'AAAA', + 'ResourceRecords': [{ + 'Value': '2001:0db8:3c4d:0015:0000:0000:1a2f:1a4b' + }], + 'TTL': 61, + 'HealthCheckId': '33', + }, { + # default value pool + 'Name': '_octodns-default-value.cname.unit.tests.', + 'Type': 'CNAME', + 'GeoLocation': { + 'CountryCode': '*', + }, + 'ResourceRecords': [{ + 'Value': 'cname.unit.tests.', + }], + 'TTL': 61, + 'HealthCheckId': '33', + }, { + # different record + 'Name': '_octodns-two-value.other.unit.tests.', + 'Type': 'CNAME', + 'GeoLocation': { + 'CountryCode': '*', + }, + 'ResourceRecords': [{ + 'Value': 'cname.unit.tests.', + }], + 'TTL': 61, + 'HealthCheckId': '33', + }, { + # same everything, but different type + 'Name': '_octodns-one-value.cname.unit.tests.', + 'Type': 'AAAA', + 'ResourceRecords': [{ + 'Value': '2001:0db8:3c4d:0015:0000:0000:1a2f:1a4b' + }], + 'TTL': 61, + 'HealthCheckId': '33', + }, { + # same everything, sub + 'Name': '_octodns-one-value.sub.cname.unit.tests.', + 'Type': 'CNAME', + 'ResourceRecords': [{ + 'Value': 'cname.unit.tests.', + }], + 'TTL': 61, + 'HealthCheckId': '33', + }, { + # match + 'Name': '_octodns-one-value.cname.unit.tests.', + 'Type': 'CNAME', + 'ResourceRecords': [{ + 'Value': 'one.cname.unit.tests.', + }], + 'TTL': 61, + 'HealthCheckId': '42', + }], + 'IsTruncated': False, + 'MaxItems': '100', + } + stubber.add_response('list_resource_record_sets', + list_resource_record_sets_resp, + {'HostedZoneId': 'z42'}) + + stubber.add_response('list_health_checks', { + 'HealthChecks': [{ + 'Id': '42', + 'CallerReference': self.caller_ref, + 'HealthCheckConfig': { + 'Type': 'HTTPS', + 'FullyQualifiedDomainName': 'one.cname.unit.tests.', + 'ResourcePath': '/_dns', + 'Type': 'HTTPS', + 'Port': 443, + 'MeasureLatency': True, + 'RequestInterval': 10, + }, + 'HealthCheckVersion': 2, + }], + 'IsTruncated': False, + 'MaxItems': '100', + 'Marker': '', + }) + extra = provider._extra_changes(desired=desired, changes=[]) + self.assertEquals(0, len(extra)) + stubber.assert_no_pending_responses() + + # change b/c of healthcheck path + record._octodns['healthcheck'] = { + 'path': '/_ready' + } + extra = provider._extra_changes(desired=desired, changes=[]) + self.assertEquals(1, len(extra)) + stubber.assert_no_pending_responses() + + # no change b/c healthcheck host ignored for dynamic cname + record._octodns['healthcheck'] = { + 'host': 'foo.bar.io' + } + extra = provider._extra_changes(desired=desired, changes=[]) + self.assertEquals(0, len(extra)) + stubber.assert_no_pending_responses() + def _get_test_plan(self, max_changes): provider = Route53Provider('test', 'abc', '123', max_changes) From 5f57b52d07f5a75777e36837a76828c3af9c48a3 Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Tue, 18 May 2021 10:12:59 -0700 Subject: [PATCH 62/92] map Oceania to Australia/Pacific in Azure --- octodns/provider/azuredns.py | 21 +++++++++++++++++++-- tests/test_octodns_provider_azuredns.py | 6 +++--- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 5c2d23c..8eeba02 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -655,12 +655,22 @@ class AzureProvider(BaseProvider): geos = rule.setdefault('geos', []) for code in geo_map: if code.startswith('GEO-'): - geos.append(code[len('GEO-'):]) + # continent + if code == 'GEO-AP': + # Azure uses Australia/Pacific (AP) instead of + # Oceania https://docs.microsoft.com/en-us/azure/ + # traffic-manager/ + # traffic-manager-geographic-regions + geos.append('OC') + else: + geos.append(code[len('GEO-'):]) elif '-' in code: + # state country, province = code.split('-', 1) country = GeoCodes.country_to_code(country) geos.append('{}-{}'.format(country, province)) else: + # country geos.append(GeoCodes.country_to_code(code)) # second level priority profile @@ -872,15 +882,22 @@ class AzureProvider(BaseProvider): if len(rule_geos) > 0: for geo in rule_geos: if '-' in geo: + # country or state geos.append(geo.split('-', 1)[-1]) else: - geos.append('GEO-{}'.format(geo)) + # continent if geo == 'AS': # Middle East is part of Asia in octoDNS, but # Azure treats it as a separate "group", so let's # add it in the list of geo mappings. We will drop # it when we later parse the list of regions. geos.append('GEO-ME') + elif geo == 'OC': + # Azure uses Australia/Pacific (AP) instead of + # Oceania + geo = 'AP' + + geos.append('GEO-{}'.format(geo)) else: geos.append('WORLD') geo_endpoints.append(Endpoint( diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index 609fc5c..e8e5281 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -573,7 +573,7 @@ class TestAzureDnsProvider(TestCase): }, }, 'rules': [ - {'geos': ['AF', 'EU-DE', 'NA-US-CA'], 'pool': 'one'}, + {'geos': ['AF', 'EU-DE', 'NA-US-CA', 'OC'], 'pool': 'one'}, {'pool': 'two'}, ], }, @@ -694,7 +694,7 @@ class TestAzureDnsProvider(TestCase): monitor_config=monitor, endpoints=[ Endpoint( - geo_mapping=['GEO-AF', 'DE', 'US-CA'], + geo_mapping=['GEO-AF', 'DE', 'US-CA', 'GEO-AP'], name='rule-one', type=nested, target_resource_id=id_format.format('rule-one'), @@ -944,7 +944,7 @@ class TestAzureDnsProvider(TestCase): }, }, 'rules': [ - {'geos': ['AF', 'EU-DE', 'NA-US-CA'], 'pool': 'one'}, + {'geos': ['AF', 'EU-DE', 'NA-US-CA', 'OC'], 'pool': 'one'}, {'pool': 'two'}, ], }) From 62122e442931cf30f7b98cd91280419a3c58fac7 Mon Sep 17 00:00:00 2001 From: Tom Kaminski Date: Thu, 20 May 2021 15:24:34 -0500 Subject: [PATCH 63/92] More explicit check for CNAME records when checking for extra updates --- octodns/provider/route53.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index a3b3a57..ad59eb0 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -1253,15 +1253,11 @@ class Route53Provider(BaseProvider): return self._gen_mods('DELETE', existing_records, existing_rrsets) def _extra_changes_update_needed(self, record, rrset): - value = rrset['ResourceRecords'][0]['Value'] - - try: - ip_address(text_type(value)) - # We're working with an IP + if record._type == 'CNAME': + # For CNAME, healthcheck host by default points to the CNAME value + healthcheck_host = rrset['ResourceRecords'][0]['Value'] + else: healthcheck_host = record.healthcheck_host - except (AddressValueError, ValueError): - # This isn't an IP, host is the value - healthcheck_host = value healthcheck_path = record.healthcheck_path healthcheck_protocol = record.healthcheck_protocol From eb14873abbd6acbf08ce8ee08eaf70aa691343a7 Mon Sep 17 00:00:00 2001 From: Sham Date: Mon, 24 May 2021 19:01:17 -0700 Subject: [PATCH 64/92] Allow the option to not pass Host header in healthchecks --- octodns/provider/ns1.py | 4 +++- octodns/provider/route53.py | 2 +- tests/test_octodns_provider_ns1.py | 4 ++++ tests/test_octodns_provider_route53.py | 30 ++++++++++++++++++++++++++ 4 files changed, 38 insertions(+), 2 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index a910459..8408682 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -929,7 +929,9 @@ class Ns1Provider(BaseProvider): if record.healthcheck_protocol != 'TCP': # IF it's HTTP we need to send the request string path = record.healthcheck_path - host = record.healthcheck_host + # if host header is explicitly set to null in the yaml, + # just pass the domain (value) as the host header + host = record.healthcheck_host or value request = r'GET {path} HTTP/1.0\r\nHost: {host}\r\n' \ r'User-agent: NS1\r\n\r\n'.format(path=path, host=host) ret['config']['send'] = request diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index ad59eb0..0331847 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -1131,7 +1131,7 @@ class Route53Provider(BaseProvider): 'Type': healthcheck_protocol, } if healthcheck_protocol != 'TCP': - config['FullyQualifiedDomainName'] = healthcheck_host + config['FullyQualifiedDomainName'] = healthcheck_host or value config['ResourcePath'] = healthcheck_path if value: config['IPAddress'] = value diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index f54c7bd..4ae4757 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -757,6 +757,10 @@ class TestNs1ProviderDynamic(TestCase): self.assertFalse(monitor['config']['ssl']) self.assertEquals('host:unit.tests type:A', monitor['notes']) + record._octodns['healthcheck']['host'] = None + monitor = provider._monitor_gen(record, value) + self.assertTrue(r'\nHost: 3.4.5.6\r' in monitor['config']['send']) + record._octodns['healthcheck']['protocol'] = 'HTTPS' monitor = provider._monitor_gen(record, value) self.assertTrue(monitor['config']['ssl']) diff --git a/tests/test_octodns_provider_route53.py b/tests/test_octodns_provider_route53.py index 1e7210d..1bf3332 100644 --- a/tests/test_octodns_provider_route53.py +++ b/tests/test_octodns_provider_route53.py @@ -1166,6 +1166,31 @@ class TestRoute53Provider(TestCase): }) stubber.add_response('change_tags_for_resource', {}) + health_check_config = { + 'EnableSNI': False, + 'FailureThreshold': 6, + 'FullyQualifiedDomainName': '4.2.3.4', + 'IPAddress': '4.2.3.4', + 'MeasureLatency': True, + 'Port': 8080, + 'RequestInterval': 10, + 'ResourcePath': '/_status', + 'Type': 'HTTP' + } + stubber.add_response('create_health_check', { + 'HealthCheck': { + 'Id': '43', + 'CallerReference': self.caller_ref, + 'HealthCheckConfig': health_check_config, + 'HealthCheckVersion': 1, + }, + 'Location': 'http://url', + }, { + 'CallerReference': ANY, + 'HealthCheckConfig': health_check_config, + }) + stubber.add_response('change_tags_for_resource', {}) + record = Record.new(self.expected, '', { 'ttl': 61, 'type': 'A', @@ -1191,6 +1216,11 @@ class TestRoute53Provider(TestCase): # when allowed to create we do id = provider.get_health_check_id(record, value, True) self.assertEquals('42', id) + + # when allowed to create and when host is None + record._octodns['healthcheck']['host'] = None + id = provider.get_health_check_id(record, value, True) + self.assertEquals('43', id) stubber.assert_no_pending_responses() # A CNAME style healthcheck, without a value From f00766a77909c216249b72dcb4396ced16909ac2 Mon Sep 17 00:00:00 2001 From: Stan Brinkerhoff Date: Wed, 26 May 2021 22:45:07 -0400 Subject: [PATCH 65/92] Add support for ALIAS type --- octodns/provider/ultra.py | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/octodns/provider/ultra.py b/octodns/provider/ultra.py index 03b70de..f65c519 100644 --- a/octodns/provider/ultra.py +++ b/octodns/provider/ultra.py @@ -36,12 +36,12 @@ class UltraProvider(BaseProvider): ''' Neustar UltraDNS provider - Documentation for Ultra REST API requires a login: - https://portal.ultradns.com/static/docs/REST-API_User_Guide.pdf - Implemented to the May 20, 2020 version of the document (dated on page ii) - Also described as Version 2.83.0 (title page) + Documentation for Ultra REST API: + https://ultra-portalstatic.ultradns.com/static/docs/REST-API_User_Guide.pdf + Implemented to the May 26, 2021 version of the document (dated on page ii) + Also described as Version 3.18.0 (title page) - Tested against 3.0.0-20200627220036.81047f5 + Tested against 3.20.1-20210521075351.36b9297 As determined by querying https://api.ultradns.com/version ultra: @@ -57,6 +57,7 @@ class UltraProvider(BaseProvider): RECORDS_TO_TYPE = { 'A (1)': 'A', 'AAAA (28)': 'AAAA', + 'APEXALIAS (65282)': 'ALIAS', 'CAA (257)': 'CAA', 'CNAME (5)': 'CNAME', 'MX (15)': 'MX', @@ -72,6 +73,7 @@ class UltraProvider(BaseProvider): SUPPORTS_GEO = False SUPPORTS_DYNAMIC = False TIMEOUT = 5 + ZONE_REQUEST_LIMIT = 1000 def _request(self, method, path, params=None, data=None, json=None, json_response=True): @@ -151,7 +153,7 @@ class UltraProvider(BaseProvider): def zones(self): if self._zones is None: offset = 0 - limit = 100 + limit = self.ZONE_REQUEST_LIMIT zones = [] paging = True while paging: @@ -401,8 +403,15 @@ class UltraProvider(BaseProvider): def _gen_data(self, record): zone_name = self._remove_prefix(record.fqdn, record.name + '.') + + # UltraDNS treats the `APEXALIAS` type as the octodns `ALIAS`. + if record._type == "ALIAS": + record_type = "APEXALIAS" + else: + record_type = record._type + path = '/v2/zones/{}/rrsets/{}/{}'.format(zone_name, - record._type, + record_type, record.fqdn) contents_for = getattr(self, '_contents_for_{}'.format(record._type)) return path, contents_for(record) @@ -444,7 +453,12 @@ class UltraProvider(BaseProvider): existing._type == self.RECORDS_TO_TYPE[record['rrtype']]: zone_name = self._remove_prefix(existing.fqdn, existing.name + '.') - path = '/v2/zones/{}/rrsets/{}/{}'.format(zone_name, + + # UltraDNS treats the `APEXALIAS` type as the octodns `ALIAS`. + if existing._type == "ALIAS": + existing._type = "APEXALIAS" + + path = '/v2/zones/{}/rrsets/{}/{ }'.format(zone_name, existing._type, existing.fqdn) self._delete(path, json_response=False) From 4b6fd8b4a1e19fe889943aab705ecf23b6c19495 Mon Sep 17 00:00:00 2001 From: sbrinkerhoff Date: Thu, 27 May 2021 17:59:10 -0400 Subject: [PATCH 66/92] Remove type mutation Co-authored-by: Ross McFarland --- octodns/provider/ultra.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/octodns/provider/ultra.py b/octodns/provider/ultra.py index f65c519..a2eae3c 100644 --- a/octodns/provider/ultra.py +++ b/octodns/provider/ultra.py @@ -455,10 +455,11 @@ class UltraProvider(BaseProvider): existing.name + '.') # UltraDNS treats the `APEXALIAS` type as the octodns `ALIAS`. - if existing._type == "ALIAS": - existing._type = "APEXALIAS" + existing_type = existing._type + if existing_type == "ALIAS": + existing_type = "APEXALIAS" path = '/v2/zones/{}/rrsets/{}/{ }'.format(zone_name, - existing._type, + existing_type, existing.fqdn) self._delete(path, json_response=False) From e816e8e49b6c467cdf6a911b9d0a27bd2c8424e6 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Thu, 27 May 2021 15:19:33 -0700 Subject: [PATCH 67/92] Fixed extraneous whitespace in format in UltraDNS --- octodns/provider/ultra.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/ultra.py b/octodns/provider/ultra.py index a2eae3c..55c2c34 100644 --- a/octodns/provider/ultra.py +++ b/octodns/provider/ultra.py @@ -459,7 +459,7 @@ class UltraProvider(BaseProvider): if existing_type == "ALIAS": existing_type = "APEXALIAS" - path = '/v2/zones/{}/rrsets/{}/{ }'.format(zone_name, + path = '/v2/zones/{}/rrsets/{}/{}'.format(zone_name, existing_type, existing.fqdn) self._delete(path, json_response=False) From 60f916cd2400a27f23fb365f90978e6968cc088a Mon Sep 17 00:00:00 2001 From: Stan Brinkerhoff Date: Fri, 28 May 2021 00:52:48 -0400 Subject: [PATCH 68/92] add contents for alias method, test for alias --- octodns/provider/ultra.py | 4 +++- tests/test_octodns_provider_ultra.py | 15 ++++++++++++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/octodns/provider/ultra.py b/octodns/provider/ultra.py index 55c2c34..bc855d4 100644 --- a/octodns/provider/ultra.py +++ b/octodns/provider/ultra.py @@ -73,7 +73,7 @@ class UltraProvider(BaseProvider): SUPPORTS_GEO = False SUPPORTS_DYNAMIC = False TIMEOUT = 5 - ZONE_REQUEST_LIMIT = 1000 + ZONE_REQUEST_LIMIT = 100 def _request(self, method, path, params=None, data=None, json=None, json_response=True): @@ -213,6 +213,7 @@ class UltraProvider(BaseProvider): _data_for_PTR = _data_for_single _data_for_CNAME = _data_for_single + _data_for_ALIAS = _data_for_single def _data_for_CAA(self, _type, records): return { @@ -376,6 +377,7 @@ class UltraProvider(BaseProvider): } _contents_for_PTR = _contents_for_CNAME + _contents_for_ALIAS = _contents_for_CNAME def _contents_for_SRV(self, record): return { diff --git a/tests/test_octodns_provider_ultra.py b/tests/test_octodns_provider_ultra.py index b6d1017..52e0307 100644 --- a/tests/test_octodns_provider_ultra.py +++ b/tests/test_octodns_provider_ultra.py @@ -274,7 +274,7 @@ class TestUltraProvider(TestCase): self.assertTrue(provider.populate(zone)) self.assertEquals('octodns1.test.', zone.name) - self.assertEquals(11, len(zone.records)) + self.assertEquals(12, len(zone.records)) self.assertEquals(4, mock.call_count) def test_apply(self): @@ -352,8 +352,8 @@ class TestUltraProvider(TestCase): })) plan = provider.plan(wanted) - self.assertEquals(10, len(plan.changes)) - self.assertEquals(10, provider.apply(plan)) + self.assertEquals(11, len(plan.changes)) + self.assertEquals(11, provider.apply(plan)) self.assertTrue(plan.exists) provider._request.assert_has_calls([ @@ -492,6 +492,15 @@ class TestUltraProvider(TestCase): Record.new(zone, 'txt', {'ttl': 60, 'type': 'TXT', 'values': ['abc', 'def']})), + + # ALIAS + ('', 'ALIAS', + '/v2/zones/unit.tests./rrsets/APEXALIAS/unit.tests.', + {'ttl': 60, 'rdata': ['target.unit.tests.']}, + Record.new(zone, '', + {'ttl': 60, 'type': 'ALIAS', + 'value': 'target.unit.tests.'})), + ): # Validate path and payload based on record meet expectations path, payload = provider._gen_data(expected_record) From 6e71f2df76d454578bac1a84bf8920058350743f Mon Sep 17 00:00:00 2001 From: Stan Brinkerhoff Date: Fri, 28 May 2021 00:54:57 -0400 Subject: [PATCH 69/92] update test record --- tests/fixtures/ultra-records-page-2.json | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/fixtures/ultra-records-page-2.json b/tests/fixtures/ultra-records-page-2.json index abdc44f..274d95e 100644 --- a/tests/fixtures/ultra-records-page-2.json +++ b/tests/fixtures/ultra-records-page-2.json @@ -32,7 +32,16 @@ "rdata": [ "www.octodns1.test." ] + }, + { + "ownerName": "host1.octodns1.test.", + "rrtype": "RRSET (70)", + "ttl": 3600, + "rdata": [ + "E3B0C44298FC1C149AFBF4C8996FB92427AE41E4649B934CA495991B7852B855" + ] } + ], "resultInfo": { "totalCount": 13, From e97675fe3ddaa02ec4baa145fc7254f9b616fc28 Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Fri, 28 May 2021 12:48:09 -0700 Subject: [PATCH 70/92] check dns config when comparing profiles for equality --- octodns/provider/azuredns.py | 10 +++++++++- tests/test_octodns_provider_azuredns.py | 20 +++++++++++++------- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 8eeba02..65d9967 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -284,10 +284,18 @@ def _profile_is_match(have, desired): # compare basic attributes if have.name != desired.name or \ have.traffic_routing_method != desired.traffic_routing_method or \ - have.dns_config.ttl != desired.dns_config.ttl or \ len(have.endpoints) != len(desired.endpoints): return False + # compare dns config + dns_have = have.dns_config + dns_desired = desired.dns_config + if dns_have.ttl != dns_desired.ttl or \ + dns_have.relative_name is None or \ + dns_desired.relative_name is None or \ + dns_have.relative_name != dns_desired.relative_name: + return False + # compare monitoring configuration monitor_have = have.monitor_config monitor_desired = desired.monitor_config diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index e8e5281..0a1c366 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -453,6 +453,7 @@ class Test_ProfileIsMatch(TestCase): name = 'foo-unit-tests', ttl = 60, method = 'Geographic', + dns_name = None, monitor_proto = 'HTTPS', monitor_port = 4443, monitor_path = '/_ping', @@ -465,7 +466,7 @@ class Test_ProfileIsMatch(TestCase): weight = 1, priority = 1, ): - dns = DnsConfig(ttl=ttl) + dns = DnsConfig(relative_name=(dns_name or name), ttl=ttl) return Profile( name=name, traffic_routing_method=method, dns_config=dns, monitor_config=MonitorConfig( @@ -488,6 +489,7 @@ class Test_ProfileIsMatch(TestCase): self.assertFalse(is_match(profile(), profile(name='two'))) self.assertFalse(is_match(profile(), profile(endpoints=2))) + self.assertFalse(is_match(profile(), profile(dns_name='two'))) self.assertFalse(is_match(profile(), profile(monitor_proto='HTTP'))) self.assertFalse(is_match(profile(), profile(endpoint_name='a'))) self.assertFalse(is_match(profile(), profile(endpoint_type='b'))) @@ -596,7 +598,6 @@ class TestAzureDnsProvider(TestCase): id_format = base_id + '{}--' + suffix name_format = '{}--' + suffix - dns = DnsConfig(ttl=60) header = MonitorConfigCustomHeadersItem(name='Host', value='foo.unit.tests') monitor = MonitorConfig(protocol='HTTPS', port=4443, path='/_ping', @@ -604,12 +605,12 @@ class TestAzureDnsProvider(TestCase): external = 'Microsoft.Network/trafficManagerProfiles/externalEndpoints' nested = 'Microsoft.Network/trafficManagerProfiles/nestedEndpoints' - return [ + profiles = [ Profile( id=id_format.format('pool-two'), name=name_format.format('pool-two'), traffic_routing_method='Weighted', - dns_config=dns, + dns_config=DnsConfig(ttl=60), monitor_config=monitor, endpoints=[ Endpoint( @@ -630,7 +631,7 @@ class TestAzureDnsProvider(TestCase): id=id_format.format('rule-one'), name=name_format.format('rule-one'), traffic_routing_method='Priority', - dns_config=dns, + dns_config=DnsConfig(ttl=60), monitor_config=monitor, endpoints=[ Endpoint( @@ -663,7 +664,7 @@ class TestAzureDnsProvider(TestCase): id=id_format.format('rule-two'), name=name_format.format('rule-two'), traffic_routing_method='Priority', - dns_config=dns, + dns_config=DnsConfig(ttl=60), monitor_config=monitor, endpoints=[ Endpoint( @@ -690,7 +691,7 @@ class TestAzureDnsProvider(TestCase): id=base_id + suffix, name=suffix, traffic_routing_method='Geographic', - dns_config=dns, + dns_config=DnsConfig(ttl=60), monitor_config=monitor, endpoints=[ Endpoint( @@ -709,6 +710,11 @@ class TestAzureDnsProvider(TestCase): ), ] + for profile in profiles: + profile.dns_config.relative_name = profile.name + + return profiles + def _get_dynamic_package(self): '''Convenience function to setup a sample dynamic record. ''' From 18c0e5675978507063178cf9a83622ae25fe054b Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Fri, 28 May 2021 12:59:27 -0700 Subject: [PATCH 71/92] log.debug the reason for profile mismatch --- octodns/provider/azuredns.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 65d9967..689088c 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -281,11 +281,20 @@ def _profile_is_match(have, desired): if have is None or desired is None: return False + log = logging.getLogger('azuredns._profile_is_match').debug + + def false(have, desired, name=None): + prefix = 'profile={}'.format(name) if name else '' + attr = have.__class__.__name__ + log('%s have.%s = %s', prefix, attr, have) + log('%s desired.%s = %s', prefix, attr, desired) + return False + # compare basic attributes if have.name != desired.name or \ have.traffic_routing_method != desired.traffic_routing_method or \ len(have.endpoints) != len(desired.endpoints): - return False + return false(have, desired) # compare dns config dns_have = have.dns_config @@ -294,7 +303,7 @@ def _profile_is_match(have, desired): dns_have.relative_name is None or \ dns_desired.relative_name is None or \ dns_have.relative_name != dns_desired.relative_name: - return False + return false(dns_have, dns_desired, have.name) # compare monitoring configuration monitor_have = have.monitor_config @@ -303,7 +312,7 @@ def _profile_is_match(have, desired): monitor_have.port != monitor_desired.port or \ monitor_have.path != monitor_desired.path or \ monitor_have.custom_headers != monitor_desired.custom_headers: - return False + return false(monitor_have, monitor_desired, have.name) # compare endpoints method = have.traffic_routing_method @@ -321,26 +330,26 @@ def _profile_is_match(have, desired): for have_endpoint, desired_endpoint in endpoints: if have_endpoint.name != desired_endpoint.name or \ have_endpoint.type != desired_endpoint.type: - return False + return false(have_endpoint, desired_endpoint, have.name) target_type = have_endpoint.type.split('/')[-1] if target_type == 'externalEndpoints': # compare value, weight, priority if have_endpoint.target != desired_endpoint.target: - return False + return false(have_endpoint, desired_endpoint, have.name) if method == 'Weighted' and \ have_endpoint.weight != desired_endpoint.weight: - return False + return false(have_endpoint, desired_endpoint, have.name) elif target_type == 'nestedEndpoints': # compare targets if have_endpoint.target_resource_id != \ desired_endpoint.target_resource_id: - return False + return false(have_endpoint, desired_endpoint, have.name) # compare geos if method == 'Geographic': have_geos = sorted(have_endpoint.geo_mapping) desired_geos = sorted(desired_endpoint.geo_mapping) if have_geos != desired_geos: - return False + return false(have_endpoint, desired_endpoint, have.name) else: # unexpected, give up return False From aaffdb1388a0569710ca5f0063236d977fdeb9f8 Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Fri, 28 May 2021 13:23:35 -0700 Subject: [PATCH 72/92] minimize Azure Traffic Manager hops --- octodns/provider/azuredns.py | 259 +++++++---- tests/test_octodns_provider_azuredns.py | 585 ++++++++++++++++++------ 2 files changed, 625 insertions(+), 219 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 689088c..777e15b 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -646,14 +646,23 @@ class AzureProvider(BaseProvider): pools = defaultdict(lambda: {'fallback': None, 'values': []}) rules = [] - # top level geo profile - geo_profile = self._get_tm_profile_by_id(azrecord.target_resource.id) - for geo_ep in geo_profile.endpoints: + # top level profile + root_profile = self._get_tm_profile_by_id(azrecord.target_resource.id) + if root_profile.traffic_routing_method != 'Geographic': + # This record does not use geo fencing, so we skip the Geographic + # profile hop; let's pretend to be a geo-profile's only endpoint + geo_ep = Endpoint(target_resource_id=root_profile.id) + geo_ep.target_resource = root_profile + endpoints = [geo_ep] + else: + endpoints = root_profile.endpoints + + for geo_ep in endpoints: rule = {} # resolve list of regions - geo_map = list(geo_ep.geo_mapping) - if geo_map != ['WORLD']: + geo_map = list(geo_ep.geo_mapping or []) + if geo_map and geo_map != ['WORLD']: if 'GEO-ME' in geo_map: # Azure treats Middle East as a separate group, but # its part of Asia in octoDNS, so we need to remove GEO-ME @@ -662,7 +671,7 @@ class AzureProvider(BaseProvider): # profile was generated by octoDNS if 'GEO-AS' not in geo_map: msg = 'Profile={} for record {}: '.format( - geo_profile.name, azrecord.fqdn) + root_profile.name, azrecord.fqdn) msg += 'Middle East (GEO-ME) is not supported by ' + \ 'octoDNS. It needs to be either paired ' + \ 'with Asia (GEO-AS) or expanded into ' + \ @@ -690,18 +699,35 @@ class AzureProvider(BaseProvider): # country geos.append(GeoCodes.country_to_code(code)) - # second level priority profile + # build fallback chain from second level priority profile + if geo_ep.target_resource_id: + target = geo_ep.target_resource + if target.traffic_routing_method == 'Priority': + rule_endpoints = target.endpoints + rule_endpoints.sort(key=lambda e: e.priority) + else: + # Weighted + geo_ep.name = target.endpoints[0].name.split('--', 1)[0] + rule_endpoints = [geo_ep] + else: + # this geo directly points to the default, so we skip the + # Priority profile hop and directly use an external endpoint; + # let's pretend to be a Priority profile's only endpoint + rule_endpoints = [geo_ep] + pool = None - rule_endpoints = geo_ep.target_resource.endpoints - rule_endpoints = sorted(rule_endpoints, key=lambda e: e.priority) for rule_ep in rule_endpoints: pool_name = rule_ep.name # last/default pool - if pool_name == '--default--': + if pool_name.endswith('--default--'): default.add(rule_ep.target) - # this should be the last one, so let's break here - break + if pool_name == '--default--': + # this should be the last one, so let's break here + break + # last pool is a single value pool and its value is same + # as record's default value + pool_name = pool_name[:-len('--default--')] # set first priority endpoint as the rule's primary pool if 'pool' not in rule: @@ -711,32 +737,32 @@ class AzureProvider(BaseProvider): # set current pool as fallback of the previous pool pool['fallback'] = pool_name + if pool_name in pools: + # we've already populated the pool + continue + + # populate the pool from Weighted profile + # these should be leaf node entries with no further nesting pool = pools[pool_name] endpoints = [] - # these should be leaf node entries with no further nesting + if rule_ep.target_resource_id: # third (and last) level weighted RR profile endpoints = rule_ep.target_resource.endpoints else: - # single-value pool + # single-value pool, so we skip the Weighted profile hop + # and directly use an external endpoint; let's pretend to + # be a Weighted profile's only endpoint endpoints = [rule_ep] for pool_ep in endpoints: val = pool_ep.target - value_dict = { + pool['values'].append({ 'value': _check_endswith_dot(val), 'weight': pool_ep.weight or 1, - } - if value_dict not in pool['values']: - pool['values'].append(value_dict) - - if 'pool' not in rule or not default: - # this will happen if the priority profile does not have - # enough endpoints - msg = 'Expected at least 2 endpoints in {}, got {}'.format( - geo_ep.target_resource.name, len(rule_endpoints) - ) - raise AzureException(msg) + }) + if pool_ep.name.endswith('--default--'): + default.add(val) rules.append(rule) @@ -809,7 +835,7 @@ class AzureProvider(BaseProvider): elif ep.target: ep.type = endpoint_type_prefix + 'externalEndpoints' else: - msg = ('Invalid endpoint {} in profile {}, needs to have' + + msg = ('Invalid endpoint {} in profile {}, needs to have ' + 'either target or target_resource_id').format( ep.name, name) raise AzureException(msg) @@ -828,39 +854,83 @@ class AzureProvider(BaseProvider): location='global', ) + def _update_tm_name(self, profile, new_name): + profile.name = new_name + profile.id = self._profile_name_to_id(new_name) + profile.dns_config.relative_name = new_name + + return profile + def _generate_traffic_managers(self, record): traffic_managers = [] pools = record.dynamic.pools + default = record.value[:-1] tm_suffix = _traffic_manager_suffix(record) profile = self._generate_tm_profile geo_endpoints = [] + pool_profiles = {} for rule in record.dynamic.rules: + # Prepare the list of Traffic manager geos + rule_geos = rule.data.get('geos', []) + geos = [] + for geo in rule_geos: + if '-' in geo: + # country/state + geos.append(geo.split('-', 1)[-1]) + else: + # continent + if geo == 'AS': + # Middle East is part of Asia in octoDNS, but + # Azure treats it as a separate "group", so let's + # add it in the list of geo mappings. We will drop + # it when we later parse the list of regions. + geos.append('GEO-ME') + elif geo == 'OC': + # Azure uses Australia/Pacific (AP) instead of + # Oceania + geo = 'AP' + + geos.append('GEO-{}'.format(geo)) + if not geos: + geos.append('WORLD') + pool_name = rule.data['pool'] rule_endpoints = [] priority = 1 + default_seen = False while pool_name: # iterate until we reach end of fallback chain + default_seen = False pool = pools[pool_name].data - profile_name = 'pool-{}--{}'.format(pool_name, tm_suffix) if len(pool['values']) > 1: # create Weighted profile for multi-value pool - endpoints = [] - for val in pool['values']: - target = val['value'] - # strip trailing dot from CNAME value - target = target[:-1] - endpoints.append(Endpoint( - name=target, - target=target, - weight=val.get('weight', 1), - )) - pool_profile = profile(profile_name, 'Weighted', endpoints, - record) - traffic_managers.append(pool_profile) + pool_profile = pool_profiles.get(pool_name) + if pool_profile is None: + endpoints = [] + for val in pool['values']: + target = val['value'] + # strip trailing dot from CNAME value + target = target[:-1] + ep_name = '{}--{}'.format(pool_name, target) + if target == default: + # mark default + ep_name += '--default--' + default_seen = True + endpoints.append(Endpoint( + name=ep_name, + target=target, + weight=val.get('weight', 1), + )) + profile_name = 'pool-{}--{}'.format( + pool_name, tm_suffix) + pool_profile = profile(profile_name, 'Weighted', + endpoints, record) + traffic_managers.append(pool_profile) + pool_profiles[pool_name] = pool_profile # append pool to endpoint list of fallback rule profile rule_endpoints.append(Endpoint( @@ -869,8 +939,15 @@ class AzureProvider(BaseProvider): priority=priority, )) else: - # add single-value pool as an external endpoint + # Skip Weighted profile hop for single-value pool + # append its value as an external endpoint to fallback + # rule profile target = pool['values'][0]['value'][:-1] + ep_name = pool_name + if target == default: + # mark default + ep_name += '--default--' + default_seen = True rule_endpoints.append(Endpoint( name=pool_name, target=target, @@ -880,51 +957,61 @@ class AzureProvider(BaseProvider): priority += 1 pool_name = pool.get('fallback') - # append default profile to the end - rule_endpoints.append(Endpoint( - name='--default--', - target=record.value[:-1], - priority=priority, - )) - # create rule profile with fallback chain - rule_profile_name = 'rule-{}--{}'.format(rule.data['pool'], - tm_suffix) - rule_profile = profile(rule_profile_name, 'Priority', - rule_endpoints, record) - traffic_managers.append(rule_profile) - - # append rule profile to top-level geo profile - rule_geos = rule.data.get('geos', []) - geos = [] - if len(rule_geos) > 0: - for geo in rule_geos: - if '-' in geo: - # country or state - geos.append(geo.split('-', 1)[-1]) - else: - # continent - if geo == 'AS': - # Middle East is part of Asia in octoDNS, but - # Azure treats it as a separate "group", so let's - # add it in the list of geo mappings. We will drop - # it when we later parse the list of regions. - geos.append('GEO-ME') - elif geo == 'OC': - # Azure uses Australia/Pacific (AP) instead of - # Oceania - geo = 'AP' - - geos.append('GEO-{}'.format(geo)) + # append default endpoint unless it is already included in + # last pool of rule profile + if not default_seen: + rule_endpoints.append(Endpoint( + name='--default--', + target=default, + priority=priority, + )) + + if len(rule_endpoints) > 1: + # create rule profile with fallback chain + rule_profile_name = 'rule-{}--{}'.format( + rule.data['pool'], tm_suffix) + rule_profile = profile(rule_profile_name, 'Priority', + rule_endpoints, record) + traffic_managers.append(rule_profile) + + # append rule profile to top-level geo profile + geo_endpoints.append(Endpoint( + name='rule-{}'.format(rule.data['pool']), + target_resource_id=rule_profile.id, + geo_mapping=geos, + )) else: - geos.append('WORLD') - geo_endpoints.append(Endpoint( - name='rule-{}'.format(rule.data['pool']), - target_resource_id=rule_profile.id, - geo_mapping=geos, - )) - - geo_profile = profile(tm_suffix, 'Geographic', geo_endpoints, record) - traffic_managers.append(geo_profile) + # Priority profile has only one endpoint; skip the hop and + # append its only endpoint to the top-level profile + rule_ep = rule_endpoints[0] + if rule_ep.target_resource_id: + # point directly to the Weighted pool profile + geo_endpoints.append(Endpoint( + name='rule-{}'.format(rule.data['pool']), + target_resource_id=rule_ep.target_resource_id, + geo_mapping=geos, + )) + else: + # just add the value of single-value pool + geo_endpoints.append(Endpoint( + name=rule_ep.name + '--default--', + target=rule_ep.target, + geo_mapping=geos, + )) + + if len(geo_endpoints) == 1 and \ + geo_endpoints[0].geo_mapping == ['WORLD'] and \ + geo_endpoints[0].target_resource_id: + # Single WORLD rule does not require a Geographic profile, use + # the target profile as the root profile + target_profile_id = geo_endpoints[0].target_resource_id + profile_map = dict((tm.id, tm) for tm in traffic_managers) + target_profile = profile_map[target_profile_id] + self._update_tm_name(target_profile, tm_suffix) + else: + geo_profile = profile(tm_suffix, 'Geographic', geo_endpoints, + record) + traffic_managers.append(geo_profile) return traffic_managers diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index 0a1c366..2c998b6 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -614,13 +614,13 @@ class TestAzureDnsProvider(TestCase): monitor_config=monitor, endpoints=[ Endpoint( - name='two1.unit.tests', + name='two--two1.unit.tests', type=external, target='two1.unit.tests', weight=3, ), Endpoint( - name='two2.unit.tests', + name='two--two2.unit.tests', type=external, target='two2.unit.tests', weight=4, @@ -847,121 +847,6 @@ class TestAzureDnsProvider(TestCase): self.assertEquals(len(zone.records), 17) self.assertTrue(exists) - def test_populate_dynamic(self): - # Middle east without Asia raises exception - provider, zone, record = self._get_dynamic_package() - tm_suffix = _traffic_manager_suffix(record) - tm_id = provider._profile_name_to_id - tm_list = provider._tm_client.profiles.list_by_resource_group - rule_name = 'rule-one--{}'.format(tm_suffix) - nested = 'Microsoft.Network/trafficManagerProfiles/nestedEndpoints' - tm_list.return_value = [ - Profile( - id=tm_id(tm_suffix), - name=tm_suffix, - traffic_routing_method='Geographic', - endpoints=[ - Endpoint( - geo_mapping=['GEO-ME'], - ), - ], - ), - ] - azrecord = RecordSet( - ttl=60, - target_resource=SubResource(id=tm_id(tm_suffix)), - ) - azrecord.name = record.name or '@' - azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) - with self.assertRaises(AzureException) as ctx: - provider._populate_record(zone, azrecord) - self.assertTrue(text_type(ctx).startswith( - 'Middle East (GEO-ME) is not supported' - )) - - # empty priority profile raises exception - provider, zone, record = self._get_dynamic_package() - tm_list = provider._tm_client.profiles.list_by_resource_group - rule_name = 'rule-one--{}'.format(tm_suffix) - nested = 'Microsoft.Network/trafficManagerProfiles/nestedEndpoints' - tm_list.return_value = [ - Profile( - id=tm_id(rule_name), - name=rule_name, - traffic_routing_method='Priority', - endpoints=[], - ), - Profile( - id=tm_id(tm_suffix), - name=tm_suffix, - traffic_routing_method='Geographic', - endpoints=[ - Endpoint( - geo_mapping=['WORLD'], - name='rule-one', - type=nested, - target_resource_id=tm_id(rule_name), - ), - ], - ), - ] - with self.assertRaises(AzureException) as ctx: - provider._populate_record(zone, azrecord) - self.assertTrue(text_type(ctx).startswith( - 'Expected at least 2 endpoints' - )) - - # valid set of profiles produce expected dynamic record - provider, zone, record = self._get_dynamic_package() - root_profile_id = provider._profile_name_to_id( - _traffic_manager_suffix(record) - ) - azrecord = RecordSet( - ttl=60, - target_resource=SubResource(id=root_profile_id), - ) - azrecord.name = record.name or '@' - azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) - - record = provider._populate_record(zone, azrecord) - self.assertEqual(record.name, 'foo') - self.assertEqual(record.ttl, 60) - self.assertEqual(record.value, 'default.unit.tests.') - self.assertEqual(record.dynamic._data(), { - 'pools': { - 'one': { - 'values': [ - {'value': 'one.unit.tests.', 'weight': 1}, - ], - 'fallback': 'two', - }, - 'two': { - 'values': [ - {'value': 'two1.unit.tests.', 'weight': 3}, - {'value': 'two2.unit.tests.', 'weight': 4}, - ], - 'fallback': 'three', - }, - 'three': { - 'values': [ - {'value': 'three.unit.tests.', 'weight': 1}, - ], - 'fallback': None, - }, - }, - 'rules': [ - {'geos': ['AF', 'EU-DE', 'NA-US-CA', 'OC'], 'pool': 'one'}, - {'pool': 'two'}, - ], - }) - - # valid profiles with Middle East test case - geo_profile = provider._get_tm_for_dynamic_record(record) - geo_profile.endpoints[0].geo_mapping.extend(['GEO-ME', 'GEO-AS']) - record = provider._populate_record(zone, azrecord) - self.assertIn('AS', record.dynamic.rules[0].data['geos']) - self.assertNotIn('ME', record.dynamic.rules[0].data['geos']) - def test_populate_zone(self): provider = self._get_provider() @@ -1114,6 +999,7 @@ class TestAzureDnsProvider(TestCase): '/providers/Microsoft.Network/trafficManagerProfiles/' + name self.assertEqual(profile.id, expected_id) self.assertEqual(profile.name, name) + self.assertEqual(profile.name, profile.dns_config.relative_name) self.assertEqual(profile.traffic_routing_method, routing) self.assertEqual(profile.dns_config.ttl, record.ttl) self.assertEqual(len(profile.endpoints), len(endpoints)) @@ -1127,33 +1013,451 @@ class TestAzureDnsProvider(TestCase): 'Microsoft.Network/trafficManagerProfiles/nestedEndpoints' ) - def test_generate_traffic_managers(self): + def test_dynamic_record(self): provider, zone, record = self._get_dynamic_package() profiles = provider._generate_traffic_managers(record) - deduped = [] - seen = set() - for profile in profiles: - if profile.name not in seen: - deduped.append(profile) - seen.add(profile.name) # check that every profile is a match with what we expect expected_profiles = self._get_tm_profiles(provider) - self.assertEqual(len(expected_profiles), len(deduped)) - for have, expected in zip(deduped, expected_profiles): + self.assertEqual(len(expected_profiles), len(profiles)) + for have, expected in zip(profiles, expected_profiles): self.assertTrue(_profile_is_match(have, expected)) + # check that dynamic record is populated back from profiles + azrecord = RecordSet( + ttl=60, + target_resource=SubResource(id=profiles[-1].id), + ) + azrecord.name = record.name or '@' + azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) + record2 = provider._populate_record(zone, azrecord) + self.assertEqual(record2.dynamic._data(), record.dynamic._data()) + + def test_generate_traffic_managers_middle_east(self): # check Asia/Middle East test case + provider, zone, record = self._get_dynamic_package() record.dynamic._data()['rules'][0]['geos'].append('AS') profiles = provider._generate_traffic_managers(record) - geo_profile_name = _traffic_manager_suffix(record) - geo_profile = next( - profile - for profile in profiles - if profile.name == geo_profile_name + self.assertIn('GEO-ME', profiles[-1].endpoints[0].geo_mapping) + self.assertIn('GEO-AS', profiles[-1].endpoints[0].geo_mapping) + + def test_populate_dynamic_middle_east(self): + # Middle east without Asia raises exception + provider, zone, record = self._get_dynamic_package() + tm_suffix = _traffic_manager_suffix(record) + tm_id = provider._profile_name_to_id + tm_list = provider._tm_client.profiles.list_by_resource_group + tm_list.return_value = [ + Profile( + id=tm_id(tm_suffix), + name=tm_suffix, + traffic_routing_method='Geographic', + endpoints=[ + Endpoint( + geo_mapping=['GEO-ME'], + ), + ], + ), + ] + azrecord = RecordSet( + ttl=60, + target_resource=SubResource(id=tm_id(tm_suffix)), + ) + azrecord.name = record.name or '@' + azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) + with self.assertRaises(AzureException) as ctx: + provider._populate_record(zone, azrecord) + self.assertTrue(text_type(ctx).startswith( + 'Middle East (GEO-ME) is not supported' + )) + + # valid profiles with Middle East test case + provider, zone, record = self._get_dynamic_package() + geo_profile = provider._get_tm_for_dynamic_record(record) + geo_profile.endpoints[0].geo_mapping.extend(['GEO-ME', 'GEO-AS']) + record = provider._populate_record(zone, azrecord) + self.assertIn('AS', record.dynamic.rules[0].data['geos']) + self.assertNotIn('ME', record.dynamic.rules[0].data['geos']) + + def test_dynamic_no_geo(self): + # test that traffic managers are generated as expected + provider, zone, record = self._get_dynamic_package() + external = 'Microsoft.Network/trafficManagerProfiles/externalEndpoints' + + record = Record.new(zone, 'foo', data={ + 'type': 'CNAME', + 'ttl': 60, + 'value': 'default.unit.tests.', + 'dynamic': { + 'pools': { + 'one': { + 'values': [ + {'value': 'one.unit.tests.'}, + ], + }, + }, + 'rules': [ + {'pool': 'one'}, + ], + } + }) + profiles = provider._generate_traffic_managers(record) + + self.assertEqual(len(profiles), 1) + self.assertTrue(_profile_is_match(profiles[0], Profile( + name='foo-unit-tests', + traffic_routing_method='Priority', + dns_config=DnsConfig( + relative_name='foo-unit-tests', ttl=60), + monitor_config=_get_monitor(record), + endpoints=[ + Endpoint( + name='one', + type=external, + target='one.unit.tests', + priority=1, + ), + Endpoint( + name='--default--', + type=external, + target='default.unit.tests', + priority=2, + ), + ], + ))) + + # test that same record gets populated back from traffic managers + tm_list = provider._tm_client.profiles.list_by_resource_group + tm_list.return_value = profiles + azrecord = RecordSet( + ttl=60, + target_resource=SubResource(id=profiles[0].id), + ) + azrecord.name = record.name or '@' + azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) + record2 = provider._populate_record(zone, azrecord) + self.assertEqual(record2.dynamic._data(), record.dynamic._data()) + + def test_dynamic_fallback_is_default(self): + # test that traffic managers are generated as expected + provider, zone, record = self._get_dynamic_package() + external = 'Microsoft.Network/trafficManagerProfiles/externalEndpoints' + + record = Record.new(zone, 'foo', data={ + 'type': 'CNAME', + 'ttl': 60, + 'value': 'default.unit.tests.', + 'dynamic': { + 'pools': { + 'def': { + 'values': [ + {'value': 'default.unit.tests.'}, + ], + }, + }, + 'rules': [ + {'geos': ['AF'], 'pool': 'def'}, + ], + } + }) + profiles = provider._generate_traffic_managers(record) + + self.assertEqual(len(profiles), 1) + self.assertTrue(_profile_is_match(profiles[0], Profile( + name='foo-unit-tests', + traffic_routing_method='Geographic', + dns_config=DnsConfig( + relative_name='foo-unit-tests', ttl=60), + monitor_config=_get_monitor(record), + endpoints=[ + Endpoint( + name='def--default--', + type=external, + target='default.unit.tests', + geo_mapping=['GEO-AF'], + ), + ], + ))) + + # test that same record gets populated back from traffic managers + tm_list = provider._tm_client.profiles.list_by_resource_group + tm_list.return_value = profiles + azrecord = RecordSet( + ttl=60, + target_resource=SubResource(id=profiles[0].id), + ) + azrecord.name = record.name or '@' + azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) + record2 = provider._populate_record(zone, azrecord) + self.assertEqual(record2.dynamic._data(), record.dynamic._data()) + + def test_dynamic_pool_contains_default(self): + # test that traffic managers are generated as expected + provider, zone, record = self._get_dynamic_package() + tm_id = provider._profile_name_to_id + external = 'Microsoft.Network/trafficManagerProfiles/externalEndpoints' + nested = 'Microsoft.Network/trafficManagerProfiles/nestedEndpoints' + + record = Record.new(zone, 'foo', data={ + 'type': 'CNAME', + 'ttl': 60, + 'value': 'default.unit.tests.', + 'dynamic': { + 'pools': { + 'rr': { + 'values': [ + {'value': 'one.unit.tests.'}, + {'value': 'two.unit.tests.'}, + {'value': 'default.unit.tests.'}, + {'value': 'final.unit.tests.'}, + ], + }, + }, + 'rules': [ + {'geos': ['AF'], 'pool': 'rr'}, + ], + } + }) + profiles = provider._generate_traffic_managers(record) + + self.assertEqual(len(profiles), 2) + self.assertTrue(_profile_is_match(profiles[0], Profile( + name='pool-rr--foo-unit-tests', + traffic_routing_method='Weighted', + dns_config=DnsConfig( + relative_name='pool-rr--foo-unit-tests', ttl=60), + monitor_config=_get_monitor(record), + endpoints=[ + Endpoint( + name='rr--one.unit.tests', + type=external, + target='one.unit.tests', + weight=1, + ), + Endpoint( + name='rr--two.unit.tests', + type=external, + target='two.unit.tests', + weight=1, + ), + Endpoint( + name='rr--default.unit.tests--default--', + type=external, + target='default.unit.tests', + weight=1, + ), + Endpoint( + name='rr--final.unit.tests', + type=external, + target='final.unit.tests', + weight=1, + ), + ], + ))) + self.assertTrue(_profile_is_match(profiles[1], Profile( + name='foo-unit-tests', + traffic_routing_method='Geographic', + dns_config=DnsConfig( + relative_name='foo-unit-tests', ttl=60), + monitor_config=_get_monitor(record), + endpoints=[ + Endpoint( + name='rule-rr', + type=nested, + target_resource_id=tm_id('pool-rr--foo-unit-tests'), + geo_mapping=['GEO-AF'], + ), + ], + ))) + + # test that same record gets populated back from traffic managers + tm_list = provider._tm_client.profiles.list_by_resource_group + tm_list.return_value = profiles + azrecord = RecordSet( + ttl=60, + target_resource=SubResource(id=profiles[1].id), + ) + azrecord.name = record.name or '@' + azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) + record2 = provider._populate_record(zone, azrecord) + self.assertEqual(record2.dynamic._data(), record.dynamic._data()) + + def test_dynamic_pool_contains_default_no_geo(self): + # test that traffic managers are generated as expected + provider, zone, record = self._get_dynamic_package() + external = 'Microsoft.Network/trafficManagerProfiles/externalEndpoints' + + record = Record.new(zone, 'foo', data={ + 'type': 'CNAME', + 'ttl': 60, + 'value': 'default.unit.tests.', + 'dynamic': { + 'pools': { + 'rr': { + 'values': [ + {'value': 'one.unit.tests.'}, + {'value': 'two.unit.tests.'}, + {'value': 'default.unit.tests.'}, + {'value': 'final.unit.tests.'}, + ], + }, + }, + 'rules': [ + {'pool': 'rr'}, + ], + } + }) + profiles = provider._generate_traffic_managers(record) + + self.assertEqual(len(profiles), 1) + self.assertTrue(_profile_is_match(profiles[0], Profile( + name='foo-unit-tests', + traffic_routing_method='Weighted', + dns_config=DnsConfig( + relative_name='foo-unit-tests', ttl=60), + monitor_config=_get_monitor(record), + endpoints=[ + Endpoint( + name='rr--one.unit.tests', + type=external, + target='one.unit.tests', + weight=1, + ), + Endpoint( + name='rr--two.unit.tests', + type=external, + target='two.unit.tests', + weight=1, + ), + Endpoint( + name='rr--default.unit.tests--default--', + type=external, + target='default.unit.tests', + weight=1, + ), + Endpoint( + name='rr--final.unit.tests', + type=external, + target='final.unit.tests', + weight=1, + ), + ], + ))) + + # test that same record gets populated back from traffic managers + tm_list = provider._tm_client.profiles.list_by_resource_group + tm_list.return_value = profiles + azrecord = RecordSet( + ttl=60, + target_resource=SubResource(id=profiles[0].id), ) - self.assertIn('GEO-ME', geo_profile.endpoints[0].geo_mapping) - self.assertIn('GEO-AS', geo_profile.endpoints[0].geo_mapping) + azrecord.name = record.name or '@' + azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) + record2 = provider._populate_record(zone, azrecord) + self.assertEqual(record2.dynamic._data(), record.dynamic._data()) + + def test_dynamic_last_pool_contains_default_no_geo(self): + # test that traffic managers are generated as expected + provider, zone, record = self._get_dynamic_package() + tm_id = provider._profile_name_to_id + external = 'Microsoft.Network/trafficManagerProfiles/externalEndpoints' + nested = 'Microsoft.Network/trafficManagerProfiles/nestedEndpoints' + + record = Record.new(zone, 'foo', data={ + 'type': 'CNAME', + 'ttl': 60, + 'value': 'default.unit.tests.', + 'dynamic': { + 'pools': { + 'cloud': { + 'values': [ + {'value': 'cloud.unit.tests.'}, + ], + 'fallback': 'rr', + }, + 'rr': { + 'values': [ + {'value': 'one.unit.tests.'}, + {'value': 'two.unit.tests.'}, + {'value': 'default.unit.tests.'}, + {'value': 'final.unit.tests.'}, + ], + }, + }, + 'rules': [ + {'pool': 'cloud'}, + ], + } + }) + profiles = provider._generate_traffic_managers(record) + + self.assertEqual(len(profiles), 2) + self.assertTrue(_profile_is_match(profiles[0], Profile( + name='pool-rr--foo-unit-tests', + traffic_routing_method='Weighted', + dns_config=DnsConfig( + relative_name='pool-rr--foo-unit-tests', ttl=60), + monitor_config=_get_monitor(record), + endpoints=[ + Endpoint( + name='rr--one.unit.tests', + type=external, + target='one.unit.tests', + weight=1, + ), + Endpoint( + name='rr--two.unit.tests', + type=external, + target='two.unit.tests', + weight=1, + ), + Endpoint( + name='rr--default.unit.tests--default--', + type=external, + target='default.unit.tests', + weight=1, + ), + Endpoint( + name='rr--final.unit.tests', + type=external, + target='final.unit.tests', + weight=1, + ), + ], + ))) + self.assertTrue(_profile_is_match(profiles[1], Profile( + name='foo-unit-tests', + traffic_routing_method='Priority', + dns_config=DnsConfig( + relative_name='foo-unit-tests', ttl=60), + monitor_config=_get_monitor(record), + endpoints=[ + Endpoint( + name='cloud', + type=external, + target='cloud.unit.tests', + priority=1, + ), + Endpoint( + name='rr', + type=nested, + target_resource_id=tm_id('pool-rr--foo-unit-tests'), + priority=2, + ), + ], + ))) + + # test that same record gets populated back from traffic managers + tm_list = provider._tm_client.profiles.list_by_resource_group + tm_list.return_value = profiles + azrecord = RecordSet( + ttl=60, + target_resource=SubResource(id=profiles[1].id), + ) + azrecord.name = record.name or '@' + azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) + record2 = provider._populate_record(zone, azrecord) + self.assertEqual(record2.dynamic._data(), record.dynamic._data()) def test_sync_traffic_managers(self): provider, zone, record = self._get_dynamic_package() @@ -1194,6 +1498,21 @@ class TestAzureDnsProvider(TestCase): ) self.assertEqual(new_profile.endpoints[0].weight, 14) + @patch( + 'octodns.provider.azuredns.AzureProvider._generate_traffic_managers') + def test_sync_traffic_managers_duplicate(self, mock_gen_tms): + provider, zone, record = self._get_dynamic_package() + tm_sync = provider._tm_client.profiles.create_or_update + + # change and duplicate profiles + profile = self._get_tm_profiles(provider)[0] + profile.name = 'changing_this_to_trigger_sync' + mock_gen_tms.return_value = [profile, profile] + provider._sync_traffic_managers(record) + + # it should only be called once for duplicate profiles + tm_sync.assert_called_once() + def test_find_traffic_managers(self): provider, zone, record = self._get_dynamic_package() From f90d261133d11e10aff322a9526493315b503584 Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Wed, 9 Jun 2021 15:12:04 -0700 Subject: [PATCH 73/92] drop Azure DNS env vars in scripts --- script/coverage | 4 ++++ script/test | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/script/coverage b/script/coverage index bd6e4c9..db8e219 100755 --- a/script/coverage +++ b/script/coverage @@ -25,6 +25,10 @@ export DYN_CUSTOMER= export DYN_PASSWORD= export DYN_USERNAME= export GOOGLE_APPLICATION_CREDENTIALS= +export ARM_CLIENT_ID= +export ARM_CLIENT_SECRET= +export ARM_TENANT_ID= +export ARM_SUBSCRIPTION_ID= # Don't allow disabling coverage grep -r -I --line-number "# pragma: +no.*cover" octodns && { diff --git a/script/test b/script/test index 41edfd8..98bae20 100755 --- a/script/test +++ b/script/test @@ -25,5 +25,9 @@ export DYN_CUSTOMER= export DYN_PASSWORD= export DYN_USERNAME= export GOOGLE_APPLICATION_CREDENTIALS= +export ARM_CLIENT_ID= +export ARM_CLIENT_SECRET= +export ARM_TENANT_ID= +export ARM_SUBSCRIPTION_ID= nosetests "$@" From 5a943f0b13c08796137637dd55149fc2878ddc72 Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Wed, 9 Jun 2021 16:01:18 -0700 Subject: [PATCH 74/92] handle broken alias to ATM for dynamic records --- octodns/provider/azuredns.py | 9 +++++++-- tests/test_octodns_provider_azuredns.py | 21 +++++++++++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 777e15b..082fd42 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -613,8 +613,13 @@ class AzureProvider(BaseProvider): :type return: dict ''' - if azrecord.cname_record is None and azrecord.target_resource.id: - return self._data_for_dynamic(azrecord) + if azrecord.cname_record is None: + if azrecord.target_resource.id: + return self._data_for_dynamic(azrecord) + else: + # dynamic record alias is broken, return dummy value and apply + # will likely overwrite/fix it + return {'value': 'iam.invalid.'} return {'value': _check_endswith_dot(azrecord.cname_record.cname)} diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index 2c998b6..0332a72 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -1677,6 +1677,27 @@ class TestAzureDnsProvider(TestCase): dns_update.assert_not_called() tm_delete.assert_called_once() + # both are dynamic but alias is broken + provider, existing, record1 = self._get_dynamic_package() + azrecord = RecordSet( + ttl=record1.ttl, target_resource=SubResource(id=None)) + azrecord.name = record1.name or '@' + azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record1._type) + + record2 = provider._populate_record(zone, azrecord) + self.assertEqual(record2.value, 'iam.invalid.') + + change = Update(record2, record1) + provider._apply_Update(change) + tm_sync, dns_update, tm_delete = ( + provider._tm_client.profiles.create_or_update, + provider._dns_client.record_sets.create_or_update, + provider._tm_client.profiles.delete + ) + tm_sync.assert_not_called() + dns_update.assert_called_once() + tm_delete.assert_not_called() + def test_apply_delete_dynamic(self): provider, existing, record = self._get_dynamic_package() provider._populate_traffic_managers() From 7e0c6296fbcd62243f0b0d673b177e9c6a548ea1 Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Wed, 9 Jun 2021 17:25:06 -0700 Subject: [PATCH 75/92] ensure dynamic records map to unique ATM profiles --- octodns/provider/azuredns.py | 77 ++++++++----- tests/test_octodns_provider_azuredns.py | 145 ++++++++++++++++-------- 2 files changed, 147 insertions(+), 75 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 082fd42..bd555a7 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -259,8 +259,21 @@ def _parse_azure_type(string): return string.split('/')[-1] -def _traffic_manager_suffix(record): - return record.fqdn[:-1].replace('.', '-') +def _root_traffic_manager_name(record): + # ATM names can only have letters, numbers and hyphens + # replace dots with double hyphens to ensure unique mapping, + # hoping that real life FQDNs won't have double hyphens + return record.fqdn[:-1].replace('.', '--') + + +def _rule_traffic_manager_name(pool, record): + prefix = _root_traffic_manager_name(record) + return '{}-rule-{}'.format(prefix, pool) + + +def _pool_traffic_manager_name(pool, record): + prefix = _root_traffic_manager_name(record) + return '{}-pool-{}'.format(prefix, pool) def _get_monitor(record): @@ -533,7 +546,7 @@ class AzureProvider(BaseProvider): return self._get_tm_profile_by_id(profile_id) def _get_tm_for_dynamic_record(self, record): - name = _traffic_manager_suffix(record) + name = _root_traffic_manager_name(record) return self._get_tm_profile_by_name(name) def populate(self, zone, target=False, lenient=False): @@ -799,6 +812,7 @@ class AzureProvider(BaseProvider): raise AzureException(msg) log = self.log.info + seen_profiles = {} extra = [] for record in desired.records: if not getattr(record, 'dynamic', False): @@ -813,6 +827,16 @@ class AzureProvider(BaseProvider): profiles = self._generate_traffic_managers(record) for profile in profiles: name = profile.name + if name in seen_profiles: + # exit if a possible collision is detected, even though + # we've tried to ensure unique mapping + msg = 'Collision in Traffic Manager names detected' + msg = '{}: {} and {} both want to use {}'.format( + msg, seen_profiles[name], record.fqdn, name) + raise AzureException(msg) + else: + seen_profiles[name] = record.fqdn + active.add(name) existing_profile = self._get_tm_profile_by_name(name) if not _profile_is_match(existing_profile, profile): @@ -831,7 +855,14 @@ class AzureProvider(BaseProvider): return extra - def _generate_tm_profile(self, name, routing, endpoints, record): + def _generate_tm_profile(self, routing, endpoints, record, label=None): + # figure out profile name and Traffic Manager FQDN + name = _root_traffic_manager_name(record) + if routing == 'Weighted': + name = _pool_traffic_manager_name(label, record) + elif routing == 'Priority': + name = _rule_traffic_manager_name(label, record) + # set appropriate endpoint types endpoint_type_prefix = 'Microsoft.Network/trafficManagerProfiles/' for ep in endpoints: @@ -859,10 +890,10 @@ class AzureProvider(BaseProvider): location='global', ) - def _update_tm_name(self, profile, new_name): - profile.name = new_name - profile.id = self._profile_name_to_id(new_name) - profile.dns_config.relative_name = new_name + def _convert_tm_to_root(self, profile, record): + profile.name = _root_traffic_manager_name(record) + profile.id = self._profile_name_to_id(profile.name) + profile.dns_config.relative_name = profile.name return profile @@ -871,7 +902,6 @@ class AzureProvider(BaseProvider): pools = record.dynamic.pools default = record.value[:-1] - tm_suffix = _traffic_manager_suffix(record) profile = self._generate_tm_profile geo_endpoints = [] @@ -930,10 +960,8 @@ class AzureProvider(BaseProvider): target=target, weight=val.get('weight', 1), )) - profile_name = 'pool-{}--{}'.format( - pool_name, tm_suffix) - pool_profile = profile(profile_name, 'Weighted', - endpoints, record) + pool_profile = profile( + 'Weighted', endpoints, record, pool_name) traffic_managers.append(pool_profile) pool_profiles[pool_name] = pool_profile @@ -973,10 +1001,8 @@ class AzureProvider(BaseProvider): if len(rule_endpoints) > 1: # create rule profile with fallback chain - rule_profile_name = 'rule-{}--{}'.format( - rule.data['pool'], tm_suffix) - rule_profile = profile(rule_profile_name, 'Priority', - rule_endpoints, record) + rule_profile = profile( + 'Priority', rule_endpoints, record, rule.data['pool']) traffic_managers.append(rule_profile) # append rule profile to top-level geo profile @@ -1009,13 +1035,9 @@ class AzureProvider(BaseProvider): geo_endpoints[0].target_resource_id: # Single WORLD rule does not require a Geographic profile, use # the target profile as the root profile - target_profile_id = geo_endpoints[0].target_resource_id - profile_map = dict((tm.id, tm) for tm in traffic_managers) - target_profile = profile_map[target_profile_id] - self._update_tm_name(target_profile, tm_suffix) + self._convert_tm_to_root(traffic_managers[-1], record) else: - geo_profile = profile(tm_suffix, 'Geographic', geo_endpoints, - record) + geo_profile = profile('Geographic', geo_endpoints, record) traffic_managers.append(geo_profile) return traffic_managers @@ -1047,14 +1069,15 @@ class AzureProvider(BaseProvider): return seen def _find_traffic_managers(self, record): - tm_suffix = _traffic_manager_suffix(record) + tm_prefix = _root_traffic_manager_name(record) profiles = set() for profile_id in self._traffic_managers: - # match existing profiles with record's suffix + # match existing profiles with record's prefix name = profile_id.split('/')[-1] - if name == tm_suffix or \ - name.endswith('--{}'.format(tm_suffix)): + if name == tm_prefix or \ + name.startswith('{}-pool-'.format(tm_prefix)) or \ + name.startswith('{}-rule-'.format(tm_prefix)): profiles.add(name) return profiles diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index 0332a72..c7840d6 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -7,7 +7,7 @@ from __future__ import absolute_import, division, print_function, \ from octodns.record import Create, Update, Delete, Record from octodns.provider.azuredns import _AzureRecord, AzureProvider, \ - _check_endswith_dot, _parse_azure_type, _traffic_manager_suffix, \ + _check_endswith_dot, _parse_azure_type, _root_traffic_manager_name, \ _get_monitor, _profile_is_match, AzureException from octodns.zone import Zone from octodns.provider.base import Plan @@ -402,12 +402,12 @@ class Test_CheckEndswithDot(TestCase): self.assertEquals(expected, _check_endswith_dot(test)) -class Test_TrafficManagerSuffix(TestCase): - def test_traffic_manager_suffix(self): +class Test_RootTrafficManagerName(TestCase): + def test_root_traffic_manager_name(self): test = Record.new(zone, 'foo', data={ 'ttl': 60, 'type': 'CNAME', 'value': 'default.unit.tests.', }) - self.assertEqual(_traffic_manager_suffix(test), 'foo-unit-tests') + self.assertEqual(_root_traffic_manager_name(test), 'foo--unit--tests') class Test_GetMonitor(TestCase): @@ -594,9 +594,9 @@ class TestAzureDnsProvider(TestCase): base_id = '/subscriptions/' + sub + \ '/resourceGroups/' + rg + \ '/providers/Microsoft.Network/trafficManagerProfiles/' - suffix = 'foo-unit-tests' - id_format = base_id + '{}--' + suffix - name_format = '{}--' + suffix + prefix = 'foo--unit--tests' + name_format = prefix + '-{}' + id_format = base_id + name_format header = MonitorConfigCustomHeadersItem(name='Host', value='foo.unit.tests') @@ -688,8 +688,8 @@ class TestAzureDnsProvider(TestCase): ], ), Profile( - id=base_id + suffix, - name=suffix, + id=base_id + prefix, + name=prefix, traffic_routing_method='Geographic', dns_config=DnsConfig(ttl=60), monitor_config=monitor, @@ -899,10 +899,10 @@ class TestAzureDnsProvider(TestCase): # test unused TM produces the extra change for clean up sample_profile = self._get_tm_profiles(provider)[0] tm_id = provider._profile_name_to_id - root_profile_name = _traffic_manager_suffix(record) + root_profile_name = _root_traffic_manager_name(record) extra_profile = Profile( - id=tm_id('random--{}'.format(root_profile_name)), - name='random--{}'.format(root_profile_name), + id=tm_id('{}-pool-random'.format(root_profile_name)), + name='{}-pool-random'.format(root_profile_name), traffic_routing_method='Weighted', dns_config=sample_profile.dns_config, monitor_config=sample_profile.monitor_config, @@ -968,14 +968,40 @@ class TestAzureDnsProvider(TestCase): }) desired.add_record(a_dynamic) changes.append(Create(a_dynamic)) - with self.assertRaises(AzureException): + with self.assertRaises(AzureException) as ctx: + provider._extra_changes(existing, desired, changes) + self.assertTrue(text_type(ctx).endswith( + 'must be of type CNAME' + )) + desired._remove_record(a_dynamic) + + # test colliding ATM names throws exception + record1 = Record.new(desired, 'sub.www', data={ + 'type': record._type, + 'ttl': record.ttl, + 'value': record.value, + 'dynamic': record.dynamic._data(), + }) + record2 = Record.new(desired, 'sub--www', data={ + 'type': record._type, + 'ttl': record.ttl, + 'value': record.value, + 'dynamic': record.dynamic._data(), + }) + desired.add_record(record1) + desired.add_record(record2) + changes = [Create(record1), Create(record2)] + with self.assertRaises(AzureException) as ctx: provider._extra_changes(existing, desired, changes) + self.assertTrue(text_type(ctx).startswith( + 'Collision in Traffic Manager' + )) def test_generate_tm_profile(self): provider, zone, record = self._get_dynamic_package() profile_gen = provider._generate_tm_profile - name = 'foobar' + label = 'foobar' routing = 'Priority' endpoints = [ Endpoint(target='one.unit.tests'), @@ -985,20 +1011,22 @@ class TestAzureDnsProvider(TestCase): # invalid endpoint raises exception with self.assertRaises(AzureException): - profile_gen(name, routing, endpoints, record) + profile_gen(routing, endpoints, record, label) # regular test endpoints.pop() - profile = profile_gen(name, routing, endpoints, record) + profile = profile_gen(routing, endpoints, record, label) # implicitly tests _profile_name_to_id sub = provider._dns_client_subscription_id rg = provider._resource_group + expected_name = 'foo--unit--tests-rule-foobar' expected_id = '/subscriptions/' + sub + \ '/resourceGroups/' + rg + \ - '/providers/Microsoft.Network/trafficManagerProfiles/' + name + '/providers/Microsoft.Network/trafficManagerProfiles/' + \ + expected_name self.assertEqual(profile.id, expected_id) - self.assertEqual(profile.name, name) + self.assertEqual(profile.name, expected_name) self.assertEqual(profile.name, profile.dns_config.relative_name) self.assertEqual(profile.traffic_routing_method, routing) self.assertEqual(profile.dns_config.ttl, record.ttl) @@ -1044,7 +1072,7 @@ class TestAzureDnsProvider(TestCase): def test_populate_dynamic_middle_east(self): # Middle east without Asia raises exception provider, zone, record = self._get_dynamic_package() - tm_suffix = _traffic_manager_suffix(record) + tm_suffix = _root_traffic_manager_name(record) tm_id = provider._profile_name_to_id tm_list = provider._tm_client.profiles.list_by_resource_group tm_list.return_value = [ @@ -1105,10 +1133,10 @@ class TestAzureDnsProvider(TestCase): self.assertEqual(len(profiles), 1) self.assertTrue(_profile_is_match(profiles[0], Profile( - name='foo-unit-tests', + name='foo--unit--tests', traffic_routing_method='Priority', dns_config=DnsConfig( - relative_name='foo-unit-tests', ttl=60), + relative_name='foo--unit--tests', ttl=60), monitor_config=_get_monitor(record), endpoints=[ Endpoint( @@ -1164,10 +1192,10 @@ class TestAzureDnsProvider(TestCase): self.assertEqual(len(profiles), 1) self.assertTrue(_profile_is_match(profiles[0], Profile( - name='foo-unit-tests', + name='foo--unit--tests', traffic_routing_method='Geographic', dns_config=DnsConfig( - relative_name='foo-unit-tests', ttl=60), + relative_name='foo--unit--tests', ttl=60), monitor_config=_get_monitor(record), endpoints=[ Endpoint( @@ -1222,10 +1250,10 @@ class TestAzureDnsProvider(TestCase): self.assertEqual(len(profiles), 2) self.assertTrue(_profile_is_match(profiles[0], Profile( - name='pool-rr--foo-unit-tests', + name='foo--unit--tests-pool-rr', traffic_routing_method='Weighted', dns_config=DnsConfig( - relative_name='pool-rr--foo-unit-tests', ttl=60), + relative_name='foo--unit--tests-pool-rr', ttl=60), monitor_config=_get_monitor(record), endpoints=[ Endpoint( @@ -1255,16 +1283,16 @@ class TestAzureDnsProvider(TestCase): ], ))) self.assertTrue(_profile_is_match(profiles[1], Profile( - name='foo-unit-tests', + name='foo--unit--tests', traffic_routing_method='Geographic', dns_config=DnsConfig( - relative_name='foo-unit-tests', ttl=60), + relative_name='foo--unit--tests', ttl=60), monitor_config=_get_monitor(record), endpoints=[ Endpoint( name='rule-rr', type=nested, - target_resource_id=tm_id('pool-rr--foo-unit-tests'), + target_resource_id=tm_id(profiles[0].name), geo_mapping=['GEO-AF'], ), ], @@ -1311,10 +1339,10 @@ class TestAzureDnsProvider(TestCase): self.assertEqual(len(profiles), 1) self.assertTrue(_profile_is_match(profiles[0], Profile( - name='foo-unit-tests', + name='foo--unit--tests', traffic_routing_method='Weighted', dns_config=DnsConfig( - relative_name='foo-unit-tests', ttl=60), + relative_name='foo--unit--tests', ttl=60), monitor_config=_get_monitor(record), endpoints=[ Endpoint( @@ -1393,10 +1421,10 @@ class TestAzureDnsProvider(TestCase): self.assertEqual(len(profiles), 2) self.assertTrue(_profile_is_match(profiles[0], Profile( - name='pool-rr--foo-unit-tests', + name='foo--unit--tests-pool-rr', traffic_routing_method='Weighted', dns_config=DnsConfig( - relative_name='pool-rr--foo-unit-tests', ttl=60), + relative_name='foo--unit--tests-pool-rr', ttl=60), monitor_config=_get_monitor(record), endpoints=[ Endpoint( @@ -1426,10 +1454,10 @@ class TestAzureDnsProvider(TestCase): ], ))) self.assertTrue(_profile_is_match(profiles[1], Profile( - name='foo-unit-tests', + name='foo--unit--tests', traffic_routing_method='Priority', dns_config=DnsConfig( - relative_name='foo-unit-tests', ttl=60), + relative_name='foo--unit--tests', ttl=60), monitor_config=_get_monitor(record), endpoints=[ Endpoint( @@ -1441,7 +1469,7 @@ class TestAzureDnsProvider(TestCase): Endpoint( name='rr', type=nested, - target_resource_id=tm_id('pool-rr--foo-unit-tests'), + target_resource_id=tm_id(profiles[0].name), priority=2, ), ], @@ -1459,16 +1487,37 @@ class TestAzureDnsProvider(TestCase): record2 = provider._populate_record(zone, azrecord) self.assertEqual(record2.dynamic._data(), record.dynamic._data()) + def test_dynamic_unique_traffic_managers(self): + record = self._get_dynamic_record(zone) + data = { + 'type': record._type, + 'ttl': record.ttl, + 'value': record.value, + 'dynamic': record.dynamic._data() + } + record_names = [ + 'www.foo', 'www-foo' + ] + provider = self._get_provider() + + seen = set() + for name in record_names: + record = Record.new(zone, name, data=data) + tms = provider._generate_traffic_managers(record) + for tm in tms: + self.assertNotIn(tm.name, seen) + seen.add(tm.name) + def test_sync_traffic_managers(self): provider, zone, record = self._get_dynamic_package() provider._populate_traffic_managers() tm_sync = provider._tm_client.profiles.create_or_update - suffix = 'foo-unit-tests' + prefix = 'foo--unit--tests' expected_seen = { - suffix, 'pool-two--{}'.format(suffix), - 'rule-one--{}'.format(suffix), 'rule-two--{}'.format(suffix), + prefix, '{}-pool-two'.format(prefix), + '{}-rule-one'.format(prefix), '{}-rule-two'.format(prefix), } # test no change @@ -1494,7 +1543,7 @@ class TestAzureDnsProvider(TestCase): # test that new profile was successfully inserted in cache new_profile = provider._get_tm_profile_by_name( - 'pool-two--{}'.format(suffix) + '{}-pool-two'.format(prefix) ) self.assertEqual(new_profile.endpoints[0].weight, 14) @@ -1524,11 +1573,11 @@ class TestAzureDnsProvider(TestCase): 'ttl': record.ttl, 'value': record.value, }) - suffix2 = _traffic_manager_suffix(record2) + prefix2 = _root_traffic_manager_name(record2) tm_id = provider._profile_name_to_id extra_profile = Profile( - id=tm_id('random--{}'.format(suffix2)), - name='random--{}'.format(suffix2), + id=tm_id('{}-pool-random'.format(prefix2)), + name='{}-pool-random'.format(prefix2), traffic_routing_method='Weighted', dns_config=sample_profile.dns_config, monitor_config=sample_profile.monitor_config, @@ -1539,10 +1588,10 @@ class TestAzureDnsProvider(TestCase): provider._populate_traffic_managers() # implicitly asserts that non-matching profile is not included - suffix = _traffic_manager_suffix(record) + prefix = _root_traffic_manager_name(record) self.assertEqual(provider._find_traffic_managers(record), { - suffix, 'pool-two--{}'.format(suffix), - 'rule-one--{}'.format(suffix), 'rule-two--{}'.format(suffix), + prefix, '{}-pool-two'.format(prefix), + '{}-rule-one'.format(prefix), '{}-rule-two'.format(prefix), }) def test_traffic_manager_gc(self): @@ -1655,10 +1704,10 @@ class TestAzureDnsProvider(TestCase): provider, existing, dynamic_record = self._get_dynamic_package() sample_profile = self._get_tm_profiles(provider)[0] tm_id = provider._profile_name_to_id - root_profile_name = _traffic_manager_suffix(dynamic_record) + root_profile_name = _root_traffic_manager_name(dynamic_record) extra_profile = Profile( - id=tm_id('random--{}'.format(root_profile_name)), - name='random--{}'.format(root_profile_name), + id=tm_id('{}-pool-random'.format(root_profile_name)), + name='{}-pool-random'.format(root_profile_name), traffic_routing_method='Weighted', dns_config=sample_profile.dns_config, monitor_config=sample_profile.monitor_config, From 6734448462b79977ef0d0fa1391c716ad63bee6c Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Fri, 11 Jun 2021 11:43:35 -0700 Subject: [PATCH 76/92] log warning when dynamic CNAME has broken alias --- octodns/provider/azuredns.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index bd555a7..0fd3dae 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -632,6 +632,10 @@ class AzureProvider(BaseProvider): else: # dynamic record alias is broken, return dummy value and apply # will likely overwrite/fix it + self.log.warn('_data_for_CNAME: Missing Traffic Manager ' + 'alias for dynamic CNAME record %s, forcing ' + 're-link by setting an invalid value', + azrecord.fqdn) return {'value': 'iam.invalid.'} return {'value': _check_endswith_dot(azrecord.cname_record.cname)} From fec0aea14426b7f8cb9845063ec72bd4d89f6f8d Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Mon, 14 Jun 2021 13:56:19 -0700 Subject: [PATCH 77/92] minor clean up of azuredns tests --- tests/test_octodns_provider_azuredns.py | 26 ++++++++++++------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index c7840d6..81a3084 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -1109,7 +1109,7 @@ class TestAzureDnsProvider(TestCase): def test_dynamic_no_geo(self): # test that traffic managers are generated as expected - provider, zone, record = self._get_dynamic_package() + provider = self._get_provider() external = 'Microsoft.Network/trafficManagerProfiles/externalEndpoints' record = Record.new(zone, 'foo', data={ @@ -1159,7 +1159,7 @@ class TestAzureDnsProvider(TestCase): tm_list.return_value = profiles azrecord = RecordSet( ttl=60, - target_resource=SubResource(id=profiles[0].id), + target_resource=SubResource(id=profiles[-1].id), ) azrecord.name = record.name or '@' azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) @@ -1168,7 +1168,7 @@ class TestAzureDnsProvider(TestCase): def test_dynamic_fallback_is_default(self): # test that traffic managers are generated as expected - provider, zone, record = self._get_dynamic_package() + provider = self._get_provider() external = 'Microsoft.Network/trafficManagerProfiles/externalEndpoints' record = Record.new(zone, 'foo', data={ @@ -1212,7 +1212,7 @@ class TestAzureDnsProvider(TestCase): tm_list.return_value = profiles azrecord = RecordSet( ttl=60, - target_resource=SubResource(id=profiles[0].id), + target_resource=SubResource(id=profiles[-1].id), ) azrecord.name = record.name or '@' azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) @@ -1221,8 +1221,7 @@ class TestAzureDnsProvider(TestCase): def test_dynamic_pool_contains_default(self): # test that traffic managers are generated as expected - provider, zone, record = self._get_dynamic_package() - tm_id = provider._profile_name_to_id + provider = self._get_provider() external = 'Microsoft.Network/trafficManagerProfiles/externalEndpoints' nested = 'Microsoft.Network/trafficManagerProfiles/nestedEndpoints' @@ -1292,7 +1291,7 @@ class TestAzureDnsProvider(TestCase): Endpoint( name='rule-rr', type=nested, - target_resource_id=tm_id(profiles[0].name), + target_resource_id=profiles[0].id, geo_mapping=['GEO-AF'], ), ], @@ -1303,7 +1302,7 @@ class TestAzureDnsProvider(TestCase): tm_list.return_value = profiles azrecord = RecordSet( ttl=60, - target_resource=SubResource(id=profiles[1].id), + target_resource=SubResource(id=profiles[-1].id), ) azrecord.name = record.name or '@' azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) @@ -1312,7 +1311,7 @@ class TestAzureDnsProvider(TestCase): def test_dynamic_pool_contains_default_no_geo(self): # test that traffic managers are generated as expected - provider, zone, record = self._get_dynamic_package() + provider = self._get_provider() external = 'Microsoft.Network/trafficManagerProfiles/externalEndpoints' record = Record.new(zone, 'foo', data={ @@ -1377,7 +1376,7 @@ class TestAzureDnsProvider(TestCase): tm_list.return_value = profiles azrecord = RecordSet( ttl=60, - target_resource=SubResource(id=profiles[0].id), + target_resource=SubResource(id=profiles[-1].id), ) azrecord.name = record.name or '@' azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) @@ -1386,8 +1385,7 @@ class TestAzureDnsProvider(TestCase): def test_dynamic_last_pool_contains_default_no_geo(self): # test that traffic managers are generated as expected - provider, zone, record = self._get_dynamic_package() - tm_id = provider._profile_name_to_id + provider = self._get_provider() external = 'Microsoft.Network/trafficManagerProfiles/externalEndpoints' nested = 'Microsoft.Network/trafficManagerProfiles/nestedEndpoints' @@ -1469,7 +1467,7 @@ class TestAzureDnsProvider(TestCase): Endpoint( name='rr', type=nested, - target_resource_id=tm_id(profiles[0].name), + target_resource_id=profiles[0].id, priority=2, ), ], @@ -1480,7 +1478,7 @@ class TestAzureDnsProvider(TestCase): tm_list.return_value = profiles azrecord = RecordSet( ttl=60, - target_resource=SubResource(id=profiles[1].id), + target_resource=SubResource(id=profiles[-1].id), ) azrecord.name = record.name or '@' azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) From 725189af49791686542c5f580b2c30660ea653bb Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Mon, 14 Jun 2021 13:58:20 -0700 Subject: [PATCH 78/92] Handle re-used pools in Azure DNS dynamic records --- octodns/provider/azuredns.py | 28 ++++++++++- tests/test_octodns_provider_azuredns.py | 67 +++++++++++++++++++++++++ 2 files changed, 93 insertions(+), 2 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 0fd3dae..93478b8 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -717,6 +717,8 @@ class AzureProvider(BaseProvider): country, province = code.split('-', 1) country = GeoCodes.country_to_code(country) geos.append('{}-{}'.format(country, province)) + elif code == 'WORLD': + geos.append(code) else: # country geos.append(GeoCodes.country_to_code(code)) @@ -788,6 +790,13 @@ class AzureProvider(BaseProvider): rules.append(rule) + # add separate rule for re-used world pool + for rule in list(rules): + geos = rule.get('geos', []) + if len(geos) > 1 and 'WORLD' in geos: + geos.remove('WORLD') + rules.append({'pool': rule['pool']}) + # Order and convert to a list default = sorted(default) @@ -904,14 +913,29 @@ class AzureProvider(BaseProvider): def _generate_traffic_managers(self, record): traffic_managers = [] pools = record.dynamic.pools + rules = record.dynamic.rules default = record.value[:-1] profile = self._generate_tm_profile + # a pool can be re-used only with a world pool, record the pool + # to later consolidate it with a geo pool if one exists since we + # can't have multiple endpoints with the same target in ATM + world_pool = None + for rule in rules: + if not rule.data.get('geos', []): + world_pool = rule.data['pool'] + world_seen = False + geo_endpoints = [] pool_profiles = {} for rule in record.dynamic.rules: + pool_name = rule.data['pool'] + if pool_name == world_pool and world_seen: + # this world pool is already mentioned in another geo rule + continue + # Prepare the list of Traffic manager geos rule_geos = rule.data.get('geos', []) geos = [] @@ -933,10 +957,10 @@ class AzureProvider(BaseProvider): geo = 'AP' geos.append('GEO-{}'.format(geo)) - if not geos: + if not geos or pool_name == world_pool: geos.append('WORLD') + world_seen = True - pool_name = rule.data['pool'] rule_endpoints = [] priority = 1 default_seen = False diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index 81a3084..56a783d 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -1506,6 +1506,73 @@ class TestAzureDnsProvider(TestCase): self.assertNotIn(tm.name, seen) seen.add(tm.name) + def test_dynamic_reused_pool(self): + # test that traffic managers are generated as expected + provider = self._get_provider() + nested = 'Microsoft.Network/trafficManagerProfiles/nestedEndpoints' + + record = Record.new(zone, 'foo', data={ + 'type': 'CNAME', + 'ttl': 60, + 'value': 'default.unit.tests.', + 'dynamic': { + 'pools': { + 'iad': { + 'values': [ + {'value': 'iad.unit.tests.'}, + ], + 'fallback': 'lhr', + }, + 'lhr': { + 'values': [ + {'value': 'lhr.unit.tests.'}, + ], + }, + }, + 'rules': [ + {'geos': ['EU'], 'pool': 'iad'}, + {'geos': ['EU-GB'], 'pool': 'lhr'}, + {'pool': 'lhr'}, + ], + } + }) + profiles = provider._generate_traffic_managers(record) + + self.assertEqual(len(profiles), 3) + self.assertTrue(_profile_is_match(profiles[-1], Profile( + name='foo--unit--tests', + traffic_routing_method='Geographic', + dns_config=DnsConfig( + relative_name='foo--unit--tests', ttl=record.ttl), + monitor_config=_get_monitor(record), + endpoints=[ + Endpoint( + name='rule-iad', + type=nested, + target_resource_id=profiles[0].id, + geo_mapping=['GEO-EU'], + ), + Endpoint( + name='rule-lhr', + type=nested, + target_resource_id=profiles[1].id, + geo_mapping=['GB', 'WORLD'], + ), + ], + ))) + + # test that same record gets populated back from traffic managers + tm_list = provider._tm_client.profiles.list_by_resource_group + tm_list.return_value = profiles + azrecord = RecordSet( + ttl=60, + target_resource=SubResource(id=profiles[-1].id), + ) + azrecord.name = record.name or '@' + azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) + record2 = provider._populate_record(zone, azrecord) + self.assertEqual(record2.dynamic._data(), record.dynamic._data()) + def test_sync_traffic_managers(self): provider, zone, record = self._get_dynamic_package() provider._populate_traffic_managers() From a31fa189c131b8e45b5cddb309945e1a4bb7e3d7 Mon Sep 17 00:00:00 2001 From: Ella <72365100+eilla1@users.noreply.github.com> Date: Wed, 16 Jun 2021 00:32:43 -0700 Subject: [PATCH 79/92] fix: update syntax for commands for example: `$ octodns-sync --config-file=./config/production.yaml` was throwing an error. Then, when i did `$ octodns-dump`, it prompted me to put it in this format: `$ octodns-sync --config-file CONFIG_FILE` (without the = sign) --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 2ce9445..eb7789b 100644 --- a/README.md +++ b/README.md @@ -122,7 +122,7 @@ Further information can be found in [Records Documentation](/docs/records.md). We're ready to do a dry-run with our new setup to see what changes it would make. Since we're pretending here we'll act like there are no existing records for `example.com.` in our accounts on either provider. ```shell -$ octodns-sync --config-file=./config/production.yaml +$ octodns-sync --config-file ./config/production.yaml ... ******************************************************************************** * example.com. @@ -146,7 +146,7 @@ There will be other logging information presented on the screen, but successful Now it's time to tell OctoDNS to make things happen. We'll invoke it again with the same options and add a `--doit` on the end to tell it this time we actually want it to try and make the specified changes. ```shell -$ octodns-sync --config-file=./config/production.yaml --doit +$ octodns-sync --config-file ./config/production.yaml --doit ... ``` @@ -177,7 +177,7 @@ If that goes smoothly, you again see the expected changes, and verify them with Very few situations will involve starting with a blank slate which is why there's tooling built in to pull existing data out of providers into a matching config file. ```shell -$ octodns-dump --config-file=config/production.yaml --output-dir=tmp/ example.com. route53 +$ octodns-dump --config-file ./config/production.yaml --output-dir tmp/ example.com. route53 2017-03-15T13:33:34 INFO Manager __init__: config_file=tmp/production.yaml 2017-03-15T13:33:34 INFO Manager dump: zone=example.com., sources=('route53',) 2017-03-15T13:33:36 INFO Route53Provider[route53] populate: found 64 records From fb805ced62e649dd42529081055cf38bcb616bc8 Mon Sep 17 00:00:00 2001 From: Ella <72365100+eilla1@users.noreply.github.com> Date: Wed, 16 Jun 2021 00:39:01 -0700 Subject: [PATCH 80/92] chore: add alt text to images --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index eb7789b..3ad16d3 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ - +OctoDNS Logo ## DNS as code - Tools for managing DNS across multiple providers @@ -158,17 +158,17 @@ In the above case we manually ran OctoDNS from the command line. That works and The first step is to create a PR with your changes. -![](/docs/assets/pr.png) +![GitHub user interface of a pull request](/docs/assets/pr.png) Assuming the code tests and config validation statuses are green the next step is to do a noop deploy and verify that the changes OctoDNS plans to make are the ones you expect. -![](/docs/assets/noop.png) +![Output of a noop deployment command](/docs/assets/noop.png) After that comes a set of reviews. One from a teammate who should have full context on what you're trying to accomplish and visibility in to the changes you're making to do it. The other is from a member of the team here at GitHub that owns DNS, mostly as a sanity check and to make sure that best practices are being followed. As much of that as possible is baked into `octodns-validate`. After the reviews it's time to branch deploy the change. -![](/docs/assets/deploy.png) +![Output of a deployment command](/docs/assets/deploy.png) If that goes smoothly, you again see the expected changes, and verify them with `dig` and/or `octodns-report` you're good to hit the merge button. If there are problems you can quickly do a `.deploy dns/master` to go back to the previous state. From 501ae886738522b13e3c9e8c5f564c48d5c135d8 Mon Sep 17 00:00:00 2001 From: Ella <72365100+eilla1@users.noreply.github.com> Date: Wed, 16 Jun 2021 15:25:14 -0700 Subject: [PATCH 81/92] chore: use = --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 3ad16d3..91e04eb 100644 --- a/README.md +++ b/README.md @@ -122,7 +122,7 @@ Further information can be found in [Records Documentation](/docs/records.md). We're ready to do a dry-run with our new setup to see what changes it would make. Since we're pretending here we'll act like there are no existing records for `example.com.` in our accounts on either provider. ```shell -$ octodns-sync --config-file ./config/production.yaml +$ octodns-sync --config-file=./config/production.yaml ... ******************************************************************************** * example.com. @@ -146,7 +146,7 @@ There will be other logging information presented on the screen, but successful Now it's time to tell OctoDNS to make things happen. We'll invoke it again with the same options and add a `--doit` on the end to tell it this time we actually want it to try and make the specified changes. ```shell -$ octodns-sync --config-file ./config/production.yaml --doit +$ octodns-sync --config-file=./config/production.yaml --doit ... ``` @@ -177,7 +177,7 @@ If that goes smoothly, you again see the expected changes, and verify them with Very few situations will involve starting with a blank slate which is why there's tooling built in to pull existing data out of providers into a matching config file. ```shell -$ octodns-dump --config-file ./config/production.yaml --output-dir tmp/ example.com. route53 +$ octodns-dump --config-file=config/production.yaml --output-dir=tmp/ example.com. route53 2017-03-15T13:33:34 INFO Manager __init__: config_file=tmp/production.yaml 2017-03-15T13:33:34 INFO Manager dump: zone=example.com., sources=('route53',) 2017-03-15T13:33:36 INFO Route53Provider[route53] populate: found 64 records From 794fca0ee765b8ae8912ef557c66484f8253959f Mon Sep 17 00:00:00 2001 From: Iordanis Grigoriou Date: Thu, 17 Jun 2021 16:36:46 +0200 Subject: [PATCH 82/92] Fix typo --- octodns/provider/dyn.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/octodns/provider/dyn.py b/octodns/provider/dyn.py index da56a2e..2fbcadb 100644 --- a/octodns/provider/dyn.py +++ b/octodns/provider/dyn.py @@ -604,7 +604,7 @@ class DynProvider(BaseProvider): return record - def _is_traffic_director_dyanmic(self, td, rulesets): + def _is_traffic_director_dynamic(self, td, rulesets): for ruleset in rulesets: try: pieces = ruleset.label.split(':') @@ -632,7 +632,7 @@ class DynProvider(BaseProvider): continue # critical to call rulesets once, each call loads them :-( rulesets = td.rulesets - if self._is_traffic_director_dyanmic(td, rulesets): + if self._is_traffic_director_dynamic(td, rulesets): record = \ self._populate_dynamic_traffic_director(zone, fqdn, _type, td, From f147a3ab0fd2ddacc07964872c82d8c458e9e8ff Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Mon, 21 Jun 2021 15:36:57 -0700 Subject: [PATCH 83/92] Fix endpoint naming --- octodns/provider/azuredns.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 6629d6c..fcc9f54 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -1010,7 +1010,7 @@ class AzureProvider(BaseProvider): ep_name += '--default--' default_seen = True rule_endpoints.append(Endpoint( - name=pool_name, + name=ep_name, target=target, priority=priority, )) @@ -1053,7 +1053,7 @@ class AzureProvider(BaseProvider): else: # just add the value of single-value pool geo_endpoints.append(Endpoint( - name=rule_ep.name + '--default--', + name=rule_ep.name, target=rule_ep.target, geo_mapping=geos, )) From e7524ec1ad8341608c9ca1f96161adc05186ae53 Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Mon, 21 Jun 2021 15:46:27 -0700 Subject: [PATCH 84/92] Partial support for dynamic A+AAAA records on Azure --- octodns/provider/azuredns.py | 123 +++++-- tests/test_octodns_provider_azuredns.py | 412 +++++++++++++++++++++++- 2 files changed, 506 insertions(+), 29 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index fcc9f54..2790536 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -125,6 +125,9 @@ class _AzureRecord(object): self.params['ttl'] = record.ttl def _params_for_A(self, data, key_name, azure_class): + if self._record.dynamic and self.traffic_manager: + return {'target_resource': self.traffic_manager} + try: values = data['values'] except KeyError: @@ -132,6 +135,9 @@ class _AzureRecord(object): return {key_name: [azure_class(ipv4_address=v) for v in values]} def _params_for_AAAA(self, data, key_name, azure_class): + if self._record.dynamic and self.traffic_manager: + return {'target_resource': self.traffic_manager} + try: values = data['values'] except KeyError: @@ -263,7 +269,10 @@ def _root_traffic_manager_name(record): # ATM names can only have letters, numbers and hyphens # replace dots with double hyphens to ensure unique mapping, # hoping that real life FQDNs won't have double hyphens - return record.fqdn[:-1].replace('.', '--') + name = record.fqdn[:-1].replace('.', '--') + if record._type != 'CNAME': + name += '-{}'.format(record._type) + return name def _rule_traffic_manager_name(pool, record): @@ -608,9 +617,33 @@ class AzureProvider(BaseProvider): lenient=lenient) def _data_for_A(self, azrecord): + if azrecord.a_records is None: + if azrecord.target_resource.id: + return self._data_for_dynamic(azrecord) + else: + # dynamic record alias is broken, return dummy value and apply + # will likely overwrite/fix it + self.log.warn('_data_for_A: Missing Traffic Manager ' + 'alias for dynamic A record %s, forcing ' + 're-link by setting an invalid value', + azrecord.fqdn) + return {'values': ['255.255.255.255']} + return {'values': [ar.ipv4_address for ar in azrecord.a_records]} def _data_for_AAAA(self, azrecord): + if azrecord.aaaa_records is None: + if azrecord.target_resource.id: + return self._data_for_dynamic(azrecord) + else: + # dynamic record alias is broken, return dummy value and apply + # will likely overwrite/fix it + self.log.warn('_data_for_AAAA: Missing Traffic Manager ' + 'alias for dynamic AAAA record %s, forcing ' + 're-link by setting an invalid value', + azrecord.fqdn) + return {'values': ['::1']} + return {'values': [ar.ipv6_address for ar in azrecord.aaaa_records]} def _data_for_CAA(self, azrecord): @@ -667,6 +700,7 @@ class AzureProvider(BaseProvider): default = set() pools = defaultdict(lambda: {'fallback': None, 'values': []}) rules = [] + typ = _parse_azure_type(azrecord.type) # top level profile root_profile = self._get_tm_profile_by_id(azrecord.target_resource.id) @@ -781,8 +815,10 @@ class AzureProvider(BaseProvider): for pool_ep in endpoints: val = pool_ep.target + if typ == 'CNAME': + val = _check_endswith_dot(val) pool['values'].append({ - 'value': _check_endswith_dot(val), + 'value': val, 'weight': pool_ep.weight or 1, }) if pool_ep.name.endswith('--default--'): @@ -805,23 +841,58 @@ class AzureProvider(BaseProvider): 'pools': pools, 'rules': rules, }, - 'value': _check_endswith_dot(default[0]), } + if typ == 'CNAME': + data['value'] = _check_endswith_dot(default[0]) + else: + data['values'] = default + return data def _extra_changes(self, existing, desired, changes): changed = set() - # Abort if there are non-CNAME dynamic records + # Abort if there are unsupported dynamic records for change in changes: record = change.record changed.add(record) typ = record._type dynamic = getattr(record, 'dynamic', False) - if dynamic and typ != 'CNAME': - msg = '{}: Dynamic records in Azure must be of type CNAME' - msg = msg.format(record.fqdn) + if not dynamic: + continue + if typ in ['A', 'AAAA']: + data = dynamic._data() + values = set(record.values) + pools = data['pools'].values() + seen_values = set() + rr = False + fallback = False + for pool in pools: + vals = pool['values'] + if len(vals) > 1: + rr = True + pool_values = set(val['value'] for val in vals) + if pool.get('fallback'): + fallback = True + seen_values.update(pool_values) + + if values != seen_values: + msg = ('{} {}: All pool values of A/AAAA dynamic records ' + 'must be included in top-level \'values\'.' + .format(record.fqdn, record._type)) + raise AzureException(msg) + + geo = any(r.get('geos') for r in data['rules']) + + if [rr, fallback, geo].count(True) > 1: + msg = ('{} {}: A/AAAA dynamic records must use at most ' + 'one of round-robin, fallback and geo-fencing') + msg = msg.format(record.fqdn, record._type) + raise AzureException(msg) + elif typ != 'CNAME': + msg = ('{}: Dynamic records in Azure must be of type ' + 'A/AAAA/CNAME').format(record.fqdn) raise AzureException(msg) log = self.log.info @@ -833,11 +904,20 @@ class AzureProvider(BaseProvider): continue # let's walk through and show what will be changed even if - # the record is already be in list of changes + # the record is already in list of changes added = (record in changed) active = set() profiles = self._generate_traffic_managers(record) + + # this should not happen with above checks, still adding to block + # undesired changes + if record._type in ['A', 'AAAA'] and len(profiles) > 1: + msg = ('Unknown error: {} {} needs more than 1 Traffic ' + 'Managers which is not supported for A/AAAA dynamic ' + 'records').format(record.fqdn, record._type) + raise AzureException(msg) + for profile in profiles: name = profile.name if name in seen_profiles: @@ -871,9 +951,9 @@ class AzureProvider(BaseProvider): def _generate_tm_profile(self, routing, endpoints, record, label=None): # figure out profile name and Traffic Manager FQDN name = _root_traffic_manager_name(record) - if routing == 'Weighted': + if routing == 'Weighted' and label: name = _pool_traffic_manager_name(label, record) - elif routing == 'Priority': + elif routing == 'Priority' and label: name = _rule_traffic_manager_name(label, record) # set appropriate endpoint types @@ -895,7 +975,7 @@ class AzureProvider(BaseProvider): name=name, traffic_routing_method=routing, dns_config=DnsConfig( - relative_name=name, + relative_name=name.lower(), ttl=record.ttl, ), monitor_config=_get_monitor(record), @@ -906,7 +986,7 @@ class AzureProvider(BaseProvider): def _convert_tm_to_root(self, profile, record): profile.name = _root_traffic_manager_name(record) profile.id = self._profile_name_to_id(profile.name) - profile.dns_config.relative_name = profile.name + profile.dns_config.relative_name = profile.name.lower() return profile @@ -914,8 +994,12 @@ class AzureProvider(BaseProvider): traffic_managers = [] pools = record.dynamic.pools rules = record.dynamic.rules + typ = record._type - default = record.value[:-1] + if typ == 'CNAME': + defaults = [record.value[:-1]] + else: + defaults = record.values profile = self._generate_tm_profile # a pool can be re-used only with a world pool, record the pool @@ -977,9 +1061,10 @@ class AzureProvider(BaseProvider): for val in pool['values']: target = val['value'] # strip trailing dot from CNAME value - target = target[:-1] + if typ == 'CNAME': + target = target[:-1] ep_name = '{}--{}'.format(pool_name, target) - if target == default: + if target in defaults: # mark default ep_name += '--default--' default_seen = True @@ -1003,9 +1088,11 @@ class AzureProvider(BaseProvider): # Skip Weighted profile hop for single-value pool # append its value as an external endpoint to fallback # rule profile - target = pool['values'][0]['value'][:-1] + target = pool['values'][0]['value'] + if typ == 'CNAME': + target = target[:-1] ep_name = pool_name - if target == default: + if target in defaults: # mark default ep_name += '--default--' default_seen = True @@ -1023,7 +1110,7 @@ class AzureProvider(BaseProvider): if not default_seen: rule_endpoints.append(Endpoint( name='--default--', - target=default, + target=defaults[0], priority=priority, )) diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index 3248b97..41454e9 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -4,6 +4,7 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals +from logging import debug from octodns.record import Create, Update, Delete, Record from octodns.provider.azuredns import _AzureRecord, AzureProvider, \ @@ -883,12 +884,13 @@ class TestAzureDnsProvider(TestCase): # test simple records produce no extra changes desired = Zone(name=existing.name, sub_zones=[]) - desired.add_record(Record.new(desired, 'simple', data={ + simple = Record.new(desired, 'simple', data={ 'type': record._type, 'ttl': record.ttl, 'value': record.value, - })) - extra = provider._extra_changes(desired, desired, []) + }) + desired.add_record(simple) + extra = provider._extra_changes(desired, desired, [Create(simple)]) self.assertEqual(len(extra), 0) # test an unchanged dynamic record produces no extra changes @@ -952,28 +954,28 @@ class TestAzureDnsProvider(TestCase): self.assertIsInstance(extra, Update) self.assertEqual(extra.new, update_dynamic) - # test non-CNAME dynamic record throws exception - a_dynamic = Record.new(desired, record.name + '3', data={ - 'type': 'A', + # test dynamic record of unsupported type throws exception + unsupported_dynamic = Record.new(desired, record.name + '3', data={ + 'type': 'DNAME', 'ttl': record.ttl, - 'values': ['1.1.1.1'], + 'value': 'default.unit.tests.', 'dynamic': { 'pools': { - 'one': {'values': [{'value': '2.2.2.2'}]}, + 'one': {'values': [{'value': 'one.unit.tests.'}]}, }, 'rules': [ {'pool': 'one'}, ], }, }) - desired.add_record(a_dynamic) - changes.append(Create(a_dynamic)) + desired.add_record(unsupported_dynamic) + changes = [Create(unsupported_dynamic)] with self.assertRaises(AzureException) as ctx: provider._extra_changes(existing, desired, changes) self.assertTrue(text_type(ctx).endswith( 'must be of type CNAME' )) - desired._remove_record(a_dynamic) + desired._remove_record(unsupported_dynamic) # test colliding ATM names throws exception record1 = Record.new(desired, 'sub.www', data={ @@ -997,6 +999,100 @@ class TestAzureDnsProvider(TestCase): 'Collision in Traffic Manager' )) + def test_extra_changes_invalid_dynamic_a(self): + provider = self._get_provider() + + desired = Zone(name=zone.name, sub_zones=[]) + + # too many test case combinations, here's a method to generate them + def record_data(all_values=True, rr=True, fallback=True, geo=True): + data = { + 'type': 'A', + 'ttl': 60, + 'dynamic': { + 'pools': { + 'one': { + 'values': [ + {'value': '11.11.11.11'}, + {'value': '12.12.12.12'}, + ], + 'fallback': 'two', + }, + 'two': { + 'values': [ + {'value': '2.2.2.2'}, + ], + }, + }, + 'rules': [ + {'geos': ['EU'], 'pool': 'two'}, + {'pool': 'one'}, + ], + } + } + dynamic = data['dynamic'] + if not rr: + dynamic['pools']['one']['values'].pop() + if not fallback: + dynamic['pools']['one'].pop('fallback') + if not geo: + rule = dynamic['rules'].pop(0) + if not fallback: + dynamic['pools'].pop(rule['pool']) + # put all pool values in default + data['values'] = [ + v['value'] + for p in dynamic['pools'].values() + for v in p['values'] + ] + if not all_values: + rm = list(dynamic['pools'].values())[0]['values'][0]['value'] + data['values'].remove(rm) + return data + + # test all combinations + values = [True, False] + combos = [ + [arg1, arg2, arg3, arg4] + for arg1 in values + for arg2 in values + for arg3 in values + for arg4 in values + ] + for all_values, rr, fallback, geo in combos: + args = [all_values, rr, fallback, geo] + + if not any(args): + # all False, invalid use-case + continue + + debug('[all_values, rr, fallback, geo] = %s', args) + data = record_data(*args) + record = Record.new(desired, 'foo', data) + + features = args[1:] + if all_values and features.count(True) <= 1: + # assert does not raise exception + provider._extra_changes(zone, desired, [Create(record)]) + continue + + with self.assertRaises(AzureException) as ctx: + msg = text_type(ctx) + provider._extra_changes(zone, desired, [Create(record)]) + if not all_values: + self.assertTrue('included in top-level \'values\'' in msg) + else: + self.assertTrue('at most one of' in msg) + + # multiple profiles should also raise + record = Record.new(desired, 'foo', + record_data(True, True, True, True)) + desired.add_record(record) + with self.assertRaises(AzureException) as ctx: + # bypass above check by setting changes to empty + provider._extra_changes(zone, desired, []) + self.assertTrue('more than 1 Traffic Managers' in text_type(ctx)) + def test_generate_tm_profile(self): provider, zone, record = self._get_dynamic_package() profile_gen = provider._generate_tm_profile @@ -1573,6 +1669,300 @@ class TestAzureDnsProvider(TestCase): record2 = provider._populate_record(zone, azrecord) self.assertEqual(record2.dynamic._data(), record.dynamic._data()) + def test_dynamic_a_geo(self): + provider = self._get_provider() + external = 'Microsoft.Network/trafficManagerProfiles/externalEndpoints' + + record = Record.new(zone, 'foo', data={ + 'type': 'A', + 'ttl': 60, + 'values': ['1.1.1.1', '2.2.2.2', '3.3.3.3'], + 'dynamic': { + 'pools': { + 'one': { + 'values': [ + {'value': '1.1.1.1'}, + ], + }, + 'two': { + 'values': [ + {'value': '2.2.2.2'}, + ], + }, + 'three': { + 'values': [ + {'value': '3.3.3.3'}, + ], + }, + }, + 'rules': [ + {'geos': ['AS'], 'pool': 'one'}, + {'geos': ['AF'], 'pool': 'two'}, + {'pool': 'three'}, + ], + } + }) + + # test that extra_changes doesn't complain + changes = [Create(record)] + provider._extra_changes(zone, zone, changes) + + profiles = provider._generate_traffic_managers(record) + + self.assertEqual(len(profiles), 1) + self.assertTrue(_profile_is_match(profiles[0], Profile( + name='foo--unit--tests-A', + traffic_routing_method='Geographic', + dns_config=DnsConfig( + relative_name='foo--unit--tests-a', ttl=record.ttl), + monitor_config=_get_monitor(record), + endpoints=[ + Endpoint( + name='one--default--', + type=external, + target='1.1.1.1', + geo_mapping=['GEO-AS'], + ), + Endpoint( + name='two--default--', + type=external, + target='2.2.2.2', + geo_mapping=['GEO-AF'], + ), + Endpoint( + name='three--default--', + type=external, + target='3.3.3.3', + geo_mapping=['WORLD'], + ), + ], + ))) + + # test that the record and ATM profile gets created + tm_sync = provider._tm_client.profiles.create_or_update + create = provider._dns_client.record_sets.create_or_update + provider._apply_Create(changes[0]) + # A dynamic record can only have 1 profile + tm_sync.assert_called_once() + create.assert_called_once() + + # test broken alias + azrecord = RecordSet( + ttl=60, target_resource=SubResource(id=None)) + azrecord.name = record.name or '@' + azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) + record2 = provider._populate_record(zone, azrecord) + self.assertEqual(record2.values, ['255.255.255.255']) + + # test that same record gets populated back from traffic managers + tm_list = provider._tm_client.profiles.list_by_resource_group + tm_list.return_value = profiles + azrecord = RecordSet( + ttl=60, + target_resource=SubResource(id=profiles[-1].id), + ) + azrecord.name = record.name or '@' + azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) + record2 = provider._populate_record(zone, azrecord) + self.assertEqual(record2.dynamic._data(), record.dynamic._data()) + + def test_dynamic_a_fallback(self): + provider = self._get_provider() + external = 'Microsoft.Network/trafficManagerProfiles/externalEndpoints' + + record = Record.new(zone, 'foo', data={ + 'type': 'A', + 'ttl': 60, + 'values': ['8.8.8.8'], + 'dynamic': { + 'pools': { + 'one': { + 'values': [ + {'value': '1.1.1.1'}, + ], + 'fallback': 'two', + }, + 'two': { + 'values': [ + {'value': '2.2.2.2'}, + ], + }, + }, + 'rules': [ + {'pool': 'one'}, + ], + } + }) + profiles = provider._generate_traffic_managers(record) + + self.assertEqual(len(profiles), 1) + self.assertTrue(_profile_is_match(profiles[0], Profile( + name='foo--unit--tests-A', + traffic_routing_method='Priority', + dns_config=DnsConfig( + relative_name='foo--unit--tests-a', ttl=record.ttl), + monitor_config=_get_monitor(record), + endpoints=[ + Endpoint( + name='one', + type=external, + target='1.1.1.1', + priority=1, + ), + Endpoint( + name='two', + type=external, + target='2.2.2.2', + priority=2, + ), + Endpoint( + name='--default--', + type=external, + target='8.8.8.8', + priority=3, + ), + ], + ))) + + # test that same record gets populated back from traffic managers + tm_list = provider._tm_client.profiles.list_by_resource_group + tm_list.return_value = profiles + azrecord = RecordSet( + ttl=60, + target_resource=SubResource(id=profiles[-1].id), + ) + azrecord.name = record.name or '@' + azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) + record2 = provider._populate_record(zone, azrecord) + self.assertEqual(record2.dynamic._data(), record.dynamic._data()) + + def test_dynamic_a_weighted_rr(self): + provider = self._get_provider() + external = 'Microsoft.Network/trafficManagerProfiles/externalEndpoints' + + record = Record.new(zone, 'foo', data={ + 'type': 'A', + 'ttl': 60, + 'values': ['1.1.1.1', '8.8.8.8'], + 'dynamic': { + 'pools': { + 'one': { + 'values': [ + {'value': '1.1.1.1', 'weight': 11}, + {'value': '8.8.8.8', 'weight': 8}, + ], + }, + }, + 'rules': [ + {'pool': 'one'}, + ], + } + }) + profiles = provider._generate_traffic_managers(record) + + self.assertEqual(len(profiles), 1) + self.assertTrue(_profile_is_match(profiles[0], Profile( + name='foo--unit--tests-A', + traffic_routing_method='Weighted', + dns_config=DnsConfig( + relative_name='foo--unit--tests-a', ttl=record.ttl), + monitor_config=_get_monitor(record), + endpoints=[ + Endpoint( + name='one--1.1.1.1--default--', + type=external, + target='1.1.1.1', + weight=11, + ), + Endpoint( + name='one--8.8.8.8--default--', + type=external, + target='8.8.8.8', + weight=8, + ), + ], + ))) + + # test that same record gets populated back from traffic managers + tm_list = provider._tm_client.profiles.list_by_resource_group + tm_list.return_value = profiles + azrecord = RecordSet( + ttl=60, + target_resource=SubResource(id=profiles[-1].id), + ) + azrecord.name = record.name or '@' + azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) + record2 = provider._populate_record(zone, azrecord) + self.assertEqual(record2.dynamic._data(), record.dynamic._data()) + + def test_dynamic_aaaa(self): + provider = self._get_provider() + external = 'Microsoft.Network/trafficManagerProfiles/externalEndpoints' + + record = Record.new(zone, 'foo', data={ + 'type': 'AAAA', + 'ttl': 60, + 'values': ['1::1'], + 'dynamic': { + 'pools': { + 'one': { + 'values': [ + {'value': '1::1'}, + ], + }, + }, + 'rules': [ + {'pool': 'one'}, + ], + } + }) + profiles = provider._generate_traffic_managers(record) + + self.assertEqual(len(profiles), 1) + self.assertTrue(_profile_is_match(profiles[0], Profile( + name='foo--unit--tests-AAAA', + traffic_routing_method='Geographic', + dns_config=DnsConfig( + relative_name='foo--unit--tests-aaaa', ttl=record.ttl), + monitor_config=_get_monitor(record), + endpoints=[ + Endpoint( + name='one--default--', + type=external, + target='1::1', + geo_mapping=['WORLD'], + ), + ], + ))) + + # test that the record and ATM profile gets created + tm_sync = provider._tm_client.profiles.create_or_update + create = provider._dns_client.record_sets.create_or_update + provider._apply_Create(Create(record)) + # A dynamic record can only have 1 profile + tm_sync.assert_called_once() + create.assert_called_once() + + # test broken alias + azrecord = RecordSet( + ttl=60, target_resource=SubResource(id=None)) + azrecord.name = record.name or '@' + azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) + record2 = provider._populate_record(zone, azrecord) + self.assertEqual(record2.values, ['::1']) + + # test that same record gets populated back from traffic managers + tm_list = provider._tm_client.profiles.list_by_resource_group + tm_list.return_value = profiles + azrecord = RecordSet( + ttl=60, + target_resource=SubResource(id=profiles[-1].id), + ) + azrecord.name = record.name or '@' + azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) + record2 = provider._populate_record(zone, azrecord) + self.assertEqual(record2.dynamic._data(), record.dynamic._data()) + def test_sync_traffic_managers(self): provider, zone, record = self._get_dynamic_package() provider._populate_traffic_managers() From c0161fb228fdb3191225b9bcc39ba1036b173c44 Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Tue, 22 Jun 2021 15:48:38 -0700 Subject: [PATCH 85/92] Split out dynamic record validty check for readability --- octodns/provider/azuredns.py | 92 +++++++++++++------------ tests/test_octodns_provider_azuredns.py | 41 +++++++++-- 2 files changed, 83 insertions(+), 50 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 2790536..4551c6c 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -299,6 +299,47 @@ def _get_monitor(record): return monitor +def _check_valid_dynamic(record): + typ = record._type + dynamic = record.dynamic + if typ in ['A', 'AAAA']: + # A/AAAA records cannot be aliased to Traffic Managers that contain + # other nested Traffic Managers. Due to this limitation, A/AAAA + # dynamic records can do only one of geo-fencing, fallback and + # weighted RR. So let's validate that the record adheres to this + # limitation. + data = dynamic._data() + values = set(record.values) + pools = data['pools'].values() + seen_values = set() + rr = False + fallback = False + for pool in pools: + vals = pool['values'] + if len(vals) > 1: + rr = True + pool_values = set(val['value'] for val in vals) + if pool.get('fallback'): + fallback = True + seen_values.update(pool_values) + + if values != seen_values: + msg = ('{} {}: All pool values of A/AAAA dynamic records must be ' + 'included in top-level \'values\'.') + raise AzureException(msg.format(record.fqdn, record._type)) + + geo = any(r.get('geos') for r in data['rules']) + + if [rr, fallback, geo].count(True) > 1: + msg = ('{} {}: A/AAAA dynamic records must use at most one of ' + 'round-robin, fallback and geo-fencing') + raise AzureException(msg.format(record.fqdn, record._type)) + elif typ != 'CNAME': + # dynamic records of unsupported type + msg = '{}: Dynamic records in Azure must be of type A/AAAA/CNAME' + raise AzureException(msg.format(record.fqdn)) + + def _profile_is_match(have, desired): if have is None or desired is None: return False @@ -851,49 +892,7 @@ class AzureProvider(BaseProvider): return data def _extra_changes(self, existing, desired, changes): - changed = set() - - # Abort if there are unsupported dynamic records - for change in changes: - record = change.record - changed.add(record) - typ = record._type - dynamic = getattr(record, 'dynamic', False) - if not dynamic: - continue - if typ in ['A', 'AAAA']: - data = dynamic._data() - values = set(record.values) - pools = data['pools'].values() - seen_values = set() - rr = False - fallback = False - for pool in pools: - vals = pool['values'] - if len(vals) > 1: - rr = True - pool_values = set(val['value'] for val in vals) - if pool.get('fallback'): - fallback = True - seen_values.update(pool_values) - - if values != seen_values: - msg = ('{} {}: All pool values of A/AAAA dynamic records ' - 'must be included in top-level \'values\'.' - .format(record.fqdn, record._type)) - raise AzureException(msg) - - geo = any(r.get('geos') for r in data['rules']) - - if [rr, fallback, geo].count(True) > 1: - msg = ('{} {}: A/AAAA dynamic records must use at most ' - 'one of round-robin, fallback and geo-fencing') - msg = msg.format(record.fqdn, record._type) - raise AzureException(msg) - elif typ != 'CNAME': - msg = ('{}: Dynamic records in Azure must be of type ' - 'A/AAAA/CNAME').format(record.fqdn) - raise AzureException(msg) + changed = set(c.record for c in changes) log = self.log.info seen_profiles = {} @@ -903,6 +902,9 @@ class AzureProvider(BaseProvider): # Already changed, or not dynamic, no need to check it continue + # Abort if there are unsupported dynamic record configurations + _check_valid_dynamic(record) + # let's walk through and show what will be changed even if # the record is already in list of changes added = (record in changed) @@ -910,8 +912,8 @@ class AzureProvider(BaseProvider): active = set() profiles = self._generate_traffic_managers(record) - # this should not happen with above checks, still adding to block - # undesired changes + # this should not happen with above check, check again here to + # prevent undesired changes if record._type in ['A', 'AAAA'] and len(profiles) > 1: msg = ('Unknown error: {} {} needs more than 1 Traffic ' 'Managers which is not supported for A/AAAA dynamic ' diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index 41454e9..16fa5b1 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -1002,8 +1002,6 @@ class TestAzureDnsProvider(TestCase): def test_extra_changes_invalid_dynamic_a(self): provider = self._get_provider() - desired = Zone(name=zone.name, sub_zones=[]) - # too many test case combinations, here's a method to generate them def record_data(all_values=True, rr=True, fallback=True, geo=True): data = { @@ -1068,7 +1066,9 @@ class TestAzureDnsProvider(TestCase): debug('[all_values, rr, fallback, geo] = %s', args) data = record_data(*args) + desired = Zone(name=zone.name, sub_zones=[]) record = Record.new(desired, 'foo', data) + desired.add_record(record) features = args[1:] if all_values and features.count(True) <= 1: @@ -1084,9 +1084,40 @@ class TestAzureDnsProvider(TestCase): else: self.assertTrue('at most one of' in msg) - # multiple profiles should also raise - record = Record.new(desired, 'foo', - record_data(True, True, True, True)) + @patch( + 'octodns.provider.azuredns._check_valid_dynamic') + def test_extra_changes_dynamic_a_multiple_profiles(self, mock_cvd): + provider = self._get_provider() + + # bypass validity check to trigger mutliple-profiles check + mock_cvd.return_value = True + + desired = Zone(name=zone.name, sub_zones=[]) + record = Record.new(desired, 'foo', { + 'type': 'A', + 'ttl': 60, + 'values': ['11.11.11.11', '12.12.12.12', '2.2.2.2'], + 'dynamic': { + 'pools': { + 'one': { + 'values': [ + {'value': '11.11.11.11'}, + {'value': '12.12.12.12'}, + ], + 'fallback': 'two', + }, + 'two': { + 'values': [ + {'value': '2.2.2.2'}, + ], + }, + }, + 'rules': [ + {'geos': ['EU'], 'pool': 'two'}, + {'pool': 'one'}, + ], + } + }) desired.add_record(record) with self.assertRaises(AzureException) as ctx: # bypass above check by setting changes to empty From 568e6860d32fbcbaa5e8487646c4c6ff57fd868e Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Tue, 22 Jun 2021 15:56:08 -0700 Subject: [PATCH 86/92] drop comment --- tests/test_octodns_provider_azuredns.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index 16fa5b1..b577aa4 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -1120,8 +1120,7 @@ class TestAzureDnsProvider(TestCase): }) desired.add_record(record) with self.assertRaises(AzureException) as ctx: - # bypass above check by setting changes to empty - provider._extra_changes(zone, desired, []) + provider._extra_changes(zone, desired, [Create(record)]) self.assertTrue('more than 1 Traffic Managers' in text_type(ctx)) def test_generate_tm_profile(self): From 500097542be7bef9d2ca61a3ec16ae16e5277eec Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Tue, 22 Jun 2021 16:11:28 -0700 Subject: [PATCH 87/92] drop unneeded line break --- tests/test_octodns_provider_azuredns.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index b577aa4..697b30c 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -1084,8 +1084,7 @@ class TestAzureDnsProvider(TestCase): else: self.assertTrue('at most one of' in msg) - @patch( - 'octodns.provider.azuredns._check_valid_dynamic') + @patch('octodns.provider.azuredns._check_valid_dynamic') def test_extra_changes_dynamic_a_multiple_profiles(self, mock_cvd): provider = self._get_provider() From a8a9c7c6d12fa2aa4462db1e1042eae151d28476 Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Wed, 23 Jun 2021 10:52:14 -0700 Subject: [PATCH 88/92] better test method names --- tests/test_octodns_provider_azuredns.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index 697b30c..eb74007 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -999,7 +999,7 @@ class TestAzureDnsProvider(TestCase): 'Collision in Traffic Manager' )) - def test_extra_changes_invalid_dynamic_a(self): + def test_extra_changes_invalid_dynamic_A(self): provider = self._get_provider() # too many test case combinations, here's a method to generate them @@ -1085,7 +1085,7 @@ class TestAzureDnsProvider(TestCase): self.assertTrue('at most one of' in msg) @patch('octodns.provider.azuredns._check_valid_dynamic') - def test_extra_changes_dynamic_a_multiple_profiles(self, mock_cvd): + def test_extra_changes_dynamic_A_multiple_profiles(self, mock_cvd): provider = self._get_provider() # bypass validity check to trigger mutliple-profiles check @@ -1698,7 +1698,7 @@ class TestAzureDnsProvider(TestCase): record2 = provider._populate_record(zone, azrecord) self.assertEqual(record2.dynamic._data(), record.dynamic._data()) - def test_dynamic_a_geo(self): + def test_dynamic_A_geo(self): provider = self._get_provider() external = 'Microsoft.Network/trafficManagerProfiles/externalEndpoints' @@ -1795,7 +1795,7 @@ class TestAzureDnsProvider(TestCase): record2 = provider._populate_record(zone, azrecord) self.assertEqual(record2.dynamic._data(), record.dynamic._data()) - def test_dynamic_a_fallback(self): + def test_dynamic_A_fallback(self): provider = self._get_provider() external = 'Microsoft.Network/trafficManagerProfiles/externalEndpoints' @@ -1865,7 +1865,7 @@ class TestAzureDnsProvider(TestCase): record2 = provider._populate_record(zone, azrecord) self.assertEqual(record2.dynamic._data(), record.dynamic._data()) - def test_dynamic_a_weighted_rr(self): + def test_dynamic_A_weighted_rr(self): provider = self._get_provider() external = 'Microsoft.Network/trafficManagerProfiles/externalEndpoints' @@ -1924,7 +1924,7 @@ class TestAzureDnsProvider(TestCase): record2 = provider._populate_record(zone, azrecord) self.assertEqual(record2.dynamic._data(), record.dynamic._data()) - def test_dynamic_aaaa(self): + def test_dynamic_AAAA(self): provider = self._get_provider() external = 'Microsoft.Network/trafficManagerProfiles/externalEndpoints' From fe7f57e2adeb42cd080680b4ec6bcb089fd7a6df Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Wed, 23 Jun 2021 11:26:57 -0700 Subject: [PATCH 89/92] update README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 91e04eb..3f36e73 100644 --- a/README.md +++ b/README.md @@ -192,7 +192,7 @@ The above command pulled the existing data out of Route53 and placed the results | Provider | Requirements | Record Support | Dynamic | Notes | |--|--|--|--|--| -| [AzureProvider](/octodns/provider/azuredns.py) | azure-identity, azure-mgmt-dns, azure-mgmt-trafficmanager | A, AAAA, CAA, CNAME, MX, NS, PTR, SRV, TXT | Alpha (CNAMEs only) | | +| [AzureProvider](/octodns/provider/azuredns.py) | azure-identity, azure-mgmt-dns, azure-mgmt-trafficmanager | A, AAAA, CAA, CNAME, MX, NS, PTR, SRV, TXT | Alpha (CNAMEs and partial A/AAAA) | | | [Akamai](/octodns/provider/edgedns.py) | edgegrid-python | A, AAAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, SSHFP, TXT | No | | | [CloudflareProvider](/octodns/provider/cloudflare.py) | | A, AAAA, ALIAS, CAA, CNAME, LOC, MX, NS, PTR, SPF, SRV, TXT | No | CAA tags restricted | | [ConstellixProvider](/octodns/provider/constellix.py) | | A, AAAA, ALIAS (ANAME), CAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | CAA tags restricted | From c3f0bf677a9775bcb55ce3a25b7f622deba6798d Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Wed, 23 Jun 2021 18:49:19 -0700 Subject: [PATCH 90/92] Validate processor config sections --- octodns/manager.py | 11 +++++++++-- tests/config/unknown-processor.yaml | 17 +++++++++++++++++ tests/test_octodns_manager.py | 5 +++++ 3 files changed, 31 insertions(+), 2 deletions(-) create mode 100644 tests/config/unknown-processor.yaml diff --git a/octodns/manager.py b/octodns/manager.py index c7173c6..dcbf083 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -573,8 +573,15 @@ class Manager(object): if isinstance(source, YamlProvider): source.populate(zone) - # TODO: validate - # processors = config.get('processors', []) + # check that processors are in order if any are specified + processors = config.get('processors', []) + try: + # same as above, but for processors this time + for processor in processors: + collected.append(self.processors[processor]) + except KeyError: + raise ManagerException('Zone {}, unknown processor: {}' + .format(zone_name, processor)) def get_zone(self, zone_name): if not zone_name[-1] == '.': diff --git a/tests/config/unknown-processor.yaml b/tests/config/unknown-processor.yaml new file mode 100644 index 0000000..4aff713 --- /dev/null +++ b/tests/config/unknown-processor.yaml @@ -0,0 +1,17 @@ +manager: + max_workers: 2 +providers: + in: + class: octodns.provider.yaml.YamlProvider + directory: tests/config + dump: + class: octodns.provider.yaml.YamlProvider + directory: env/YAML_TMP_DIR +zones: + unit.tests.: + sources: + - in + processors: + - missing + targets: + - dump diff --git a/tests/test_octodns_manager.py b/tests/test_octodns_manager.py index 069bc0b..8bada06 100644 --- a/tests/test_octodns_manager.py +++ b/tests/test_octodns_manager.py @@ -339,6 +339,11 @@ class TestManager(TestCase): Manager(get_config_filename('simple-alias-zone.yaml')) \ .validate_configs() + with self.assertRaises(ManagerException) as ctx: + Manager(get_config_filename('unknown-processor.yaml')) \ + .validate_configs() + self.assertTrue('unknown processor' in text_type(ctx.exception)) + def test_get_zone(self): Manager(get_config_filename('simple.yaml')).get_zone('unit.tests.') From 8f3ee159c25389bf961a2c5e6401f28a3931412a Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Wed, 23 Jun 2021 18:56:20 -0700 Subject: [PATCH 91/92] Changelog entry for processors --- CHANGELOG.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d16544..0b9e498 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,13 @@ +## v0.9.13 - 2021-..-.. - + +#### Noteworthy changes + +* Alpha support for Processors has been added. Processors allow for hooking + into the source, target, and planing process to make nearly arbitrary changes + to data. See the [octodns/processors/](/octodns/processors) directory for + examples. The change has been designed to have no impact on the process + unless the `processors` key is present in zone configs. + ## v0.9.12 - 2021-04-30 - Enough time has passed #### Noteworthy changes From a7659c308669fa163ff647afb85f544a9b64033f Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Wed, 23 Jun 2021 19:02:26 -0700 Subject: [PATCH 92/92] processor not processors --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b9e498..f70d67c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ * Alpha support for Processors has been added. Processors allow for hooking into the source, target, and planing process to make nearly arbitrary changes - to data. See the [octodns/processors/](/octodns/processors) directory for + to data. See the [octodns/processor/](/octodns/processor) directory for examples. The change has been designed to have no impact on the process unless the `processors` key is present in zone configs.