From c89664a9f10864007d4f376abb0af342d6232828 Mon Sep 17 00:00:00 2001 From: Piotr Pieprzycki Date: Wed, 15 Jul 2020 15:59:55 +0300 Subject: [PATCH 001/358] Add support for long txt records. Skip Alias records --- octodns/provider/azuredns.py | 53 ++++++++++++++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 3 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 3d8122a..eb822cb 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -28,6 +28,25 @@ def unescape_semicolon(s): return s.replace('\\;', ';') + +def azure_chunked_value(val): + CHUNK_SIZE = 255 + val_replace = val.replace('"', '\\"') + value = unescape_semicolon(val_replace) + if len(val) > CHUNK_SIZE: + vs = [value[i:i + CHUNK_SIZE] + for i in range(0, len(value), CHUNK_SIZE)] + else: + vs = value + return vs + + +def azure_chunked_values(s): + values = [] + for v in s: + values.append(azure_chunked_value(v)) + return values + class _AzureRecord(object): '''Wrapper for OctoDNS record for AzureProvider to make dns_client calls. @@ -72,6 +91,8 @@ class _AzureRecord(object): :type return: _AzureRecord ''' + self.log = logging.getLogger('AzureRecord') + self.resource_group = resource_group self.zone_name = record.zone.name[:len(record.zone.name) - 1] self.relative_record_set_name = record.name or '@' @@ -91,6 +112,9 @@ class _AzureRecord(object): self.params = self.params(record.data, key_name, azure_class) self.params['ttl'] = record.ttl + + + def _params_for_A(self, data, key_name, azure_class): try: values = data['values'] @@ -161,12 +185,22 @@ class _AzureRecord(object): values = [data['value']] return {key_name: [azure_class(ptrdname=v) for v in values]} + def _params_for_TXT(self, data, key_name, azure_class): + + params = [] try: # API for TxtRecord has list of str, even for singleton - values = [unescape_semicolon(v) for v in data['values']] + values = [v for v in azure_chunked_values(data['values'])] except KeyError: - values = [unescape_semicolon(data['value'])] - return {key_name: [azure_class(value=[v]) for v in values]} + values = [azure_chunked_value(data['value'])] + + for v in values: + if isinstance(v, list): + params.append(azure_class(value=v)) + else: + params.append(azure_class(value=[v])) + return {key_name: params} + def _equals(self, b): '''Checks whether two records are equal by comparing all fields. @@ -387,17 +421,30 @@ class AzureProvider(BaseProvider): for azrecord in _records: record_name = azrecord.name if azrecord.name != '@' else '' typ = _parse_azure_type(azrecord.type) + + if typ in ['A', 'CNAME']: + if self._check_for_alias(azrecord, typ): + self.log.info( + 'This entry is an Azure alias. Skipping. zone=%s record=%s, type=%s', zone_name, record_name, typ) + continue + data = getattr(self, '_data_for_{}'.format(typ)) data = data(azrecord) data['type'] = typ data['ttl'] = azrecord.ttl record = Record.new(zone, record_name, data, source=self) + zone.add_record(record, lenient=lenient) self.log.info('populate: found %s records, exists=%s', len(zone.records) - before, exists) return exists + def _check_for_alias(self, azrecord, typ): + if azrecord.target_resource.id and not azrecord.arecords and not azrecord.arecords and not azrecord.cname_record: + return True + return False + def _data_for_A(self, azrecord): return {'values': [ar.ipv4_address for ar in azrecord.arecords]} From 41d7ef660110ce635cf59cdf1141f6a6fbffa235 Mon Sep 17 00:00:00 2001 From: Piotr Pieprzycki Date: Wed, 15 Jul 2020 17:07:45 +0300 Subject: [PATCH 002/358] Fix blank lines --- octodns/provider/azuredns.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index eb822cb..0d20f57 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -28,14 +28,13 @@ def unescape_semicolon(s): return s.replace('\\;', ';') - def azure_chunked_value(val): CHUNK_SIZE = 255 val_replace = val.replace('"', '\\"') value = unescape_semicolon(val_replace) if len(val) > CHUNK_SIZE: vs = [value[i:i + CHUNK_SIZE] - for i in range(0, len(value), CHUNK_SIZE)] + for i in range(0, len(value), CHUNK_SIZE)] else: vs = value return vs @@ -47,6 +46,7 @@ def azure_chunked_values(s): values.append(azure_chunked_value(v)) return values + class _AzureRecord(object): '''Wrapper for OctoDNS record for AzureProvider to make dns_client calls. @@ -112,9 +112,6 @@ class _AzureRecord(object): self.params = self.params(record.data, key_name, azure_class) self.params['ttl'] = record.ttl - - - def _params_for_A(self, data, key_name, azure_class): try: values = data['values'] @@ -185,9 +182,8 @@ class _AzureRecord(object): values = [data['value']] return {key_name: [azure_class(ptrdname=v) for v in values]} - def _params_for_TXT(self, data, key_name, azure_class): - + params = [] try: # API for TxtRecord has list of str, even for singleton values = [v for v in azure_chunked_values(data['values'])] @@ -201,7 +197,6 @@ class _AzureRecord(object): params.append(azure_class(value=[v])) return {key_name: params} - def _equals(self, b): '''Checks whether two records are equal by comparing all fields. :param b: Another _AzureRecord object From 2680b024bd4af29ef83d070b7b8c6d3978953d4f Mon Sep 17 00:00:00 2001 From: Piotr Pieprzycki Date: Wed, 15 Jul 2020 22:36:30 +0300 Subject: [PATCH 003/358] Fix long lines. Change log to debug --- octodns/provider/azuredns.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 0d20f57..bd5c226 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -419,8 +419,9 @@ class AzureProvider(BaseProvider): if typ in ['A', 'CNAME']: if self._check_for_alias(azrecord, typ): - self.log.info( - 'This entry is an Azure alias. Skipping. zone=%s record=%s, type=%s', zone_name, record_name, typ) + self.log.debug( + 'This entry is an Azure alias. Skipping. zone=%s record=%s, type=%s', + zone_name, record_name, typ) continue data = getattr(self, '_data_for_{}'.format(typ)) @@ -436,7 +437,8 @@ class AzureProvider(BaseProvider): return exists def _check_for_alias(self, azrecord, typ): - if azrecord.target_resource.id and not azrecord.arecords and not azrecord.arecords and not azrecord.cname_record: + if (azrecord.target_resource.id and not azrecord.arecords + and not azrecord.arecords and not azrecord.cname_record): return True return False From cbcd0b3f00c2a596c95b1c39646b2dbc92717f75 Mon Sep 17 00:00:00 2001 From: Piotr Pieprzycki Date: Wed, 15 Jul 2020 22:55:48 +0300 Subject: [PATCH 004/358] Fix long lines. Change log to debug --- octodns/provider/azuredns.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index bd5c226..cdaa88f 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -420,7 +420,7 @@ class AzureProvider(BaseProvider): if typ in ['A', 'CNAME']: if self._check_for_alias(azrecord, typ): self.log.debug( - 'This entry is an Azure alias. Skipping. zone=%s record=%s, type=%s', + 'Skipping - ALIAS. zone=%s record=%s, type=%s', zone_name, record_name, typ) continue @@ -437,8 +437,8 @@ class AzureProvider(BaseProvider): return exists def _check_for_alias(self, azrecord, typ): - if (azrecord.target_resource.id and not azrecord.arecords - and not azrecord.arecords and not azrecord.cname_record): + if (azrecord.target_resource.id and not azrecord.arecords and not + azrecord.arecords and not azrecord.cname_record): return True return False From 71296189198eead1f249fae4dd49129193b2d0d7 Mon Sep 17 00:00:00 2001 From: Piotr Pieprzycki Date: Wed, 15 Jul 2020 23:52:02 +0300 Subject: [PATCH 005/358] Add SubResource in tests --- tests/test_octodns_provider_azuredns.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index 1769cef..ab66359 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -13,7 +13,7 @@ from octodns.provider.base import Plan from azure.mgmt.dns.models import ARecord, AaaaRecord, CaaRecord, \ CnameRecord, MxRecord, SrvRecord, NsRecord, PtrRecord, TxtRecord, \ - RecordSet, SoaRecord, Zone as AzureZone + RecordSet, SoaRecord, SubResource, Zone as AzureZone from msrestazure.azure_exceptions import CloudError from unittest import TestCase @@ -358,19 +358,23 @@ class TestAzureDnsProvider(TestCase): rs = [] recordSet = RecordSet(arecords=[ARecord(ipv4_address='1.1.1.1')]) recordSet.name, recordSet.ttl, recordSet.type = 'a1', 0, 'A' + recordSet.target_resource = SubResource() rs.append(recordSet) recordSet = RecordSet(arecords=[ARecord(ipv4_address='1.1.1.1'), ARecord(ipv4_address='2.2.2.2')]) recordSet.name, recordSet.ttl, recordSet.type = 'a2', 1, 'A' + recordSet.target_resource = SubResource() rs.append(recordSet) aaaa1 = AaaaRecord(ipv6_address='1:1ec:1::1') recordSet = RecordSet(aaaa_records=[aaaa1]) recordSet.name, recordSet.ttl, recordSet.type = 'aaaa1', 2, 'AAAA' + recordSet.target_resource = SubResource() rs.append(recordSet) aaaa2 = AaaaRecord(ipv6_address='1:1ec:1::2') recordSet = RecordSet(aaaa_records=[aaaa1, aaaa2]) recordSet.name, recordSet.ttl, recordSet.type = 'aaaa2', 3, 'AAAA' + recordSet.target_resource = SubResource() rs.append(recordSet) recordSet = RecordSet(caa_records=[CaaRecord(flags=0, tag='issue', @@ -388,9 +392,11 @@ class TestAzureDnsProvider(TestCase): cname1 = CnameRecord(cname='cname.unit.test.') recordSet = RecordSet(cname_record=cname1) recordSet.name, recordSet.ttl, recordSet.type = 'cname1', 5, 'CNAME' + recordSet.target_resource = SubResource() rs.append(recordSet) recordSet = RecordSet(cname_record=None) recordSet.name, recordSet.ttl, recordSet.type = 'cname2', 6, 'CNAME' + recordSet.target_resource = SubResource() rs.append(recordSet) recordSet = RecordSet(mx_records=[MxRecord(preference=10, exchange='mx1.unit.test.')]) @@ -434,10 +440,12 @@ class TestAzureDnsProvider(TestCase): rs.append(recordSet) recordSet = RecordSet(txt_records=[TxtRecord(value='sample text1')]) recordSet.name, recordSet.ttl, recordSet.type = 'txt1', 15, 'TXT' + recordSet.target_resource = SubResource() rs.append(recordSet) recordSet = RecordSet(txt_records=[TxtRecord(value='sample text1'), TxtRecord(value='sample text2')]) recordSet.name, recordSet.ttl, recordSet.type = 'txt2', 16, 'TXT' + recordSet.target_resource = SubResource() rs.append(recordSet) recordSet = RecordSet(soa_record=[SoaRecord()]) recordSet.name, recordSet.ttl, recordSet.type = '', 17, 'SOA' From 5a75890021c37d19b25dea7b9cf1080302841a69 Mon Sep 17 00:00:00 2001 From: Piotr Pieprzycki Date: Thu, 16 Jul 2020 00:10:39 +0300 Subject: [PATCH 006/358] Add SubResource in tests --- tests/test_octodns_provider_azuredns.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index ab66359..b7d8c11 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -450,6 +450,18 @@ class TestAzureDnsProvider(TestCase): recordSet = RecordSet(soa_record=[SoaRecord()]) recordSet.name, recordSet.ttl, recordSet.type = '', 17, 'SOA' rs.append(recordSet) + long_txt = "v=spf1 ip4:10.10.0.0/24 ip4:10.10.1.0/24 ip4:10.10.2.0/24" + long_txt += " ip4:10.10.3.0/24 ip4:10.10.4.0/24 ip4:10.10.5.0/24 " + long_txt += " 10.6.0/24 ip4:10.10.7.0/24 ip4:10.10.8.0/24 ip4:10.10.9.0/24" + long_txt += " ip4:10.10.10.0/24 ip4:10.10.11.0/24 ip4:10.10.12.0/24" + long_txt += " ip4:10.10.13.0/24 ip4:10.10.14.0/24 ip4:10.10.15.0/24" + long_txt += " ip4:10.10.16.0/24 ip4:10.10.17.0/24 ip4:10.10.18.0/24" + long_txt += " ip4:10.10.19.0/24 ip4:10.10.20.0/24 ip4:10.10.21.0/24 ~all" + recordSet = RecordSet(txt_records=[TxtRecord(value='sample value1'), + TxtRecord(value=long_txt)]) + recordSet.name, recordSet.ttl, recordSet.type = 'txt2', 18, 'TXT' + recordSet.target_resource = SubResource() + rs.append(recordSet) record_list = provider._dns_client.record_sets.list_by_dns_zone record_list.return_value = rs From 224c90784a2b081447b1fcae1085c5e3c7fa6108 Mon Sep 17 00:00:00 2001 From: Piotr Pieprzycki Date: Thu, 16 Jul 2020 00:16:07 +0300 Subject: [PATCH 007/358] Add SubResource in tests --- tests/test_octodns_provider_azuredns.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index b7d8c11..622d0df 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -452,11 +452,11 @@ class TestAzureDnsProvider(TestCase): rs.append(recordSet) long_txt = "v=spf1 ip4:10.10.0.0/24 ip4:10.10.1.0/24 ip4:10.10.2.0/24" long_txt += " ip4:10.10.3.0/24 ip4:10.10.4.0/24 ip4:10.10.5.0/24 " - long_txt += " 10.6.0/24 ip4:10.10.7.0/24 ip4:10.10.8.0/24 ip4:10.10.9.0/24" + long_txt += " 10.6.0/24 ip4:10.10.7.0/24 ip4:10.10.8.0/24 " long_txt += " ip4:10.10.10.0/24 ip4:10.10.11.0/24 ip4:10.10.12.0/24" long_txt += " ip4:10.10.13.0/24 ip4:10.10.14.0/24 ip4:10.10.15.0/24" long_txt += " ip4:10.10.16.0/24 ip4:10.10.17.0/24 ip4:10.10.18.0/24" - long_txt += " ip4:10.10.19.0/24 ip4:10.10.20.0/24 ip4:10.10.21.0/24 ~all" + long_txt += " ip4:10.10.19.0/24 ip4:10.10.20.0/24 ~all" recordSet = RecordSet(txt_records=[TxtRecord(value='sample value1'), TxtRecord(value=long_txt)]) recordSet.name, recordSet.ttl, recordSet.type = 'txt2', 18, 'TXT' From 53bcb86240ae1724e0e4ab789c2432a740b3f543 Mon Sep 17 00:00:00 2001 From: Piotr Pieprzycki Date: Thu, 16 Jul 2020 00:19:46 +0300 Subject: [PATCH 008/358] Fix record name in test zone --- tests/test_octodns_provider_azuredns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index 622d0df..e1fcf3c 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -459,7 +459,7 @@ class TestAzureDnsProvider(TestCase): long_txt += " ip4:10.10.19.0/24 ip4:10.10.20.0/24 ~all" recordSet = RecordSet(txt_records=[TxtRecord(value='sample value1'), TxtRecord(value=long_txt)]) - recordSet.name, recordSet.ttl, recordSet.type = 'txt2', 18, 'TXT' + recordSet.name, recordSet.ttl, recordSet.type = 'txt3', 18, 'TXT' recordSet.target_resource = SubResource() rs.append(recordSet) From 450cbe020847b0db442243f8c112ff3be64fdc3f Mon Sep 17 00:00:00 2001 From: Piotr Pieprzycki Date: Thu, 16 Jul 2020 00:23:44 +0300 Subject: [PATCH 009/358] Test increase number of zones --- tests/test_octodns_provider_azuredns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index e1fcf3c..e14fe38 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -469,7 +469,7 @@ class TestAzureDnsProvider(TestCase): exists = provider.populate(zone) self.assertTrue(exists) - self.assertEquals(len(zone.records), 18) + self.assertEquals(len(zone.records), 19) def test_populate_zone(self): provider = self._get_provider() From 4e056d315dcdb84efe02fb80a3bde8dc5249f5c8 Mon Sep 17 00:00:00 2001 From: Arunothia Marappan Date: Thu, 16 Jul 2020 16:41:53 -0700 Subject: [PATCH 010/358] Forcing delete to happen before create --- octodns/provider/azuredns.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 3d8122a..3909ca4 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -497,6 +497,10 @@ class AzureProvider(BaseProvider): azure_zone_name = desired.name[:len(desired.name) - 1] self._check_zone(azure_zone_name, create=True) + # Force the operation order to be Update() -> Delete() -> Create() + # This will help avoid problems in updating a CNAME record into an A record. + changes.reverse() + for change in changes: class_name = change.__class__.__name__ getattr(self, '_apply_{}'.format(class_name))(change) From b67dac5a55c8bdf98bfc8565c1a424f9b067524e Mon Sep 17 00:00:00 2001 From: Arunothia Marappan Date: Thu, 16 Jul 2020 16:46:44 -0700 Subject: [PATCH 011/358] Reducing comment line length --- octodns/provider/azuredns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 3909ca4..7259bfe 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -498,7 +498,7 @@ class AzureProvider(BaseProvider): self._check_zone(azure_zone_name, create=True) # Force the operation order to be Update() -> Delete() -> Create() - # This will help avoid problems in updating a CNAME record into an A record. + # Helps avoid problems in updating a CNAME record into an A record. changes.reverse() for change in changes: From 9b619c5ef21b3aad2576f44abc077961ff4a9f5b Mon Sep 17 00:00:00 2001 From: Arunothia Marappan Date: Thu, 16 Jul 2020 17:07:33 -0700 Subject: [PATCH 012/358] Update comment --- octodns/provider/azuredns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 7259bfe..6fcf015 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -497,7 +497,7 @@ class AzureProvider(BaseProvider): azure_zone_name = desired.name[:len(desired.name) - 1] self._check_zone(azure_zone_name, create=True) - # Force the operation order to be Update() -> Delete() -> Create() + # Force the operation order to be Delete() before Create() # Helps avoid problems in updating a CNAME record into an A record. changes.reverse() From b926d78c5c182a13df03566ea9327dffdc9fb29f Mon Sep 17 00:00:00 2001 From: Jonathan Leroy Date: Mon, 3 Aug 2020 00:47:22 +0200 Subject: [PATCH 013/358] Add support for zones aliases This commit adds support for zones aliases. This allows to define one or multiple zone as aliases of an existing zone without using workarounds like simlinks and miltiple "zones" entries in the configuration file. An alias zone is share all of its content with it parent zone, only the name of the zone is different. ``` zones: example.com.: aliases: - example.net. - example.org. sources: - in targets: - out ``` Known issues: - No documentation, - Only the `octodns-sync` and `octodns-validate` commands supports aliases zones at this time, I added a loop in the manager init function which convert all alias zone to "real" ones during config validation, however I'm not sure this is the right approach. Comments welcome. --- octodns/manager.py | 34 ++++++++++++++++++++++++------ octodns/provider/yaml.py | 15 +++++++------ octodns/zone.py | 3 ++- tests/config/bad-zone-aliases.yaml | 17 +++++++++++++++ tests/config/simple-aliases.yaml | 17 +++++++++++++++ tests/test_octodns_manager.py | 14 ++++++++++-- 6 files changed, 84 insertions(+), 16 deletions(-) create mode 100644 tests/config/bad-zone-aliases.yaml create mode 100644 tests/config/simple-aliases.yaml diff --git a/octodns/manager.py b/octodns/manager.py index 0665938..2e1f6df 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -121,6 +121,20 @@ class Manager(object): raise ManagerException('Incorrect provider config for {}' .format(provider_name)) + for zone_name, zone_config in self.config['zones'].copy().items(): + if 'aliases' in zone_config: + for alias in zone_config['aliases']: + if alias in self.config['zones']: + self.log.exception('Invalid zone alias') + raise ManagerException('Invalid zone alias {}: ' + 'this zone already exists' + .format(alias)) + + self.config['zones'][alias] = zone_config + self.config['zones'][alias]['template_zone'] = zone_name + + del self.config['zones'][zone_name]['aliases'] + zone_tree = {} # sort by reversed strings so that parent zones always come first for name in sorted(self.config['zones'].keys(), key=lambda s: s[::-1]): @@ -222,12 +236,14 @@ class Manager(object): self.log.debug('configured_sub_zones: subs=%s', sub_zone_names) return set(sub_zone_names) - def _populate_and_plan(self, zone_name, sources, targets, lenient=False): + def _populate_and_plan(self, zone_name, template_zone, sources, targets, + lenient=False): - self.log.debug('sync: populating, zone=%s, lenient=%s', - zone_name, lenient) + self.log.debug('sync: populating, zone=%s, template=%s, lenient=%s', + zone_name, template_zone, lenient) zone = Zone(zone_name, - sub_zones=self.configured_sub_zones(zone_name)) + sub_zones=self.configured_sub_zones(zone_name), + template_zone=template_zone) for source in sources: try: source.populate(zone, lenient=lenient) @@ -269,6 +285,7 @@ class Manager(object): for zone_name, config in zones: self.log.info('sync: zone=%s', zone_name) lenient = config.get('lenient', False) + template_zone = config.get('template_zone', zone_name) try: sources = config['sources'] except KeyError: @@ -318,8 +335,9 @@ class Manager(object): .format(zone_name, target)) futures.append(self._executor.submit(self._populate_and_plan, - zone_name, sources, - targets, lenient=lenient)) + zone_name, template_zone, + sources, targets, + lenient=lenient)) # Wait on all results and unpack/flatten them in to a list of target & # plan pairs. @@ -413,7 +431,9 @@ class Manager(object): def validate_configs(self): for zone_name, config in self.config['zones'].items(): - zone = Zone(zone_name, self.configured_sub_zones(zone_name)) + template_zone = config.get('template_zone', zone_name) + zone = Zone(zone_name, self.configured_sub_zones(zone_name), + template_zone) try: sources = config['sources'] diff --git a/octodns/provider/yaml.py b/octodns/provider/yaml.py index 10add5a..878d7d5 100644 --- a/octodns/provider/yaml.py +++ b/octodns/provider/yaml.py @@ -139,8 +139,8 @@ class YamlProvider(BaseProvider): filename) def populate(self, zone, target=False, lenient=False): - self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, - target, lenient) + self.log.debug('populate: name=%s, template=%s, target=%s, lenient=%s', + zone.name, zone.template_zone, target, lenient) if target: # When acting as a target we ignore any existing records so that we @@ -148,7 +148,9 @@ class YamlProvider(BaseProvider): return False before = len(zone.records) - filename = join(self.directory, '{}yaml'.format(zone.name)) + filename = join(self.directory, '{}yaml'.format(zone.template_zone + if zone.template_zone + else zone.name)) self._populate_from_file(filename, zone, lenient) self.log.info('populate: found %s records, exists=False', @@ -243,11 +245,12 @@ class SplitYamlProvider(YamlProvider): super(SplitYamlProvider, self).__init__(id, directory, *args, **kwargs) def _zone_directory(self, zone): - return join(self.directory, zone.name) + return join(self.directory, zone.template_zone if zone.template_zone + else zone.name) def populate(self, zone, target=False, lenient=False): - self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, - target, lenient) + self.log.debug('populate: name=%s, template=%s, target=%s, lenient=%s', + zone.name, zone.template_zone, target, lenient) if target: # When acting as a target we ignore any existing records so that we diff --git a/octodns/zone.py b/octodns/zone.py index 5f099ac..7a5aaa6 100644 --- a/octodns/zone.py +++ b/octodns/zone.py @@ -35,13 +35,14 @@ def _is_eligible(record): class Zone(object): log = getLogger('Zone') - def __init__(self, name, sub_zones): + def __init__(self, name, sub_zones, template_zone=None): if not name[-1] == '.': raise Exception('Invalid zone name {}, missing ending dot' .format(name)) # Force everything to lowercase just to be safe self.name = text_type(name).lower() if name else name self.sub_zones = sub_zones + self.template_zone = template_zone # We're grouping by node, it allows us to efficiently search for # duplicates and detect when CNAMEs co-exist with other records self._records = defaultdict(set) diff --git a/tests/config/bad-zone-aliases.yaml b/tests/config/bad-zone-aliases.yaml new file mode 100644 index 0000000..4c47e3c --- /dev/null +++ b/tests/config/bad-zone-aliases.yaml @@ -0,0 +1,17 @@ +manager: + max_workers: 2 +providers: + in: + class: octodns.provider.yaml.YamlProvider + directory: tests/config + dump: + class: octodns.provider.yaml.YamlProvider + directory: env/YAML_TMP_DIR +zones: + unit.tests.: + aliases: + - unit.tests. + sources: + - in + targets: + - dump diff --git a/tests/config/simple-aliases.yaml b/tests/config/simple-aliases.yaml new file mode 100644 index 0000000..07a2d74 --- /dev/null +++ b/tests/config/simple-aliases.yaml @@ -0,0 +1,17 @@ +manager: + max_workers: 2 +providers: + in: + class: octodns.provider.yaml.YamlProvider + directory: tests/config + dump: + class: octodns.provider.yaml.YamlProvider + directory: env/YAML_TMP_DIR +zones: + unit.tests.: + aliases: + - unit-alias.tests. + sources: + - in + targets: + - dump diff --git a/tests/test_octodns_manager.py b/tests/test_octodns_manager.py index 581689a..052238f 100644 --- a/tests/test_octodns_manager.py +++ b/tests/test_octodns_manager.py @@ -290,7 +290,8 @@ class TestManager(TestCase): pass # This should be ok, we'll fall back to not passing it - manager._populate_and_plan('unit.tests.', [NoLenient()], []) + manager._populate_and_plan('unit.tests.', 'unit.tests.', + [NoLenient()], []) class NoZone(SimpleProvider): @@ -299,7 +300,16 @@ class TestManager(TestCase): # This will blow up, we don't fallback for source with self.assertRaises(TypeError): - manager._populate_and_plan('unit.tests.', [NoZone()], []) + manager._populate_and_plan('unit.tests.', 'unit.tests.', + [NoZone()], []) + + def test_zone_aliases(self): + Manager(get_config_filename('simple-aliases.yaml')).validate_configs() + + with self.assertRaises(ManagerException) as ctx: + Manager(get_config_filename('bad-zone-aliases.yaml')) \ + .validate_configs() + self.assertTrue('Invalid zone alias' in text_type(ctx.exception)) class TestMainThreadExecutor(TestCase): From b4da48b860d5b51698e46002088dbbf766d539c4 Mon Sep 17 00:00:00 2001 From: Piotr Pieprzycki Date: Tue, 11 Aug 2020 04:47:05 -0400 Subject: [PATCH 014/358] Add tests for long txt record, test alias entries --- octodns/provider/azuredns.py | 19 +++++----- tests/test_octodns_provider_azuredns.py | 47 ++++++++++++++++++++++--- 2 files changed, 53 insertions(+), 13 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index cdaa88f..cd65fd8 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -263,6 +263,13 @@ def _parse_azure_type(string): return string.split('/')[len(string.split('/')) - 1] +def _check_for_alias(azrecord): + if (azrecord.target_resource.id and not azrecord.arecords and not + azrecord.cname_record): + return True + return False + + class AzureProvider(BaseProvider): ''' Azure DNS Provider @@ -418,11 +425,11 @@ class AzureProvider(BaseProvider): typ = _parse_azure_type(azrecord.type) if typ in ['A', 'CNAME']: - if self._check_for_alias(azrecord, typ): + if _check_for_alias(azrecord): self.log.debug( 'Skipping - ALIAS. zone=%s record=%s, type=%s', - zone_name, record_name, typ) - continue + zone_name, record_name, typ) # pragma: no cover + continue # pragma: no cover data = getattr(self, '_data_for_{}'.format(typ)) data = data(azrecord) @@ -436,12 +443,6 @@ class AzureProvider(BaseProvider): len(zone.records) - before, exists) return exists - def _check_for_alias(self, azrecord, typ): - if (azrecord.target_resource.id and not azrecord.arecords and not - azrecord.arecords and not azrecord.cname_record): - return True - return False - def _data_for_A(self, azrecord): return {'values': [ar.ipv4_address for ar in azrecord.arecords]} diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index e14fe38..d0ecdf3 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -7,7 +7,7 @@ from __future__ import absolute_import, division, print_function, \ from octodns.record import Create, Delete, Record from octodns.provider.azuredns import _AzureRecord, AzureProvider, \ - _check_endswith_dot, _parse_azure_type + _check_endswith_dot, _parse_azure_type, _check_for_alias from octodns.zone import Zone from octodns.provider.base import Plan @@ -134,6 +134,18 @@ octo_records.append(Record.new(zone, 'txt2', { 'type': 'TXT', 'values': ['txt multiple test', 'txt multiple test 2']})) +long_txt = "v=spf1 ip4:10.10.0.0/24 ip4:10.10.1.0/24 ip4:10.10.2.0/24" +long_txt += " ip4:10.10.3.0/24 ip4:10.10.4.0/24 ip4:10.10.5.0/24 " +long_txt += " 10.6.0/24 ip4:10.10.7.0/24 ip4:10.10.8.0/24 " +long_txt += " ip4:10.10.10.0/24 ip4:10.10.11.0/24 ip4:10.10.12.0/24" +long_txt += " ip4:10.10.13.0/24 ip4:10.10.14.0/24 ip4:10.10.15.0/24" +long_txt += " ip4:10.10.16.0/24 ip4:10.10.17.0/24 ip4:10.10.18.0/24" +long_txt += " ip4:10.10.19.0/24 ip4:10.10.20.0/24 ~all" +octo_records.append(Record.new(zone, 'txt3', { + 'ttl': 10, + 'type': 'TXT', + 'values': ['txt multiple test', long_txt]})) + azure_records = [] _base0 = _AzureRecord('TestAzure', octo_records[0]) _base0.zone_name = 'unit.tests' @@ -306,6 +318,22 @@ _base17.params['txt_records'] = [TxtRecord(value=['txt multiple test']), TxtRecord(value=['txt multiple test 2'])] azure_records.append(_base17) +long_txt_az1 = "v=spf1 ip4:10.10.0.0/24 ip4:10.10.1.0/24 ip4:10.10.2.0/24" +long_txt_az1 += " ip4:10.10.3.0/24 ip4:10.10.4.0/24 ip4:10.10.5.0/24 " +long_txt_az1 += " 10.6.0/24 ip4:10.10.7.0/24 ip4:10.10.8.0/24 " +long_txt_az1 += " ip4:10.10.10.0/24 ip4:10.10.11.0/24 ip4:10.10.12.0/24" +long_txt_az1 += " ip4:10.10.13.0/24 ip4:10.10.14.0/24 ip4:10.10." +long_txt_az2 = "15.0/24 ip4:10.10.16.0/24 ip4:10.10.17.0/24 ip4:10.10.18.0/24" +long_txt_az2 += " ip4:10.10.19.0/24 ip4:10.10.20.0/24 ~all" +_base18 = _AzureRecord('TestAzure', octo_records[18]) +_base18.zone_name = 'unit.tests' +_base18.relative_record_set_name = 'txt3' +_base18.record_type = 'TXT' +_base18.params['ttl'] = 10 +_base18.params['txt_records'] = [TxtRecord(value=['txt multiple test']), + TxtRecord(value=[long_txt_az1, long_txt_az2])] +azure_records.append(_base18) + class Test_AzureRecord(TestCase): def test_azure_record(self): @@ -333,6 +361,17 @@ class Test_CheckEndswithDot(TestCase): self.assertEquals(expected, _check_endswith_dot(test)) +class Test_CheckAzureAlias(TestCase): + def test_check_for_alias(self): + alias_record = type('C', (object,), {}) + alias_record.target_resource = type('C', (object,), {}) + alias_record.target_resource.id = "/subscriptions/x/resourceGroups/y/z" + alias_record.arecords = None + alias_record.cname_record = None + + self.assertEquals(_check_for_alias(alias_record), True) + + class TestAzureDnsProvider(TestCase): def _provider(self): return self._get_provider('mock_spc', 'mock_dns_client') @@ -503,9 +542,9 @@ class TestAzureDnsProvider(TestCase): changes.append(Create(i)) deletes.append(Delete(i)) - self.assertEquals(18, provider.apply(Plan(None, zone, + self.assertEquals(19, provider.apply(Plan(None, zone, changes, True))) - self.assertEquals(18, provider.apply(Plan(zone, zone, + self.assertEquals(19, provider.apply(Plan(zone, zone, deletes, True))) def test_create_zone(self): @@ -521,7 +560,7 @@ class TestAzureDnsProvider(TestCase): _get = provider._dns_client.zones.get _get.side_effect = CloudError(Mock(status=404), err_msg) - self.assertEquals(18, provider.apply(Plan(None, desired, changes, + self.assertEquals(19, provider.apply(Plan(None, desired, changes, True))) def test_check_zone_no_create(self): From 2442c2390431457d0db6dcbaa5fdb91139c0dabc Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Tue, 1 Sep 2020 22:53:51 +0700 Subject: [PATCH 015/358] MANIFEST.in: Add test data Fixes https://github.com/github/octodns/issues/608 --- MANIFEST.in | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index cda90ed..2b82e59 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,7 +1,11 @@ include README.md include CONTRIBUTING.md include LICENSE -include docs/* -include octodns/* +include requirements.txt +include requirements-dev.txt +recursive-include docs *.png *.md include script/* -include tests/* +recursive-include tests *.py +recursive-include tests *.json *.txt *.yaml +recursive-include tests/zones *. +recursive-include tests/zones/tinydns *.* From 50f739495d10310cfb6b8425eda1a26b2b94b40c Mon Sep 17 00:00:00 2001 From: ftm-qsc <54101720+ftm-qsc@users.noreply.github.com> Date: Wed, 14 Oct 2020 20:45:49 +0200 Subject: [PATCH 016/358] docs: fixed small typo in geo_records.md Did you mean 'strongly'? --- docs/geo_records.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/geo_records.md b/docs/geo_records.md index ba99260..3777564 100644 --- a/docs/geo_records.md +++ b/docs/geo_records.md @@ -1,6 +1,6 @@ ## Geo Record Support -Note: Geo DNS records are still supported for the time being, but it is still strongy encouraged that you look at [Dynamic Records](/docs/dynamic_records.md) instead as they are a superset of functionality. +Note: Geo DNS records are still supported for the time being, but it is still strongly encouraged that you look at [Dynamic Records](/docs/dynamic_records.md) instead as they are a superset of functionality. GeoDNS is currently supported for `A` and `AAAA` records on the Dyn (via Traffic Directors) and Route53 providers. Records with geo information pushed to providers without support for them will be managed as non-geo records using the base values. From 7bf0b31367e51e99598a914e7c97cbf831e617eb Mon Sep 17 00:00:00 2001 From: Jonathan Leroy Date: Tue, 20 Oct 2020 19:54:35 +0200 Subject: [PATCH 017/358] Revert "Add support for zones aliases" This reverts commit b926d78c5c182a13df03566ea9327dffdc9fb29f. --- octodns/manager.py | 34 ++++++------------------------ octodns/provider/yaml.py | 15 ++++++------- octodns/zone.py | 3 +-- tests/config/bad-zone-aliases.yaml | 17 --------------- tests/config/simple-aliases.yaml | 17 --------------- tests/test_octodns_manager.py | 14 ++---------- 6 files changed, 16 insertions(+), 84 deletions(-) delete mode 100644 tests/config/bad-zone-aliases.yaml delete mode 100644 tests/config/simple-aliases.yaml diff --git a/octodns/manager.py b/octodns/manager.py index 137f13b..288645f 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -121,20 +121,6 @@ class Manager(object): raise ManagerException('Incorrect provider config for {}' .format(provider_name)) - for zone_name, zone_config in self.config['zones'].copy().items(): - if 'aliases' in zone_config: - for alias in zone_config['aliases']: - if alias in self.config['zones']: - self.log.exception('Invalid zone alias') - raise ManagerException('Invalid zone alias {}: ' - 'this zone already exists' - .format(alias)) - - self.config['zones'][alias] = zone_config - self.config['zones'][alias]['template_zone'] = zone_name - - del self.config['zones'][zone_name]['aliases'] - zone_tree = {} # sort by reversed strings so that parent zones always come first for name in sorted(self.config['zones'].keys(), key=lambda s: s[::-1]): @@ -236,14 +222,12 @@ class Manager(object): self.log.debug('configured_sub_zones: subs=%s', sub_zone_names) return set(sub_zone_names) - def _populate_and_plan(self, zone_name, template_zone, sources, targets, - lenient=False): + def _populate_and_plan(self, zone_name, sources, targets, lenient=False): - self.log.debug('sync: populating, zone=%s, template=%s, lenient=%s', - zone_name, template_zone, lenient) + self.log.debug('sync: populating, zone=%s, lenient=%s', + zone_name, lenient) zone = Zone(zone_name, - sub_zones=self.configured_sub_zones(zone_name), - template_zone=template_zone) + sub_zones=self.configured_sub_zones(zone_name)) for source in sources: try: source.populate(zone, lenient=lenient) @@ -285,7 +269,6 @@ class Manager(object): for zone_name, config in zones: self.log.info('sync: zone=%s', zone_name) lenient = config.get('lenient', False) - template_zone = config.get('template_zone', zone_name) try: sources = config['sources'] except KeyError: @@ -341,9 +324,8 @@ class Manager(object): .format(zone_name, target)) futures.append(self._executor.submit(self._populate_and_plan, - zone_name, template_zone, - sources, targets, - lenient=lenient)) + zone_name, sources, + targets, lenient=lenient)) # Wait on all results and unpack/flatten them in to a list of target & # plan pairs. @@ -437,9 +419,7 @@ class Manager(object): def validate_configs(self): for zone_name, config in self.config['zones'].items(): - template_zone = config.get('template_zone', zone_name) - zone = Zone(zone_name, self.configured_sub_zones(zone_name), - template_zone) + zone = Zone(zone_name, self.configured_sub_zones(zone_name)) try: sources = config['sources'] diff --git a/octodns/provider/yaml.py b/octodns/provider/yaml.py index 878d7d5..10add5a 100644 --- a/octodns/provider/yaml.py +++ b/octodns/provider/yaml.py @@ -139,8 +139,8 @@ class YamlProvider(BaseProvider): filename) def populate(self, zone, target=False, lenient=False): - self.log.debug('populate: name=%s, template=%s, target=%s, lenient=%s', - zone.name, zone.template_zone, target, lenient) + self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, + target, lenient) if target: # When acting as a target we ignore any existing records so that we @@ -148,9 +148,7 @@ class YamlProvider(BaseProvider): return False before = len(zone.records) - filename = join(self.directory, '{}yaml'.format(zone.template_zone - if zone.template_zone - else zone.name)) + filename = join(self.directory, '{}yaml'.format(zone.name)) self._populate_from_file(filename, zone, lenient) self.log.info('populate: found %s records, exists=False', @@ -245,12 +243,11 @@ class SplitYamlProvider(YamlProvider): super(SplitYamlProvider, self).__init__(id, directory, *args, **kwargs) def _zone_directory(self, zone): - return join(self.directory, zone.template_zone if zone.template_zone - else zone.name) + return join(self.directory, zone.name) def populate(self, zone, target=False, lenient=False): - self.log.debug('populate: name=%s, template=%s, target=%s, lenient=%s', - zone.name, zone.template_zone, target, lenient) + self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, + target, lenient) if target: # When acting as a target we ignore any existing records so that we diff --git a/octodns/zone.py b/octodns/zone.py index 7a5aaa6..5f099ac 100644 --- a/octodns/zone.py +++ b/octodns/zone.py @@ -35,14 +35,13 @@ def _is_eligible(record): class Zone(object): log = getLogger('Zone') - def __init__(self, name, sub_zones, template_zone=None): + def __init__(self, name, sub_zones): if not name[-1] == '.': raise Exception('Invalid zone name {}, missing ending dot' .format(name)) # Force everything to lowercase just to be safe self.name = text_type(name).lower() if name else name self.sub_zones = sub_zones - self.template_zone = template_zone # We're grouping by node, it allows us to efficiently search for # duplicates and detect when CNAMEs co-exist with other records self._records = defaultdict(set) diff --git a/tests/config/bad-zone-aliases.yaml b/tests/config/bad-zone-aliases.yaml deleted file mode 100644 index 4c47e3c..0000000 --- a/tests/config/bad-zone-aliases.yaml +++ /dev/null @@ -1,17 +0,0 @@ -manager: - max_workers: 2 -providers: - in: - class: octodns.provider.yaml.YamlProvider - directory: tests/config - dump: - class: octodns.provider.yaml.YamlProvider - directory: env/YAML_TMP_DIR -zones: - unit.tests.: - aliases: - - unit.tests. - sources: - - in - targets: - - dump diff --git a/tests/config/simple-aliases.yaml b/tests/config/simple-aliases.yaml deleted file mode 100644 index 07a2d74..0000000 --- a/tests/config/simple-aliases.yaml +++ /dev/null @@ -1,17 +0,0 @@ -manager: - max_workers: 2 -providers: - in: - class: octodns.provider.yaml.YamlProvider - directory: tests/config - dump: - class: octodns.provider.yaml.YamlProvider - directory: env/YAML_TMP_DIR -zones: - unit.tests.: - aliases: - - unit-alias.tests. - sources: - - in - targets: - - dump diff --git a/tests/test_octodns_manager.py b/tests/test_octodns_manager.py index b6f9cbb..9956790 100644 --- a/tests/test_octodns_manager.py +++ b/tests/test_octodns_manager.py @@ -298,8 +298,7 @@ class TestManager(TestCase): pass # This should be ok, we'll fall back to not passing it - manager._populate_and_plan('unit.tests.', 'unit.tests.', - [NoLenient()], []) + manager._populate_and_plan('unit.tests.', [NoLenient()], []) class NoZone(SimpleProvider): @@ -308,16 +307,7 @@ class TestManager(TestCase): # This will blow up, we don't fallback for source with self.assertRaises(TypeError): - manager._populate_and_plan('unit.tests.', 'unit.tests.', - [NoZone()], []) - - def test_zone_aliases(self): - Manager(get_config_filename('simple-aliases.yaml')).validate_configs() - - with self.assertRaises(ManagerException) as ctx: - Manager(get_config_filename('bad-zone-aliases.yaml')) \ - .validate_configs() - self.assertTrue('Invalid zone alias' in text_type(ctx.exception)) + manager._populate_and_plan('unit.tests.', [NoZone()], []) class TestMainThreadExecutor(TestCase): From f2a6f870b40d317b95dbd3e12e8d6ddb0324f33b Mon Sep 17 00:00:00 2001 From: Jonathan Leroy Date: Tue, 20 Oct 2020 22:18:48 +0200 Subject: [PATCH 018/358] Make each alias zone reference its target zone instead of listing all aliases zones in the target zone configuration --- octodns/manager.py | 39 ++++++++++++++++++++++----- octodns/provider/yaml.py | 5 ++-- octodns/zone.py | 4 ++- tests/config/simple-alias-zone.yaml | 19 +++++++++++++ tests/config/unknown-source-zone.yaml | 13 +++++++++ tests/test_octodns_manager.py | 19 +++++++++++-- 6 files changed, 87 insertions(+), 12 deletions(-) create mode 100644 tests/config/simple-alias-zone.yaml create mode 100644 tests/config/unknown-source-zone.yaml diff --git a/octodns/manager.py b/octodns/manager.py index 288645f..613be29 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -121,6 +121,23 @@ class Manager(object): raise ManagerException('Incorrect provider config for {}' .format(provider_name)) + for zone_name, zone_config in self.config['zones'].copy().items(): + if 'alias' in zone_config: + source_zone = zone_config['alias'] + # Check that the source zone is defined. + if source_zone not in self.config['zones']: + self.log.exception('Invalid alias zone') + raise ManagerException('Invalid alias zone {}: ' + 'source zone {} does not exist' + .format(zone_name, source_zone)) + self.config['zones'][zone_name] = \ + self.config['zones'][source_zone] + self.config['zones'][zone_name]['is_alias'] = True + self.config['zones'][zone_name]['file'] = source_zone + else: + self.config['zones'][zone_name]['is_alias'] = False + self.config['zones'][zone_name]['file'] = zone_name + zone_tree = {} # sort by reversed strings so that parent zones always come first for name in sorted(self.config['zones'].keys(), key=lambda s: s[::-1]): @@ -222,12 +239,14 @@ class Manager(object): self.log.debug('configured_sub_zones: subs=%s', sub_zone_names) return set(sub_zone_names) - def _populate_and_plan(self, zone_name, sources, targets, lenient=False): + def _populate_and_plan(self, zone_name, file, is_alias, sources, targets, + lenient=False): - self.log.debug('sync: populating, zone=%s, lenient=%s', - zone_name, lenient) + self.log.debug('sync: populating, zone=%s, file=%s, is_alias=%s, ' + 'lenient=%s', zone_name, file, is_alias, lenient) zone = Zone(zone_name, - sub_zones=self.configured_sub_zones(zone_name)) + sub_zones=self.configured_sub_zones(zone_name), file=file, + is_alias=is_alias) for source in sources: try: source.populate(zone, lenient=lenient) @@ -268,6 +287,8 @@ class Manager(object): futures = [] for zone_name, config in zones: self.log.info('sync: zone=%s', zone_name) + file = config.get('file') + is_alias = config.get('is_alias') lenient = config.get('lenient', False) try: sources = config['sources'] @@ -324,8 +345,9 @@ class Manager(object): .format(zone_name, target)) futures.append(self._executor.submit(self._populate_and_plan, - zone_name, sources, - targets, lenient=lenient)) + zone_name, file, is_alias, + sources, targets, + lenient=lenient)) # Wait on all results and unpack/flatten them in to a list of target & # plan pairs. @@ -419,7 +441,10 @@ class Manager(object): def validate_configs(self): for zone_name, config in self.config['zones'].items(): - zone = Zone(zone_name, self.configured_sub_zones(zone_name)) + file = config.get('file', False) + is_alias = config.get('is_alias', False) + zone = Zone(zone_name, self.configured_sub_zones(zone_name), + file, is_alias) try: sources = config['sources'] diff --git a/octodns/provider/yaml.py b/octodns/provider/yaml.py index 10add5a..e486982 100644 --- a/octodns/provider/yaml.py +++ b/octodns/provider/yaml.py @@ -139,7 +139,8 @@ class YamlProvider(BaseProvider): filename) def populate(self, zone, target=False, lenient=False): - self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, + self.log.debug('populate: name=%s, file=%s, is_alias:%s, target=%s, ' + 'lenient=%s', zone.name, zone.file, zone.is_alias, target, lenient) if target: @@ -148,7 +149,7 @@ class YamlProvider(BaseProvider): return False before = len(zone.records) - filename = join(self.directory, '{}yaml'.format(zone.name)) + filename = join(self.directory, '{}yaml'.format(zone.file)) self._populate_from_file(filename, zone, lenient) self.log.info('populate: found %s records, exists=False', diff --git a/octodns/zone.py b/octodns/zone.py index 5f099ac..0a78f72 100644 --- a/octodns/zone.py +++ b/octodns/zone.py @@ -35,13 +35,15 @@ def _is_eligible(record): class Zone(object): log = getLogger('Zone') - def __init__(self, name, sub_zones): + def __init__(self, name, sub_zones, file=None, is_alias=False): if not name[-1] == '.': raise Exception('Invalid zone name {}, missing ending dot' .format(name)) # Force everything to lowercase just to be safe self.name = text_type(name).lower() if name else name self.sub_zones = sub_zones + self.file = text_type(file if file else name).lower() + self.is_alias = is_alias # We're grouping by node, it allows us to efficiently search for # duplicates and detect when CNAMEs co-exist with other records self._records = defaultdict(set) diff --git a/tests/config/simple-alias-zone.yaml b/tests/config/simple-alias-zone.yaml new file mode 100644 index 0000000..32154d5 --- /dev/null +++ b/tests/config/simple-alias-zone.yaml @@ -0,0 +1,19 @@ +manager: + max_workers: 2 +providers: + in: + class: octodns.provider.yaml.YamlProvider + directory: tests/config + dump: + class: octodns.provider.yaml.YamlProvider + directory: env/YAML_TMP_DIR +zones: + unit.tests.: + sources: + - in + targets: + - dump + + alias.tests.: + alias: unit.tests. + diff --git a/tests/config/unknown-source-zone.yaml b/tests/config/unknown-source-zone.yaml new file mode 100644 index 0000000..313853e --- /dev/null +++ b/tests/config/unknown-source-zone.yaml @@ -0,0 +1,13 @@ +manager: + max_workers: 2 +providers: + in: + class: octodns.provider.yaml.YamlProvider + directory: tests/config + dump: + class: octodns.provider.yaml.YamlProvider + directory: env/YAML_TMP_DIR +zones: + unit.tests.: + alias: unit-source.tests. + diff --git a/tests/test_octodns_manager.py b/tests/test_octodns_manager.py index 9956790..1b8752e 100644 --- a/tests/test_octodns_manager.py +++ b/tests/test_octodns_manager.py @@ -298,7 +298,8 @@ class TestManager(TestCase): pass # This should be ok, we'll fall back to not passing it - manager._populate_and_plan('unit.tests.', [NoLenient()], []) + manager._populate_and_plan('unit.tests.', None, False, + [NoLenient()], []) class NoZone(SimpleProvider): @@ -307,7 +308,21 @@ class TestManager(TestCase): # This will blow up, we don't fallback for source with self.assertRaises(TypeError): - manager._populate_and_plan('unit.tests.', [NoZone()], []) + manager._populate_and_plan('unit.tests.', None, False, + [NoZone()], []) + + def test_alias_zones(self): + with TemporaryDirectory() as tmpdir: + environ['YAML_TMP_DIR'] = tmpdir.dirname + + Manager(get_config_filename('simple-alias-zone.yaml')) \ + .validate_configs() + + with self.assertRaises(ManagerException) as ctx: + Manager(get_config_filename('unknown-source-zone.yaml')) \ + .validate_configs() + self.assertTrue('Invalid alias zone' in + text_type(ctx.exception)) class TestMainThreadExecutor(TestCase): From 06c18f406317918536bd389738af68dbc4bb11ac Mon Sep 17 00:00:00 2001 From: Jonathan Leroy Date: Wed, 21 Oct 2020 19:11:02 +0200 Subject: [PATCH 019/358] Add zones aliases support to octodns-report command --- octodns/cmds/report.py | 2 +- octodns/manager.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/octodns/cmds/report.py b/octodns/cmds/report.py index 3a26052..1ccad33 100755 --- a/octodns/cmds/report.py +++ b/octodns/cmds/report.py @@ -56,7 +56,7 @@ def main(): except KeyError as e: raise Exception('Unknown source: {}'.format(e.args[0])) - zone = Zone(args.zone, manager.configured_sub_zones(args.zone)) + zone = manager.get_zone(args.zone) for source in sources: source.populate(zone) diff --git a/octodns/manager.py b/octodns/manager.py index 613be29..d131e78 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -467,3 +467,18 @@ class Manager(object): for source in sources: if isinstance(source, YamlProvider): source.populate(zone) + + def get_zone(self, zone_name): + if not zone_name[-1] == '.': + raise Exception('Invalid zone name {}, missing ending dot' + .format(zone_name)) + + for name, config in self.config['zones'].items(): + if name == zone_name: + file = config.get('file', False) + is_alias = config.get('is_alias', False) + + return Zone(name, self.configured_sub_zones(name), + file, is_alias) + + raise ManagerException('Unkown zone name {}'.format(zone_name)) From 12c3aa64a83a01f9f18a815db9a8694be971f992 Mon Sep 17 00:00:00 2001 From: Jonathan Leroy Date: Wed, 21 Oct 2020 19:11:25 +0200 Subject: [PATCH 020/358] Add zones aliases support to octodns-compare command --- octodns/manager.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/octodns/manager.py b/octodns/manager.py index d131e78..6fd4239 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -399,12 +399,11 @@ class Manager(object): except KeyError as e: raise ManagerException('Unknown source: {}'.format(e.args[0])) - sub_zones = self.configured_sub_zones(zone) - za = Zone(zone, sub_zones) + za = self.get_zone(zone) for source in a: source.populate(za) - zb = Zone(zone, sub_zones) + zb = self.get_zone(zone) for source in b: source.populate(zb) From 94a8b67a3be7885c5a75ae22be455c63ba560562 Mon Sep 17 00:00:00 2001 From: Jonathan Leroy Date: Wed, 21 Oct 2020 19:18:27 +0200 Subject: [PATCH 021/358] Fixes linting errors --- octodns/cmds/report.py | 1 - octodns/manager.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/octodns/cmds/report.py b/octodns/cmds/report.py index 1ccad33..d0b82c0 100755 --- a/octodns/cmds/report.py +++ b/octodns/cmds/report.py @@ -17,7 +17,6 @@ from six import text_type from octodns.cmds.args import ArgumentParser from octodns.manager import Manager -from octodns.zone import Zone class AsyncResolver(Resolver): diff --git a/octodns/manager.py b/octodns/manager.py index 6fd4239..7015ad8 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -478,6 +478,6 @@ class Manager(object): is_alias = config.get('is_alias', False) return Zone(name, self.configured_sub_zones(name), - file, is_alias) + file, is_alias) raise ManagerException('Unkown zone name {}'.format(zone_name)) From 1f60a6af5e650fcdba44435038dd8d132600b2dc Mon Sep 17 00:00:00 2001 From: Jonathan Leroy Date: Wed, 21 Oct 2020 19:24:49 +0200 Subject: [PATCH 022/358] Fixes typo in manager.get_zone() --- octodns/manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/manager.py b/octodns/manager.py index 7015ad8..eab7ac4 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -480,4 +480,4 @@ class Manager(object): return Zone(name, self.configured_sub_zones(name), file, is_alias) - raise ManagerException('Unkown zone name {}'.format(zone_name)) + raise ManagerException('Unknown zone name {}'.format(zone_name)) From 897a033443c8c66a32209a610be613173b361b06 Mon Sep 17 00:00:00 2001 From: Jonathan Leroy Date: Wed, 21 Oct 2020 20:02:12 +0200 Subject: [PATCH 023/358] Add tests for Manager.get_zones() --- octodns/manager.py | 4 ++-- tests/test_octodns_manager.py | 16 ++++++++++++++-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/octodns/manager.py b/octodns/manager.py index eab7ac4..eff3a74 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -469,8 +469,8 @@ class Manager(object): def get_zone(self, zone_name): if not zone_name[-1] == '.': - raise Exception('Invalid zone name {}, missing ending dot' - .format(zone_name)) + raise ManagerException('Invalid zone name {}, missing ending dot' + .format(zone_name)) for name, config in self.config['zones'].items(): if name == zone_name: diff --git a/tests/test_octodns_manager.py b/tests/test_octodns_manager.py index 1b8752e..b493540 100644 --- a/tests/test_octodns_manager.py +++ b/tests/test_octodns_manager.py @@ -286,6 +286,18 @@ class TestManager(TestCase): .validate_configs() self.assertTrue('unknown source' in text_type(ctx.exception)) + def test_get_zone(self): + Manager(get_config_filename('simple.yaml')).get_zone('unit.tests.') + + with self.assertRaises(ManagerException) as ctx: + Manager(get_config_filename('simple.yaml')).get_zone('unit.tests') + self.assertTrue('missing ending dot' in text_type(ctx.exception)) + + with self.assertRaises(ManagerException) as ctx: + Manager(get_config_filename('simple.yaml')) \ + .get_zone('unknown-zone.tests.') + self.assertTrue('Unknown zone name' in text_type(ctx.exception)) + def test_populate_lenient_fallback(self): with TemporaryDirectory() as tmpdir: environ['YAML_TMP_DIR'] = tmpdir.dirname @@ -321,8 +333,8 @@ class TestManager(TestCase): with self.assertRaises(ManagerException) as ctx: Manager(get_config_filename('unknown-source-zone.yaml')) \ .validate_configs() - self.assertTrue('Invalid alias zone' in - text_type(ctx.exception)) + self.assertTrue('Invalid alias zone' in + text_type(ctx.exception)) class TestMainThreadExecutor(TestCase): From fd136b42d1583a1376103967af644ed9a9ec627e Mon Sep 17 00:00:00 2001 From: Jonathan Leroy Date: Sun, 25 Oct 2020 01:08:08 +0200 Subject: [PATCH 024/358] Add support for Gandi LiveDNS --- octodns/provider/gandi.py | 334 +++++++++++++++++++++++++ tests/fixtures/gandi-default-zone.json | 93 +++++++ tests/fixtures/gandi-no-changes.json | 127 ++++++++++ tests/test_octodns_provider_gandi.py | 312 +++++++++++++++++++++++ 4 files changed, 866 insertions(+) create mode 100644 octodns/provider/gandi.py create mode 100644 tests/fixtures/gandi-default-zone.json create mode 100644 tests/fixtures/gandi-no-changes.json create mode 100644 tests/test_octodns_provider_gandi.py diff --git a/octodns/provider/gandi.py b/octodns/provider/gandi.py new file mode 100644 index 0000000..e11e9a3 --- /dev/null +++ b/octodns/provider/gandi.py @@ -0,0 +1,334 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from collections import defaultdict +from requests import Session +import logging + +from ..record import Record +from .base import BaseProvider + + +class GandiClientException(Exception): + pass + + +class GandiClientBadRequest(GandiClientException): + + def __init__(self, r): + super(GandiClientBadRequest, self).__init__(r.text) + + +class GandiClientUnauthorized(GandiClientException): + + def __init__(self, r): + super(GandiClientUnauthorized, self).__init__(r.text) + + +class GandiClientForbidden(GandiClientException): + + def __init__(self, r): + super(GandiClientForbidden, self).__init__(r.text) + + +class GandiClientNotFound(GandiClientException): + + def __init__(self, r): + super(GandiClientNotFound, self).__init__(r.text) + + +class GandiClient(object): + + def __init__(self, token): + session = Session() + session.headers.update({'Authorization': 'Apikey {}'.format(token)}) + self._session = session + self.endpoint = 'https://api.gandi.net/v5' + + def _request(self, method, path, params={}, data=None): + url = '{}{}'.format(self.endpoint, path) + r = self._session.request(method, url, params=params, json=data) + if r.status_code == 400: + raise GandiClientBadRequest(r) + if r.status_code == 401: + raise GandiClientUnauthorized(r) + elif r.status_code == 403: + raise GandiClientForbidden(r) + elif r.status_code == 404: + raise GandiClientNotFound(r) + r.raise_for_status() + return r + + def zone_records(self, zone_name): + records = self._request('GET', '/livedns/domains/{}/records' + .format(zone_name)).json() + + for record in records: + if record['rrset_name'] == '@': + record['rrset_name'] = '' + + return records + + def record_create(self, zone_name, data): + self._request('POST', '/livedns/domains/{}/records'.format(zone_name), + data=data) + + def record_delete(self, zone_name, record_name, record_type): + self._request('DELETE', '/livedns/domains/{}/records/{}/{}' + .format(zone_name, record_name, record_type)) + + +class GandiProvider(BaseProvider): + ''' + Gandi provider using API v5. + + gandi: + class: octodns.provider.gandi.GandiProvider + # Your API key (required) + token: XXXXXXXXXXXX + ''' + + SUPPORTS_GEO = False + SUPPORTS_DYNAMIC = False + SUPPORTS = set((['A', 'AAAA', 'ALIAS', 'CAA', 'CNAME', 'DNAME', + 'MX', 'NS', 'PTR', 'SPF', 'SRV', 'SSHFP', 'TXT'])) + + def __init__(self, id, token, *args, **kwargs): + self.log = logging.getLogger('GandiProvider[{}]'.format(id)) + self.log.debug('__init__: id=%s, token=***', id) + super(GandiProvider, self).__init__(id, *args, **kwargs) + self._client = GandiClient(token) + + self._zone_records = {} + + def _data_for_multiple(self, _type, records): + return { + 'ttl': records[0]['rrset_ttl'], + 'type': _type, + 'values': [v.replace(';', '\\;') for v in + records[0]['rrset_values']] if _type == 'TXT' else + records[0]['rrset_values'] + } + + _data_for_A = _data_for_multiple + _data_for_AAAA = _data_for_multiple + _data_for_TXT = _data_for_multiple + _data_for_SPF = _data_for_multiple + _data_for_NS = _data_for_multiple + + def _data_for_CAA(self, _type, records): + values = [] + for record in records[0]['rrset_values']: + flags, tag, value = record.split(' ') + values.append({ + 'flags': flags, + 'tag': tag, + # Remove quotes around value. + 'value': value[1:-1], + }) + + return { + 'ttl': records[0]['rrset_ttl'], + 'type': _type, + 'values': values + } + + def _data_for_single(self, _type, records): + return { + 'ttl': records[0]['rrset_ttl'], + 'type': _type, + 'value': records[0]['rrset_values'][0] + } + + _data_for_ALIAS = _data_for_single + _data_for_CNAME = _data_for_single + _data_for_DNAME = _data_for_single + _data_for_PTR = _data_for_single + + def _data_for_MX(self, _type, records): + values = [] + for record in records[0]['rrset_values']: + priority, server = record.split(' ') + values.append({ + 'preference': priority, + 'exchange': server + }) + + return { + 'ttl': records[0]['rrset_ttl'], + 'type': _type, + 'values': values + } + + def _data_for_SRV(self, _type, records): + values = [] + for record in records[0]['rrset_values']: + priority, weight, port, target = record.split(' ', 3) + values.append({ + 'priority': priority, + 'weight': weight, + 'port': port, + 'target': target + }) + + return { + 'ttl': records[0]['rrset_ttl'], + 'type': _type, + 'values': values + } + + def _data_for_SSHFP(self, _type, records): + values = [] + for record in records[0]['rrset_values']: + algorithm, fingerprint_type, fingerprint = record.split(' ', 2) + values.append({ + 'algorithm': algorithm, + 'fingerprint': fingerprint, + 'fingerprint_type': fingerprint_type + }) + + return { + 'ttl': records[0]['rrset_ttl'], + 'type': _type, + 'values': values + } + + def zone_records(self, zone): + if zone.name not in self._zone_records: + try: + self._zone_records[zone.name] = \ + self._client.zone_records(zone.name[:-1]) + except GandiClientNotFound: + return [] + + return self._zone_records[zone.name] + + def populate(self, zone, target=False, lenient=False): + self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, + target, lenient) + + values = defaultdict(lambda: defaultdict(list)) + for record in self.zone_records(zone): + _type = record['rrset_type'] + if _type not in self.SUPPORTS: + continue + values[record['rrset_name']][record['rrset_type']].append(record) + + before = len(zone.records) + for name, types in values.items(): + for _type, records in types.items(): + data_for = getattr(self, '_data_for_{}'.format(_type)) + record = Record.new(zone, name, data_for(_type, records), + source=self, lenient=lenient) + zone.add_record(record, lenient=lenient) + + exists = zone.name in self._zone_records + self.log.info('populate: found %s records, exists=%s', + len(zone.records) - before, exists) + return exists + + def _record_name(self, name): + return name if name else '@' + + def _params_for_multiple(self, record): + return { + 'rrset_name': self._record_name(record.name), + 'rrset_ttl': record.ttl, + 'rrset_type': record._type, + 'rrset_values': [v.replace('\\;', ';') for v in + record.values] if record._type == 'TXT' + else record.values + } + + _params_for_A = _params_for_multiple + _params_for_AAAA = _params_for_multiple + _params_for_NS = _params_for_multiple + _params_for_TXT = _params_for_multiple + _params_for_SPF = _params_for_multiple + + def _params_for_CAA(self, record): + return { + 'rrset_name': self._record_name(record.name), + 'rrset_ttl': record.ttl, + 'rrset_type': record._type, + 'rrset_values': ['{} {} "{}"'.format(v.flags, v.tag, v.value) + for v in record.values] + } + + def _params_for_single(self, record): + return { + 'rrset_name': self._record_name(record.name), + 'rrset_ttl': record.ttl, + 'rrset_type': record._type, + 'rrset_values': [record.value] + } + + _params_for_ALIAS = _params_for_single + _params_for_CNAME = _params_for_single + _params_for_DNAME = _params_for_single + _params_for_PTR = _params_for_single + + def _params_for_MX(self, record): + return { + 'rrset_name': self._record_name(record.name), + 'rrset_ttl': record.ttl, + 'rrset_type': record._type, + 'rrset_values': ['{} {}'.format(v.preference, v.exchange) + for v in record.values] + } + + def _params_for_SRV(self, record): + return { + 'rrset_name': self._record_name(record.name), + 'rrset_ttl': record.ttl, + 'rrset_type': record._type, + 'rrset_values': ['{} {} {} {}'.format(v.priority, v.weight, v.port, + v.target) for v in record.values] + } + + def _params_for_SSHFP(self, record): + return { + 'rrset_name': self._record_name(record.name), + 'rrset_ttl': record.ttl, + 'rrset_type': record._type, + 'rrset_values': ['{} {} {}'.format(v.algorithm, v.fingerprint_type, + v.fingerprint) for v in record.values] + } + + def _apply_create(self, change): + new = change.new + data = getattr(self, '_params_for_{}'.format(new._type))(new) + self._client.record_create(new.zone.name[:-1], data) + + def _apply_update(self, change): + self._apply_delete(change) + self._apply_create(change) + + def _apply_delete(self, change): + existing = change.existing + zone = existing.zone + self._client.record_delete(zone.name[:-1], + self._record_name(existing.name), + existing._type) + + def _apply(self, plan): + desired = plan.desired + changes = plan.changes + self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name, + len(changes)) + + # Force records deletion to be done before creation in order to avoid + # "CNAME record must be the only record" error when an existing CNAME + # record is replaced by an A/AAAA record. + changes.reverse() + + for change in changes: + class_name = change.__class__.__name__ + getattr(self, '_apply_{}'.format(class_name.lower()))(change) + + # Clear out the cache if any + self._zone_records.pop(desired.name, None) diff --git a/tests/fixtures/gandi-default-zone.json b/tests/fixtures/gandi-default-zone.json new file mode 100644 index 0000000..deb4cb8 --- /dev/null +++ b/tests/fixtures/gandi-default-zone.json @@ -0,0 +1,93 @@ +[ + { + "rrset_type": "A", + "rrset_ttl": 10800, + "rrset_name": "", + "rrset_href": "https://api.gandi.net/v5/livedns/domains/unit.tests/records/%40/A", + "rrset_values": [ + "217.70.184.38" + ] + }, + { + "rrset_type": "MX", + "rrset_ttl": 10800, + "rrset_name": "", + "rrset_href": "https://api.gandi.net/v5/livedns/domains/unit.tests/records/%40/MX", + "rrset_values": [ + "10 spool.mail.gandi.net.", + "50 fb.mail.gandi.net." + ] + }, + { + "rrset_type": "TXT", + "rrset_ttl": 10800, + "rrset_name": "", + "rrset_href": "https://api.gandi.net/v5/livedns/domains/unit.tests/records/%40/TXT", + "rrset_values": [ + "\"v=spf1 include:_mailcust.gandi.net ?all\"" + ] + }, + { + "rrset_type": "CNAME", + "rrset_ttl": 10800, + "rrset_name": "webmail", + "rrset_href": "https://api.gandi.net/v5/livedns/domains/unit.tests/records/webmail/CNAME", + "rrset_values": [ + "webmail.gandi.net." + ] + }, + { + "rrset_type": "CNAME", + "rrset_ttl": 10800, + "rrset_name": "www", + "rrset_href": "https://api.gandi.net/v5/livedns/domains/unit.tests/records/www/CNAME", + "rrset_values": [ + "webredir.vip.gandi.net." + ] + }, + { + "rrset_type": "SRV", + "rrset_ttl": 10800, + "rrset_name": "_imap._tcp", + "rrset_href": "https://api.gandi.net/v5/livedns/domains/unit.tests/records/_imap._tcp/SRV", + "rrset_values": [ + "0 0 0 ." + ] + }, + { + "rrset_type": "SRV", + "rrset_ttl": 10800, + "rrset_name": "_imaps._tcp", + "rrset_href": "https://api.gandi.net/v5/livedns/domains/unit.tests/records/_imaps._tcp/SRV", + "rrset_values": [ + "0 1 993 mail.gandi.net." + ] + }, + { + "rrset_type": "SRV", + "rrset_ttl": 10800, + "rrset_name": "_pop3._tcp", + "rrset_href": "https://api.gandi.net/v5/livedns/domains/unit.tests/records/_pop3._tcp/SRV", + "rrset_values": [ + "0 0 0 ." + ] + }, + { + "rrset_type": "SRV", + "rrset_ttl": 10800, + "rrset_name": "_pop3s._tcp", + "rrset_href": "https://api.gandi.net/v5/livedns/domains/unit.tests/records/_pop3s._tcp/SRV", + "rrset_values": [ + "10 1 995 mail.gandi.net." + ] + }, + { + "rrset_type": "SRV", + "rrset_ttl": 10800, + "rrset_name": "_submission._tcp", + "rrset_href": "https://api.gandi.net/v5/livedns/domains/unit.tests/records/_submission._tcp/SRV", + "rrset_values": [ + "0 1 465 mail.gandi.net." + ] + } +] diff --git a/tests/fixtures/gandi-no-changes.json b/tests/fixtures/gandi-no-changes.json new file mode 100644 index 0000000..9bff3cb --- /dev/null +++ b/tests/fixtures/gandi-no-changes.json @@ -0,0 +1,127 @@ +[ + { + "rrset_type": "A", + "rrset_ttl": 300, + "rrset_name": "", + "rrset_href": "https://api.gandi.net/v5/livedns/domains/reductioncarbone.fr/records/%40/A", + "rrset_values": [ + "1.2.3.4", + "1.2.3.5" + ] + }, + { + "rrset_type": "CAA", + "rrset_ttl": 3600, + "rrset_name": "", + "rrset_href": "https://api.gandi.net/v5/livedns/domains/reductioncarbone.fr/records/%40/CAA", + "rrset_values": [ + "0 issue \"ca.unit.tests\"" + ] + }, + { + "rrset_type": "SSHFP", + "rrset_ttl": 3600, + "rrset_name": "", + "rrset_href": "https://api.gandi.net/v5/livedns/domains/reductioncarbone.fr/records/%40/SSHFP", + "rrset_values": [ + "1 1 7491973e5f8b39d5327cd4e08bc81b05f7710b49", + "1 1 bf6b6825d2977c511a475bbefb88aad54a92ac73" + ] + }, + { + "rrset_type": "AAAA", + "rrset_ttl": 600, + "rrset_name": "aaaa", + "rrset_href": "https://api.gandi.net/v5/livedns/domains/reductioncarbone.fr/records/aaaa/AAAA", + "rrset_values": [ + "2601:644:500:e210:62f8:1dff:feb8:947a" + ] + }, + { + "rrset_type": "CNAME", + "rrset_ttl": 300, + "rrset_name": "cname", + "rrset_href": "https://api.gandi.net/v5/livedns/domains/reductioncarbone.fr/records/cname/CNAME", + "rrset_values": [ + "unit.tests." + ] + }, + { + "rrset_type": "CNAME", + "rrset_ttl": 3600, + "rrset_name": "excluded", + "rrset_href": "https://api.gandi.net/v5/livedns/domains/reductioncarbone.fr/records/excluded/CNAME", + "rrset_values": [ + "unit.tests." + ] + }, + { + "rrset_type": "MX", + "rrset_ttl": 300, + "rrset_name": "mx", + "rrset_href": "https://api.gandi.net/v5/livedns/domains/reductioncarbone.fr/records/mx/MX", + "rrset_values": [ + "10 smtp-4.unit.tests.", + "20 smtp-2.unit.tests.", + "30 smtp-3.unit.tests.", + "40 smtp-1.unit.tests." + ] + }, + { + "rrset_type": "PTR", + "rrset_ttl": 300, + "rrset_name": "ptr", + "rrset_href": "https://api.gandi.net/v5/livedns/domains/reductioncarbone.fr/records/ptr/PTR", + "rrset_values": [ + "foo.bar.com." + ] + }, + { + "rrset_type": "SPF", + "rrset_ttl": 600, + "rrset_name": "spf", + "rrset_href": "https://api.gandi.net/v5/livedns/domains/reductioncarbone.fr/records/spf/SPF", + "rrset_values": [ + "\"v=spf1 ip4:192.168.0.1/16-all\"" + ] + }, + { + "rrset_type": "TXT", + "rrset_ttl": 600, + "rrset_name": "txt", + "rrset_href": "https://api.gandi.net/v5/livedns/domains/reductioncarbone.fr/records/txt/TXT", + "rrset_values": [ + "\"Bah bah black sheep\"", + "\"have you any wool.\"", + "\"v=DKIM1;k=rsa;s=email;h=sha256;p=A/kinda+of/long/string+with+numb3rs\"" + ] + }, + { + "rrset_type": "A", + "rrset_ttl": 300, + "rrset_name": "www", + "rrset_href": "https://api.gandi.net/v5/livedns/domains/reductioncarbone.fr/records/www/A", + "rrset_values": [ + "2.2.3.6" + ] + }, + { + "rrset_type": "A", + "rrset_ttl": 300, + "rrset_name": "www.sub", + "rrset_href": "https://api.gandi.net/v5/livedns/domains/reductioncarbone.fr/records/www.sub/A", + "rrset_values": [ + "2.2.3.6" + ] + }, + { + "rrset_type": "SRV", + "rrset_ttl": 600, + "rrset_name": "_srv._tcp", + "rrset_href": "https://api.gandi.net/v5/livedns/domains/reductioncarbone.fr/records/_srv._tcp/SRV", + "rrset_values": [ + "10 20 30 foo-1.unit.tests.", + "12 20 30 foo-2.unit.tests." + ] + } + ] diff --git a/tests/test_octodns_provider_gandi.py b/tests/test_octodns_provider_gandi.py new file mode 100644 index 0000000..8152fcf --- /dev/null +++ b/tests/test_octodns_provider_gandi.py @@ -0,0 +1,312 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from mock import Mock, call +from os.path import dirname, join +from requests import HTTPError +from requests_mock import ANY, mock as requests_mock +from six import text_type +from unittest import TestCase + +from octodns.record import Record +from octodns.provider.gandi import GandiProvider, GandiClientBadRequest, \ + GandiClientUnauthorized, GandiClientForbidden, GandiClientNotFound +from octodns.provider.yaml import YamlProvider +from octodns.zone import Zone + + +class TestGandiProvider(TestCase): + expected = Zone('unit.tests.', []) + source = YamlProvider('test', join(dirname(__file__), 'config')) + source.populate(expected) + + # We remove this record from the test zone as Gandi API reject it + # (rightfully). + expected._remove_record(Record.new(expected, 'sub', { + 'ttl': 1800, + 'type': 'NS', + 'values': [ + '6.2.3.4.', + '7.2.3.4.' + ] + })) + + def test_populate(self): + + provider = GandiProvider('test_id', 'token') + + # 400 - Bad Request. + with requests_mock() as mock: + mock.get(ANY, status_code=400, + text='{"status": "error", "errors": [{"location": ' + '"body", "name": "items", "description": ' + '"\'6.2.3.4.\': invalid hostname (param: ' + '{\'rrset_type\': u\'NS\', \'rrset_ttl\': 3600, ' + '\'rrset_name\': u\'sub\', \'rrset_values\': ' + '[u\'6.2.3.4.\', u\'7.2.3.4.\']})"}, {"location": ' + '"body", "name": "items", "description": ' + '"\'7.2.3.4.\': invalid hostname (param: ' + '{\'rrset_type\': u\'NS\', \'rrset_ttl\': 3600, ' + '\'rrset_name\': u\'sub\', \'rrset_values\': ' + '[u\'6.2.3.4.\', u\'7.2.3.4.\']})"}]}') + + with self.assertRaises(GandiClientBadRequest) as ctx: + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertIn('"status": "error"', text_type(ctx.exception)) + + # 401 - Unauthorized. + with requests_mock() as mock: + mock.get(ANY, status_code=401, + text='{"code":401,"message":"The server could not verify ' + 'that you authorized to access the document you ' + 'requested. Either you supplied the wrong ' + 'credentials (e.g., bad api key), or your access ' + 'token has expired","object":"HTTPUnauthorized",' + '"cause":"Unauthorized"}') + + with self.assertRaises(GandiClientUnauthorized) as ctx: + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertIn('"cause":"Unauthorized"', text_type(ctx.exception)) + + # 403 - Forbidden. + with requests_mock() as mock: + mock.get(ANY, status_code=403, + text='{"code":403,"message":"Access was denied to this ' + 'resource.","object":"HTTPForbidden","cause":' + '"Forbidden"}') + + with self.assertRaises(GandiClientForbidden) as ctx: + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertIn('"cause":"Forbidden"', text_type(ctx.exception)) + + # General error + with requests_mock() as mock: + mock.get(ANY, status_code=502, text='Things caught fire') + + with self.assertRaises(HTTPError) as ctx: + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals(502, ctx.exception.response.status_code) + + # Non-existent zone doesn't populate anything + with requests_mock() as mock: + mock.get(ANY, status_code=404, + text='{"message": "Domain `foo.bar` not found"}') + + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals(set(), zone.records) + + # No diffs == no changes + with requests_mock() as mock: + base = 'https://api.gandi.net/v5/livedns/domains/unit.tests' \ + '/records' + with open('tests/fixtures/gandi-no-changes.json') as fh: + mock.get(base, text=fh.read()) + + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals(13, len(zone.records)) + changes = self.expected.changes(zone, provider) + self.assertEquals(0, len(changes)) + + del provider._zone_records[zone.name] + + # Default Gandi zone file. + with requests_mock() as mock: + base = 'https://api.gandi.net/v5/livedns/domains/unit.tests' \ + '/records' + with open('tests/fixtures/gandi-default-zone.json') as fh: + mock.get(base, text=fh.read()) + + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals(10, len(zone.records)) + changes = self.expected.changes(zone, provider) + self.assertEquals(22, len(changes)) + + # 2nd populate makes no network calls/all from cache + again = Zone('unit.tests.', []) + provider.populate(again) + self.assertEquals(10, len(again.records)) + + # bust the cache + del provider._zone_records[zone.name] + + def test_apply(self): + provider = GandiProvider('test_id', 'token') + + resp = Mock() + resp.json = Mock() + provider._client._request = Mock(return_value=resp) + + # non-existent domain + resp.json.side_effect = [ + GandiClientNotFound(resp), # no zone in populate + GandiClientNotFound(resp), # no domain during apply + ] + plan = provider.plan(self.expected) + + # No root NS, no ignored, no excluded + n = len(self.expected.records) - 4 + self.assertEquals(n, len(plan.changes)) + self.assertEquals(n, provider.apply(plan)) + self.assertFalse(plan.exists) + + provider._client._request.assert_has_calls([ + call('GET', '/livedns/domains/unit.tests/records'), + call('POST', '/livedns/domains/unit.tests/records', data={ + 'rrset_name': 'www.sub', + 'rrset_ttl': 300, + 'rrset_type': 'A', + 'rrset_values': ['2.2.3.6'] + }), + call('POST', '/livedns/domains/unit.tests/records', data={ + 'rrset_name': 'www', + 'rrset_ttl': 300, + 'rrset_type': 'A', + 'rrset_values': ['2.2.3.6'] + }), + call('POST', '/livedns/domains/unit.tests/records', data={ + 'rrset_name': 'txt', + 'rrset_ttl': 600, + 'rrset_type': 'TXT', + 'rrset_values': [ + 'Bah bah black sheep', + 'have you any wool.', + 'v=DKIM1;k=rsa;s=email;h=sha256;p=A/kinda+of/long/string' + '+with+numb3rs' + ] + }), + call('POST', '/livedns/domains/unit.tests/records', data={ + 'rrset_name': 'spf', + 'rrset_ttl': 600, + 'rrset_type': 'SPF', + 'rrset_values': ['v=spf1 ip4:192.168.0.1/16-all'] + }), + call('POST', '/livedns/domains/unit.tests/records', data={ + 'rrset_name': 'ptr', + 'rrset_ttl': 300, + 'rrset_type': 'PTR', + 'rrset_values': ['foo.bar.com.'] + }), + call('POST', '/livedns/domains/unit.tests/records', data={ + 'rrset_name': 'mx', + 'rrset_ttl': 300, + 'rrset_type': 'MX', + 'rrset_values': [ + '10 smtp-4.unit.tests.', + '20 smtp-2.unit.tests.', + '30 smtp-3.unit.tests.', + '40 smtp-1.unit.tests.' + ] + }), + call('POST', '/livedns/domains/unit.tests/records', data={ + 'rrset_name': 'excluded', + 'rrset_ttl': 3600, + 'rrset_type': 'CNAME', + 'rrset_values': ['unit.tests.'] + }), + call('POST', '/livedns/domains/unit.tests/records', data={ + 'rrset_name': 'cname', + 'rrset_ttl': 300, + 'rrset_type': 'CNAME', + 'rrset_values': ['unit.tests.'] + }), + call('POST', '/livedns/domains/unit.tests/records', data={ + 'rrset_name': 'aaaa', + 'rrset_ttl': 600, + 'rrset_type': 'AAAA', + 'rrset_values': ['2601:644:500:e210:62f8:1dff:feb8:947a'] + }), + call('POST', '/livedns/domains/unit.tests/records', data={ + 'rrset_name': '_srv._tcp', + 'rrset_ttl': 600, + 'rrset_type': 'SRV', + 'rrset_values': [ + '10 20 30 foo-1.unit.tests.', + '12 20 30 foo-2.unit.tests.' + ] + }), + call('POST', '/livedns/domains/unit.tests/records', data={ + 'rrset_name': '@', + 'rrset_ttl': 3600, + 'rrset_type': 'SSHFP', + 'rrset_values': [ + '1 1 7491973e5f8b39d5327cd4e08bc81b05f7710b49', + '1 1 bf6b6825d2977c511a475bbefb88aad54a92ac73' + ] + }), + call('POST', '/livedns/domains/unit.tests/records', data={ + 'rrset_name': '@', + 'rrset_ttl': 3600, + 'rrset_type': 'CAA', + 'rrset_values': ['0 issue "ca.unit.tests"'] + }), + call('POST', '/livedns/domains/unit.tests/records', data={ + 'rrset_name': '@', + 'rrset_ttl': 300, + 'rrset_type': 'A', + 'rrset_values': ['1.2.3.4', '1.2.3.5'] + }) + ]) + # expected number of total calls + self.assertEquals(14, provider._client._request.call_count) + + provider._client._request.reset_mock() + + # delete 1 and update 1 + provider._client.zone_records = Mock(return_value=[ + { + 'rrset_name': 'www', + 'rrset_ttl': 300, + 'rrset_type': 'A', + 'rrset_values': ['1.2.3.4'] + }, + { + 'rrset_name': 'www', + 'rrset_ttl': 300, + 'rrset_type': 'A', + 'rrset_values': ['2.2.3.4'] + }, + { + 'rrset_name': 'ttl', + 'rrset_ttl': 600, + 'rrset_type': 'A', + 'rrset_values': ['3.2.3.4'] + } + ]) + + # Domain exists, we don't care about return + resp.json.side_effect = ['{}'] + + wanted = Zone('unit.tests.', []) + wanted.add_record(Record.new(wanted, 'ttl', { + 'ttl': 300, + 'type': 'A', + 'value': '3.2.3.4' + })) + + plan = provider.plan(wanted) + self.assertTrue(plan.exists) + self.assertEquals(2, len(plan.changes)) + self.assertEquals(2, provider.apply(plan)) + + # recreate for update, and deletes for the 2 parts of the other + provider._client._request.assert_has_calls([ + call('DELETE', '/livedns/domains/unit.tests/records/www/A'), + call('DELETE', '/livedns/domains/unit.tests/records/ttl/A'), + call('POST', '/livedns/domains/unit.tests/records', data={ + 'rrset_name': 'ttl', + 'rrset_ttl': 300, + 'rrset_type': 'A', + 'rrset_values': ['3.2.3.4'] + }) + ], any_order=True) From 3f852442648e6e4b2971ebf1cddea5ad53fce103 Mon Sep 17 00:00:00 2001 From: Jonathan Leroy Date: Mon, 26 Oct 2020 20:30:15 +0100 Subject: [PATCH 025/358] Fixes incorrect domain name in gandi-no-changes.json --- tests/fixtures/gandi-no-changes.json | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/tests/fixtures/gandi-no-changes.json b/tests/fixtures/gandi-no-changes.json index 9bff3cb..4646327 100644 --- a/tests/fixtures/gandi-no-changes.json +++ b/tests/fixtures/gandi-no-changes.json @@ -3,7 +3,7 @@ "rrset_type": "A", "rrset_ttl": 300, "rrset_name": "", - "rrset_href": "https://api.gandi.net/v5/livedns/domains/reductioncarbone.fr/records/%40/A", + "rrset_href": "https://api.gandi.net/v5/livedns/domains/unit.tests/records/%40/A", "rrset_values": [ "1.2.3.4", "1.2.3.5" @@ -13,7 +13,7 @@ "rrset_type": "CAA", "rrset_ttl": 3600, "rrset_name": "", - "rrset_href": "https://api.gandi.net/v5/livedns/domains/reductioncarbone.fr/records/%40/CAA", + "rrset_href": "https://api.gandi.net/v5/livedns/domains/unit.tests/records/%40/CAA", "rrset_values": [ "0 issue \"ca.unit.tests\"" ] @@ -22,7 +22,7 @@ "rrset_type": "SSHFP", "rrset_ttl": 3600, "rrset_name": "", - "rrset_href": "https://api.gandi.net/v5/livedns/domains/reductioncarbone.fr/records/%40/SSHFP", + "rrset_href": "https://api.gandi.net/v5/livedns/domains/unit.tests/records/%40/SSHFP", "rrset_values": [ "1 1 7491973e5f8b39d5327cd4e08bc81b05f7710b49", "1 1 bf6b6825d2977c511a475bbefb88aad54a92ac73" @@ -32,7 +32,7 @@ "rrset_type": "AAAA", "rrset_ttl": 600, "rrset_name": "aaaa", - "rrset_href": "https://api.gandi.net/v5/livedns/domains/reductioncarbone.fr/records/aaaa/AAAA", + "rrset_href": "https://api.gandi.net/v5/livedns/domains/unit.tests/records/aaaa/AAAA", "rrset_values": [ "2601:644:500:e210:62f8:1dff:feb8:947a" ] @@ -41,7 +41,7 @@ "rrset_type": "CNAME", "rrset_ttl": 300, "rrset_name": "cname", - "rrset_href": "https://api.gandi.net/v5/livedns/domains/reductioncarbone.fr/records/cname/CNAME", + "rrset_href": "https://api.gandi.net/v5/livedns/domains/unit.tests/records/cname/CNAME", "rrset_values": [ "unit.tests." ] @@ -50,7 +50,7 @@ "rrset_type": "CNAME", "rrset_ttl": 3600, "rrset_name": "excluded", - "rrset_href": "https://api.gandi.net/v5/livedns/domains/reductioncarbone.fr/records/excluded/CNAME", + "rrset_href": "https://api.gandi.net/v5/livedns/domains/unit.tests/records/excluded/CNAME", "rrset_values": [ "unit.tests." ] @@ -59,7 +59,7 @@ "rrset_type": "MX", "rrset_ttl": 300, "rrset_name": "mx", - "rrset_href": "https://api.gandi.net/v5/livedns/domains/reductioncarbone.fr/records/mx/MX", + "rrset_href": "https://api.gandi.net/v5/livedns/domains/unit.tests/records/mx/MX", "rrset_values": [ "10 smtp-4.unit.tests.", "20 smtp-2.unit.tests.", @@ -71,7 +71,7 @@ "rrset_type": "PTR", "rrset_ttl": 300, "rrset_name": "ptr", - "rrset_href": "https://api.gandi.net/v5/livedns/domains/reductioncarbone.fr/records/ptr/PTR", + "rrset_href": "https://api.gandi.net/v5/livedns/domains/unit.tests/records/ptr/PTR", "rrset_values": [ "foo.bar.com." ] @@ -80,7 +80,7 @@ "rrset_type": "SPF", "rrset_ttl": 600, "rrset_name": "spf", - "rrset_href": "https://api.gandi.net/v5/livedns/domains/reductioncarbone.fr/records/spf/SPF", + "rrset_href": "https://api.gandi.net/v5/livedns/domains/unit.tests/records/spf/SPF", "rrset_values": [ "\"v=spf1 ip4:192.168.0.1/16-all\"" ] @@ -89,7 +89,7 @@ "rrset_type": "TXT", "rrset_ttl": 600, "rrset_name": "txt", - "rrset_href": "https://api.gandi.net/v5/livedns/domains/reductioncarbone.fr/records/txt/TXT", + "rrset_href": "https://api.gandi.net/v5/livedns/domains/unit.tests/records/txt/TXT", "rrset_values": [ "\"Bah bah black sheep\"", "\"have you any wool.\"", @@ -100,7 +100,7 @@ "rrset_type": "A", "rrset_ttl": 300, "rrset_name": "www", - "rrset_href": "https://api.gandi.net/v5/livedns/domains/reductioncarbone.fr/records/www/A", + "rrset_href": "https://api.gandi.net/v5/livedns/domains/unit.tests/records/www/A", "rrset_values": [ "2.2.3.6" ] @@ -109,7 +109,7 @@ "rrset_type": "A", "rrset_ttl": 300, "rrset_name": "www.sub", - "rrset_href": "https://api.gandi.net/v5/livedns/domains/reductioncarbone.fr/records/www.sub/A", + "rrset_href": "https://api.gandi.net/v5/livedns/domains/unit.tests/records/www.sub/A", "rrset_values": [ "2.2.3.6" ] @@ -118,7 +118,7 @@ "rrset_type": "SRV", "rrset_ttl": 600, "rrset_name": "_srv._tcp", - "rrset_href": "https://api.gandi.net/v5/livedns/domains/reductioncarbone.fr/records/_srv._tcp/SRV", + "rrset_href": "https://api.gandi.net/v5/livedns/domains/unit.tests/records/_srv._tcp/SRV", "rrset_values": [ "10 20 30 foo-1.unit.tests.", "12 20 30 foo-2.unit.tests." From de51e5f531f85909d8ad066e3d5e495f8fe9a740 Mon Sep 17 00:00:00 2001 From: Jonathan Leroy Date: Mon, 26 Oct 2020 22:18:35 +0100 Subject: [PATCH 026/358] Add support for DNAME records --- docs/records.md | 1 + octodns/provider/yaml.py | 4 +- octodns/record/__init__.py | 10 ++ tests/config/dynamic.tests.yaml | 34 ++++++ tests/config/split/dynamic.tests./dname.yaml | 42 +++++++ tests/config/split/unit.tests./dname.yaml | 5 + tests/config/unit.tests.yaml | 4 + tests/test_octodns_manager.py | 14 +-- tests/test_octodns_provider_constellix.py | 2 +- tests/test_octodns_provider_digitalocean.py | 2 +- tests/test_octodns_provider_dnsimple.py | 2 +- tests/test_octodns_provider_dnsmadeeasy.py | 2 +- tests/test_octodns_provider_easydns.py | 2 +- tests/test_octodns_provider_powerdns.py | 4 +- tests/test_octodns_provider_yaml.py | 43 ++++--- tests/test_octodns_record.py | 117 ++++++++++++++++++- 16 files changed, 249 insertions(+), 39 deletions(-) create mode 100644 tests/config/split/dynamic.tests./dname.yaml create mode 100644 tests/config/split/unit.tests./dname.yaml diff --git a/docs/records.md b/docs/records.md index 609383c..a28e86f 100644 --- a/docs/records.md +++ b/docs/records.md @@ -7,6 +7,7 @@ OctoDNS supports the following record types: * `A` * `AAAA` * `CNAME` +* `DNAME` * `MX` * `NAPTR` * `NS` diff --git a/octodns/provider/yaml.py b/octodns/provider/yaml.py index 10add5a..55a1632 100644 --- a/octodns/provider/yaml.py +++ b/octodns/provider/yaml.py @@ -104,8 +104,8 @@ class YamlProvider(BaseProvider): ''' SUPPORTS_GEO = True SUPPORTS_DYNAMIC = True - SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CAA', 'CNAME', 'MX', 'NAPTR', 'NS', - 'PTR', 'SSHFP', 'SPF', 'SRV', 'TXT')) + SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CAA', 'CNAME', 'DNAME', 'MX', + 'NAPTR', 'NS', 'PTR', 'SSHFP', 'SPF', 'SRV', 'TXT')) def __init__(self, id, directory, default_ttl=3600, enforce_order=True, populate_should_replace=False, *args, **kwargs): diff --git a/octodns/record/__init__.py b/octodns/record/__init__.py index 849e035..08ec2ee 100644 --- a/octodns/record/__init__.py +++ b/octodns/record/__init__.py @@ -95,6 +95,7 @@ class Record(EqualityTupleMixin): 'ALIAS': AliasRecord, 'CAA': CaaRecord, 'CNAME': CnameRecord, + 'DNAME': DnameRecord, 'MX': MxRecord, 'NAPTR': NaptrRecord, 'NS': NsRecord, @@ -759,6 +760,10 @@ class CnameValue(_TargetValue): pass +class DnameValue(_TargetValue): + pass + + class ARecord(_DynamicMixin, _GeoMixin, Record): _type = 'A' _value_type = Ipv4List @@ -842,6 +847,11 @@ class CnameRecord(_DynamicMixin, _ValueMixin, Record): return reasons +class DnameRecord(_DynamicMixin, _ValueMixin, Record): + _type = 'DNAME' + _value_type = DnameValue + + class MxValue(EqualityTupleMixin): @classmethod diff --git a/tests/config/dynamic.tests.yaml b/tests/config/dynamic.tests.yaml index 4bd97a7..5595a6d 100644 --- a/tests/config/dynamic.tests.yaml +++ b/tests/config/dynamic.tests.yaml @@ -109,6 +109,40 @@ cname: - pool: iad type: CNAME value: target.unit.tests. +dname: + dynamic: + pools: + ams: + values: + - value: target-ams.unit.tests. + iad: + values: + - value: target-iad.unit.tests. + lax: + values: + - value: target-lax.unit.tests. + sea: + values: + - value: target-sea-1.unit.tests. + weight: 10 + - value: target-sea-2.unit.tests. + weight: 14 + rules: + - geos: + - EU-GB + pool: lax + - geos: + - EU + pool: ams + - geos: + - NA-US-CA + - NA-US-NC + - NA-US-OR + - NA-US-WA + pool: sea + - pool: iad + type: DNAME + value: target.unit.tests. real-ish-a: dynamic: pools: diff --git a/tests/config/split/dynamic.tests./dname.yaml b/tests/config/split/dynamic.tests./dname.yaml new file mode 100644 index 0000000..45c33fe --- /dev/null +++ b/tests/config/split/dynamic.tests./dname.yaml @@ -0,0 +1,42 @@ +--- +dname: + dynamic: + pools: + ams: + fallback: null + values: + - value: target-ams.unit.tests. + weight: 1 + iad: + fallback: null + values: + - value: target-iad.unit.tests. + weight: 1 + lax: + fallback: null + values: + - value: target-lax.unit.tests. + weight: 1 + sea: + fallback: null + values: + - value: target-sea-1.unit.tests. + weight: 10 + - value: target-sea-2.unit.tests. + weight: 14 + rules: + - geos: + - EU-GB + pool: lax + - geos: + - EU + pool: ams + - geos: + - NA-US-CA + - NA-US-NC + - NA-US-OR + - NA-US-WA + pool: sea + - pool: iad + type: DNAME + value: target.unit.tests. diff --git a/tests/config/split/unit.tests./dname.yaml b/tests/config/split/unit.tests./dname.yaml new file mode 100644 index 0000000..7cd1755 --- /dev/null +++ b/tests/config/split/unit.tests./dname.yaml @@ -0,0 +1,5 @@ +--- +dname: + ttl: 300 + type: DNAME + value: unit.tests. diff --git a/tests/config/unit.tests.yaml b/tests/config/unit.tests.yaml index 1da2465..7b84ac9 100644 --- a/tests/config/unit.tests.yaml +++ b/tests/config/unit.tests.yaml @@ -56,6 +56,10 @@ cname: ttl: 300 type: CNAME value: unit.tests. +dname: + ttl: 300 + type: DNAME + value: unit.tests. excluded: octodns: excluded: diff --git a/tests/test_octodns_manager.py b/tests/test_octodns_manager.py index 9956790..7d25048 100644 --- a/tests/test_octodns_manager.py +++ b/tests/test_octodns_manager.py @@ -118,12 +118,12 @@ class TestManager(TestCase): environ['YAML_TMP_DIR'] = tmpdir.dirname tc = Manager(get_config_filename('simple.yaml')) \ .sync(dry_run=False) - self.assertEquals(21, tc) + self.assertEquals(22, tc) # try with just one of the zones tc = Manager(get_config_filename('simple.yaml')) \ .sync(dry_run=False, eligible_zones=['unit.tests.']) - self.assertEquals(15, tc) + self.assertEquals(16, tc) # the subzone, with 2 targets tc = Manager(get_config_filename('simple.yaml')) \ @@ -138,18 +138,18 @@ class TestManager(TestCase): # Again with force tc = Manager(get_config_filename('simple.yaml')) \ .sync(dry_run=False, force=True) - self.assertEquals(21, tc) + self.assertEquals(22, tc) # Again with max_workers = 1 tc = Manager(get_config_filename('simple.yaml'), max_workers=1) \ .sync(dry_run=False, force=True) - self.assertEquals(21, tc) + self.assertEquals(22, tc) # Include meta tc = Manager(get_config_filename('simple.yaml'), max_workers=1, include_meta=True) \ .sync(dry_run=False, force=True) - self.assertEquals(25, tc) + self.assertEquals(26, tc) def test_eligible_sources(self): with TemporaryDirectory() as tmpdir: @@ -183,13 +183,13 @@ class TestManager(TestCase): fh.write('---\n{}') changes = manager.compare(['in'], ['dump'], 'unit.tests.') - self.assertEquals(15, len(changes)) + self.assertEquals(16, len(changes)) # Compound sources with varying support changes = manager.compare(['in', 'nosshfp'], ['dump'], 'unit.tests.') - self.assertEquals(14, len(changes)) + self.assertEquals(15, len(changes)) with self.assertRaises(ManagerException) as ctx: manager.compare(['nope'], ['dump'], 'unit.tests.') diff --git a/tests/test_octodns_provider_constellix.py b/tests/test_octodns_provider_constellix.py index 151d0d4..c19ae29 100644 --- a/tests/test_octodns_provider_constellix.py +++ b/tests/test_octodns_provider_constellix.py @@ -138,7 +138,7 @@ class TestConstellixProvider(TestCase): plan = provider.plan(self.expected) # No root NS, no ignored, no excluded, no unsupported - n = len(self.expected.records) - 5 + n = len(self.expected.records) - 6 self.assertEquals(n, len(plan.changes)) self.assertEquals(n, provider.apply(plan)) diff --git a/tests/test_octodns_provider_digitalocean.py b/tests/test_octodns_provider_digitalocean.py index ebb5319..0ad8f72 100644 --- a/tests/test_octodns_provider_digitalocean.py +++ b/tests/test_octodns_provider_digitalocean.py @@ -163,7 +163,7 @@ class TestDigitalOceanProvider(TestCase): plan = provider.plan(self.expected) # No root NS, no ignored, no excluded, no unsupported - n = len(self.expected.records) - 7 + n = len(self.expected.records) - 8 self.assertEquals(n, len(plan.changes)) self.assertEquals(n, provider.apply(plan)) self.assertFalse(plan.exists) diff --git a/tests/test_octodns_provider_dnsimple.py b/tests/test_octodns_provider_dnsimple.py index b918962..92f32b1 100644 --- a/tests/test_octodns_provider_dnsimple.py +++ b/tests/test_octodns_provider_dnsimple.py @@ -137,7 +137,7 @@ class TestDnsimpleProvider(TestCase): plan = provider.plan(self.expected) # No root NS, no ignored, no excluded - n = len(self.expected.records) - 3 + n = len(self.expected.records) - 4 self.assertEquals(n, len(plan.changes)) self.assertEquals(n, provider.apply(plan)) self.assertFalse(plan.exists) diff --git a/tests/test_octodns_provider_dnsmadeeasy.py b/tests/test_octodns_provider_dnsmadeeasy.py index ba61b94..50fa576 100644 --- a/tests/test_octodns_provider_dnsmadeeasy.py +++ b/tests/test_octodns_provider_dnsmadeeasy.py @@ -140,7 +140,7 @@ class TestDnsMadeEasyProvider(TestCase): plan = provider.plan(self.expected) # No root NS, no ignored, no excluded, no unsupported - n = len(self.expected.records) - 5 + n = len(self.expected.records) - 6 self.assertEquals(n, len(plan.changes)) self.assertEquals(n, provider.apply(plan)) diff --git a/tests/test_octodns_provider_easydns.py b/tests/test_octodns_provider_easydns.py index 2681bf4..8df0e22 100644 --- a/tests/test_octodns_provider_easydns.py +++ b/tests/test_octodns_provider_easydns.py @@ -374,7 +374,7 @@ class TestEasyDNSProvider(TestCase): plan = provider.plan(self.expected) # No root NS, no ignored, no excluded, no unsupported - n = len(self.expected.records) - 6 + n = len(self.expected.records) - 7 self.assertEquals(n, len(plan.changes)) self.assertEquals(n, provider.apply(plan)) self.assertFalse(plan.exists) diff --git a/tests/test_octodns_provider_powerdns.py b/tests/test_octodns_provider_powerdns.py index fd877ef..c9b1d08 100644 --- a/tests/test_octodns_provider_powerdns.py +++ b/tests/test_octodns_provider_powerdns.py @@ -171,7 +171,7 @@ class TestPowerDnsProvider(TestCase): expected = Zone('unit.tests.', []) source = YamlProvider('test', join(dirname(__file__), 'config')) source.populate(expected) - expected_n = len(expected.records) - 2 + expected_n = len(expected.records) - 3 self.assertEquals(16, expected_n) # No diffs == no changes @@ -277,7 +277,7 @@ class TestPowerDnsProvider(TestCase): expected = Zone('unit.tests.', []) source = YamlProvider('test', join(dirname(__file__), 'config')) source.populate(expected) - self.assertEquals(18, len(expected.records)) + self.assertEquals(19, len(expected.records)) # A small change to a single record with requests_mock() as mock: diff --git a/tests/test_octodns_provider_yaml.py b/tests/test_octodns_provider_yaml.py index f858c05..7b285ec 100644 --- a/tests/test_octodns_provider_yaml.py +++ b/tests/test_octodns_provider_yaml.py @@ -35,10 +35,10 @@ class TestYamlProvider(TestCase): # without it we see everything source.populate(zone) - self.assertEquals(18, len(zone.records)) + self.assertEquals(19, len(zone.records)) source.populate(dynamic_zone) - self.assertEquals(5, len(dynamic_zone.records)) + self.assertEquals(6, len(dynamic_zone.records)) # Assumption here is that a clean round-trip means that everything # worked as expected, data that went in came back out and could be @@ -58,21 +58,21 @@ class TestYamlProvider(TestCase): # We add everything plan = target.plan(zone) - self.assertEquals(15, len([c for c in plan.changes + self.assertEquals(16, len([c for c in plan.changes if isinstance(c, Create)])) self.assertFalse(isfile(yaml_file)) # Now actually do it - self.assertEquals(15, target.apply(plan)) + self.assertEquals(16, target.apply(plan)) self.assertTrue(isfile(yaml_file)) # Dynamic plan plan = target.plan(dynamic_zone) - self.assertEquals(5, len([c for c in plan.changes + self.assertEquals(6, len([c for c in plan.changes if isinstance(c, Create)])) self.assertFalse(isfile(dynamic_yaml_file)) # Apply it - self.assertEquals(5, target.apply(plan)) + self.assertEquals(6, target.apply(plan)) self.assertTrue(isfile(dynamic_yaml_file)) # There should be no changes after the round trip @@ -87,7 +87,7 @@ class TestYamlProvider(TestCase): # A 2nd sync should still create everything plan = target.plan(zone) - self.assertEquals(15, len([c for c in plan.changes + self.assertEquals(16, len([c for c in plan.changes if isinstance(c, Create)])) with open(yaml_file) as fh: @@ -109,6 +109,7 @@ class TestYamlProvider(TestCase): # these are stored as singular 'value' self.assertTrue('value' in data.pop('aaaa')) self.assertTrue('value' in data.pop('cname')) + self.assertTrue('value' in data.pop('dname')) self.assertTrue('value' in data.pop('included')) self.assertTrue('value' in data.pop('ptr')) self.assertTrue('value' in data.pop('spf')) @@ -136,6 +137,10 @@ class TestYamlProvider(TestCase): self.assertTrue('value' in dyna) # self.assertTrue('dynamic' in dyna) + dyna = data.pop('dname') + self.assertTrue('value' in dyna) + # self.assertTrue('dynamic' in dyna) + dyna = data.pop('real-ish-a') self.assertTrue('values' in dyna) # self.assertTrue('dynamic' in dyna) @@ -237,10 +242,10 @@ class TestSplitYamlProvider(TestCase): # without it we see everything source.populate(zone) - self.assertEquals(18, len(zone.records)) + self.assertEquals(19, len(zone.records)) source.populate(dynamic_zone) - self.assertEquals(5, len(dynamic_zone.records)) + self.assertEquals(6, len(dynamic_zone.records)) with TemporaryDirectory() as td: # Add some subdirs to make sure that it can create them @@ -251,20 +256,20 @@ class TestSplitYamlProvider(TestCase): # We add everything plan = target.plan(zone) - self.assertEquals(15, len([c for c in plan.changes + self.assertEquals(16, len([c for c in plan.changes if isinstance(c, Create)])) self.assertFalse(isdir(zone_dir)) # Now actually do it - self.assertEquals(15, target.apply(plan)) + self.assertEquals(16, target.apply(plan)) # Dynamic plan plan = target.plan(dynamic_zone) - self.assertEquals(5, len([c for c in plan.changes + self.assertEquals(6, len([c for c in plan.changes if isinstance(c, Create)])) self.assertFalse(isdir(dynamic_zone_dir)) # Apply it - self.assertEquals(5, target.apply(plan)) + self.assertEquals(6, target.apply(plan)) self.assertTrue(isdir(dynamic_zone_dir)) # There should be no changes after the round trip @@ -279,7 +284,7 @@ class TestSplitYamlProvider(TestCase): # A 2nd sync should still create everything plan = target.plan(zone) - self.assertEquals(15, len([c for c in plan.changes + self.assertEquals(16, len([c for c in plan.changes if isinstance(c, Create)])) yaml_file = join(zone_dir, '$unit.tests.yaml') @@ -302,8 +307,8 @@ class TestSplitYamlProvider(TestCase): self.assertTrue('values' in data.pop(record_name)) # These are stored as singular "value." Again, check each file. - for record_name in ('aaaa', 'cname', 'included', 'ptr', 'spf', - 'www.sub', 'www'): + for record_name in ('aaaa', 'cname', 'dname', 'included', 'ptr', + 'spf', 'www.sub', 'www'): yaml_file = join(zone_dir, '{}.yaml'.format(record_name)) self.assertTrue(isfile(yaml_file)) with open(yaml_file) as fh: @@ -322,7 +327,7 @@ class TestSplitYamlProvider(TestCase): self.assertTrue('dynamic' in dyna) # Singular again. - for record_name in ('cname', 'simple-weighted'): + for record_name in ('cname', 'dname', 'simple-weighted'): yaml_file = join( dynamic_zone_dir, '{}.yaml'.format(record_name)) self.assertTrue(isfile(yaml_file)) @@ -386,7 +391,7 @@ class TestOverridingYamlProvider(TestCase): # Load the base, should see the 5 records base.populate(zone) got = {r.name: r for r in zone.records} - self.assertEquals(5, len(got)) + self.assertEquals(6, len(got)) # We get the "dynamic" A from the bae config self.assertTrue('dynamic' in got['a'].data) # No added @@ -395,7 +400,7 @@ class TestOverridingYamlProvider(TestCase): # Load the overrides, should replace one and add 1 override.populate(zone) got = {r.name: r for r in zone.records} - self.assertEquals(6, len(got)) + self.assertEquals(7, len(got)) # 'a' was replaced with a generic record self.assertEquals({ 'ttl': 3600, diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index 08a3e7a..10e9575 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -9,10 +9,10 @@ from six import text_type from unittest import TestCase from octodns.record import ARecord, AaaaRecord, AliasRecord, CaaRecord, \ - CaaValue, CnameRecord, Create, Delete, GeoValue, MxRecord, MxValue, \ - NaptrRecord, NaptrValue, NsRecord, PtrRecord, Record, SshfpRecord, \ - SshfpValue, SpfRecord, SrvRecord, SrvValue, TxtRecord, Update, \ - ValidationError, _Dynamic, _DynamicPool, _DynamicRule + CaaValue, CnameRecord, DnameRecord, Create, Delete, GeoValue, MxRecord, \ + MxValue, NaptrRecord, NaptrValue, NsRecord, PtrRecord, Record, \ + SshfpRecord, SshfpValue, SpfRecord, SrvRecord, SrvValue, TxtRecord, \ + Update, ValidationError, _Dynamic, _DynamicPool, _DynamicRule from octodns.zone import Zone from helpers import DynamicProvider, GeoProvider, SimpleProvider @@ -55,6 +55,19 @@ class TestRecord(TestCase): }) self.assertEquals(upper_record.value, lower_record.value) + def test_dname_lowering_value(self): + upper_record = DnameRecord(self.zone, 'DnameUppwerValue', { + 'ttl': 30, + 'type': 'DNAME', + 'value': 'GITHUB.COM', + }) + lower_record = DnameRecord(self.zone, 'DnameLowerValue', { + 'ttl': 30, + 'type': 'DNAME', + 'value': 'github.com', + }) + self.assertEquals(upper_record.value, lower_record.value) + def test_ptr_lowering_value(self): upper_record = PtrRecord(self.zone, 'PtrUppwerValue', { 'ttl': 30, @@ -362,6 +375,10 @@ class TestRecord(TestCase): self.assertSingleValue(CnameRecord, 'target.foo.com.', 'other.foo.com.') + def test_dname(self): + self.assertSingleValue(DnameRecord, 'target.foo.com.', + 'other.foo.com.') + def test_mx(self): a_values = [{ 'preference': 10, @@ -1825,6 +1842,31 @@ class TestRecordValidation(TestCase): self.assertEquals(['CNAME value "foo.bar.com" missing trailing .'], ctx.exception.reasons) + def test_DNAME(self): + # A valid DNAME record. + Record.new(self.zone, 'sub', { + 'type': 'DNAME', + 'ttl': 600, + 'value': 'foo.bar.com.', + }) + + # A DNAME record can be present at the zone APEX. + Record.new(self.zone, '', { + 'type': 'DNAME', + 'ttl': 600, + 'value': 'foo.bar.com.', + }) + + # missing trailing . + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, 'www', { + 'type': 'DNAME', + 'ttl': 600, + 'value': 'foo.bar.com', + }) + self.assertEquals(['DNAME value "foo.bar.com" missing trailing .'], + ctx.exception.reasons) + def test_MX(self): # doesn't blow up Record.new(self.zone, '', { @@ -2628,6 +2670,73 @@ class TestDynamicRecords(TestCase): self.assertTrue(rules) self.assertEquals(cname_data['dynamic']['rules'][0], rules[0].data) + def test_simple_dname_weighted(self): + dname_data = { + 'dynamic': { + 'pools': { + 'one': { + 'values': [{ + 'value': 'one.dname.target.', + }], + }, + 'two': { + 'values': [{ + 'value': 'two.dname.target.', + }], + }, + 'three': { + 'values': [{ + 'weight': 12, + 'value': 'three-1.dname.target.', + }, { + 'weight': 32, + 'value': 'three-2.dname.target.', + }] + }, + }, + 'rules': [{ + 'geos': ['AF', 'EU'], + 'pool': 'three', + }, { + 'geos': ['NA-US-CA'], + 'pool': 'two', + }, { + 'pool': 'one', + }], + }, + 'ttl': 60, + 'value': 'dname.target.', + } + dname = DnameRecord(self.zone, 'weighted', dname_data) + self.assertEquals('DNAME', dname._type) + self.assertEquals(dname_data['ttl'], dname.ttl) + self.assertEquals(dname_data['value'], dname.value) + + dynamic = dname.dynamic + self.assertTrue(dynamic) + + pools = dynamic.pools + self.assertTrue(pools) + self.assertEquals({ + 'value': 'one.dname.target.', + 'weight': 1, + }, pools['one'].data['values'][0]) + self.assertEquals({ + 'value': 'two.dname.target.', + 'weight': 1, + }, pools['two'].data['values'][0]) + self.assertEquals([{ + 'value': 'three-1.dname.target.', + 'weight': 12, + }, { + 'value': 'three-2.dname.target.', + 'weight': 32, + }], pools['three'].data['values']) + + rules = dynamic.rules + self.assertTrue(rules) + self.assertEquals(dname_data['dynamic']['rules'][0], rules[0].data) + def test_dynamic_validation(self): # Missing pools a_data = { From bfaafeb61b92d284e835367c9de296035fcdb489 Mon Sep 17 00:00:00 2001 From: Jonathan Leroy Date: Mon, 26 Oct 2020 23:10:36 +0100 Subject: [PATCH 027/358] Fixes value of "rrset_name" parameter for domain APEX --- tests/fixtures/gandi-default-zone.json | 6 +++--- tests/fixtures/gandi-no-changes.json | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/fixtures/gandi-default-zone.json b/tests/fixtures/gandi-default-zone.json index deb4cb8..254b7c1 100644 --- a/tests/fixtures/gandi-default-zone.json +++ b/tests/fixtures/gandi-default-zone.json @@ -2,7 +2,7 @@ { "rrset_type": "A", "rrset_ttl": 10800, - "rrset_name": "", + "rrset_name": "@", "rrset_href": "https://api.gandi.net/v5/livedns/domains/unit.tests/records/%40/A", "rrset_values": [ "217.70.184.38" @@ -11,7 +11,7 @@ { "rrset_type": "MX", "rrset_ttl": 10800, - "rrset_name": "", + "rrset_name": "@", "rrset_href": "https://api.gandi.net/v5/livedns/domains/unit.tests/records/%40/MX", "rrset_values": [ "10 spool.mail.gandi.net.", @@ -21,7 +21,7 @@ { "rrset_type": "TXT", "rrset_ttl": 10800, - "rrset_name": "", + "rrset_name": "@", "rrset_href": "https://api.gandi.net/v5/livedns/domains/unit.tests/records/%40/TXT", "rrset_values": [ "\"v=spf1 include:_mailcust.gandi.net ?all\"" diff --git a/tests/fixtures/gandi-no-changes.json b/tests/fixtures/gandi-no-changes.json index 4646327..1154628 100644 --- a/tests/fixtures/gandi-no-changes.json +++ b/tests/fixtures/gandi-no-changes.json @@ -2,7 +2,7 @@ { "rrset_type": "A", "rrset_ttl": 300, - "rrset_name": "", + "rrset_name": "@", "rrset_href": "https://api.gandi.net/v5/livedns/domains/unit.tests/records/%40/A", "rrset_values": [ "1.2.3.4", @@ -12,7 +12,7 @@ { "rrset_type": "CAA", "rrset_ttl": 3600, - "rrset_name": "", + "rrset_name": "@", "rrset_href": "https://api.gandi.net/v5/livedns/domains/unit.tests/records/%40/CAA", "rrset_values": [ "0 issue \"ca.unit.tests\"" @@ -21,7 +21,7 @@ { "rrset_type": "SSHFP", "rrset_ttl": 3600, - "rrset_name": "", + "rrset_name": "@", "rrset_href": "https://api.gandi.net/v5/livedns/domains/unit.tests/records/%40/SSHFP", "rrset_values": [ "1 1 7491973e5f8b39d5327cd4e08bc81b05f7710b49", From 7161baa2628fd28d3eb2c9ceec6953a9afae359b Mon Sep 17 00:00:00 2001 From: Jonathan Leroy Date: Mon, 26 Oct 2020 23:23:32 +0100 Subject: [PATCH 028/358] Fixes code coverage for unsupported records types --- tests/fixtures/gandi-default-zone.json | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/fixtures/gandi-default-zone.json b/tests/fixtures/gandi-default-zone.json index 254b7c1..9b07b8e 100644 --- a/tests/fixtures/gandi-default-zone.json +++ b/tests/fixtures/gandi-default-zone.json @@ -89,5 +89,14 @@ "rrset_values": [ "0 1 465 mail.gandi.net." ] + }, + { + "rrset_type": "CDS", + "rrset_ttl": 10800, + "rrset_name": "sub", + "rrset_href": "https://api.gandi.net/v5/livedns/domains/unit.tests/records/sub/CDS", + "rrset_values": [ + "32128 13 1 6823D9BB1B03DF714DD0EB163E20B341C96D18C0" + ] } ] From 6d17b4671ab964d1dada7319e77f4de12438de02 Mon Sep 17 00:00:00 2001 From: Jonathan Leroy Date: Tue, 27 Oct 2020 11:23:22 +0100 Subject: [PATCH 029/358] Handle domains not registred at Gandi or not using Gandi's DNS --- octodns/provider/gandi.py | 32 ++++++++++++++++++++++++++++ tests/fixtures/gandi-zone.json | 7 ++++++ tests/test_octodns_provider_gandi.py | 32 +++++++++++++++++++--------- 3 files changed, 61 insertions(+), 10 deletions(-) create mode 100644 tests/fixtures/gandi-zone.json diff --git a/octodns/provider/gandi.py b/octodns/provider/gandi.py index e11e9a3..1f89a80 100644 --- a/octodns/provider/gandi.py +++ b/octodns/provider/gandi.py @@ -41,6 +41,12 @@ class GandiClientNotFound(GandiClientException): super(GandiClientNotFound, self).__init__(r.text) +class GandiClientUnknownDomainName(GandiClientException): + + def __init__(self, msg): + super(GandiClientUnknownDomainName, self).__init__(msg) + + class GandiClient(object): def __init__(self, token): @@ -63,6 +69,16 @@ class GandiClient(object): r.raise_for_status() return r + def zone(self, zone_name): + return self._request('GET', '/livedns/domains/{}' + .format(zone_name)).json() + + def zone_create(self, zone_name): + return self._request('POST', '/livedns/domains', data={ + 'fqdn': zone_name, + 'zone': {} + }).json() + def zone_records(self, zone_name): records = self._request('GET', '/livedns/domains/{}/records' .format(zone_name)).json() @@ -318,9 +334,25 @@ class GandiProvider(BaseProvider): def _apply(self, plan): desired = plan.desired changes = plan.changes + zone = desired.name[:-1] self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name, len(changes)) + try: + self._client.zone(zone) + except GandiClientNotFound: + self.log.info('_apply: no existing zone, trying to create it') + try: + self._client.zone_create(zone) + self.log.info('_apply: zone has been successfully created') + except GandiClientNotFound: + raise GandiClientUnknownDomainName('This domain is not ' + 'registred at Gandi. ' + 'Please register or ' + 'transfer it here ' + 'to be able to manage its ' + 'DNS zone.') + # Force records deletion to be done before creation in order to avoid # "CNAME record must be the only record" error when an existing CNAME # record is replaced by an A/AAAA record. diff --git a/tests/fixtures/gandi-zone.json b/tests/fixtures/gandi-zone.json new file mode 100644 index 0000000..e132f4c --- /dev/null +++ b/tests/fixtures/gandi-zone.json @@ -0,0 +1,7 @@ +{ + "domain_keys_href": "https://api.gandi.net/v5/livedns/domains/unit.tests/keys", + "fqdn": "unit.tests", + "automatic_snapshots": true, + "domain_records_href": "https://api.gandi.net/v5/livedns/domains/unit.tests/records", + "domain_href": "https://api.gandi.net/v5/livedns/domains/unit.tests" +} \ No newline at end of file diff --git a/tests/test_octodns_provider_gandi.py b/tests/test_octodns_provider_gandi.py index 8152fcf..7448666 100644 --- a/tests/test_octodns_provider_gandi.py +++ b/tests/test_octodns_provider_gandi.py @@ -86,6 +86,18 @@ class TestGandiProvider(TestCase): provider.populate(zone) self.assertIn('"cause":"Forbidden"', text_type(ctx.exception)) + # 404 - Not Found. + with requests_mock() as mock: + mock.get(ANY, status_code=404, + text='{"code": 404, "message": "The resource could not ' + 'be found.", "object": "HTTPNotFound", "cause": ' + '"Not Found"}') + + with self.assertRaises(GandiClientNotFound) as ctx: + zone = Zone('unit.tests.', []) + provider._client.zone(zone) + self.assertIn('"cause": "Not Found"', text_type(ctx.exception)) + # General error with requests_mock() as mock: mock.get(ANY, status_code=502, text='Things caught fire') @@ -95,15 +107,6 @@ class TestGandiProvider(TestCase): provider.populate(zone) self.assertEquals(502, ctx.exception.response.status_code) - # Non-existent zone doesn't populate anything - with requests_mock() as mock: - mock.get(ANY, status_code=404, - text='{"message": "Domain `foo.bar` not found"}') - - zone = Zone('unit.tests.', []) - provider.populate(zone) - self.assertEquals(set(), zone.records) - # No diffs == no changes with requests_mock() as mock: base = 'https://api.gandi.net/v5/livedns/domains/unit.tests' \ @@ -147,10 +150,14 @@ class TestGandiProvider(TestCase): resp.json = Mock() provider._client._request = Mock(return_value=resp) + with open('tests/fixtures/gandi-zone.json') as fh: + zone = fh.read() + # non-existent domain resp.json.side_effect = [ GandiClientNotFound(resp), # no zone in populate GandiClientNotFound(resp), # no domain during apply + zone ] plan = provider.plan(self.expected) @@ -162,6 +169,11 @@ class TestGandiProvider(TestCase): provider._client._request.assert_has_calls([ call('GET', '/livedns/domains/unit.tests/records'), + call('GET', '/livedns/domains/unit.tests'), + call('POST', '/livedns/domains', data={ + 'fqdn': 'unit.tests', + 'zone': {} + }), call('POST', '/livedns/domains/unit.tests/records', data={ 'rrset_name': 'www.sub', 'rrset_ttl': 300, @@ -258,7 +270,7 @@ class TestGandiProvider(TestCase): }) ]) # expected number of total calls - self.assertEquals(14, provider._client._request.call_count) + self.assertEquals(16, provider._client._request.call_count) provider._client._request.reset_mock() From b280449969c5af0fe31eee1c8139e0995e54892f Mon Sep 17 00:00:00 2001 From: Jonathan Leroy Date: Tue, 27 Oct 2020 11:25:55 +0100 Subject: [PATCH 030/358] Add record targets normalizaltion --- octodns/provider/gandi.py | 8 ++++++++ .../{gandi-default-zone.json => gandi-records.json} | 9 +++++++++ tests/test_octodns_provider_gandi.py | 8 ++++---- 3 files changed, 21 insertions(+), 4 deletions(-) rename tests/fixtures/{gandi-default-zone.json => gandi-records.json} (92%) diff --git a/octodns/provider/gandi.py b/octodns/provider/gandi.py index 1f89a80..dcc222d 100644 --- a/octodns/provider/gandi.py +++ b/octodns/provider/gandi.py @@ -87,6 +87,14 @@ class GandiClient(object): if record['rrset_name'] == '@': record['rrset_name'] = '' + # Change relative targets to absolute ones. + if record['rrset_type'] in ['ALIAS', 'CNAME', 'DNAME', 'MX', + 'NS', 'SRV']: + for i, value in enumerate(record['rrset_values']): + if not value.endswith('.'): + record['rrset_values'][i] = '{}.{}.'.format( + value, zone_name) + return records def record_create(self, zone_name, data): diff --git a/tests/fixtures/gandi-default-zone.json b/tests/fixtures/gandi-records.json similarity index 92% rename from tests/fixtures/gandi-default-zone.json rename to tests/fixtures/gandi-records.json index 9b07b8e..01d30f7 100644 --- a/tests/fixtures/gandi-default-zone.json +++ b/tests/fixtures/gandi-records.json @@ -98,5 +98,14 @@ "rrset_values": [ "32128 13 1 6823D9BB1B03DF714DD0EB163E20B341C96D18C0" ] + }, + { + "rrset_type": "CNAME", + "rrset_ttl": 10800, + "rrset_name": "relative", + "rrset_href": "https://api.gandi.net/v5/livedns/domains/unit.tests/records/relative/CNAME", + "rrset_values": [ + "target" + ] } ] diff --git a/tests/test_octodns_provider_gandi.py b/tests/test_octodns_provider_gandi.py index 7448666..a818919 100644 --- a/tests/test_octodns_provider_gandi.py +++ b/tests/test_octodns_provider_gandi.py @@ -126,19 +126,19 @@ class TestGandiProvider(TestCase): with requests_mock() as mock: base = 'https://api.gandi.net/v5/livedns/domains/unit.tests' \ '/records' - with open('tests/fixtures/gandi-default-zone.json') as fh: + with open('tests/fixtures/gandi-records.json') as fh: mock.get(base, text=fh.read()) zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals(10, len(zone.records)) + self.assertEquals(11, len(zone.records)) changes = self.expected.changes(zone, provider) - self.assertEquals(22, len(changes)) + self.assertEquals(23, len(changes)) # 2nd populate makes no network calls/all from cache again = Zone('unit.tests.', []) provider.populate(again) - self.assertEquals(10, len(again.records)) + self.assertEquals(11, len(again.records)) # bust the cache del provider._zone_records[zone.name] From eec4c4f81c100c44e3f3550ecae6c541394068c6 Mon Sep 17 00:00:00 2001 From: Jonathan Leroy Date: Tue, 27 Oct 2020 20:31:57 +0100 Subject: [PATCH 031/358] Remove support for dynamic DNAME records as no provider currently support them --- tests/config/dynamic.tests.yaml | 34 ---------- tests/config/split/dynamic.tests./dname.yaml | 42 ------------ tests/test_octodns_provider_yaml.py | 24 +++---- tests/test_octodns_record.py | 67 -------------------- 4 files changed, 10 insertions(+), 157 deletions(-) delete mode 100644 tests/config/split/dynamic.tests./dname.yaml diff --git a/tests/config/dynamic.tests.yaml b/tests/config/dynamic.tests.yaml index 5595a6d..4bd97a7 100644 --- a/tests/config/dynamic.tests.yaml +++ b/tests/config/dynamic.tests.yaml @@ -109,40 +109,6 @@ cname: - pool: iad type: CNAME value: target.unit.tests. -dname: - dynamic: - pools: - ams: - values: - - value: target-ams.unit.tests. - iad: - values: - - value: target-iad.unit.tests. - lax: - values: - - value: target-lax.unit.tests. - sea: - values: - - value: target-sea-1.unit.tests. - weight: 10 - - value: target-sea-2.unit.tests. - weight: 14 - rules: - - geos: - - EU-GB - pool: lax - - geos: - - EU - pool: ams - - geos: - - NA-US-CA - - NA-US-NC - - NA-US-OR - - NA-US-WA - pool: sea - - pool: iad - type: DNAME - value: target.unit.tests. real-ish-a: dynamic: pools: diff --git a/tests/config/split/dynamic.tests./dname.yaml b/tests/config/split/dynamic.tests./dname.yaml deleted file mode 100644 index 45c33fe..0000000 --- a/tests/config/split/dynamic.tests./dname.yaml +++ /dev/null @@ -1,42 +0,0 @@ ---- -dname: - dynamic: - pools: - ams: - fallback: null - values: - - value: target-ams.unit.tests. - weight: 1 - iad: - fallback: null - values: - - value: target-iad.unit.tests. - weight: 1 - lax: - fallback: null - values: - - value: target-lax.unit.tests. - weight: 1 - sea: - fallback: null - values: - - value: target-sea-1.unit.tests. - weight: 10 - - value: target-sea-2.unit.tests. - weight: 14 - rules: - - geos: - - EU-GB - pool: lax - - geos: - - EU - pool: ams - - geos: - - NA-US-CA - - NA-US-NC - - NA-US-OR - - NA-US-WA - pool: sea - - pool: iad - type: DNAME - value: target.unit.tests. diff --git a/tests/test_octodns_provider_yaml.py b/tests/test_octodns_provider_yaml.py index 7b285ec..15e90da 100644 --- a/tests/test_octodns_provider_yaml.py +++ b/tests/test_octodns_provider_yaml.py @@ -38,7 +38,7 @@ class TestYamlProvider(TestCase): self.assertEquals(19, len(zone.records)) source.populate(dynamic_zone) - self.assertEquals(6, len(dynamic_zone.records)) + self.assertEquals(5, len(dynamic_zone.records)) # Assumption here is that a clean round-trip means that everything # worked as expected, data that went in came back out and could be @@ -68,11 +68,11 @@ class TestYamlProvider(TestCase): # Dynamic plan plan = target.plan(dynamic_zone) - self.assertEquals(6, len([c for c in plan.changes + self.assertEquals(5, len([c for c in plan.changes if isinstance(c, Create)])) self.assertFalse(isfile(dynamic_yaml_file)) # Apply it - self.assertEquals(6, target.apply(plan)) + self.assertEquals(5, target.apply(plan)) self.assertTrue(isfile(dynamic_yaml_file)) # There should be no changes after the round trip @@ -137,10 +137,6 @@ class TestYamlProvider(TestCase): self.assertTrue('value' in dyna) # self.assertTrue('dynamic' in dyna) - dyna = data.pop('dname') - self.assertTrue('value' in dyna) - # self.assertTrue('dynamic' in dyna) - dyna = data.pop('real-ish-a') self.assertTrue('values' in dyna) # self.assertTrue('dynamic' in dyna) @@ -245,7 +241,7 @@ class TestSplitYamlProvider(TestCase): self.assertEquals(19, len(zone.records)) source.populate(dynamic_zone) - self.assertEquals(6, len(dynamic_zone.records)) + self.assertEquals(5, len(dynamic_zone.records)) with TemporaryDirectory() as td: # Add some subdirs to make sure that it can create them @@ -265,11 +261,11 @@ class TestSplitYamlProvider(TestCase): # Dynamic plan plan = target.plan(dynamic_zone) - self.assertEquals(6, len([c for c in plan.changes + self.assertEquals(5, len([c for c in plan.changes if isinstance(c, Create)])) self.assertFalse(isdir(dynamic_zone_dir)) # Apply it - self.assertEquals(6, target.apply(plan)) + self.assertEquals(5, target.apply(plan)) self.assertTrue(isdir(dynamic_zone_dir)) # There should be no changes after the round trip @@ -327,7 +323,7 @@ class TestSplitYamlProvider(TestCase): self.assertTrue('dynamic' in dyna) # Singular again. - for record_name in ('cname', 'dname', 'simple-weighted'): + for record_name in ('cname', 'simple-weighted'): yaml_file = join( dynamic_zone_dir, '{}.yaml'.format(record_name)) self.assertTrue(isfile(yaml_file)) @@ -391,8 +387,8 @@ class TestOverridingYamlProvider(TestCase): # Load the base, should see the 5 records base.populate(zone) got = {r.name: r for r in zone.records} - self.assertEquals(6, len(got)) - # We get the "dynamic" A from the bae config + self.assertEquals(5, len(got)) + # We get the "dynamic" A from the base config self.assertTrue('dynamic' in got['a'].data) # No added self.assertFalse('added' in got) @@ -400,7 +396,7 @@ class TestOverridingYamlProvider(TestCase): # Load the overrides, should replace one and add 1 override.populate(zone) got = {r.name: r for r in zone.records} - self.assertEquals(7, len(got)) + self.assertEquals(6, len(got)) # 'a' was replaced with a generic record self.assertEquals({ 'ttl': 3600, diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index 10e9575..524f8f2 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -2670,73 +2670,6 @@ class TestDynamicRecords(TestCase): self.assertTrue(rules) self.assertEquals(cname_data['dynamic']['rules'][0], rules[0].data) - def test_simple_dname_weighted(self): - dname_data = { - 'dynamic': { - 'pools': { - 'one': { - 'values': [{ - 'value': 'one.dname.target.', - }], - }, - 'two': { - 'values': [{ - 'value': 'two.dname.target.', - }], - }, - 'three': { - 'values': [{ - 'weight': 12, - 'value': 'three-1.dname.target.', - }, { - 'weight': 32, - 'value': 'three-2.dname.target.', - }] - }, - }, - 'rules': [{ - 'geos': ['AF', 'EU'], - 'pool': 'three', - }, { - 'geos': ['NA-US-CA'], - 'pool': 'two', - }, { - 'pool': 'one', - }], - }, - 'ttl': 60, - 'value': 'dname.target.', - } - dname = DnameRecord(self.zone, 'weighted', dname_data) - self.assertEquals('DNAME', dname._type) - self.assertEquals(dname_data['ttl'], dname.ttl) - self.assertEquals(dname_data['value'], dname.value) - - dynamic = dname.dynamic - self.assertTrue(dynamic) - - pools = dynamic.pools - self.assertTrue(pools) - self.assertEquals({ - 'value': 'one.dname.target.', - 'weight': 1, - }, pools['one'].data['values'][0]) - self.assertEquals({ - 'value': 'two.dname.target.', - 'weight': 1, - }, pools['two'].data['values'][0]) - self.assertEquals([{ - 'value': 'three-1.dname.target.', - 'weight': 12, - }, { - 'value': 'three-2.dname.target.', - 'weight': 32, - }], pools['three'].data['values']) - - rules = dynamic.rules - self.assertTrue(rules) - self.assertEquals(dname_data['dynamic']['rules'][0], rules[0].data) - def test_dynamic_validation(self): # Missing pools a_data = { From 3acea0d89d88a7f921429ab77e3efe9cd3fde392 Mon Sep 17 00:00:00 2001 From: Jonathan Leroy Date: Sat, 31 Oct 2020 01:09:37 +0100 Subject: [PATCH 032/358] Handle multiples sources on aliased zones --- octodns/manager.py | 143 +++++++++++++++++++++------------------ octodns/provider/yaml.py | 5 +- octodns/zone.py | 4 +- 3 files changed, 82 insertions(+), 70 deletions(-) diff --git a/octodns/manager.py b/octodns/manager.py index eff3a74..0517c6e 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -121,23 +121,6 @@ class Manager(object): raise ManagerException('Incorrect provider config for {}' .format(provider_name)) - for zone_name, zone_config in self.config['zones'].copy().items(): - if 'alias' in zone_config: - source_zone = zone_config['alias'] - # Check that the source zone is defined. - if source_zone not in self.config['zones']: - self.log.exception('Invalid alias zone') - raise ManagerException('Invalid alias zone {}: ' - 'source zone {} does not exist' - .format(zone_name, source_zone)) - self.config['zones'][zone_name] = \ - self.config['zones'][source_zone] - self.config['zones'][zone_name]['is_alias'] = True - self.config['zones'][zone_name]['file'] = source_zone - else: - self.config['zones'][zone_name]['is_alias'] = False - self.config['zones'][zone_name]['file'] = zone_name - zone_tree = {} # sort by reversed strings so that parent zones always come first for name in sorted(self.config['zones'].keys(), key=lambda s: s[::-1]): @@ -239,23 +222,32 @@ class Manager(object): self.log.debug('configured_sub_zones: subs=%s', sub_zone_names) return set(sub_zone_names) - def _populate_and_plan(self, zone_name, file, is_alias, sources, targets, + def _populate_and_plan(self, zone_name, sources, targets, desired=None, lenient=False): - self.log.debug('sync: populating, zone=%s, file=%s, is_alias=%s, ' - 'lenient=%s', zone_name, file, is_alias, lenient) + self.log.debug('sync: populating, zone=%s, lenient=%s', + zone_name, lenient) zone = Zone(zone_name, - sub_zones=self.configured_sub_zones(zone_name), file=file, - is_alias=is_alias) - for source in sources: - try: - source.populate(zone, lenient=lenient) - except TypeError as e: - if "keyword argument 'lenient'" not in text_type(e): - raise - self.log.warn(': provider %s does not accept lenient param', - source.__class__.__name__) - source.populate(zone) + sub_zones=self.configured_sub_zones(zone_name)) + + if not desired: + for source in sources: + try: + source.populate(zone, lenient=lenient) + except TypeError as e: + if "keyword argument 'lenient'" not in text_type(e): + raise + self.log.warn(': provider %s does not accept lenient ' + 'param', source.__class__.__name__) + source.populate(zone) + + else: + for _, records in desired._records.items(): + for record in records: + d = record.data + d['type'] = record._type + r = Record.new(zone, record.name, d, source=record.source) + zone.add_record(r, lenient=lenient) self.log.debug('sync: planning, zone=%s', zone_name) plans = [] @@ -284,11 +276,22 @@ class Manager(object): if eligible_zones: zones = [z for z in zones if z[0] in eligible_zones] + aliased_zones = {} futures = [] for zone_name, config in zones: self.log.info('sync: zone=%s', zone_name) - file = config.get('file') - is_alias = config.get('is_alias') + if 'alias' in config: + source_zone = config['alias'] + # Check that the source zone is defined. + if source_zone not in self.config['zones']: + self.log.exception('Invalid alias zone') + raise ManagerException('Invalid alias zone {}: ' + 'source zone {} does not exist' + .format(zone_name, source_zone)) + + aliased_zones[zone_name] = source_zone + continue + lenient = config.get('lenient', False) try: sources = config['sources'] @@ -345,14 +348,32 @@ class Manager(object): .format(zone_name, target)) futures.append(self._executor.submit(self._populate_and_plan, - zone_name, file, is_alias, - sources, targets, - lenient=lenient)) + zone_name, sources, + targets, lenient=lenient)) # Wait on all results and unpack/flatten them in to a list of target & # plan pairs. plans = [p for f in futures for p in f.result()] + # Populate aliases zones. + futures = [] + for zone_name, zone_source in aliased_zones.items(): + plan = [p for t, p in plans if p.desired.name == zone_source] + if not plan: + continue + + source_config = self.config['zones'][zone_source] + futures.append(self._executor.submit( + self._populate_and_plan, + zone_name, + [self.providers[s] for s in source_config['sources']], + [self.providers[t] for t in source_config['targets']], + desired=plan[0].desired, + lenient=lenient + )) + + plans += [p for f in futures for p in f.result()] + # Best effort sort plans children first so that we create/update # children zones before parents which should allow us to more safely # extract things into sub-zones. Combining a child back into a parent @@ -440,32 +461,30 @@ class Manager(object): def validate_configs(self): for zone_name, config in self.config['zones'].items(): - file = config.get('file', False) - is_alias = config.get('is_alias', False) - zone = Zone(zone_name, self.configured_sub_zones(zone_name), - file, is_alias) + zone = Zone(zone_name, self.configured_sub_zones(zone_name)) - try: - sources = config['sources'] - except KeyError: - raise ManagerException('Zone {} is missing sources' - .format(zone_name)) + if not config.get('alias'): + try: + sources = config['sources'] + except KeyError: + raise ManagerException('Zone {} is missing sources' + .format(zone_name)) - try: - # rather than using a list comprehension, we break this loop - # out so that the `except` block below can reference the - # `source` - collected = [] - for source in sources: - collected.append(self.providers[source]) - sources = collected - except KeyError: - raise ManagerException('Zone {}, unknown source: {}' - .format(zone_name, source)) + try: + # rather than using a list comprehension, we break this + # loop out so that the `except` block below can reference + # the `source` + collected = [] + for source in sources: + collected.append(self.providers[source]) + sources = collected + except KeyError: + raise ManagerException('Zone {}, unknown source: {}' + .format(zone_name, source)) - for source in sources: - if isinstance(source, YamlProvider): - source.populate(zone) + for source in sources: + if isinstance(source, YamlProvider): + source.populate(zone) def get_zone(self, zone_name): if not zone_name[-1] == '.': @@ -474,10 +493,6 @@ class Manager(object): for name, config in self.config['zones'].items(): if name == zone_name: - file = config.get('file', False) - is_alias = config.get('is_alias', False) - - return Zone(name, self.configured_sub_zones(name), - file, is_alias) + return Zone(name, self.configured_sub_zones(name)) raise ManagerException('Unknown zone name {}'.format(zone_name)) diff --git a/octodns/provider/yaml.py b/octodns/provider/yaml.py index f1b921b..55a1632 100644 --- a/octodns/provider/yaml.py +++ b/octodns/provider/yaml.py @@ -139,8 +139,7 @@ class YamlProvider(BaseProvider): filename) def populate(self, zone, target=False, lenient=False): - self.log.debug('populate: name=%s, file=%s, is_alias:%s, target=%s, ' - 'lenient=%s', zone.name, zone.file, zone.is_alias, + self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, target, lenient) if target: @@ -149,7 +148,7 @@ class YamlProvider(BaseProvider): return False before = len(zone.records) - filename = join(self.directory, '{}yaml'.format(zone.file)) + filename = join(self.directory, '{}yaml'.format(zone.name)) self._populate_from_file(filename, zone, lenient) self.log.info('populate: found %s records, exists=False', diff --git a/octodns/zone.py b/octodns/zone.py index 0a78f72..5f099ac 100644 --- a/octodns/zone.py +++ b/octodns/zone.py @@ -35,15 +35,13 @@ def _is_eligible(record): class Zone(object): log = getLogger('Zone') - def __init__(self, name, sub_zones, file=None, is_alias=False): + def __init__(self, name, sub_zones): if not name[-1] == '.': raise Exception('Invalid zone name {}, missing ending dot' .format(name)) # Force everything to lowercase just to be safe self.name = text_type(name).lower() if name else name self.sub_zones = sub_zones - self.file = text_type(file if file else name).lower() - self.is_alias = is_alias # We're grouping by node, it allows us to efficiently search for # duplicates and detect when CNAMEs co-exist with other records self._records = defaultdict(set) From 0b3a99bb8ce593f16a7849b3eb5e8b8e361670c4 Mon Sep 17 00:00:00 2001 From: Jonathan Leroy Date: Sat, 31 Oct 2020 09:38:35 +0100 Subject: [PATCH 033/358] Implement Record.copy() function Flip if in _populate_and_plan() --- octodns/manager.py | 15 ++++++--------- octodns/record/__init__.py | 9 +++++++++ 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/octodns/manager.py b/octodns/manager.py index 0517c6e..d312fad 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -230,7 +230,12 @@ class Manager(object): zone = Zone(zone_name, sub_zones=self.configured_sub_zones(zone_name)) - if not desired: + if desired: + for _, records in desired._records.items(): + for record in records: + zone.add_record(record.copy(zone=zone), lenient=lenient) + + else: for source in sources: try: source.populate(zone, lenient=lenient) @@ -241,14 +246,6 @@ class Manager(object): 'param', source.__class__.__name__) source.populate(zone) - else: - for _, records in desired._records.items(): - for record in records: - d = record.data - d['type'] = record._type - r = Record.new(zone, record.name, d, source=record.source) - zone.add_record(r, lenient=lenient) - self.log.debug('sync: planning, zone=%s', zone_name) plans = [] diff --git a/octodns/record/__init__.py b/octodns/record/__init__.py index 08ec2ee..6c4e79f 100644 --- a/octodns/record/__init__.py +++ b/octodns/record/__init__.py @@ -151,6 +151,7 @@ class Record(EqualityTupleMixin): # force everything lower-case just to be safe self.name = text_type(name).lower() if name else name self.source = source + self._raw_data = data self.ttl = int(data['ttl']) self._octodns = data.get('octodns', {}) @@ -219,6 +220,14 @@ class Record(EqualityTupleMixin): if self.ttl != other.ttl: return Update(self, other) + def copy(self, zone=None): + return Record( + zone if zone else self.zone, + self.name, + self._raw_data, + self.source + ) + # NOTE: we're using __hash__ and ordering methods that consider Records # equivalent if they have the same name & _type. Values are ignored. This # is useful when computing diffs/changes. From 8679bb4899b0d622a8f304235a13f9d8f8fc5711 Mon Sep 17 00:00:00 2001 From: Jonathan Leroy Date: Sat, 31 Oct 2020 09:41:27 +0100 Subject: [PATCH 034/358] Remove sources argument when calling _populate_and_plan() for an alias zone --- octodns/manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/manager.py b/octodns/manager.py index d312fad..45a16eb 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -363,7 +363,7 @@ class Manager(object): futures.append(self._executor.submit( self._populate_and_plan, zone_name, - [self.providers[s] for s in source_config['sources']], + [], [self.providers[t] for t in source_config['targets']], desired=plan[0].desired, lenient=lenient From 6f01a543df9b470b54575e1a00cbc5aa7c83a7a9 Mon Sep 17 00:00:00 2001 From: Jonathan Leroy Date: Sat, 31 Oct 2020 09:43:23 +0100 Subject: [PATCH 035/358] Implement configuration validation for alias zones --- octodns/manager.py | 46 +++++++++++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 19 deletions(-) diff --git a/octodns/manager.py b/octodns/manager.py index 45a16eb..a807a5c 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -460,28 +460,36 @@ class Manager(object): for zone_name, config in self.config['zones'].items(): zone = Zone(zone_name, self.configured_sub_zones(zone_name)) - if not config.get('alias'): - try: - sources = config['sources'] - except KeyError: - raise ManagerException('Zone {} is missing sources' - .format(zone_name)) + source_zone = config.get('alias') + if source_zone: + if source_zone not in self.config['zones']: + self.log.exception('Invalid alias zone') + raise ManagerException('Invalid alias zone {}: ' + 'source zone {} does not exist' + .format(zone_name, source_zone)) + continue - try: - # rather than using a list comprehension, we break this - # loop out so that the `except` block below can reference - # the `source` - collected = [] - for source in sources: - collected.append(self.providers[source]) - sources = collected - except KeyError: - raise ManagerException('Zone {}, unknown source: {}' - .format(zone_name, source)) + try: + sources = config['sources'] + except KeyError: + raise ManagerException('Zone {} is missing sources' + .format(zone_name)) + try: + # rather than using a list comprehension, we break this + # loop out so that the `except` block below can reference + # the `source` + collected = [] for source in sources: - if isinstance(source, YamlProvider): - source.populate(zone) + collected.append(self.providers[source]) + sources = collected + except KeyError: + raise ManagerException('Zone {}, unknown source: {}' + .format(zone_name, source)) + + for source in sources: + if isinstance(source, YamlProvider): + source.populate(zone) def get_zone(self, zone_name): if not zone_name[-1] == '.': From 4fb102e4be973609f6e356387072a30f9b2cff36 Mon Sep 17 00:00:00 2001 From: Jonathan Leroy Date: Sat, 31 Oct 2020 09:44:06 +0100 Subject: [PATCH 036/358] Fixes tests related to _populate_and_plan() --- tests/test_octodns_manager.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/test_octodns_manager.py b/tests/test_octodns_manager.py index cc39c87..b99e460 100644 --- a/tests/test_octodns_manager.py +++ b/tests/test_octodns_manager.py @@ -310,8 +310,7 @@ class TestManager(TestCase): pass # This should be ok, we'll fall back to not passing it - manager._populate_and_plan('unit.tests.', None, False, - [NoLenient()], []) + manager._populate_and_plan('unit.tests.', [NoLenient()], []) class NoZone(SimpleProvider): @@ -320,8 +319,7 @@ class TestManager(TestCase): # This will blow up, we don't fallback for source with self.assertRaises(TypeError): - manager._populate_and_plan('unit.tests.', None, False, - [NoZone()], []) + manager._populate_and_plan('unit.tests.', [NoZone()], []) def test_alias_zones(self): with TemporaryDirectory() as tmpdir: From a1e62281f6cd7cdf265930520b05ab7149f445df Mon Sep 17 00:00:00 2001 From: Jonathan Leroy Date: Sat, 31 Oct 2020 10:54:17 +0100 Subject: [PATCH 037/358] Fixes record copy when record is a child class of Record and as no record type specified in its data --- octodns/record/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/octodns/record/__init__.py b/octodns/record/__init__.py index 6c4e79f..ae9ccb1 100644 --- a/octodns/record/__init__.py +++ b/octodns/record/__init__.py @@ -221,7 +221,11 @@ class Record(EqualityTupleMixin): return Update(self, other) def copy(self, zone=None): - return Record( + _type = getattr(self, '_type') + if not self._raw_data.get('type'): + self._raw_data['type'] = _type + + return Record.new( zone if zone else self.zone, self.name, self._raw_data, From a2aa98377d512d1f36880bbe0e2778a78436015b Mon Sep 17 00:00:00 2001 From: Jonathan Leroy Date: Sat, 31 Oct 2020 10:57:14 +0100 Subject: [PATCH 038/358] Add tests for Record.copy() --- tests/test_octodns_record.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index 524f8f2..a286efb 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -813,6 +813,39 @@ class TestRecord(TestCase): }) self.assertTrue('Unknown record type' in text_type(ctx.exception)) + def test_record_copy(self): + a = Record.new(self.zone, 'a', { + 'ttl': 44, + 'type': 'A', + 'value': '1.2.3.4', + }) + + # Identical copy. + b = a.copy() + self.assertIsInstance(b, ARecord) + self.assertEquals('unit.tests.', b.zone.name) + self.assertEquals('a', b.name) + self.assertEquals('A', b._type) + self.assertEquals(['1.2.3.4'], b.values) + + # Copy with another zone object. + c_zone = Zone('other.tests.', []) + c = a.copy(c_zone) + self.assertIsInstance(c, ARecord) + self.assertEquals('other.tests.', c.zone.name) + self.assertEquals('a', c.name) + self.assertEquals('A', c._type) + self.assertEquals(['1.2.3.4'], c.values) + + # Record with no record type specified in data. + d_data = { + 'ttl': 600, + 'values': ['just a test'] + } + d = TxtRecord(self.zone, 'txt', d_data) + d.copy() + self.assertEquals('TXT', d._type) + def test_change(self): existing = Record.new(self.zone, 'txt', { 'ttl': 44, From b0da090723a54821d989939d2cede1aac353feff Mon Sep 17 00:00:00 2001 From: Jonathan Leroy Date: Sat, 31 Oct 2020 14:09:54 +0100 Subject: [PATCH 039/358] Add test for alias zones --- tests/config/unknown-source-zone.yaml | 9 +++++++-- tests/test_octodns_manager.py | 15 +++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/tests/config/unknown-source-zone.yaml b/tests/config/unknown-source-zone.yaml index 313853e..a3940ff 100644 --- a/tests/config/unknown-source-zone.yaml +++ b/tests/config/unknown-source-zone.yaml @@ -9,5 +9,10 @@ providers: directory: env/YAML_TMP_DIR zones: unit.tests.: - alias: unit-source.tests. - + sources: + - in + targets: + - dump + + alias.tests.: + alias: does-not-exists.tests. diff --git a/tests/test_octodns_manager.py b/tests/test_octodns_manager.py index b99e460..084ad08 100644 --- a/tests/test_octodns_manager.py +++ b/tests/test_octodns_manager.py @@ -167,6 +167,21 @@ class TestManager(TestCase): .sync(eligible_targets=['foo']) self.assertEquals(0, tc) + def test_aliases(self): + with TemporaryDirectory() as tmpdir: + environ['YAML_TMP_DIR'] = tmpdir.dirname + # Only allow a target that doesn't exist + tc = Manager(get_config_filename('simple-alias-zone.yaml')) \ + .sync() + self.assertEquals(0, tc) + + with self.assertRaises(ManagerException) as ctx: + tc = Manager(get_config_filename('unknown-source-zone.yaml')) \ + .sync() + self.assertEquals('Invalid alias zone alias.tests.: source zone ' + 'does-not-exists.tests. does not exist', + text_type(ctx.exception)) + def test_compare(self): with TemporaryDirectory() as tmpdir: environ['YAML_TMP_DIR'] = tmpdir.dirname From a6d8848fad8d773f5f281376b235aa90d82737ae Mon Sep 17 00:00:00 2001 From: Jonathan Leroy Date: Sat, 31 Oct 2020 14:19:43 +0100 Subject: [PATCH 040/358] Fixes linting issues --- tests/test_octodns_manager.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_octodns_manager.py b/tests/test_octodns_manager.py index 084ad08..c9693c1 100644 --- a/tests/test_octodns_manager.py +++ b/tests/test_octodns_manager.py @@ -172,14 +172,14 @@ class TestManager(TestCase): environ['YAML_TMP_DIR'] = tmpdir.dirname # Only allow a target that doesn't exist tc = Manager(get_config_filename('simple-alias-zone.yaml')) \ - .sync() + .sync() self.assertEquals(0, tc) with self.assertRaises(ManagerException) as ctx: tc = Manager(get_config_filename('unknown-source-zone.yaml')) \ - .sync() + .sync() self.assertEquals('Invalid alias zone alias.tests.: source zone ' - 'does-not-exists.tests. does not exist', + 'does-not-exists.tests. does not exist', text_type(ctx.exception)) def test_compare(self): From 6b568f5c9dd06ad70f50bdefbf48122c3030f5ac Mon Sep 17 00:00:00 2001 From: Jonathan Leroy Date: Sat, 31 Oct 2020 19:07:34 +0100 Subject: [PATCH 041/358] Compare alias zones content with the one of its parent zone, even if there is no changes in the parent zone --- octodns/manager.py | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/octodns/manager.py b/octodns/manager.py index a807a5c..4680640 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -261,7 +261,8 @@ class Manager(object): if plan: plans.append((target, plan)) - return plans + # Return the zone as it's the desired state + return plans, zone def sync(self, eligible_zones=[], eligible_sources=[], eligible_targets=[], dry_run=True, force=False): @@ -281,7 +282,8 @@ class Manager(object): source_zone = config['alias'] # Check that the source zone is defined. if source_zone not in self.config['zones']: - self.log.exception('Invalid alias zone') + self.log.error('Invalid alias zone {}, target {} does ' + 'not exist'.format(zone_name, source_zone)) raise ManagerException('Invalid alias zone {}: ' 'source zone {} does not exist' .format(zone_name, source_zone)) @@ -348,28 +350,32 @@ class Manager(object): zone_name, sources, targets, lenient=lenient)) - # Wait on all results and unpack/flatten them in to a list of target & - # plan pairs. - plans = [p for f in futures for p in f.result()] + # Wait on all results and unpack/flatten the plans and store the + # desired states in case we need them below + plans = [] + desired = {} + for future in futures: + ps, d = future.result() + desired[d.name] = d + for plan in ps: + plans.append(plan) # Populate aliases zones. futures = [] for zone_name, zone_source in aliased_zones.items(): - plan = [p for t, p in plans if p.desired.name == zone_source] - if not plan: - continue - source_config = self.config['zones'][zone_source] futures.append(self._executor.submit( self._populate_and_plan, zone_name, [], [self.providers[t] for t in source_config['targets']], - desired=plan[0].desired, + desired=desired[zone_source], lenient=lenient )) - plans += [p for f in futures for p in f.result()] + # Wait on results and unpack/flatten the plans, ignore the desired here + # as these are aliased zones + plans += [p for f in futures for p in f.result()[0]] # Best effort sort plans children first so that we create/update # children zones before parents which should allow us to more safely From 038ae422841fdc1659d4bb8d1579ff30bea4da56 Mon Sep 17 00:00:00 2001 From: Jonathan Leroy Date: Sat, 31 Oct 2020 20:16:26 +0100 Subject: [PATCH 042/358] Add comments and fixes some tests --- tests/test_octodns_manager.py | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/tests/test_octodns_manager.py b/tests/test_octodns_manager.py index c9693c1..3106b07 100644 --- a/tests/test_octodns_manager.py +++ b/tests/test_octodns_manager.py @@ -170,11 +170,12 @@ class TestManager(TestCase): def test_aliases(self): with TemporaryDirectory() as tmpdir: environ['YAML_TMP_DIR'] = tmpdir.dirname - # Only allow a target that doesn't exist + # Alias zones with a valid target. tc = Manager(get_config_filename('simple-alias-zone.yaml')) \ .sync() self.assertEquals(0, tc) + # Alias zone with an invalid target. with self.assertRaises(ManagerException) as ctx: tc = Manager(get_config_filename('unknown-source-zone.yaml')) \ .sync() @@ -301,6 +302,17 @@ class TestManager(TestCase): .validate_configs() self.assertTrue('unknown source' in text_type(ctx.exception)) + # Alias zone using an invalid source zone. + with self.assertRaises(ManagerException) as ctx: + Manager(get_config_filename('unknown-source-zone.yaml')) \ + .validate_configs() + self.assertTrue('Invalid alias zone' in + text_type(ctx.exception)) + + # Valid config file using an alias zone. + Manager(get_config_filename('simple-alias-zone.yaml')) \ + .validate_configs() + def test_get_zone(self): Manager(get_config_filename('simple.yaml')).get_zone('unit.tests.') @@ -336,20 +348,6 @@ class TestManager(TestCase): with self.assertRaises(TypeError): manager._populate_and_plan('unit.tests.', [NoZone()], []) - def test_alias_zones(self): - with TemporaryDirectory() as tmpdir: - environ['YAML_TMP_DIR'] = tmpdir.dirname - - Manager(get_config_filename('simple-alias-zone.yaml')) \ - .validate_configs() - - with self.assertRaises(ManagerException) as ctx: - Manager(get_config_filename('unknown-source-zone.yaml')) \ - .validate_configs() - self.assertTrue('Invalid alias zone' in - text_type(ctx.exception)) - - class TestMainThreadExecutor(TestCase): def test_success(self): From 9a4812223e6c9ef3ca1cf6feb2b6c0ea7363a2ef Mon Sep 17 00:00:00 2001 From: Jonathan Leroy Date: Sat, 31 Oct 2020 20:19:09 +0100 Subject: [PATCH 043/358] Add missing empty line --- tests/test_octodns_manager.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_octodns_manager.py b/tests/test_octodns_manager.py index 3106b07..6affa17 100644 --- a/tests/test_octodns_manager.py +++ b/tests/test_octodns_manager.py @@ -348,6 +348,7 @@ class TestManager(TestCase): with self.assertRaises(TypeError): manager._populate_and_plan('unit.tests.', [NoZone()], []) + class TestMainThreadExecutor(TestCase): def test_success(self): From fbfa46fbcc85df9af1ac2c1565289d13d7556ec7 Mon Sep 17 00:00:00 2001 From: Jonathan Leroy Date: Sat, 31 Oct 2020 22:51:09 +0100 Subject: [PATCH 044/358] Add documentation for zones aliases --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 23ac0e8..96cd5bb 100644 --- a/README.md +++ b/README.md @@ -79,6 +79,9 @@ zones: targets: - dyn - route53 + + example.net: + alias: example.com. ``` `class` is a special key that tells OctoDNS what python class should be loaded. Any other keys will be passed as configuration values to that provider. In general any sensitive or frequently rotated values should come from environmental variables. When OctoDNS sees a value that starts with `env/` it will look for that value in the process's environment and pass the result along. @@ -87,6 +90,8 @@ Further information can be found in the `docstring` of each source and provider The `max_workers` key in the `manager` section of the config enables threading to parallelize the planning portion of the sync. +In this example, `example.net` is an alias of zone `example.com`, which means they share the same sources and targets. They will therefore have identical records. + Now that we have something to tell OctoDNS about our providers & zones we need to tell it about or records. We'll keep it simple for now and just create a single `A` record at the top-level of the domain. `config/example.com.yaml` From d3be3be7342bc35b221c70e85fc289b854e869db Mon Sep 17 00:00:00 2001 From: Jonathan Leroy Date: Sat, 31 Oct 2020 23:26:27 +0100 Subject: [PATCH 045/358] Fix coverage issue --- octodns/manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/manager.py b/octodns/manager.py index 4680640..8c3f182 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -473,7 +473,7 @@ class Manager(object): raise ManagerException('Invalid alias zone {}: ' 'source zone {} does not exist' .format(zone_name, source_zone)) - continue + continue # pragma: no cover, see Python bug #2506. try: sources = config['sources'] From e524d69f631a16b5a2327735a0dffb265be00c74 Mon Sep 17 00:00:00 2001 From: Jonathan Leroy Date: Sat, 31 Oct 2020 23:32:20 +0100 Subject: [PATCH 046/358] Fixes linting issue --- octodns/manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/manager.py b/octodns/manager.py index 8c3f182..ff9e491 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -473,7 +473,7 @@ class Manager(object): raise ManagerException('Invalid alias zone {}: ' 'source zone {} does not exist' .format(zone_name, source_zone)) - continue # pragma: no cover, see Python bug #2506. + continue # pragma: no cover, see Python bug #2506. try: sources = config['sources'] From 95a71a268ebf0f29dd109d139e5e31cbdfbe97fc Mon Sep 17 00:00:00 2001 From: Jonathan Leroy Date: Sat, 31 Oct 2020 23:51:04 +0100 Subject: [PATCH 047/358] Apply workaround for python bug #2506 witout using "pragma: no cover" comment --- octodns/manager.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/octodns/manager.py b/octodns/manager.py index ff9e491..cfc9735 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -473,7 +473,10 @@ class Manager(object): raise ManagerException('Invalid alias zone {}: ' 'source zone {} does not exist' .format(zone_name, source_zone)) - continue # pragma: no cover, see Python bug #2506. + # this is just here to satisfy coverage, see + # https://github.com/nedbat/coveragepy/issues/198 + source_zone = source_zone + continue try: sources = config['sources'] From 2d4855508c89ea70933f776a2e5b967bb17f8249 Mon Sep 17 00:00:00 2001 From: Jonathan Leroy Date: Sun, 1 Nov 2020 23:58:40 +0100 Subject: [PATCH 048/358] Check that an alias zone source is not an alias zone --- octodns/manager.py | 16 ++++++++++++++++ tests/config/alias-zone-loop.yaml | 21 +++++++++++++++++++++ tests/test_octodns_manager.py | 17 ++++++++++++++++- 3 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 tests/config/alias-zone-loop.yaml diff --git a/octodns/manager.py b/octodns/manager.py index cfc9735..5f4af55 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -280,6 +280,7 @@ class Manager(object): self.log.info('sync: zone=%s', zone_name) if 'alias' in config: source_zone = config['alias'] + # Check that the source zone is defined. if source_zone not in self.config['zones']: self.log.error('Invalid alias zone {}, target {} does ' @@ -288,6 +289,14 @@ class Manager(object): 'source zone {} does not exist' .format(zone_name, source_zone)) + # Check that the source zone is not an alias zone itself. + if 'alias' in self.config['zones'][source_zone]: + self.log.error('Invalid alias zone {}, target {} is an ' + 'alias zone'.format(zone_name, source_zone)) + raise ManagerException('Invalid alias zone {}: source ' + 'zone {} is an alias zone' + .format(zone_name, source_zone)) + aliased_zones[zone_name] = source_zone continue @@ -473,6 +482,13 @@ class Manager(object): raise ManagerException('Invalid alias zone {}: ' 'source zone {} does not exist' .format(zone_name, source_zone)) + + if 'alias' in self.config['zones'][source_zone]: + self.log.exception('Invalid alias zone') + raise ManagerException('Invalid alias zone {}: ' + 'source zone {} is an alias zone' + .format(zone_name, source_zone)) + # this is just here to satisfy coverage, see # https://github.com/nedbat/coveragepy/issues/198 source_zone = source_zone diff --git a/tests/config/alias-zone-loop.yaml b/tests/config/alias-zone-loop.yaml new file mode 100644 index 0000000..df8b53f --- /dev/null +++ b/tests/config/alias-zone-loop.yaml @@ -0,0 +1,21 @@ +manager: + max_workers: 2 +providers: + in: + class: octodns.provider.yaml.YamlProvider + directory: tests/config + dump: + class: octodns.provider.yaml.YamlProvider + directory: env/YAML_TMP_DIR +zones: + unit.tests.: + sources: + - in + targets: + - dump + + alias.tests.: + alias: unit.tests. + + alias-loop.tests.: + alias: alias.tests. diff --git a/tests/test_octodns_manager.py b/tests/test_octodns_manager.py index 6affa17..dc047e8 100644 --- a/tests/test_octodns_manager.py +++ b/tests/test_octodns_manager.py @@ -183,6 +183,14 @@ class TestManager(TestCase): 'does-not-exists.tests. does not exist', text_type(ctx.exception)) + # Alias zone that points to another alias zone. + with self.assertRaises(ManagerException) as ctx: + tc = Manager(get_config_filename('alias-zone-loop.yaml')) \ + .sync() + self.assertEquals('Invalid alias zone alias-loop.tests.: source ' + 'zone alias.tests. is an alias zone', + text_type(ctx.exception)) + def test_compare(self): with TemporaryDirectory() as tmpdir: environ['YAML_TMP_DIR'] = tmpdir.dirname @@ -306,7 +314,14 @@ class TestManager(TestCase): with self.assertRaises(ManagerException) as ctx: Manager(get_config_filename('unknown-source-zone.yaml')) \ .validate_configs() - self.assertTrue('Invalid alias zone' in + self.assertTrue('does not exist' in + text_type(ctx.exception)) + + # Alias zone that points to another alias zone. + with self.assertRaises(ManagerException) as ctx: + Manager(get_config_filename('alias-zone-loop.yaml')) \ + .validate_configs() + self.assertTrue('is an alias zone' in text_type(ctx.exception)) # Valid config file using an alias zone. From 19798e3acfc65bc2c2b90495588fc6a57d9f668e Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 2 Nov 2020 07:26:07 -0800 Subject: [PATCH 049/358] Only allow ALIAS on APEX --- octodns/record/__init__.py | 8 +++++ tests/fixtures/constellix-records.json | 37 --------------------- tests/fixtures/dnsmadeeasy-records.json | 14 -------- tests/test_octodns_provider_constellix.py | 12 ++----- tests/test_octodns_provider_dnsmadeeasy.py | 12 ++----- tests/test_octodns_provider_mythicbeasts.py | 4 +-- tests/test_octodns_record.py | 14 ++++++-- 7 files changed, 28 insertions(+), 73 deletions(-) diff --git a/octodns/record/__init__.py b/octodns/record/__init__.py index 08ec2ee..d42b576 100644 --- a/octodns/record/__init__.py +++ b/octodns/record/__init__.py @@ -782,6 +782,14 @@ class AliasRecord(_ValueMixin, Record): _type = 'ALIAS' _value_type = AliasValue + @classmethod + def validate(cls, name, fqdn, data): + reasons = [] + if name != '': + reasons.append('non-root ALIAS not allowed') + reasons.extend(super(AliasRecord, cls).validate(name, fqdn, data)) + return reasons + class CaaValue(EqualityTupleMixin): # https://tools.ietf.org/html/rfc6844#page-5 diff --git a/tests/fixtures/constellix-records.json b/tests/fixtures/constellix-records.json index c1f1fb4..689fd53 100644 --- a/tests/fixtures/constellix-records.json +++ b/tests/fixtures/constellix-records.json @@ -523,43 +523,6 @@ "roundRobinFailover": [], "pools": [], "poolsDetail": [] -}, { - "id": 1808603, - "type": "ANAME", - "recordType": "aname", - "name": "sub", - "recordOption": "roundRobin", - "noAnswer": false, - "note": "", - "ttl": 1800, - "gtdRegion": 1, - "parentId": 123123, - "parent": "domain", - "source": "Domain", - "modifiedTs": 1565153387855, - "value": [{ - "value": "aname.unit.tests.", - "disableFlag": false - }], - "roundRobin": [{ - "value": "aname.unit.tests.", - "disableFlag": false - }], - "geolocation": null, - "recordFailover": { - "disabled": false, - "failoverType": 1, - "failoverTypeStr": "Normal (always lowest level)", - "values": [] - }, - "failover": { - "disabled": false, - "failoverType": 1, - "failoverTypeStr": "Normal (always lowest level)", - "values": [] - }, - "pools": [], - "poolsDetail": [] }, { "id": 1808520, "type": "A", diff --git a/tests/fixtures/dnsmadeeasy-records.json b/tests/fixtures/dnsmadeeasy-records.json index 4d3ba64..aefd6ce 100644 --- a/tests/fixtures/dnsmadeeasy-records.json +++ b/tests/fixtures/dnsmadeeasy-records.json @@ -320,20 +320,6 @@ "name": "", "value": "aname.unit.tests.", "id": 11189895, - "type": "ANAME" - }, { - "failover": false, - "monitor": false, - "sourceId": 123123, - "dynamicDns": false, - "failed": false, - "gtdLocation": "DEFAULT", - "hardLink": false, - "ttl": 1800, - "source": 1, - "name": "sub", - "value": "aname", - "id": 11189896, "type": "ANAME" }, { "failover": false, diff --git a/tests/test_octodns_provider_constellix.py b/tests/test_octodns_provider_constellix.py index c19ae29..bc17b50 100644 --- a/tests/test_octodns_provider_constellix.py +++ b/tests/test_octodns_provider_constellix.py @@ -42,12 +42,6 @@ class TestConstellixProvider(TestCase): 'value': 'aname.unit.tests.' })) - expected.add_record(Record.new(expected, 'sub', { - 'ttl': 1800, - 'type': 'ALIAS', - 'value': 'aname.unit.tests.' - })) - for record in list(expected.records): if record.name == 'sub' and record._type == 'NS': expected._remove_record(record) @@ -107,14 +101,14 @@ class TestConstellixProvider(TestCase): zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals(15, len(zone.records)) + self.assertEquals(14, len(zone.records)) changes = self.expected.changes(zone, provider) self.assertEquals(0, len(changes)) # 2nd populate makes no network calls/all from cache again = Zone('unit.tests.', []) provider.populate(again) - self.assertEquals(15, len(again.records)) + self.assertEquals(14, len(again.records)) # bust the cache del provider._zone_records[zone.name] @@ -169,7 +163,7 @@ class TestConstellixProvider(TestCase): }), ]) - self.assertEquals(18, provider._client._request.call_count) + self.assertEquals(17, provider._client._request.call_count) provider._client._request.reset_mock() diff --git a/tests/test_octodns_provider_dnsmadeeasy.py b/tests/test_octodns_provider_dnsmadeeasy.py index 50fa576..0ad059d 100644 --- a/tests/test_octodns_provider_dnsmadeeasy.py +++ b/tests/test_octodns_provider_dnsmadeeasy.py @@ -44,12 +44,6 @@ class TestDnsMadeEasyProvider(TestCase): 'value': 'aname.unit.tests.' })) - expected.add_record(Record.new(expected, 'sub', { - 'ttl': 1800, - 'type': 'ALIAS', - 'value': 'aname.unit.tests.' - })) - for record in list(expected.records): if record.name == 'sub' and record._type == 'NS': expected._remove_record(record) @@ -108,14 +102,14 @@ class TestDnsMadeEasyProvider(TestCase): zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals(15, len(zone.records)) + self.assertEquals(14, len(zone.records)) changes = self.expected.changes(zone, provider) self.assertEquals(0, len(changes)) # 2nd populate makes no network calls/all from cache again = Zone('unit.tests.', []) provider.populate(again) - self.assertEquals(15, len(again.records)) + self.assertEquals(14, len(again.records)) # bust the cache del provider._zone_records[zone.name] @@ -180,7 +174,7 @@ class TestDnsMadeEasyProvider(TestCase): 'port': 30 }), ]) - self.assertEquals(27, provider._client._request.call_count) + self.assertEquals(26, provider._client._request.call_count) provider._client._request.reset_mock() diff --git a/tests/test_octodns_provider_mythicbeasts.py b/tests/test_octodns_provider_mythicbeasts.py index 960bd65..f78cb0b 100644 --- a/tests/test_octodns_provider_mythicbeasts.py +++ b/tests/test_octodns_provider_mythicbeasts.py @@ -171,7 +171,7 @@ class TestMythicBeastsProvider(TestCase): def test_command_generation(self): zone = Zone('unit.tests.', []) - zone.add_record(Record.new(zone, 'prawf-alias', { + zone.add_record(Record.new(zone, '', { 'ttl': 60, 'type': 'ALIAS', 'value': 'alias.unit.tests.', @@ -228,7 +228,7 @@ class TestMythicBeastsProvider(TestCase): ) expected_commands = [ - 'ADD prawf-alias.unit.tests 60 ANAME alias.unit.tests.', + 'ADD unit.tests 60 ANAME alias.unit.tests.', 'ADD prawf-ns.unit.tests 300 NS alias.unit.tests.', 'ADD prawf-ns.unit.tests 300 NS alias2.unit.tests.', 'ADD prawf-a.unit.tests 60 A 1.2.3.4', diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index 524f8f2..ffca1d0 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -1710,6 +1710,16 @@ class TestRecordValidation(TestCase): 'value': 'foo.bar.com.', }) + # root only + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, 'nope', { + 'type': 'ALIAS', + 'ttl': 600, + 'value': 'foo.bar.com.', + }) + self.assertEquals(['non-root ALIAS not allowed'], + ctx.exception.reasons) + # missing value with self.assertRaises(ValidationError) as ctx: Record.new(self.zone, '', { @@ -1720,7 +1730,7 @@ class TestRecordValidation(TestCase): # missing value with self.assertRaises(ValidationError) as ctx: - Record.new(self.zone, 'www', { + Record.new(self.zone, '', { 'type': 'ALIAS', 'ttl': 600, 'value': None @@ -1729,7 +1739,7 @@ class TestRecordValidation(TestCase): # empty value with self.assertRaises(ValidationError) as ctx: - Record.new(self.zone, 'www', { + Record.new(self.zone, '', { 'type': 'ALIAS', 'ttl': 600, 'value': '' From 364b70048fbf91251438db76bed3148660cf7a83 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 2 Nov 2020 07:27:48 -0800 Subject: [PATCH 050/358] Fix coverage pragma grep --- script/coverage | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/coverage b/script/coverage index 32bdaea..bd6e4c9 100755 --- a/script/coverage +++ b/script/coverage @@ -27,7 +27,7 @@ export DYN_USERNAME= export GOOGLE_APPLICATION_CREDENTIALS= # Don't allow disabling coverage -grep -r -I --line-number "# pragma: nocover" octodns && { +grep -r -I --line-number "# pragma: +no.*cover" octodns && { echo "Code coverage should not be disabled" exit 1 } From dc9dc45ae638e3ae5384ef4583c722e2b4efdbc9 Mon Sep 17 00:00:00 2001 From: Jonathan Leroy Date: Mon, 2 Nov 2020 18:00:48 +0100 Subject: [PATCH 051/358] Fixes tests after merging of #620 --- tests/fixtures/gandi-no-changes.json | 9 +++++++++ tests/test_octodns_provider_gandi.py | 12 +++++++++--- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/tests/fixtures/gandi-no-changes.json b/tests/fixtures/gandi-no-changes.json index 1154628..b018785 100644 --- a/tests/fixtures/gandi-no-changes.json +++ b/tests/fixtures/gandi-no-changes.json @@ -46,6 +46,15 @@ "unit.tests." ] }, + { + "rrset_type": "DNAME", + "rrset_ttl": 300, + "rrset_name": "dname", + "rrset_href": "https://api.gandi.net/v5/livedns/domains/unit.tests/records/dname/DNAME", + "rrset_values": [ + "unit.tests." + ] + }, { "rrset_type": "CNAME", "rrset_ttl": 3600, diff --git a/tests/test_octodns_provider_gandi.py b/tests/test_octodns_provider_gandi.py index a818919..3cee392 100644 --- a/tests/test_octodns_provider_gandi.py +++ b/tests/test_octodns_provider_gandi.py @@ -116,7 +116,7 @@ class TestGandiProvider(TestCase): zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals(13, len(zone.records)) + self.assertEquals(14, len(zone.records)) changes = self.expected.changes(zone, provider) self.assertEquals(0, len(changes)) @@ -133,7 +133,7 @@ class TestGandiProvider(TestCase): provider.populate(zone) self.assertEquals(11, len(zone.records)) changes = self.expected.changes(zone, provider) - self.assertEquals(23, len(changes)) + self.assertEquals(24, len(changes)) # 2nd populate makes no network calls/all from cache again = Zone('unit.tests.', []) @@ -226,6 +226,12 @@ class TestGandiProvider(TestCase): 'rrset_type': 'CNAME', 'rrset_values': ['unit.tests.'] }), + call('POST', '/livedns/domains/unit.tests/records', data={ + 'rrset_name': 'dname', + 'rrset_ttl': 300, + 'rrset_type': 'DNAME', + 'rrset_values': ['unit.tests.'] + }), call('POST', '/livedns/domains/unit.tests/records', data={ 'rrset_name': 'cname', 'rrset_ttl': 300, @@ -270,7 +276,7 @@ class TestGandiProvider(TestCase): }) ]) # expected number of total calls - self.assertEquals(16, provider._client._request.call_count) + self.assertEquals(17, provider._client._request.call_count) provider._client._request.reset_mock() From 05ce1344546c3f959912dedeaf5231d894543ef2 Mon Sep 17 00:00:00 2001 From: Jonathan Leroy Date: Mon, 2 Nov 2020 18:02:15 +0100 Subject: [PATCH 052/358] Add tests for zone creation --- tests/test_octodns_provider_gandi.py | 33 +++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/tests/test_octodns_provider_gandi.py b/tests/test_octodns_provider_gandi.py index 3cee392..5871cc9 100644 --- a/tests/test_octodns_provider_gandi.py +++ b/tests/test_octodns_provider_gandi.py @@ -14,7 +14,8 @@ from unittest import TestCase from octodns.record import Record from octodns.provider.gandi import GandiProvider, GandiClientBadRequest, \ - GandiClientUnauthorized, GandiClientForbidden, GandiClientNotFound + GandiClientUnauthorized, GandiClientForbidden, GandiClientNotFound, \ + GandiClientUnknownDomainName from octodns.provider.yaml import YamlProvider from octodns.zone import Zone @@ -146,6 +147,36 @@ class TestGandiProvider(TestCase): def test_apply(self): provider = GandiProvider('test_id', 'token') + # Zone does not exists but can be created. + with requests_mock() as mock: + mock.get(ANY, status_code=404, + text='{"code": 404, "message": "The resource could not ' + 'be found.", "object": "HTTPNotFound", "cause": ' + '"Not Found"}') + mock.post(ANY, status_code=201, + text='{"message": "Domain Created"}') + + plan = provider.plan(self.expected) + provider.apply(plan) + + # Zone does not exists and can't be created. + with requests_mock() as mock: + mock.get(ANY, status_code=404, + text='{"code": 404, "message": "The resource could not ' + 'be found.", "object": "HTTPNotFound", "cause": ' + '"Not Found"}') + mock.post(ANY, status_code=404, + text='{"code": 404, "message": "The resource could not ' + 'be found.", "object": "HTTPNotFound", "cause": ' + '"Not Found"}') + + with self.assertRaises((GandiClientNotFound, + GandiClientUnknownDomainName)) as ctx: + plan = provider.plan(self.expected) + provider.apply(plan) + self.assertIn('This domain is not registred at Gandi.', + text_type(ctx.exception)) + resp = Mock() resp.json = Mock() provider._client._request = Mock(return_value=resp) From 6ebe0858811664e612bbc8eb66aa779bf79048ea Mon Sep 17 00:00:00 2001 From: Jonathan Leroy Date: Mon, 2 Nov 2020 18:04:45 +0100 Subject: [PATCH 053/358] Add GandiProvider to README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 23ac0e8..6c0982e 100644 --- a/README.md +++ b/README.md @@ -189,6 +189,7 @@ The above command pulled the existing data out of Route53 and placed the results | [EasyDNSProvider](/octodns/provider/easydns.py) | | A, AAAA, CAA, CNAME, MX, NAPTR, NS, SRV, TXT | No | | | [EtcHostsProvider](/octodns/provider/etc_hosts.py) | | A, AAAA, ALIAS, CNAME | No | | | [EnvVarSource](/octodns/source/envvar.py) | | TXT | No | read-only environment variable injection | +| [GandiProvider](/octodns/provider/gandi.py) | | A, AAAA, ALIAS, CAA, CNAME, DNAME, MX, NS, PTR, SPF, SRV, SSHFP, TXT | No | | | [GoogleCloudProvider](/octodns/provider/googlecloud.py) | google-cloud-dns | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | No | | | [MythicBeastsProvider](/octodns/provider/mythicbeasts.py) | Mythic Beasts | A, AAAA, ALIAS, CNAME, MX, NS, SRV, SSHFP, CAA, TXT | No | | | [Ns1Provider](/octodns/provider/ns1.py) | ns1-python | All | Yes | Missing `NA` geo target | From bb7a1a43b7e96cb03e712a3c7302292d5ccd72ce Mon Sep 17 00:00:00 2001 From: Jonathan Leroy Date: Mon, 2 Nov 2020 18:42:03 +0100 Subject: [PATCH 054/358] Implement suggested changes --- octodns/manager.py | 2 ++ octodns/record/__init__.py | 11 +++++------ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/octodns/manager.py b/octodns/manager.py index 5f4af55..fc05810 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -231,6 +231,8 @@ class Manager(object): sub_zones=self.configured_sub_zones(zone_name)) if desired: + # This is an alias zone, rather than populate it we'll copy the + # records over from `desired`. for _, records in desired._records.items(): for record in records: zone.add_record(record.copy(zone=zone), lenient=lenient) diff --git a/octodns/record/__init__.py b/octodns/record/__init__.py index ae9ccb1..e065620 100644 --- a/octodns/record/__init__.py +++ b/octodns/record/__init__.py @@ -151,7 +151,6 @@ class Record(EqualityTupleMixin): # force everything lower-case just to be safe self.name = text_type(name).lower() if name else name self.source = source - self._raw_data = data self.ttl = int(data['ttl']) self._octodns = data.get('octodns', {}) @@ -221,15 +220,15 @@ class Record(EqualityTupleMixin): return Update(self, other) def copy(self, zone=None): - _type = getattr(self, '_type') - if not self._raw_data.get('type'): - self._raw_data['type'] = _type + data = self.data + data['type'] = self._type return Record.new( zone if zone else self.zone, self.name, - self._raw_data, - self.source + data, + self.source, + lenient=True ) # NOTE: we're using __hash__ and ordering methods that consider Records From b017f90c669d17113ec53db055fbc3ad2ba02aee Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Tue, 3 Nov 2020 10:27:31 -0800 Subject: [PATCH 055/358] Add some docs around lenient and its uses --- docs/records.md | 38 +++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/docs/records.md b/docs/records.md index a28e86f..74c0b1c 100644 --- a/docs/records.md +++ b/docs/records.md @@ -6,15 +6,17 @@ OctoDNS supports the following record types: * `A` * `AAAA` +* `ALIAS` +* `CAA` * `CNAME` * `DNAME` * `MX` * `NAPTR` * `NS` * `PTR` -* `SSHFP` * `SPF` * `SRV` +* `SSHFP` * `TXT` Underlying provider support for each of these varies and some providers have extra requirements or limitations. In cases where a record type is not supported by a provider OctoDNS will ignore it there and continue to manage the record elsewhere. For example `SSHFP` is supported by Dyn, but not Route53. If your source data includes an SSHFP record OctoDNS will keep it in sync on Dyn, but not consider it when evaluating the state of Route53. The best way to find out what types are supported by a provider is to look for its `supports` method. If that method exists the logic will drive which records are supported and which are ignored. If the provider does not implement the method it will fall back to `BaseProvider.supports` which indicates full support. @@ -82,3 +84,37 @@ In the above example each name had a single record, but there are cases where a Each record type has a corresponding set of required data. The easiest way to determine what's required is probably to look at the record object in [`octodns/record/__init__.py`](/octodns/record/__init__.py). You may also utilize `octodns-validate` which will throw errors about what's missing when run. `type` is required for all records. `ttl` is optional. When TTL is not specified the `YamlProvider`'s default will be used. In any situation where an array of `values` can be used you can opt to go with `value` as a single item if there's only one. + +### Lenience + +octoDNS is fairly strict in terms of standards compliance and is opinionated in terms of best practices. Examples of former include SRV record naming requirements and the latter that ALIAS records are constrained to the root of zones. The strictness and support of providers varies so you may encounter existing records that fail validation when you try to dump them or you may even have use cases for which you need to create or preserve records that don't validate. octoDNS's solution to this is the `lenient` flag. + +#### octodns-dump + +If you're trying to import a zone into octoDNS config file using `octodns-dump` which fails due to validation errors you can supply the `--lenient` argument to tell octoDNS that you acknowledge that things aren't lining up with its expectations, but you'd like it to go ahead anyway. This will do its best to populate the zone and dump the results out into an octoDNS zone file and include the non-compliant bits. If you go to use that config file octoDNS will again complain about the validation problems. You can correct them in cases where that makes sense, but if you need to preserve the non-compliant records read on for options. + +#### Record level lenience + +When there are non-compliant records configured in Yaml you can add the following to tell octoDNS to do it's best to proceed with them anyway. If you use `--lenient` above to dump a zone and you'd like to sync it as-is you can mark the problematic records this way. + +```yaml +'not-root': + octodns: + lenient: true + type: ALIAS + values: something.else.com. +``` + +#### Zone level lenience + +If you'd like to enable lenience for a whole zone you can do so with the following, thought it's strongly encouraged to mark things at record level when possible. The most common case where things may need to be done at the zone level is when using something other than `YamlProvider` as a source, e.g. syncing from `Route53Provider` to `Ns1Provider` when there are non-compliant records in the zone in Route53. + +```yaml + non-compliant-zone.com.: + octodns: + lenient: true + sources: + - route53 + targets: + - ns1 +``` From 269e737812247b3763869398342e1fb54c4952a0 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Tue, 3 Nov 2020 10:35:55 -0800 Subject: [PATCH 056/358] Add a caveat emptor clause to lenient doc section. --- docs/records.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/records.md b/docs/records.md index 74c0b1c..d287d8a 100644 --- a/docs/records.md +++ b/docs/records.md @@ -89,6 +89,8 @@ Each record type has a corresponding set of required data. The easiest way to de octoDNS is fairly strict in terms of standards compliance and is opinionated in terms of best practices. Examples of former include SRV record naming requirements and the latter that ALIAS records are constrained to the root of zones. The strictness and support of providers varies so you may encounter existing records that fail validation when you try to dump them or you may even have use cases for which you need to create or preserve records that don't validate. octoDNS's solution to this is the `lenient` flag. +It's best to think of the `lenient` flag as "I know what I'm doing and accept any problems I run across." The main reason being is that some providers may allow the non-compliant setup and others may not. The behavior of the non-compliant records may even vary from one provider to another. Caveat emptor. + #### octodns-dump If you're trying to import a zone into octoDNS config file using `octodns-dump` which fails due to validation errors you can supply the `--lenient` argument to tell octoDNS that you acknowledge that things aren't lining up with its expectations, but you'd like it to go ahead anyway. This will do its best to populate the zone and dump the results out into an octoDNS zone file and include the non-compliant bits. If you go to use that config file octoDNS will again complain about the validation problems. You can correct them in cases where that makes sense, but if you need to preserve the non-compliant records read on for options. From f3e3f19cd3ac8d7910072bda89887e1d6c2d6459 Mon Sep 17 00:00:00 2001 From: Jonathan Leroy Date: Tue, 3 Nov 2020 22:59:39 +0100 Subject: [PATCH 057/358] Suppress previous exceptions before raising GandiClientUnknownDomainName exception --- octodns/provider/gandi.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/octodns/provider/gandi.py b/octodns/provider/gandi.py index dcc222d..84ff291 100644 --- a/octodns/provider/gandi.py +++ b/octodns/provider/gandi.py @@ -354,12 +354,16 @@ class GandiProvider(BaseProvider): self._client.zone_create(zone) self.log.info('_apply: zone has been successfully created') except GandiClientNotFound: - raise GandiClientUnknownDomainName('This domain is not ' - 'registred at Gandi. ' - 'Please register or ' - 'transfer it here ' - 'to be able to manage its ' - 'DNS zone.') + # We suppress existing exception before raising + # GandiClientUnknownDomainName. + e = GandiClientUnknownDomainName('This domain is not ' + 'registred at Gandi. ' + 'Please register or ' + 'transfer it here ' + 'to be able to manage its ' + 'DNS zone.') + e.__cause__ = None + raise e # Force records deletion to be done before creation in order to avoid # "CNAME record must be the only record" error when an existing CNAME From 150005d90519877801ef15ff0b7a91c31c68b419 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Thu, 5 Nov 2020 08:58:48 -0800 Subject: [PATCH 058/358] Add missing .md's remove redundant recursives --- MANIFEST.in | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 2b82e59..9e3dc38 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,11 +1,11 @@ -include README.md +include CHANGELOG.md +include CODE_OF_CONDUCT.md include CONTRIBUTING.md include LICENSE -include requirements.txt +include README.md include requirements-dev.txt -recursive-include docs *.png *.md +include requirements.txt include script/* -recursive-include tests *.py -recursive-include tests *.json *.txt *.yaml -recursive-include tests/zones *. -recursive-include tests/zones/tinydns *.* +recursive-include docs *.png *.md +recursive-include tests *.json *.py *.txt *.yaml +recursive-include tests/zones * From edf92fb1596a2f85a7062cdc191cbdc7e529111b Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Thu, 5 Nov 2020 09:45:39 -0800 Subject: [PATCH 059/358] v0.9.11 version bump and CHANGELOG update --- CHANGELOG.md | 31 ++++++++++++++++++++++++++++--- octodns/__init__.py | 2 +- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a0a92a9..061766c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,31 @@ -## v0.9.11 - 2020-??-?? - ??????????????? - -* Added support for TCP health checking to dynamic records +## v0.9.11 - 2020-11-05 - We still don't know edition + +#### Noteworthy changtes + +* ALIAS records only allowed at the root of zones - see `leient` in record docs + for work-arounds if you really need them. + +#### New Providers + +* Gandi LiveDNS +* UltraDNS +* easyDNS + +#### Stuff + +* Add support for zones aliases +* octodns-compare: Prefix filtering and status code on on mismatch +* Implement octodns-sync --source +* Adding environment variable record injection +* Add support for wildcard SRV records, as shown in RFC 2782 +* Add healthcheck option 'request_interval' for Route53 provider +* NS1 georegion, country, and catchall need to be separate groups +* Add the ability to mark a zone as lenient +* Add support for geo-targeting of CA provinces +* Update geo_data to pick up a couple renames +* Cloudflare: Add PTR Support, update rate-limit handling and pagination +* Support PowerDNS 4.3.x +* Added support for TCP health checking of dynamic records ## v0.9.10 - 2020-04-20 - Dynamic NS1 and lots of misc diff --git a/octodns/__init__.py b/octodns/__init__.py index 341f51e..3fcdaa1 100644 --- a/octodns/__init__.py +++ b/octodns/__init__.py @@ -3,4 +3,4 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -__VERSION__ = '0.9.10' +__VERSION__ = '0.9.11' From e98f21455db5d32171a17a321d851cc030eb2d25 Mon Sep 17 00:00:00 2001 From: Adam Smith Date: Thu, 12 Nov 2020 09:44:21 -0800 Subject: [PATCH 060/358] Add CAA record support to AxfrSource/ZoneFileSource --- README.md | 4 ++-- octodns/source/axfr.py | 17 ++++++++++++++++- tests/test_octodns_source_axfr.py | 6 +++--- tests/zones/unit.tests. | 4 ++++ 4 files changed, 25 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index c8ac4aa..cc69d94 100644 --- a/README.md +++ b/README.md @@ -205,8 +205,8 @@ The above command pulled the existing data out of Route53 and placed the results | [Selectel](/octodns/provider/selectel.py) | | A, AAAA, CNAME, MX, NS, SPF, SRV, TXT | No | | | [Transip](/octodns/provider/transip.py) | transip | A, AAAA, CNAME, MX, SRV, SPF, TXT, SSHFP, CAA | No | | | [UltraDns](/octodns/provider/ultra.py) | | A, AAAA, CAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | | -| [AxfrSource](/octodns/source/axfr.py) | | A, AAAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | read-only | -| [ZoneFileSource](/octodns/source/axfr.py) | | A, AAAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | read-only | +| [AxfrSource](/octodns/source/axfr.py) | | A, AAAA, CAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | read-only | +| [ZoneFileSource](/octodns/source/axfr.py) | | A, AAAA, CAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | read-only | | [TinyDnsFileSource](/octodns/source/tinydns.py) | | A, CNAME, MX, NS, PTR | No | read-only | | [YamlProvider](/octodns/provider/yaml.py) | | All | Yes | config | diff --git a/octodns/source/axfr.py b/octodns/source/axfr.py index 70569d1..2e18ef0 100644 --- a/octodns/source/axfr.py +++ b/octodns/source/axfr.py @@ -26,7 +26,7 @@ class AxfrBaseSource(BaseSource): SUPPORTS_GEO = False SUPPORTS_DYNAMIC = False - SUPPORTS = set(('A', 'AAAA', 'CNAME', 'MX', 'NS', 'PTR', 'SPF', + SUPPORTS = set(('A', 'AAAA', 'CAA', 'CNAME', 'MX', 'NS', 'PTR', 'SPF', 'SRV', 'TXT')) def __init__(self, id): @@ -43,6 +43,21 @@ class AxfrBaseSource(BaseSource): _data_for_AAAA = _data_for_multiple _data_for_NS = _data_for_multiple + def _data_for_CAA(self, _type, records): + values = [] + for record in records: + flags, tag, value = record['value'].split(' ', 2) + values.append({ + 'flags': flags, + 'tag': tag, + 'value': value.replace('"', '') + }) + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': values + } + def _data_for_MX(self, _type, records): values = [] for record in records: diff --git a/tests/test_octodns_source_axfr.py b/tests/test_octodns_source_axfr.py index bd25062..1bf3f22 100644 --- a/tests/test_octodns_source_axfr.py +++ b/tests/test_octodns_source_axfr.py @@ -34,7 +34,7 @@ class TestAxfrSource(TestCase): ] self.source.populate(got) - self.assertEquals(11, len(got.records)) + self.assertEquals(12, len(got.records)) with self.assertRaises(AxfrSourceZoneTransferFailed) as ctx: zone = Zone('unit.tests.', []) @@ -50,12 +50,12 @@ class TestZoneFileSource(TestCase): # Valid zone file in directory valid = Zone('unit.tests.', []) self.source.populate(valid) - self.assertEquals(11, len(valid.records)) + self.assertEquals(12, len(valid.records)) # 2nd populate does not read file again again = Zone('unit.tests.', []) self.source.populate(again) - self.assertEquals(11, len(again.records)) + self.assertEquals(12, len(again.records)) # bust the cache del self.source._zone_records[valid.name] diff --git a/tests/zones/unit.tests. b/tests/zones/unit.tests. index 0305e05..838de88 100644 --- a/tests/zones/unit.tests. +++ b/tests/zones/unit.tests. @@ -13,6 +13,10 @@ $ORIGIN unit.tests. under 3600 IN NS ns1.unit.tests. under 3600 IN NS ns2.unit.tests. +; CAA Records +caa 1800 IN CAA 0 issue "ca.unit.tests" +caa 1800 IN CAA 0 iodef "mailto:admin@unit.tests" + ; SRV Records _srv._tcp 600 IN SRV 10 20 30 foo-1.unit.tests. _srv._tcp 600 IN SRV 10 20 30 foo-2.unit.tests. From 9c20d0015ba6ff3194b096b91c0ae45f33fbb1a1 Mon Sep 17 00:00:00 2001 From: Guillaume Gelin Date: Mon, 16 Nov 2020 14:53:58 +0100 Subject: [PATCH 061/358] Fix name length validation Closes #626 --- octodns/record/__init__.py | 9 +++++---- tests/test_octodns_record.py | 24 ++++++++++++++++++++++-- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/octodns/record/__init__.py b/octodns/record/__init__.py index b6e15aa..c945aaf 100644 --- a/octodns/record/__init__.py +++ b/octodns/record/__init__.py @@ -126,10 +126,11 @@ class Record(EqualityTupleMixin): if n > 253: reasons.append('invalid fqdn, "{}" is too long at {} chars, max ' 'is 253'.format(fqdn, n)) - n = len(name) - if n > 63: - reasons.append('invalid name, "{}" is too long at {} chars, max ' - 'is 63'.format(name, n)) + for label in name.split('.'): + n = len(label) + if n > 63: + reasons.append('invalid label, "{}" is too long at {} chars, ' + 'max is 63'.format(label, n)) try: ttl = int(data['ttl']) if ttl < 0: diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index cdaa483..d4497e8 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -1315,7 +1315,7 @@ class TestRecordValidation(TestCase): self.assertTrue(reason.endswith('.unit.tests." is too long at 254' ' chars, max is 253')) - # label length, DNS defins max as 63 + # label length, DNS defines max as 63 with self.assertRaises(ValidationError) as ctx: # The . will put this over the edge name = 'x' * 64 @@ -1325,10 +1325,30 @@ class TestRecordValidation(TestCase): 'value': '1.2.3.4', }) reason = ctx.exception.reasons[0] - self.assertTrue(reason.startswith('invalid name, "xxxx')) + self.assertTrue(reason.startswith('invalid label, "xxxx')) self.assertTrue(reason.endswith('xxx" is too long at 64' ' chars, max is 63')) + with self.assertRaises(ValidationError) as ctx: + name = 'foo.' + 'x' * 64 + '.bar' + Record.new(self.zone, name, { + 'ttl': 300, + 'type': 'A', + 'value': '1.2.3.4', + }) + reason = ctx.exception.reasons[0] + self.assertTrue(reason.startswith('invalid label, "xxxx')) + self.assertTrue(reason.endswith('xxx" is too long at 64' + ' chars, max is 63')) + + # should not raise with dots + name = 'xxxxxxxx.' * 10 + Record.new(self.zone, name, { + 'ttl': 300, + 'type': 'A', + 'value': '1.2.3.4', + }) + # no ttl with self.assertRaises(ValidationError) as ctx: Record.new(self.zone, '', { From e02a8b3858a66a11fb11c8e6482fc8979e689093 Mon Sep 17 00:00:00 2001 From: Pieter Lexis Date: Thu, 19 Nov 2020 09:39:33 +0100 Subject: [PATCH 062/358] PowerDNS: Support pre-release versions This commit strips any superfluous -alphaN (or beta or rc) from the version number's minor number so it can be cast to an int. This will allow octodns to sync to/from PowerDNS pre-releases. --- octodns/provider/powerdns.py | 3 ++- tests/test_octodns_provider_powerdns.py | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/octodns/provider/powerdns.py b/octodns/provider/powerdns.py index bcb6980..98ab7be 100644 --- a/octodns/provider/powerdns.py +++ b/octodns/provider/powerdns.py @@ -183,7 +183,8 @@ class PowerDnsBaseProvider(BaseProvider): version = resp.json()['version'] self.log.debug('powerdns_version: got version %s from server', version) - self._powerdns_version = [int(p) for p in version.split('.')] + self._powerdns_version = [ + int(p.split('-')[0]) for p in version.split('.')[:3]] return self._powerdns_version diff --git a/tests/test_octodns_provider_powerdns.py b/tests/test_octodns_provider_powerdns.py index c9b1d08..33b5e44 100644 --- a/tests/test_octodns_provider_powerdns.py +++ b/tests/test_octodns_provider_powerdns.py @@ -82,6 +82,20 @@ class TestPowerDnsProvider(TestCase): provider._powerdns_version = None self.assertNotEquals(provider.powerdns_version, [4, 1, 10]) + # Test version detection with pre-releases + with requests_mock() as mock: + # Reset version, so detection will try again + provider._powerdns_version = None + mock.get('http://non.existent:8081/api/v1/servers/localhost', + status_code=200, json={'version': "4.4.0-alpha1"}) + self.assertEquals(provider.powerdns_version, [4, 4, 0]) + + provider._powerdns_version = None + mock.get('http://non.existent:8081/api/v1/servers/localhost', + status_code=200, + json={'version': "4.5.0-alpha0.435.master.gcb114252b"}) + self.assertEquals(provider.powerdns_version, [4, 5, 0]) + def test_provider_version_config(self): provider = PowerDnsProvider('test', 'non.existent', 'api-key', nameserver_values=['8.8.8.8.', From 5e13d5009a45b5ac6d44329497d764aaea39b764 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Thu, 19 Nov 2020 06:35:13 -0800 Subject: [PATCH 063/358] Add a comment about the `-` version split --- octodns/provider/powerdns.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/octodns/provider/powerdns.py b/octodns/provider/powerdns.py index 98ab7be..f2dd274 100644 --- a/octodns/provider/powerdns.py +++ b/octodns/provider/powerdns.py @@ -183,6 +183,8 @@ class PowerDnsBaseProvider(BaseProvider): version = resp.json()['version'] self.log.debug('powerdns_version: got version %s from server', version) + # The extra `-` split is to handle pre-release and source built + # versions like 4.5.0-alpha0.435.master.gcb114252b self._powerdns_version = [ int(p.split('-')[0]) for p in version.split('.')[:3]] From f822ef3d5fc557630c75bd3af31b0c0e39b88559 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Thu, 19 Nov 2020 06:44:44 -0800 Subject: [PATCH 064/358] Removing trailing space --- octodns/provider/powerdns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/powerdns.py b/octodns/provider/powerdns.py index f2dd274..de7743c 100644 --- a/octodns/provider/powerdns.py +++ b/octodns/provider/powerdns.py @@ -183,7 +183,7 @@ class PowerDnsBaseProvider(BaseProvider): version = resp.json()['version'] self.log.debug('powerdns_version: got version %s from server', version) - # The extra `-` split is to handle pre-release and source built + # The extra `-` split is to handle pre-release and source built # versions like 4.5.0-alpha0.435.master.gcb114252b self._powerdns_version = [ int(p.split('-')[0]) for p in version.split('.')[:3]] From b7ed4aa57f2dde650b1d55795c6c05471f0519d4 Mon Sep 17 00:00:00 2001 From: Peter Dave Hello Date: Sun, 22 Nov 2020 21:02:21 +0800 Subject: [PATCH 065/358] Improve ALIAS, CNAME, DNAME & PTR record FQDN validation Use fqdn package to help verify if the record value is really valid. The original behavior will treat value like `_.` or `.` be a valid record, which is strange, and the real world may not have those use cases at all. The RFC documents are pretty long, as I didn't read them all or enough to tell should it be valid or not by the spec, so I opened issue #612 to discuss this case and got a positive response from the main maintainer to have the change. Close #628 --- octodns/record/__init__.py | 4 ++++ requirements.txt | 1 + tests/test_octodns_record.py | 40 ++++++++++++++++++++++++++++++++++++ 3 files changed, 45 insertions(+) diff --git a/octodns/record/__init__.py b/octodns/record/__init__.py index c945aaf..f22eebf 100644 --- a/octodns/record/__init__.py +++ b/octodns/record/__init__.py @@ -10,6 +10,7 @@ from logging import getLogger import re from six import string_types, text_type +from fqdn import FQDN from ..equality import EqualityTupleMixin from .geo import GeoCodes @@ -757,6 +758,9 @@ class _TargetValue(object): reasons.append('empty value') elif not data: reasons.append('missing value') + elif not FQDN(data, allow_underscores=True).is_valid: + reasons.append('{} value "{}" is not a valid FQDN' + .format(_type, data)) elif not data.endswith('.'): reasons.append('{} value "{}" missing trailing .' .format(_type, data)) diff --git a/requirements.txt b/requirements.txt index bc9a019..8b9c052 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,6 +7,7 @@ dnspython==1.16.0 docutils==0.16 dyn==1.8.1 edgegrid-python==1.1.1 +fqdn==1.5.0 futures==3.2.0; python_version < '3.2' google-cloud-core==1.4.1 google-cloud-dns==0.32.0 diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index d4497e8..d55b3b8 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -1799,6 +1799,16 @@ class TestRecordValidation(TestCase): }) self.assertEquals(['empty value'], ctx.exception.reasons) + # not a valid FQDN + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'ALIAS', + 'ttl': 600, + 'value': '__.', + }) + self.assertEquals(['ALIAS value "__." is not a valid FQDN'], + ctx.exception.reasons) + # missing trailing . with self.assertRaises(ValidationError) as ctx: Record.new(self.zone, '', { @@ -1895,6 +1905,16 @@ class TestRecordValidation(TestCase): }) self.assertEquals(['root CNAME not allowed'], ctx.exception.reasons) + # not a valid FQDN + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, 'www', { + 'type': 'CNAME', + 'ttl': 600, + 'value': '___.', + }) + self.assertEquals(['CNAME value "___." is not a valid FQDN'], + ctx.exception.reasons) + # missing trailing . with self.assertRaises(ValidationError) as ctx: Record.new(self.zone, 'www', { @@ -1920,6 +1940,16 @@ class TestRecordValidation(TestCase): 'value': 'foo.bar.com.', }) + # not a valid FQDN + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, 'www', { + 'type': 'DNAME', + 'ttl': 600, + 'value': '.', + }) + self.assertEquals(['DNAME value "." is not a valid FQDN'], + ctx.exception.reasons) + # missing trailing . with self.assertRaises(ValidationError) as ctx: Record.new(self.zone, 'www', { @@ -2103,6 +2133,16 @@ class TestRecordValidation(TestCase): }) self.assertEquals(['missing value'], ctx.exception.reasons) + # not a valid FQDN + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'PTR', + 'ttl': 600, + 'value': '_.', + }) + self.assertEquals(['PTR value "_." is not a valid FQDN'], + ctx.exception.reasons) + # no trailing . with self.assertRaises(ValidationError) as ctx: Record.new(self.zone, '', { From 3dd7061a0c6bd50fb61f1842d684c9d378d8b94c Mon Sep 17 00:00:00 2001 From: Peter Dave Hello Date: Sun, 22 Nov 2020 21:09:45 +0800 Subject: [PATCH 066/358] Remove Azure allow empty CNAME, PTR value behavior cc #84 #628 --- octodns/provider/azuredns.py | 15 +++------------ tests/test_octodns_provider_azuredns.py | 8 +------- 2 files changed, 4 insertions(+), 19 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 3d8122a..19eb663 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -416,14 +416,8 @@ class AzureProvider(BaseProvider): :type azrecord: azure.mgmt.dns.models.RecordSet :type return: dict - - CNAME and PTR both use the catch block to catch possible empty - records. Refer to population comment. ''' - try: - return {'value': _check_endswith_dot(azrecord.cname_record.cname)} - except: - return {'value': '.'} + return {'value': _check_endswith_dot(azrecord.cname_record.cname)} def _data_for_MX(self, azrecord): return {'values': [{'preference': ar.preference, @@ -435,11 +429,8 @@ class AzureProvider(BaseProvider): return {'values': [_check_endswith_dot(val) for val in vals]} def _data_for_PTR(self, azrecord): - try: - ptrdname = azrecord.ptr_records[0].ptrdname - return {'value': _check_endswith_dot(ptrdname)} - except: - return {'value': '.'} + ptrdname = azrecord.ptr_records[0].ptrdname + return {'value': _check_endswith_dot(ptrdname)} def _data_for_SRV(self, azrecord): return {'values': [{'priority': ar.priority, 'weight': ar.weight, diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index 1769cef..3b008e5 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -389,9 +389,6 @@ class TestAzureDnsProvider(TestCase): recordSet = RecordSet(cname_record=cname1) recordSet.name, recordSet.ttl, recordSet.type = 'cname1', 5, 'CNAME' rs.append(recordSet) - recordSet = RecordSet(cname_record=None) - recordSet.name, recordSet.ttl, recordSet.type = 'cname2', 6, 'CNAME' - rs.append(recordSet) recordSet = RecordSet(mx_records=[MxRecord(preference=10, exchange='mx1.unit.test.')]) recordSet.name, recordSet.ttl, recordSet.type = 'mx1', 7, 'MX' @@ -413,9 +410,6 @@ class TestAzureDnsProvider(TestCase): recordSet = RecordSet(ptr_records=[ptr1]) recordSet.name, recordSet.ttl, recordSet.type = 'ptr1', 11, 'PTR' rs.append(recordSet) - recordSet = RecordSet(ptr_records=[PtrRecord(ptrdname=None)]) - recordSet.name, recordSet.ttl, recordSet.type = 'ptr2', 12, 'PTR' - rs.append(recordSet) recordSet = RecordSet(srv_records=[SrvRecord(priority=1, weight=2, port=3, @@ -449,7 +443,7 @@ class TestAzureDnsProvider(TestCase): exists = provider.populate(zone) self.assertTrue(exists) - self.assertEquals(len(zone.records), 18) + self.assertEquals(len(zone.records), 16) def test_populate_zone(self): provider = self._get_provider() From fa266c23d2e4cb6debab32107e7503c90a9fe375 Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Wed, 25 Nov 2020 22:28:35 +0800 Subject: [PATCH 067/358] Fix _is_valid_dkim_key for Python 3.9 compatibility in OVH provider base64.decodestring was deprecated and removed in Python 3.9 in favour of decodebytes (See https://bugs.python.org/issue39351 ) --- octodns/provider/ovh.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/octodns/provider/ovh.py b/octodns/provider/ovh.py index 54f62ac..0ee8e61 100644 --- a/octodns/provider/ovh.py +++ b/octodns/provider/ovh.py @@ -370,11 +370,16 @@ class OvhProvider(BaseProvider): @staticmethod def _is_valid_dkim_key(key): + result = True try: - base64.decodestring(bytearray(key, 'utf-8')) + decode = base64.decodestring + except AttributeError: + decode = base64.decodebytes + try: + result = decode(bytearray(key, 'utf-8')) except binascii.Error: - return False - return True + result = False + return result def get_records(self, zone_name): """ From 7f89c621a1200e81ee00c39f63b25072be93677f Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Wed, 25 Nov 2020 23:29:49 +0800 Subject: [PATCH 068/358] Address coverage for Python 2, tidy up variable names in _is_valid_dkim_key --- octodns/provider/ovh.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/octodns/provider/ovh.py b/octodns/provider/ovh.py index 0ee8e61..9f7cd9a 100644 --- a/octodns/provider/ovh.py +++ b/octodns/provider/ovh.py @@ -371,12 +371,11 @@ class OvhProvider(BaseProvider): @staticmethod def _is_valid_dkim_key(key): result = True + base64_decode = getattr(base64, 'decodestring', None) + base64_decode = getattr(base64, 'decodebytes', base64_decode) + try: - decode = base64.decodestring - except AttributeError: - decode = base64.decodebytes - try: - result = decode(bytearray(key, 'utf-8')) + result = base64_decode(bytearray(key, 'utf-8')) except binascii.Error: result = False return result From f9cf97e98a9da97b7ad1ca4498bffe70dd6125b0 Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Wed, 2 Dec 2020 11:25:01 +0800 Subject: [PATCH 069/358] Add python 3.9 to GitHub CI workflow to catch removed methods --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 726e2e8..f8b37ef 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -6,7 +6,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [2.7, 3.7] + python-version: [2.7, 3.7, 3.9] steps: - uses: actions/checkout@master - name: Setup python From e4d6084b4c85da0e97b9f88d146583aac074794d Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Thu, 3 Dec 2020 17:50:56 -0800 Subject: [PATCH 070/358] POC of processors concept that can hook in to modify zones --- octodns/manager.py | 51 ++++++++++++++++++++++++++++++---- octodns/processors/__init__.py | 17 ++++++++++++ octodns/processors/filters.py | 38 +++++++++++++++++++++++++ octodns/provider/base.py | 5 +++- tests/test_octodns_manager.py | 2 +- 5 files changed, 106 insertions(+), 7 deletions(-) create mode 100644 octodns/processors/__init__.py create mode 100644 octodns/processors/filters.py diff --git a/octodns/manager.py b/octodns/manager.py index fc05810..9116742 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -121,6 +121,25 @@ class Manager(object): raise ManagerException('Incorrect provider config for {}' .format(provider_name)) + self.processors = {} + for processor_name, processor_config in \ + self.config.get('processors', {}).items(): + try: + _class = processor_config.pop('class') + except KeyError: + self.log.exception('Invalid processor class') + raise ManagerException('Processor {} is missing class' + .format(processor_name)) + _class = self._get_named_class('processor', _class) + kwargs = self._build_kwargs(processor_config) + try: + self.processors[processor_name] = _class(processor_name, + **kwargs) + except TypeError: + self.log.exception('Invalid processor config') + raise ManagerException('Incorrect processor config for {}' + .format(processor_name)) + zone_tree = {} # sort by reversed strings so that parent zones always come first for name in sorted(self.config['zones'].keys(), key=lambda s: s[::-1]): @@ -222,8 +241,8 @@ class Manager(object): self.log.debug('configured_sub_zones: subs=%s', sub_zone_names) return set(sub_zone_names) - def _populate_and_plan(self, zone_name, sources, targets, desired=None, - lenient=False): + def _populate_and_plan(self, zone_name, processors, sources, targets, + desired=None, lenient=False): self.log.debug('sync: populating, zone=%s, lenient=%s', zone_name, lenient) @@ -248,6 +267,10 @@ class Manager(object): 'param', source.__class__.__name__) source.populate(zone) + self.log.debug('sync: processing, zone=%s', zone_name) + for processor in processors: + zone = processor.process(zone) + self.log.debug('sync: planning, zone=%s', zone_name) plans = [] @@ -259,7 +282,9 @@ class Manager(object): 'value': 'provider={}'.format(target.id) }) zone.add_record(meta, replace=True) - plan = target.plan(zone) + # TODO: if someone has overrriden plan already this will be a + # breaking change so we probably need to try both ways + plan = target.plan(zone, processors=processors) if plan: plans.append((target, plan)) @@ -315,6 +340,8 @@ class Manager(object): raise ManagerException('Zone {} is missing targets' .format(zone_name)) + processors = config.get('processors', []) + if (eligible_sources and not [s for s in sources if s in eligible_sources]): self.log.info('sync: no eligible sources, skipping') @@ -332,6 +359,15 @@ class Manager(object): self.log.info('sync: sources=%s -> targets=%s', sources, targets) + try: + collected = [] + for processor in processors: + collected.append(self.processors[processor]) + processors = collected + except KeyError: + raise ManagerException('Zone {}, unknown processor: {}' + .format(zone_name, processor)) + try: # rather than using a list comprehension, we break this loop # out so that the `except` block below can reference the @@ -358,8 +394,9 @@ class Manager(object): .format(zone_name, target)) futures.append(self._executor.submit(self._populate_and_plan, - zone_name, sources, - targets, lenient=lenient)) + zone_name, processors, + sources, targets, + lenient=lenient)) # Wait on all results and unpack/flatten the plans and store the # desired states in case we need them below @@ -378,6 +415,7 @@ class Manager(object): futures.append(self._executor.submit( self._populate_and_plan, zone_name, + processors, [], [self.providers[t] for t in source_config['targets']], desired=desired[zone_source], @@ -518,6 +556,9 @@ class Manager(object): if isinstance(source, YamlProvider): source.populate(zone) + # TODO: validate + # processors = config.get('processors', []) + def get_zone(self, zone_name): if not zone_name[-1] == '.': raise ManagerException('Invalid zone name {}, missing ending dot' diff --git a/octodns/processors/__init__.py b/octodns/processors/__init__.py new file mode 100644 index 0000000..9431e6a --- /dev/null +++ b/octodns/processors/__init__.py @@ -0,0 +1,17 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from ..zone import Zone + + +class BaseProcessor(object): + + def __init__(self, name): + self.name = name + + def _create_zone(self, zone): + return Zone(zone.name, sub_zones=zone.sub_zones) diff --git a/octodns/processors/filters.py b/octodns/processors/filters.py new file mode 100644 index 0000000..d483ab7 --- /dev/null +++ b/octodns/processors/filters.py @@ -0,0 +1,38 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from . import BaseProcessor + + +class TypeAllowlistFilter(BaseProcessor): + + def __init__(self, name, allowlist): + super(TypeAllowlistFilter, self).__init__(name) + self.allowlist = allowlist + + def process(self, zone): + ret = self._create_zone(zone) + for record in zone.records: + if record._type in self.allowlist: + ret.add_record(record) + + return ret + + +class TypeRejectlistFilter(BaseProcessor): + + def __init__(self, name, rejectlist): + super(TypeRejectlistFilter, self).__init__(name) + self.rejectlist = rejectlist + + def process(self, zone): + ret = self._create_zone(zone) + for record in zone.records: + if record._type not in self.rejectlist: + ret.add_record(record) + + return ret diff --git a/octodns/provider/base.py b/octodns/provider/base.py index ae87844..2a4ab11 100644 --- a/octodns/provider/base.py +++ b/octodns/provider/base.py @@ -44,7 +44,7 @@ class BaseProvider(BaseSource): ''' return [] - def plan(self, desired): + def plan(self, desired, processors=[]): self.log.info('plan: desired=%s', desired.name) existing = Zone(desired.name, desired.sub_zones) @@ -55,6 +55,9 @@ class BaseProvider(BaseSource): self.log.warn('Provider %s used in target mode did not return ' 'exists', self.id) + for processor in processors: + existing = processor.process(existing) + # compute the changes at the zone/record level changes = existing.changes(desired, self) diff --git a/tests/test_octodns_manager.py b/tests/test_octodns_manager.py index dc047e8..3a09809 100644 --- a/tests/test_octodns_manager.py +++ b/tests/test_octodns_manager.py @@ -352,7 +352,7 @@ class TestManager(TestCase): pass # This should be ok, we'll fall back to not passing it - manager._populate_and_plan('unit.tests.', [NoLenient()], []) + manager._populate_and_plan('unit.tests.', [], [NoLenient()], []) class NoZone(SimpleProvider): From 95d9ffc221917e592ec169fb05124f2bd29a2ed3 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Thu, 3 Dec 2020 18:12:16 -0800 Subject: [PATCH 071/358] Tell the processor when it's being called in a target context --- octodns/processors/filters.py | 4 ++-- octodns/provider/base.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/octodns/processors/filters.py b/octodns/processors/filters.py index d483ab7..09613d3 100644 --- a/octodns/processors/filters.py +++ b/octodns/processors/filters.py @@ -14,7 +14,7 @@ class TypeAllowlistFilter(BaseProcessor): super(TypeAllowlistFilter, self).__init__(name) self.allowlist = allowlist - def process(self, zone): + def process(self, zone, target=False): ret = self._create_zone(zone) for record in zone.records: if record._type in self.allowlist: @@ -29,7 +29,7 @@ class TypeRejectlistFilter(BaseProcessor): super(TypeRejectlistFilter, self).__init__(name) self.rejectlist = rejectlist - def process(self, zone): + def process(self, zone, target=False): ret = self._create_zone(zone) for record in zone.records: if record._type not in self.rejectlist: diff --git a/octodns/provider/base.py b/octodns/provider/base.py index 2a4ab11..137b664 100644 --- a/octodns/provider/base.py +++ b/octodns/provider/base.py @@ -56,7 +56,7 @@ class BaseProvider(BaseSource): 'exists', self.id) for processor in processors: - existing = processor.process(existing) + existing = processor.process(existing, target=True) # compute the changes at the zone/record level changes = existing.changes(desired, self) From f9a4239af5de677624509cda88864f759f694d79 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Thu, 3 Dec 2020 18:12:43 -0800 Subject: [PATCH 072/358] Marking side of ownership processor --- octodns/processors/ownership.py | 40 +++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 octodns/processors/ownership.py diff --git a/octodns/processors/ownership.py b/octodns/processors/ownership.py new file mode 100644 index 0000000..c132b2e --- /dev/null +++ b/octodns/processors/ownership.py @@ -0,0 +1,40 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from ..record import Record + +from . import BaseProcessor + + +class OwnershipProcessor(BaseProcessor): + + def __init__(self, name, txt_name='_owner'): + super(OwnershipProcessor, self).__init__(name) + self.txt_name = txt_name + + def add_ownerships(self, zone): + ret = self._create_zone(zone) + for record in zone.records: + ret.add_record(record) + name = '{}.{}.{}'.format(self.txt_name, record._type, record.name), + txt = Record.new(zone, name, { + 'type': 'TXT', + 'ttl': 60, + 'value': 'octodns', + }) + ret.add_record(txt) + + return ret + + def remove_unowned(self, zone): + ret = self._create_zone(zone) + return ret + + def process(self, zone, target=False): + if target: + return self.remove_unowned(zone) + return self.add_ownerships(zone) From 3c942defb5c49eef0fcf155204acda71da721cdf Mon Sep 17 00:00:00 2001 From: Peter Dave Hello Date: Thu, 3 Dec 2020 23:59:45 +0800 Subject: [PATCH 073/358] Also add Python v3.6, v3.8 on GitHub Actions for test Unlike Python v2 as v2.7 is 10 years old, Python v3 has more versions alive amoung us, it'll be great if they will all be tested. --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f8b37ef..479464f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -6,7 +6,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [2.7, 3.7, 3.9] + python-version: [2.7, 3.6, 3.7, 3.8, 3.9] steps: - uses: actions/checkout@master - name: Setup python From ac0aeef54fcf525c8fcab1b14917b95a313fa499 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 4 Dec 2020 06:59:08 -0800 Subject: [PATCH 074/358] Link to python vesrion EOL dates --- .github/workflows/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 479464f..0f62f96 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -6,6 +6,8 @@ jobs: runs-on: ubuntu-latest strategy: matrix: + # Tested versions based on dates in https://devguide.python.org/devcycle/#end-of-life-branches, + # with the addition of 2.7 b/c it's still if pretty wide active use. python-version: [2.7, 3.6, 3.7, 3.8, 3.9] steps: - uses: actions/checkout@master From 261abeb133ca90e2782f64a4fdb6d2fe3956a093 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sun, 6 Dec 2020 13:02:56 -0800 Subject: [PATCH 075/358] Sketch at process: source, target, plan setup, with ownership --- octodns/manager.py | 7 ++- octodns/processors/__init__.py | 13 ++++ octodns/processors/filters.py | 10 +++- octodns/processors/ownership.py | 103 +++++++++++++++++++++++++++----- octodns/provider/base.py | 2 +- 5 files changed, 115 insertions(+), 20 deletions(-) diff --git a/octodns/manager.py b/octodns/manager.py index 9116742..aebd0d5 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -255,7 +255,6 @@ class Manager(object): for _, records in desired._records.items(): for record in records: zone.add_record(record.copy(zone=zone), lenient=lenient) - else: for source in sources: try: @@ -267,9 +266,8 @@ class Manager(object): 'param', source.__class__.__name__) source.populate(zone) - self.log.debug('sync: processing, zone=%s', zone_name) for processor in processors: - zone = processor.process(zone) + zone = processor.process_source_zone(zone, sources=sources) self.log.debug('sync: planning, zone=%s', zone_name) plans = [] @@ -285,6 +283,9 @@ class Manager(object): # TODO: if someone has overrriden plan already this will be a # breaking change so we probably need to try both ways plan = target.plan(zone, processors=processors) + for processor in processors: + plan = processor.process_plan(plan, sources=sources, + target=target) if plan: plans.append((target, plan)) diff --git a/octodns/processors/__init__.py b/octodns/processors/__init__.py index 9431e6a..6b999e2 100644 --- a/octodns/processors/__init__.py +++ b/octodns/processors/__init__.py @@ -15,3 +15,16 @@ class BaseProcessor(object): def _create_zone(self, zone): return Zone(zone.name, sub_zones=zone.sub_zones) + + def process_source_zone(self, zone, sources): + # sources may be empty, as will be the case for aliased zones + return zone + + def process_target_zone(self, zone, target): + return zone + + def process_plan(self, plan, sources, target): + # plan may be None if no changes were detected up until now, the + # process may still create a plan. + # sources may be empty, as will be the case for aliased zones + return plan diff --git a/octodns/processors/filters.py b/octodns/processors/filters.py index 09613d3..413964f 100644 --- a/octodns/processors/filters.py +++ b/octodns/processors/filters.py @@ -14,7 +14,7 @@ class TypeAllowlistFilter(BaseProcessor): super(TypeAllowlistFilter, self).__init__(name) self.allowlist = allowlist - def process(self, zone, target=False): + def _process(self, zone, *args, **kwargs): ret = self._create_zone(zone) for record in zone.records: if record._type in self.allowlist: @@ -22,6 +22,9 @@ class TypeAllowlistFilter(BaseProcessor): return ret + process_source_zone = _process + process_target_zone = _process + class TypeRejectlistFilter(BaseProcessor): @@ -29,10 +32,13 @@ class TypeRejectlistFilter(BaseProcessor): super(TypeRejectlistFilter, self).__init__(name) self.rejectlist = rejectlist - def process(self, zone, target=False): + def _process(self, zone, *args, **kwargs): ret = self._create_zone(zone) for record in zone.records: if record._type not in self.rejectlist: ret.add_record(record) return ret + + process_source_zone = _process + process_target_zone = _process diff --git a/octodns/processors/ownership.py b/octodns/processors/ownership.py index c132b2e..27a0385 100644 --- a/octodns/processors/ownership.py +++ b/octodns/processors/ownership.py @@ -5,36 +5,111 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals +from collections import defaultdict +from pprint import pprint + +from ..provider.plan import Plan from ..record import Record from . import BaseProcessor +# Mark anything octoDNS is managing that way it can know it's safe to modify or +# delete. We'll take ownership of existing records that we're told to manage +# and thus "own" them going forward. class OwnershipProcessor(BaseProcessor): - def __init__(self, name, txt_name='_owner'): + def __init__(self, name, txt_name='_owner', txt_value='*octodns*'): super(OwnershipProcessor, self).__init__(name) self.txt_name = txt_name + self.txt_value = txt_value + self._txt_values = [txt_value] - def add_ownerships(self, zone): + def process_source_zone(self, zone, *args, **kwargs): ret = self._create_zone(zone) for record in zone.records: + # Always copy over the source records ret.add_record(record) - name = '{}.{}.{}'.format(self.txt_name, record._type, record.name), + # Then create and add an ownership TXT for each of them + record_name = record.name.replace('*', '_wildcard') + if record.name: + name = '{}.{}.{}'.format(self.txt_name, record._type, + record_name) + else: + name = '{}.{}'.format(self.txt_name, record._type) txt = Record.new(zone, name, { - 'type': 'TXT', - 'ttl': 60, - 'value': 'octodns', - }) + 'type': 'TXT', + 'ttl': 60, + 'value': self.txt_value, + }) ret.add_record(txt) return ret - def remove_unowned(self, zone): - ret = self._create_zone(zone) - return ret + def _is_ownership(self, record): + return record._type == 'TXT' and \ + record.name.startswith(self.txt_name) \ + and record.values == self._txt_values + + def process_plan(self, plan, *args, **kwargs): + if not plan: + # If we don't have any change there's nothing to do + return plan + + # First find all the ownership info + owned = defaultdict(dict) + # We need to look for ownership in both the desired and existing + # states, many things will show up in both, but that's fine. + for record in list(plan.existing.records) + list(plan.desired.records): + if self._is_ownership(record): + pieces = record.name.split('.', 2) + if len(pieces) > 2: + _, _type, name = pieces + else: + _type = pieces[1] + name = '' + name = name.replace('_wildcard', '*') + owned[name][_type.upper()] = True + + pprint(dict(owned)) + + # Cases: + # - Configured in source + # - We'll fully CRU/manage it adding ownership TXT, + # thanks to process_source_zone, if needed + # - Not in source + # - Has an ownership TXT - delete it & the ownership TXT + # - Does not have an ownership TXT - don't delete it + # - Special records like octodns-meta + # - Should be left alone and should not have ownerthis TXTs + + pprint(plan.changes) + + filtered_changes = [] + for change in plan.changes: + record = change.record + + pprint([change, + not self._is_ownership(record), + record._type not in owned[record.name], + record.name != 'octodns-meta']) + + if not self._is_ownership(record) and \ + record._type not in owned[record.name] and \ + record.name != 'octodns-meta': + # It's not an ownership TXT, it's not owned, and it's not + # special we're going to ignore it + continue + + # We own this record or owned it up until now so whatever the + # change is we should do + filtered_changes.append(change) + + pprint(filtered_changes) + + if plan.changes != filtered_changes: + return Plan(plan.existing, plan.desired, filtered_changes, + plan.exists, plan.update_pcent_threshold, + plan.delete_pcent_threshold) - def process(self, zone, target=False): - if target: - return self.remove_unowned(zone) - return self.add_ownerships(zone) + return plan diff --git a/octodns/provider/base.py b/octodns/provider/base.py index 137b664..b28dd6e 100644 --- a/octodns/provider/base.py +++ b/octodns/provider/base.py @@ -56,7 +56,7 @@ class BaseProvider(BaseSource): 'exists', self.id) for processor in processors: - existing = processor.process(existing, target=True) + existing = processor.process_target_zone(existing, target=self) # compute the changes at the zone/record level changes = existing.changes(desired, self) From 61280e1e751dbb3ce2349ce7b4a4014fddaee23d Mon Sep 17 00:00:00 2001 From: 0xflotus <0xflotus@gmail.com> Date: Tue, 8 Dec 2020 02:37:55 +0100 Subject: [PATCH 076/358] fix: error in gandi.py --- octodns/provider/gandi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/gandi.py b/octodns/provider/gandi.py index 84ff291..8401ea4 100644 --- a/octodns/provider/gandi.py +++ b/octodns/provider/gandi.py @@ -357,7 +357,7 @@ class GandiProvider(BaseProvider): # We suppress existing exception before raising # GandiClientUnknownDomainName. e = GandiClientUnknownDomainName('This domain is not ' - 'registred at Gandi. ' + 'registered at Gandi. ' 'Please register or ' 'transfer it here ' 'to be able to manage its ' From 870c1209d34a689374752439eb8f41d9d99c6e90 Mon Sep 17 00:00:00 2001 From: 0xflotus <0xflotus@gmail.com> Date: Tue, 8 Dec 2020 09:51:10 +0100 Subject: [PATCH 077/358] Update test_octodns_provider_gandi.py --- tests/test_octodns_provider_gandi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_octodns_provider_gandi.py b/tests/test_octodns_provider_gandi.py index 5871cc9..7e1c866 100644 --- a/tests/test_octodns_provider_gandi.py +++ b/tests/test_octodns_provider_gandi.py @@ -174,7 +174,7 @@ class TestGandiProvider(TestCase): GandiClientUnknownDomainName)) as ctx: plan = provider.plan(self.expected) provider.apply(plan) - self.assertIn('This domain is not registred at Gandi.', + self.assertIn('This domain is not registered at Gandi.', text_type(ctx.exception)) resp = Mock() From 040afe6fc1dcef9327ba0e68fd62d9cc8990ffb7 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Tue, 8 Dec 2020 08:13:36 -0800 Subject: [PATCH 078/358] Don't need to _wildcard empty names --- octodns/processors/ownership.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/processors/ownership.py b/octodns/processors/ownership.py index 27a0385..374e380 100644 --- a/octodns/processors/ownership.py +++ b/octodns/processors/ownership.py @@ -65,10 +65,10 @@ class OwnershipProcessor(BaseProcessor): pieces = record.name.split('.', 2) if len(pieces) > 2: _, _type, name = pieces + name = name.replace('_wildcard', '*') else: _type = pieces[1] name = '' - name = name.replace('_wildcard', '*') owned[name][_type.upper()] = True pprint(dict(owned)) From 523f1f56479a84cdd47bfc458b962544a7b04b9e Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Tue, 8 Dec 2020 08:33:04 -0800 Subject: [PATCH 079/358] s#github/octodns#octodns/octodns/g --- CHANGELOG.md | 6 +++--- CONTRIBUTING.md | 2 +- README.md | 6 +++--- setup.py | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 061766c..fb68e25 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,7 +55,7 @@ * Explicit ordering of changes by (name, type) to address inconsistent ordering for a number of providers that just convert changes into API calls as they come. Python 2 sets ordered consistently, Python 3 they do - not. https://github.com/github/octodns/pull/384/commits/7958233fccf9ea22d95e2fd06c48d7d0a4529e26 + not. https://github.com/octodns/octodns/pull/384/commits/7958233fccf9ea22d95e2fd06c48d7d0a4529e26 * Route53 `_mod_keyer` ordering wasn't 100% complete and thus unreliable and random in Python 3. This has been addressed and may result in value reordering on next plan, no actual changes in behavior should occur. @@ -152,10 +152,10 @@ recreating all health checks. This process has been tested pretty thoroughly to try and ensure a seemless upgrade without any traffic shifting around. It's probably best to take extra care when updating and to try and make sure that all health checks are passing before the first sync with `--doit`. See -[#67](https://github.com/github/octodns/pull/67) for more information. +[#67](https://github.com/octodns/octodns/pull/67) for more information. * Major update to geo healthchecks to allow configuring host (header), path, - protocol, and port [#67](https://github.com/github/octodns/pull/67) + protocol, and port [#67](https://github.com/octodns/octodns/pull/67) * SSHFP algorithm type 4 * NS1 and DNSimple support skipping unsupported record types * Revert back to old style setup.py & requirements.txt, setup.cfg was diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ea891ac..019caa3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,7 +4,7 @@ Hi there! We're thrilled that you'd like to contribute to OctoDNS. Your help is Please note that this project adheres to the [Contributor Covenant Code of Conduct](/CODE_OF_CONDUCT.md). By participating in this project you agree to abide by its terms. -If you have questions, or you'd like to check with us before embarking on a major development effort, please [open an issue](https://github.com/github/octodns/issues/new). +If you have questions, or you'd like to check with us before embarking on a major development effort, please [open an issue](https://github.com/octodns/octodns/issues/new). ## How to contribute diff --git a/README.md b/README.md index cc69d94..871a35e 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ - + ## DNS as code - Tools for managing DNS across multiple providers @@ -284,13 +284,13 @@ Please see our [contributing document](/CONTRIBUTING.md) if you would like to pa ## Getting help -If you have a problem or suggestion, please [open an issue](https://github.com/github/octodns/issues/new) in this repository, and we will do our best to help. Please note that this project adheres to the [Contributor Covenant Code of Conduct](/CODE_OF_CONDUCT.md). +If you have a problem or suggestion, please [open an issue](https://github.com/octodns/octodns/issues/new) in this repository, and we will do our best to help. Please note that this project adheres to the [Contributor Covenant Code of Conduct](/CODE_OF_CONDUCT.md). ## License OctoDNS is licensed under the [MIT license](LICENSE). -The MIT license grant is not for GitHub's trademarks, which include the logo designs. GitHub reserves all trademark and copyright rights in and to all GitHub trademarks. GitHub's logos include, for instance, the stylized designs that include "logo" in the file title in the following folder: https://github.com/github/octodns/tree/master/docs/logos/ +The MIT license grant is not for GitHub's trademarks, which include the logo designs. GitHub reserves all trademark and copyright rights in and to all GitHub trademarks. GitHub's logos include, for instance, the stylized designs that include "logo" in the file title in the following folder: https://github.com/octodns/octodns/tree/master/docs/logos/ GitHub® and its stylized versions and the Invertocat mark are GitHub's Trademarks or registered Trademarks. When using GitHub's logos, be sure to follow the GitHub logo guidelines. diff --git a/setup.py b/setup.py index 9394e7f..b25909a 100644 --- a/setup.py +++ b/setup.py @@ -81,6 +81,6 @@ setup( long_description_content_type='text/markdown', name='octodns', packages=find_packages(), - url='https://github.com/github/octodns', + url='https://github.com/octodns/octodns', version=octodns.__VERSION__, ) From 86232b48cf5739050a81b984cc2a8782fca7d229 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Thu, 10 Dec 2020 08:30:15 -0800 Subject: [PATCH 080/358] Replace some nbsp chars that have slipped into manager.py somehow --- octodns/manager.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/octodns/manager.py b/octodns/manager.py index fc05810..9283d19 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -288,7 +288,7 @@ class Manager(object): self.log.error('Invalid alias zone {}, target {} does ' 'not exist'.format(zone_name, source_zone)) raise ManagerException('Invalid alias zone {}: ' - 'source zone {} does not exist' + 'source zone {} does not exist' .format(zone_name, source_zone)) # Check that the source zone is not an alias zone itself. @@ -296,7 +296,7 @@ class Manager(object): self.log.error('Invalid alias zone {}, target {} is an ' 'alias zone'.format(zone_name, source_zone)) raise ManagerException('Invalid alias zone {}: source ' - 'zone {} is an alias zone' + 'zone {} is an alias zone' .format(zone_name, source_zone)) aliased_zones[zone_name] = source_zone @@ -482,13 +482,13 @@ class Manager(object): if source_zone not in self.config['zones']: self.log.exception('Invalid alias zone') raise ManagerException('Invalid alias zone {}: ' - 'source zone {} does not exist' + 'source zone {} does not exist' .format(zone_name, source_zone)) if 'alias' in self.config['zones'][source_zone]: self.log.exception('Invalid alias zone') raise ManagerException('Invalid alias zone {}: ' - 'source zone {} is an alias zone' + 'source zone {} is an alias zone' .format(zone_name, source_zone)) # this is just here to satisfy coverage, see From a7bb6a306c27c38749d308076925762baa5ca391 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Thu, 10 Dec 2020 08:39:31 -0800 Subject: [PATCH 081/358] Remove corresponding nbsp's from manager tests --- tests/test_octodns_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_octodns_manager.py b/tests/test_octodns_manager.py index dc047e8..4e1a756 100644 --- a/tests/test_octodns_manager.py +++ b/tests/test_octodns_manager.py @@ -180,7 +180,7 @@ class TestManager(TestCase): tc = Manager(get_config_filename('unknown-source-zone.yaml')) \ .sync() self.assertEquals('Invalid alias zone alias.tests.: source zone ' - 'does-not-exists.tests. does not exist', + 'does-not-exists.tests. does not exist', text_type(ctx.exception)) # Alias zone that points to another alias zone. @@ -188,7 +188,7 @@ class TestManager(TestCase): tc = Manager(get_config_filename('alias-zone-loop.yaml')) \ .sync() self.assertEquals('Invalid alias zone alias-loop.tests.: source ' - 'zone alias.tests. is an alias zone', + 'zone alias.tests. is an alias zone', text_type(ctx.exception)) def test_compare(self): From 9549a0dec9429c90bd6b693c08a5b193ac2e4920 Mon Sep 17 00:00:00 2001 From: Nikolay Denev Date: Thu, 10 Dec 2020 22:12:53 +0000 Subject: [PATCH 082/358] Ignore records with unsupported rrtypes and log warning. --- octodns/provider/ultra.py | 7 ++++++- tests/fixtures/ultra-records-page-1.json | 2 +- tests/fixtures/ultra-records-page-2.json | 12 ++++++++++-- tests/fixtures/ultra-zones-page-1.json | 2 +- 4 files changed, 18 insertions(+), 5 deletions(-) diff --git a/octodns/provider/ultra.py b/octodns/provider/ultra.py index eb10e0d..a6ecd23 100644 --- a/octodns/provider/ultra.py +++ b/octodns/provider/ultra.py @@ -287,7 +287,12 @@ class UltraProvider(BaseProvider): name = zone.hostname_from_fqdn(record['ownerName']) if record['rrtype'] == 'SOA (6)': continue - _type = self.RECORDS_TO_TYPE[record['rrtype']] + try: + _type = self.RECORDS_TO_TYPE[record['rrtype']] + except KeyError: + self.log.warning('populate: ignoring record with ' + 'unsupported rrtype=%s', record) + continue values[name][_type] = record for name, types in values.items(): diff --git a/tests/fixtures/ultra-records-page-1.json b/tests/fixtures/ultra-records-page-1.json index 2f5f836..8614427 100644 --- a/tests/fixtures/ultra-records-page-1.json +++ b/tests/fixtures/ultra-records-page-1.json @@ -87,7 +87,7 @@ } ], "resultInfo": { - "totalCount": 12, + "totalCount": 13, "offset": 0, "returnedCount": 10 } diff --git a/tests/fixtures/ultra-records-page-2.json b/tests/fixtures/ultra-records-page-2.json index db51828..abdc44f 100644 --- a/tests/fixtures/ultra-records-page-2.json +++ b/tests/fixtures/ultra-records-page-2.json @@ -24,11 +24,19 @@ "order": "FIXED", "description": "octodns1.test." } + }, + { + "ownerName": "octodns1.test.", + "rrtype": "APEXALIAS (65282)", + "ttl": 3600, + "rdata": [ + "www.octodns1.test." + ] } ], "resultInfo": { - "totalCount": 12, + "totalCount": 13, "offset": 10, - "returnedCount": 2 + "returnedCount": 3 } } \ No newline at end of file diff --git a/tests/fixtures/ultra-zones-page-1.json b/tests/fixtures/ultra-zones-page-1.json index ad98d48..f748d08 100644 --- a/tests/fixtures/ultra-zones-page-1.json +++ b/tests/fixtures/ultra-zones-page-1.json @@ -19,7 +19,7 @@ "dnssecStatus": "UNSIGNED", "status": "ACTIVE", "owner": "phelpstest", - "resourceRecordCount": 5, + "resourceRecordCount": 6, "lastModifiedDateTime": "2020-06-19T01:05Z" } }, From 20dc4dc6a7cbeb5502e6053a92d0f2bc092af8af Mon Sep 17 00:00:00 2001 From: Nikolay Denev <46969469+nikolay-te@users.noreply.github.com> Date: Fri, 11 Dec 2020 16:54:09 +0000 Subject: [PATCH 083/358] Update octodns/provider/ultra.py Co-authored-by: Ross McFarland --- octodns/provider/ultra.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/ultra.py b/octodns/provider/ultra.py index a6ecd23..3a4c4bf 100644 --- a/octodns/provider/ultra.py +++ b/octodns/provider/ultra.py @@ -291,7 +291,7 @@ class UltraProvider(BaseProvider): _type = self.RECORDS_TO_TYPE[record['rrtype']] except KeyError: self.log.warning('populate: ignoring record with ' - 'unsupported rrtype=%s', record) + 'unsupported rrtype, %s %s', name, record['rrtype']) continue values[name][_type] = record From 049bdb55af645292efb0344e29ee0536b9ca6660 Mon Sep 17 00:00:00 2001 From: Nikolay Denev Date: Fri, 11 Dec 2020 17:47:22 +0000 Subject: [PATCH 084/358] Shorten line --- octodns/provider/ultra.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/octodns/provider/ultra.py b/octodns/provider/ultra.py index 3a4c4bf..03b70de 100644 --- a/octodns/provider/ultra.py +++ b/octodns/provider/ultra.py @@ -291,7 +291,8 @@ class UltraProvider(BaseProvider): _type = self.RECORDS_TO_TYPE[record['rrtype']] except KeyError: self.log.warning('populate: ignoring record with ' - 'unsupported rrtype, %s %s', name, record['rrtype']) + 'unsupported rrtype, %s %s', + name, record['rrtype']) continue values[name][_type] = record From 75e75a8846720af4741adb5f9ea57d9518efe004 Mon Sep 17 00:00:00 2001 From: root Date: Sat, 12 Dec 2020 08:09:19 +0000 Subject: [PATCH 085/358] Updated doc for unsafe thresholds --- docs/records.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/records.md b/docs/records.md index d287d8a..56e2493 100644 --- a/docs/records.md +++ b/docs/records.md @@ -120,3 +120,18 @@ If you'd like to enable lenience for a whole zone you can do so with the followi targets: - ns1 ``` + +#### Restrict Record manipulations + +OctoDNS currently provides us the ability to limit the frequency of update/deletes on +DNS records by allowing us to configure a percentage of the allowed operations as a +threshold parameter. If left unconfigured, suitable defaults take over instead. +In the below example, the Dynamic provider configured accomodates only 40% of both +update and delete operations over all the records present. + +````yaml +dyn: + class: octodns.provider.dyn.DynProvider + update_pcent_threshold: 0.4 + delete_pcent_threshold: 0.4 +```` \ No newline at end of file From 77c65b042ec63564bfff0334177c99abb2f34fda Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 14 Dec 2020 06:24:17 -0800 Subject: [PATCH 086/358] Wording tweaks to threshold params doc --- docs/records.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/records.md b/docs/records.md index 56e2493..4cf1e4b 100644 --- a/docs/records.md +++ b/docs/records.md @@ -123,15 +123,15 @@ If you'd like to enable lenience for a whole zone you can do so with the followi #### Restrict Record manipulations -OctoDNS currently provides us the ability to limit the frequency of update/deletes on -DNS records by allowing us to configure a percentage of the allowed operations as a -threshold parameter. If left unconfigured, suitable defaults take over instead. -In the below example, the Dynamic provider configured accomodates only 40% of both -update and delete operations over all the records present. +OctoDNS currently provides the ability to limit the number of updates/deletes on +DNS records by configuring a percentage of allowed operations as a threshold. +If left unconfigured, suitable defaults take over instead. In the below example, +the Dyn provider is configured with limits of 40% on both update and +delete operations over all the records present. ````yaml dyn: class: octodns.provider.dyn.DynProvider update_pcent_threshold: 0.4 delete_pcent_threshold: 0.4 -```` \ No newline at end of file +```` From 2b454ccc229f14fccf2246a152a41721230a88f7 Mon Sep 17 00:00:00 2001 From: Marc 'risson' Schmitt Date: Fri, 27 Nov 2020 21:36:50 +0100 Subject: [PATCH 087/358] manager: error when an alias zone is synced without its source Signed-off-by: Marc 'risson' Schmitt --- octodns/manager.py | 9 ++++++++- tests/test_octodns_manager.py | 8 ++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/octodns/manager.py b/octodns/manager.py index 9283d19..6c05a55 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -375,12 +375,19 @@ class Manager(object): futures = [] for zone_name, zone_source in aliased_zones.items(): source_config = self.config['zones'][zone_source] + print(source_config) + try: + desired_config = desired[zone_source] + except KeyError: + raise ManagerException('Zone {} cannot be sync without zone ' + '{} sinced it is aliased' + .format(zone_name, zone_source)) futures.append(self._executor.submit( self._populate_and_plan, zone_name, [], [self.providers[t] for t in source_config['targets']], - desired=desired[zone_source], + desired=desired_config, lenient=lenient )) diff --git a/tests/test_octodns_manager.py b/tests/test_octodns_manager.py index 4e1a756..f757466 100644 --- a/tests/test_octodns_manager.py +++ b/tests/test_octodns_manager.py @@ -191,6 +191,14 @@ class TestManager(TestCase): 'zone alias.tests. is an alias zone', text_type(ctx.exception)) + # Sync an alias without the zone it refers to + with self.assertRaises(ManagerException) as ctx: + tc = Manager(get_config_filename('simple-alias-zone.yaml')) \ + .sync(eligible_zones=["alias.tests."]) + self.assertEquals('Zone alias.tests. cannot be sync without zone ' + 'unit.tests. sinced it is aliased', + text_type(ctx.exception)) + def test_compare(self): with TemporaryDirectory() as tmpdir: environ['YAML_TMP_DIR'] = tmpdir.dirname From 3e09451fd7cabe1fcdd22cec4d9ab636c7b7a56e Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 18 Dec 2020 08:50:59 -0800 Subject: [PATCH 088/358] Remove debug print. --- octodns/manager.py | 1 - 1 file changed, 1 deletion(-) diff --git a/octodns/manager.py b/octodns/manager.py index 6c05a55..9ce10ff 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -375,7 +375,6 @@ class Manager(object): futures = [] for zone_name, zone_source in aliased_zones.items(): source_config = self.config['zones'][zone_source] - print(source_config) try: desired_config = desired[zone_source] except KeyError: From bdf4a6f425e9b1f4cdcc70c4de7238159f19bbf6 Mon Sep 17 00:00:00 2001 From: Peter Dave Hello Date: Sat, 19 Dec 2020 16:45:17 +0800 Subject: [PATCH 089/358] Update development dependency twine to v3.2.0 Twine version < 2.0 was reported has a security vulnerability From Twine version 2.0+, it requires Python 3.6, but doesn't seem to have other breaking changes. Reference: - https://github.com/pypa/twine/issues/491 - https://twine.readthedocs.io/en/latest/changelog.html --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 485a33f..146d673 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,4 +5,4 @@ pycodestyle==2.6.0 pyflakes==2.2.0 readme_renderer[md]==26.0 requests_mock -twine==1.15.0 +twine==3.2.0 From 8095f4560a374724739422eb37fe52a70a806896 Mon Sep 17 00:00:00 2001 From: Peter Dave Hello Date: Mon, 21 Dec 2020 03:29:09 +0800 Subject: [PATCH 090/358] Replace legacy `...` w/ $(...) in .git_hooks_pre-commit --- .git_hooks_pre-commit | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.git_hooks_pre-commit b/.git_hooks_pre-commit index 6b3b02e..e73b892 100755 --- a/.git_hooks_pre-commit +++ b/.git_hooks_pre-commit @@ -2,9 +2,9 @@ set -e -HOOKS=`dirname $0` -GIT=`dirname $HOOKS` -ROOT=`dirname $GIT` +HOOKS=$(dirname $0) +GIT=$(dirname $HOOKS) +ROOT=$(dirname $GIT) . $ROOT/env/bin/activate $ROOT/script/lint From 0fad8fa0d976387878fcd2475be51309717ae0bc Mon Sep 17 00:00:00 2001 From: Peter Dave Hello Date: Mon, 21 Dec 2020 03:32:25 +0800 Subject: [PATCH 091/358] Quote variable to prevent globbing, word splitting in .git_hooks_pre-commit --- .git_hooks_pre-commit | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.git_hooks_pre-commit b/.git_hooks_pre-commit index e73b892..1fb621f 100755 --- a/.git_hooks_pre-commit +++ b/.git_hooks_pre-commit @@ -2,10 +2,10 @@ set -e -HOOKS=$(dirname $0) -GIT=$(dirname $HOOKS) -ROOT=$(dirname $GIT) +HOOKS=$(dirname "$0") +GIT=$(dirname "$HOOKS") +ROOT=$(dirname "$GIT") -. $ROOT/env/bin/activate -$ROOT/script/lint -$ROOT/script/coverage +. "$ROOT/env/bin/activate" +"$ROOT/script/lint" +"$ROOT/script/coverage" From 949a136f533f1ca4642edbc076a3c5fcf4599329 Mon Sep 17 00:00:00 2001 From: Arunothia Marappan Date: Fri, 25 Dec 2020 21:07:23 -0800 Subject: [PATCH 092/358] Enforcing Delete to happen before all other operations in _apply --- octodns/provider/azuredns.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 8ad6dd9..f36c25e 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -488,10 +488,15 @@ class AzureProvider(BaseProvider): azure_zone_name = desired.name[:len(desired.name) - 1] self._check_zone(azure_zone_name, create=True) - # Force the operation order to be Delete() before Create() - # Helps avoid problems in updating a CNAME record into an A record. - changes.reverse() + # Force the operation order to be Delete() before all other operations. + # Helps avoid problems in updating a CNAME record into an A record and vice-versa. for change in changes: class_name = change.__class__.__name__ - getattr(self, '_apply_{}'.format(class_name))(change) + if class_name == 'Delete': + getattr(self, '_apply_{}'.format(class_name))(change) + + for change in changes: + class_name = change.__class__.__name__ + if class_name != 'Delete': + getattr(self, '_apply_{}'.format(class_name))(change) From d28d51290b011418b240dbcba60f69616dee5f5d Mon Sep 17 00:00:00 2001 From: Arunothia Marappan Date: Fri, 25 Dec 2020 21:11:02 -0800 Subject: [PATCH 093/358] Removing space from blank line --- octodns/provider/azuredns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index f36c25e..3bbe1ea 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -495,7 +495,7 @@ class AzureProvider(BaseProvider): class_name = change.__class__.__name__ if class_name == 'Delete': getattr(self, '_apply_{}'.format(class_name))(change) - + for change in changes: class_name = change.__class__.__name__ if class_name != 'Delete': From cad48ea4e8371fb1a7f1303495d7c59a888c0fdb Mon Sep 17 00:00:00 2001 From: Arunothia Marappan Date: Fri, 25 Dec 2020 21:50:46 -0800 Subject: [PATCH 094/358] Updating lengthy comment --- octodns/provider/azuredns.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 3bbe1ea..a86d671 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -488,8 +488,12 @@ class AzureProvider(BaseProvider): azure_zone_name = desired.name[:len(desired.name) - 1] self._check_zone(azure_zone_name, create=True) - # Force the operation order to be Delete() before all other operations. - # Helps avoid problems in updating a CNAME record into an A record and vice-versa. + ''' + Force the operation order to be Delete() before all other operations. + Helps avoid problems in updating + - a CNAME record into an A record. + - an A record into a CNAME record. + ''' for change in changes: class_name = change.__class__.__name__ From cffc90607161fb1785aac5311414c3ce49c9904b Mon Sep 17 00:00:00 2001 From: Arunothia Marappan Date: Fri, 25 Dec 2020 21:53:47 -0800 Subject: [PATCH 095/358] Removing trailing whitespace in comment --- octodns/provider/azuredns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index a86d671..4be0d4d 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -490,7 +490,7 @@ class AzureProvider(BaseProvider): ''' Force the operation order to be Delete() before all other operations. - Helps avoid problems in updating + Helps avoid problems in updating - a CNAME record into an A record. - an A record into a CNAME record. ''' From d801a4005733f050163d335e4059b8d99d227540 Mon Sep 17 00:00:00 2001 From: Peter Dave Hello Date: Sun, 27 Dec 2020 04:49:11 +0800 Subject: [PATCH 096/358] Fix indent of config/example.com.yaml in README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 871a35e..a65e28f 100644 --- a/README.md +++ b/README.md @@ -102,8 +102,8 @@ Now that we have something to tell OctoDNS about our providers & zones we need t ttl: 60 type: A values: - - 1.2.3.4 - - 1.2.3.5 + - 1.2.3.4 + - 1.2.3.5 ``` Further information can be found in [Records Documentation](/docs/records.md). From 32811ed5c1906aa93afa0c83307811febfa0b531 Mon Sep 17 00:00:00 2001 From: Arunothia Marappan Date: Sat, 26 Dec 2020 15:01:14 -0800 Subject: [PATCH 097/358] Update octodns/provider/azuredns.py Co-authored-by: Ross McFarland --- octodns/provider/azuredns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 4be0d4d..d1ec333 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -498,7 +498,7 @@ class AzureProvider(BaseProvider): for change in changes: class_name = change.__class__.__name__ if class_name == 'Delete': - getattr(self, '_apply_{}'.format(class_name))(change) + self._apply_Delete(change) for change in changes: class_name = change.__class__.__name__ From 7fe72f00612186c3bbc32ce39b7e71557d831cf5 Mon Sep 17 00:00:00 2001 From: Piotr Pieprzycki Date: Mon, 4 Jan 2021 09:14:58 +0100 Subject: [PATCH 098/358] Remove blank lines --- tests/test_octodns_provider_azuredns.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index 0b94dfe..2b2c1e7 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -500,8 +500,6 @@ class TestAzureDnsProvider(TestCase): exists = provider.populate(zone) self.assertTrue(exists) - - self.assertEquals(len(zone.records), 17) def test_populate_zone(self): From 8d1f6c69e7c4c011d86ebbc51bd38b20a78bb02d Mon Sep 17 00:00:00 2001 From: Adam Smith Date: Fri, 8 Jan 2021 15:08:38 -0800 Subject: [PATCH 099/358] Add fqdn module to setup.py as dependency --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index b25909a..0b15571 100644 --- a/setup.py +++ b/setup.py @@ -69,6 +69,7 @@ setup( 'PyYaml>=4.2b1', 'dnspython>=1.15.0', 'futures>=3.2.0; python_version<"3.2"', + 'fqdn>=1.5.0', 'ipaddress>=1.0.22; python_version<"3.3"', 'natsort>=5.5.0', 'pycountry>=19.8.18', From b2eab63d54533b7dfb469b4cea8d0fcd72eaa3eb Mon Sep 17 00:00:00 2001 From: Adam Smith Date: Fri, 8 Jan 2021 19:36:58 -0500 Subject: [PATCH 100/358] ZoneFileSource: allow users to specify file extension --- octodns/source/axfr.py | 19 +++++++++++++++---- tests/test_octodns_source_axfr.py | 7 +++++++ tests/zones/unit.tests.extension | 12 ++++++++++++ 3 files changed, 34 insertions(+), 4 deletions(-) create mode 100644 tests/zones/unit.tests.extension diff --git a/octodns/source/axfr.py b/octodns/source/axfr.py index 2e18ef0..1ca2c67 100644 --- a/octodns/source/axfr.py +++ b/octodns/source/axfr.py @@ -206,17 +206,24 @@ class ZoneFileSource(AxfrBaseSource): class: octodns.source.axfr.ZoneFileSource # The directory holding the zone files # Filenames should match zone name (eg. example.com.) + # with optional extension specified with file_extension directory: ./zonefiles + # File extension on zone files + # Appended to zone name to locate file + # (optional, default None) + file_extension: zone # Should sanity checks of the origin node be done # (optional, default true) check_origin: false ''' - def __init__(self, id, directory, check_origin=True): + def __init__(self, id, directory, file_extension=None, check_origin=True): self.log = logging.getLogger('ZoneFileSource[{}]'.format(id)) - self.log.debug('__init__: id=%s, directory=%s, check_origin=%s', id, - directory, check_origin) + self.log.debug('__init__: id=%s, directory=%s, file_extension=%s, ' + 'check_origin=%s', id, + directory, file_extension, check_origin) super(ZoneFileSource, self).__init__(id) self.directory = directory + self.file_extension = file_extension self.check_origin = check_origin self._zone_records = {} @@ -225,7 +232,11 @@ class ZoneFileSource(AxfrBaseSource): zonefiles = listdir(self.directory) if zone_name in zonefiles: try: - z = dns.zone.from_file(join(self.directory, zone_name), + filename = zone_name + if self.file_extension: + filename = '{}{}'.format(zone_name, + self.file_extension.lstrip('.')) + z = dns.zone.from_file(join(self.directory, filename), zone_name, relativize=False, check_origin=self.check_origin) except DNSException as error: diff --git a/tests/test_octodns_source_axfr.py b/tests/test_octodns_source_axfr.py index 1bf3f22..f808166 100644 --- a/tests/test_octodns_source_axfr.py +++ b/tests/test_octodns_source_axfr.py @@ -45,6 +45,13 @@ class TestAxfrSource(TestCase): class TestZoneFileSource(TestCase): source = ZoneFileSource('test', './tests/zones') + source_extension = ZoneFileSource('test', './tests/zones', 'extension') + + def test_zonefiles_with_extension(self): + # Load zonefiles with a specified file extension + valid = Zone('unit.tests.', []) + self.source_extension.populate(valid) + self.assertEquals(1, len(valid.records)) def test_populate(self): # Valid zone file in directory diff --git a/tests/zones/unit.tests.extension b/tests/zones/unit.tests.extension new file mode 100644 index 0000000..2821d9a --- /dev/null +++ b/tests/zones/unit.tests.extension @@ -0,0 +1,12 @@ +$ORIGIN unit.tests. +@ 3600 IN SOA ns1.unit.tests. root.unit.tests. ( + 2018071501 ; Serial + 3600 ; Refresh (1 hour) + 600 ; Retry (10 minutes) + 604800 ; Expire (1 week) + 3600 ; NXDOMAIN ttl (1 hour) + ) + +; NS Records +@ 3600 IN NS ns1.unit.tests. +@ 3600 IN NS ns2.unit.tests. From 97feaa7823878a4f87e7caa18301eb710528e2cc Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 25 Jan 2021 15:32:30 -0800 Subject: [PATCH 101/358] Rename extention zonefile test to avoid existing unit.tests. --- tests/test_octodns_source_axfr.py | 6 +++--- .../{unit.tests.extension => ext.unit.tests.extension} | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) rename tests/zones/{unit.tests.extension => ext.unit.tests.extension} (61%) diff --git a/tests/test_octodns_source_axfr.py b/tests/test_octodns_source_axfr.py index f808166..a1d2e1c 100644 --- a/tests/test_octodns_source_axfr.py +++ b/tests/test_octodns_source_axfr.py @@ -45,12 +45,12 @@ class TestAxfrSource(TestCase): class TestZoneFileSource(TestCase): source = ZoneFileSource('test', './tests/zones') - source_extension = ZoneFileSource('test', './tests/zones', 'extension') def test_zonefiles_with_extension(self): + source = ZoneFileSource('test', './tests/zones', 'extension') # Load zonefiles with a specified file extension - valid = Zone('unit.tests.', []) - self.source_extension.populate(valid) + valid = Zone('ext.unit.tests.', []) + source.populate(valid) self.assertEquals(1, len(valid.records)) def test_populate(self): diff --git a/tests/zones/unit.tests.extension b/tests/zones/ext.unit.tests.extension similarity index 61% rename from tests/zones/unit.tests.extension rename to tests/zones/ext.unit.tests.extension index 2821d9a..2ed7ac6 100644 --- a/tests/zones/unit.tests.extension +++ b/tests/zones/ext.unit.tests.extension @@ -1,5 +1,5 @@ -$ORIGIN unit.tests. -@ 3600 IN SOA ns1.unit.tests. root.unit.tests. ( +$ORIGIN ext.unit.tests. +@ 3600 IN SOA ns1.ext.unit.tests. root.ext.unit.tests. ( 2018071501 ; Serial 3600 ; Refresh (1 hour) 600 ; Retry (10 minutes) @@ -8,5 +8,5 @@ $ORIGIN unit.tests. ) ; NS Records -@ 3600 IN NS ns1.unit.tests. -@ 3600 IN NS ns2.unit.tests. +@ 3600 IN NS ns1.ext.unit.tests. +@ 3600 IN NS ns2.ext.unit.tests. From c08d4ac88f9e8d0e02435e9ec27a57706ca62cc5 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 25 Jan 2021 15:35:37 -0800 Subject: [PATCH 102/358] Look for zone filename not zone_name in axfr directory listing --- octodns/source/axfr.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/octodns/source/axfr.py b/octodns/source/axfr.py index 1ca2c67..ed3f98f 100644 --- a/octodns/source/axfr.py +++ b/octodns/source/axfr.py @@ -229,14 +229,16 @@ class ZoneFileSource(AxfrBaseSource): self._zone_records = {} def _load_zone_file(self, zone_name): + + zone_filename = zone_name + if self.file_extension: + zone_filename = '{}{}'.format(zone_name, + self.file_extension.lstrip('.')) + zonefiles = listdir(self.directory) - if zone_name in zonefiles: + if zone_filename in zonefiles: try: - filename = zone_name - if self.file_extension: - filename = '{}{}'.format(zone_name, - self.file_extension.lstrip('.')) - z = dns.zone.from_file(join(self.directory, filename), + z = dns.zone.from_file(join(self.directory, zone_filename), zone_name, relativize=False, check_origin=self.check_origin) except DNSException as error: From ea943e606ed358cc23cd91bd0ba7f2508ab81fcf Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 25 Jan 2021 15:45:23 -0800 Subject: [PATCH 103/358] Avoid . on the end of files, but still test axfr default --- .gitignore | 3 ++- tests/test_octodns_source_axfr.py | 13 +++++++++++-- .../zones/{invalid.records. => invalid.records.tst} | 0 tests/zones/{invalid.zone. => invalid.zone.tst} | 0 tests/zones/{unit.tests. => unit.tests.tst} | 0 5 files changed, 13 insertions(+), 3 deletions(-) rename tests/zones/{invalid.records. => invalid.records.tst} (100%) rename tests/zones/{invalid.zone. => invalid.zone.tst} (100%) rename tests/zones/{unit.tests. => unit.tests.tst} (100%) diff --git a/.gitignore b/.gitignore index 715b687..5192821 100644 --- a/.gitignore +++ b/.gitignore @@ -5,8 +5,8 @@ *.pyc .coverage .env -/config/ /build/ +/config/ coverage.xml dist/ env/ @@ -14,4 +14,5 @@ htmlcov/ nosetests.xml octodns.egg-info/ output/ +tests/zones/unit.tests. tmp/ diff --git a/tests/test_octodns_source_axfr.py b/tests/test_octodns_source_axfr.py index a1d2e1c..8d0a527 100644 --- a/tests/test_octodns_source_axfr.py +++ b/tests/test_octodns_source_axfr.py @@ -9,6 +9,7 @@ import dns.zone from dns.exception import DNSException from mock import patch +from shutil import copyfile from six import text_type from unittest import TestCase @@ -21,7 +22,7 @@ from octodns.record import ValidationError class TestAxfrSource(TestCase): source = AxfrSource('test', 'localhost') - forward_zonefile = dns.zone.from_file('./tests/zones/unit.tests.', + forward_zonefile = dns.zone.from_file('./tests/zones/unit.tests.tst', 'unit.tests', relativize=False) @patch('dns.zone.from_xfr') @@ -44,7 +45,7 @@ class TestAxfrSource(TestCase): class TestZoneFileSource(TestCase): - source = ZoneFileSource('test', './tests/zones') + source = ZoneFileSource('test', './tests/zones', file_extension='tst') def test_zonefiles_with_extension(self): source = ZoneFileSource('test', './tests/zones', 'extension') @@ -53,6 +54,14 @@ class TestZoneFileSource(TestCase): source.populate(valid) self.assertEquals(1, len(valid.records)) + def test_zonefiles_without_extension(self): + copyfile('./tests/zones/unit.tests.tst', './tests/zones/unit.tests.') + source = ZoneFileSource('test', './tests/zones') + # Load zonefiles without a specified file extension + valid = Zone('unit.tests.', []) + source.populate(valid) + self.assertEquals(12, len(valid.records)) + def test_populate(self): # Valid zone file in directory valid = Zone('unit.tests.', []) diff --git a/tests/zones/invalid.records. b/tests/zones/invalid.records.tst similarity index 100% rename from tests/zones/invalid.records. rename to tests/zones/invalid.records.tst diff --git a/tests/zones/invalid.zone. b/tests/zones/invalid.zone.tst similarity index 100% rename from tests/zones/invalid.zone. rename to tests/zones/invalid.zone.tst diff --git a/tests/zones/unit.tests. b/tests/zones/unit.tests.tst similarity index 100% rename from tests/zones/unit.tests. rename to tests/zones/unit.tests.tst From 37381bd2740a899c1944c5a5a274814ee807d1ff Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 25 Jan 2021 15:50:34 -0800 Subject: [PATCH 104/358] Skip the axfr default name test if we can't create the needed tests file --- tests/test_octodns_source_axfr.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_octodns_source_axfr.py b/tests/test_octodns_source_axfr.py index 8d0a527..f732060 100644 --- a/tests/test_octodns_source_axfr.py +++ b/tests/test_octodns_source_axfr.py @@ -55,7 +55,12 @@ class TestZoneFileSource(TestCase): self.assertEquals(1, len(valid.records)) def test_zonefiles_without_extension(self): - copyfile('./tests/zones/unit.tests.tst', './tests/zones/unit.tests.') + try: + copyfile('./tests/zones/unit.tests.tst', + './tests/zones/unit.tests.') + except: + self.skipTest('Unable to create unit.tests. (ending with .) so ' + 'skipping default filename testing.') source = ZoneFileSource('test', './tests/zones') # Load zonefiles without a specified file extension valid = Zone('unit.tests.', []) From dd1dbfbfdd1d66ee4c62487a6394fa5d10f2e443 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Thu, 28 Jan 2021 12:23:13 -0800 Subject: [PATCH 105/358] Rework copyfile and skip based on feedback from windows test --- tests/test_octodns_source_axfr.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/test_octodns_source_axfr.py b/tests/test_octodns_source_axfr.py index f732060..e0871ee 100644 --- a/tests/test_octodns_source_axfr.py +++ b/tests/test_octodns_source_axfr.py @@ -9,6 +9,7 @@ import dns.zone from dns.exception import DNSException from mock import patch +from os.path import exists from shutil import copyfile from six import text_type from unittest import TestCase @@ -55,12 +56,19 @@ class TestZoneFileSource(TestCase): self.assertEquals(1, len(valid.records)) def test_zonefiles_without_extension(self): - try: - copyfile('./tests/zones/unit.tests.tst', - './tests/zones/unit.tests.') - except: + # Windows doesn't let files end with a `.` so we add a .tst to them in + # the repo and then try and create the `.` version we need for the + # default case (no extension.) + copyfile('./tests/zones/unit.tests.tst', './tests/zones/unit.tests.') + # Unfortunately copyfile silently works and create the file without + # the `.` so we have to check to see if it did that + if exists('./tests/zones/unit.tests'): + # It did so we need to skip this test, that means windows won't + # have full code coverage, but skipping the test is going out of + # our way enough for a os-specific/oddball case. self.skipTest('Unable to create unit.tests. (ending with .) so ' 'skipping default filename testing.') + source = ZoneFileSource('test', './tests/zones') # Load zonefiles without a specified file extension valid = Zone('unit.tests.', []) From 4ce2563d2ec1fdd5b6e3cfbaf4e938f17b2a26e2 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Thu, 28 Jan 2021 13:24:35 -0800 Subject: [PATCH 106/358] Remove the rest of the . ending files, clean up code and tests for better coverage --- octodns/provider/yaml.py | 6 ++-- octodns/source/axfr.py | 9 ++---- tests/config/simple-split.yaml | 3 ++ .../a.yaml | 0 .../aaaa.yaml | 0 .../cname.yaml | 0 .../real-ish-a.yaml | 0 .../simple-weighted.yaml | 0 .../split/{empty. => empty.tst}/.gitkeep | 0 .../12.yaml | 0 .../2.yaml | 0 .../test.yaml | 0 .../$unit.tests.yaml | 0 .../_srv._tcp.yaml | 0 .../{unit.tests. => unit.tests.tst}/aaaa.yaml | 0 .../cname.yaml | 0 .../dname.yaml | 0 .../excluded.yaml | 0 .../ignored.yaml | 0 .../included.yaml | 0 .../{unit.tests. => unit.tests.tst}/mx.yaml | 0 .../naptr.yaml | 0 .../{unit.tests. => unit.tests.tst}/ptr.yaml | 0 .../{unit.tests. => unit.tests.tst}/spf.yaml | 0 .../{unit.tests. => unit.tests.tst}/sub.yaml | 0 .../{unit.tests. => unit.tests.tst}/txt.yaml | 0 .../www.sub.yaml | 0 .../{unit.tests. => unit.tests.tst}/www.yaml | 0 .../{unordered. => unordered.tst}/abc.yaml | 0 .../{unordered. => unordered.tst}/xyz.yaml | 0 tests/test_octodns_provider_yaml.py | 31 ++++++++++++------- tests/test_octodns_source_axfr.py | 4 +-- 32 files changed, 30 insertions(+), 23 deletions(-) rename tests/config/split/{dynamic.tests. => dynamic.tests.tst}/a.yaml (100%) rename tests/config/split/{dynamic.tests. => dynamic.tests.tst}/aaaa.yaml (100%) rename tests/config/split/{dynamic.tests. => dynamic.tests.tst}/cname.yaml (100%) rename tests/config/split/{dynamic.tests. => dynamic.tests.tst}/real-ish-a.yaml (100%) rename tests/config/split/{dynamic.tests. => dynamic.tests.tst}/simple-weighted.yaml (100%) rename tests/config/split/{empty. => empty.tst}/.gitkeep (100%) rename tests/config/split/{subzone.unit.tests. => subzone.unit.tests.tst}/12.yaml (100%) rename tests/config/split/{subzone.unit.tests. => subzone.unit.tests.tst}/2.yaml (100%) rename tests/config/split/{subzone.unit.tests. => subzone.unit.tests.tst}/test.yaml (100%) rename tests/config/split/{unit.tests. => unit.tests.tst}/$unit.tests.yaml (100%) rename tests/config/split/{unit.tests. => unit.tests.tst}/_srv._tcp.yaml (100%) rename tests/config/split/{unit.tests. => unit.tests.tst}/aaaa.yaml (100%) rename tests/config/split/{unit.tests. => unit.tests.tst}/cname.yaml (100%) rename tests/config/split/{unit.tests. => unit.tests.tst}/dname.yaml (100%) rename tests/config/split/{unit.tests. => unit.tests.tst}/excluded.yaml (100%) rename tests/config/split/{unit.tests. => unit.tests.tst}/ignored.yaml (100%) rename tests/config/split/{unit.tests. => unit.tests.tst}/included.yaml (100%) rename tests/config/split/{unit.tests. => unit.tests.tst}/mx.yaml (100%) rename tests/config/split/{unit.tests. => unit.tests.tst}/naptr.yaml (100%) rename tests/config/split/{unit.tests. => unit.tests.tst}/ptr.yaml (100%) rename tests/config/split/{unit.tests. => unit.tests.tst}/spf.yaml (100%) rename tests/config/split/{unit.tests. => unit.tests.tst}/sub.yaml (100%) rename tests/config/split/{unit.tests. => unit.tests.tst}/txt.yaml (100%) rename tests/config/split/{unit.tests. => unit.tests.tst}/www.sub.yaml (100%) rename tests/config/split/{unit.tests. => unit.tests.tst}/www.yaml (100%) rename tests/config/split/{unordered. => unordered.tst}/abc.yaml (100%) rename tests/config/split/{unordered. => unordered.tst}/xyz.yaml (100%) diff --git a/octodns/provider/yaml.py b/octodns/provider/yaml.py index 55a1632..3deca01 100644 --- a/octodns/provider/yaml.py +++ b/octodns/provider/yaml.py @@ -239,11 +239,13 @@ class SplitYamlProvider(YamlProvider): # instead of a file matching the record name. CATCHALL_RECORD_NAMES = ('*', '') - def __init__(self, id, directory, *args, **kwargs): + def __init__(self, id, directory, extension='.', *args, **kwargs): super(SplitYamlProvider, self).__init__(id, directory, *args, **kwargs) + self.extension = extension def _zone_directory(self, zone): - return join(self.directory, zone.name) + filename = '{}{}'.format(zone.name[:-1], self.extension) + return join(self.directory, filename) def populate(self, zone, target=False, lenient=False): self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, diff --git a/octodns/source/axfr.py b/octodns/source/axfr.py index ed3f98f..e21f29f 100644 --- a/octodns/source/axfr.py +++ b/octodns/source/axfr.py @@ -216,7 +216,7 @@ class ZoneFileSource(AxfrBaseSource): # (optional, default true) check_origin: false ''' - def __init__(self, id, directory, file_extension=None, check_origin=True): + def __init__(self, id, directory, file_extension='.', check_origin=True): self.log = logging.getLogger('ZoneFileSource[{}]'.format(id)) self.log.debug('__init__: id=%s, directory=%s, file_extension=%s, ' 'check_origin=%s', id, @@ -229,12 +229,7 @@ class ZoneFileSource(AxfrBaseSource): self._zone_records = {} def _load_zone_file(self, zone_name): - - zone_filename = zone_name - if self.file_extension: - zone_filename = '{}{}'.format(zone_name, - self.file_extension.lstrip('.')) - + zone_filename = '{}{}'.format(zone_name[:-1], self.file_extension) zonefiles = listdir(self.directory) if zone_filename in zonefiles: try: diff --git a/tests/config/simple-split.yaml b/tests/config/simple-split.yaml index d106506..a798258 100644 --- a/tests/config/simple-split.yaml +++ b/tests/config/simple-split.yaml @@ -4,14 +4,17 @@ providers: in: class: octodns.provider.yaml.SplitYamlProvider directory: tests/config/split + extension: .tst dump: class: octodns.provider.yaml.SplitYamlProvider directory: env/YAML_TMP_DIR + extension: .tst # This is sort of ugly, but it shouldn't hurt anything. It'll just write out # the target file twice where it and dump are both used dump2: class: octodns.provider.yaml.SplitYamlProvider directory: env/YAML_TMP_DIR + extension: .tst simple: class: helpers.SimpleProvider geo: diff --git a/tests/config/split/dynamic.tests./a.yaml b/tests/config/split/dynamic.tests.tst/a.yaml similarity index 100% rename from tests/config/split/dynamic.tests./a.yaml rename to tests/config/split/dynamic.tests.tst/a.yaml diff --git a/tests/config/split/dynamic.tests./aaaa.yaml b/tests/config/split/dynamic.tests.tst/aaaa.yaml similarity index 100% rename from tests/config/split/dynamic.tests./aaaa.yaml rename to tests/config/split/dynamic.tests.tst/aaaa.yaml diff --git a/tests/config/split/dynamic.tests./cname.yaml b/tests/config/split/dynamic.tests.tst/cname.yaml similarity index 100% rename from tests/config/split/dynamic.tests./cname.yaml rename to tests/config/split/dynamic.tests.tst/cname.yaml diff --git a/tests/config/split/dynamic.tests./real-ish-a.yaml b/tests/config/split/dynamic.tests.tst/real-ish-a.yaml similarity index 100% rename from tests/config/split/dynamic.tests./real-ish-a.yaml rename to tests/config/split/dynamic.tests.tst/real-ish-a.yaml diff --git a/tests/config/split/dynamic.tests./simple-weighted.yaml b/tests/config/split/dynamic.tests.tst/simple-weighted.yaml similarity index 100% rename from tests/config/split/dynamic.tests./simple-weighted.yaml rename to tests/config/split/dynamic.tests.tst/simple-weighted.yaml diff --git a/tests/config/split/empty./.gitkeep b/tests/config/split/empty.tst/.gitkeep similarity index 100% rename from tests/config/split/empty./.gitkeep rename to tests/config/split/empty.tst/.gitkeep diff --git a/tests/config/split/subzone.unit.tests./12.yaml b/tests/config/split/subzone.unit.tests.tst/12.yaml similarity index 100% rename from tests/config/split/subzone.unit.tests./12.yaml rename to tests/config/split/subzone.unit.tests.tst/12.yaml diff --git a/tests/config/split/subzone.unit.tests./2.yaml b/tests/config/split/subzone.unit.tests.tst/2.yaml similarity index 100% rename from tests/config/split/subzone.unit.tests./2.yaml rename to tests/config/split/subzone.unit.tests.tst/2.yaml diff --git a/tests/config/split/subzone.unit.tests./test.yaml b/tests/config/split/subzone.unit.tests.tst/test.yaml similarity index 100% rename from tests/config/split/subzone.unit.tests./test.yaml rename to tests/config/split/subzone.unit.tests.tst/test.yaml diff --git a/tests/config/split/unit.tests./$unit.tests.yaml b/tests/config/split/unit.tests.tst/$unit.tests.yaml similarity index 100% rename from tests/config/split/unit.tests./$unit.tests.yaml rename to tests/config/split/unit.tests.tst/$unit.tests.yaml diff --git a/tests/config/split/unit.tests./_srv._tcp.yaml b/tests/config/split/unit.tests.tst/_srv._tcp.yaml similarity index 100% rename from tests/config/split/unit.tests./_srv._tcp.yaml rename to tests/config/split/unit.tests.tst/_srv._tcp.yaml diff --git a/tests/config/split/unit.tests./aaaa.yaml b/tests/config/split/unit.tests.tst/aaaa.yaml similarity index 100% rename from tests/config/split/unit.tests./aaaa.yaml rename to tests/config/split/unit.tests.tst/aaaa.yaml diff --git a/tests/config/split/unit.tests./cname.yaml b/tests/config/split/unit.tests.tst/cname.yaml similarity index 100% rename from tests/config/split/unit.tests./cname.yaml rename to tests/config/split/unit.tests.tst/cname.yaml diff --git a/tests/config/split/unit.tests./dname.yaml b/tests/config/split/unit.tests.tst/dname.yaml similarity index 100% rename from tests/config/split/unit.tests./dname.yaml rename to tests/config/split/unit.tests.tst/dname.yaml diff --git a/tests/config/split/unit.tests./excluded.yaml b/tests/config/split/unit.tests.tst/excluded.yaml similarity index 100% rename from tests/config/split/unit.tests./excluded.yaml rename to tests/config/split/unit.tests.tst/excluded.yaml diff --git a/tests/config/split/unit.tests./ignored.yaml b/tests/config/split/unit.tests.tst/ignored.yaml similarity index 100% rename from tests/config/split/unit.tests./ignored.yaml rename to tests/config/split/unit.tests.tst/ignored.yaml diff --git a/tests/config/split/unit.tests./included.yaml b/tests/config/split/unit.tests.tst/included.yaml similarity index 100% rename from tests/config/split/unit.tests./included.yaml rename to tests/config/split/unit.tests.tst/included.yaml diff --git a/tests/config/split/unit.tests./mx.yaml b/tests/config/split/unit.tests.tst/mx.yaml similarity index 100% rename from tests/config/split/unit.tests./mx.yaml rename to tests/config/split/unit.tests.tst/mx.yaml diff --git a/tests/config/split/unit.tests./naptr.yaml b/tests/config/split/unit.tests.tst/naptr.yaml similarity index 100% rename from tests/config/split/unit.tests./naptr.yaml rename to tests/config/split/unit.tests.tst/naptr.yaml diff --git a/tests/config/split/unit.tests./ptr.yaml b/tests/config/split/unit.tests.tst/ptr.yaml similarity index 100% rename from tests/config/split/unit.tests./ptr.yaml rename to tests/config/split/unit.tests.tst/ptr.yaml diff --git a/tests/config/split/unit.tests./spf.yaml b/tests/config/split/unit.tests.tst/spf.yaml similarity index 100% rename from tests/config/split/unit.tests./spf.yaml rename to tests/config/split/unit.tests.tst/spf.yaml diff --git a/tests/config/split/unit.tests./sub.yaml b/tests/config/split/unit.tests.tst/sub.yaml similarity index 100% rename from tests/config/split/unit.tests./sub.yaml rename to tests/config/split/unit.tests.tst/sub.yaml diff --git a/tests/config/split/unit.tests./txt.yaml b/tests/config/split/unit.tests.tst/txt.yaml similarity index 100% rename from tests/config/split/unit.tests./txt.yaml rename to tests/config/split/unit.tests.tst/txt.yaml diff --git a/tests/config/split/unit.tests./www.sub.yaml b/tests/config/split/unit.tests.tst/www.sub.yaml similarity index 100% rename from tests/config/split/unit.tests./www.sub.yaml rename to tests/config/split/unit.tests.tst/www.sub.yaml diff --git a/tests/config/split/unit.tests./www.yaml b/tests/config/split/unit.tests.tst/www.yaml similarity index 100% rename from tests/config/split/unit.tests./www.yaml rename to tests/config/split/unit.tests.tst/www.yaml diff --git a/tests/config/split/unordered./abc.yaml b/tests/config/split/unordered.tst/abc.yaml similarity index 100% rename from tests/config/split/unordered./abc.yaml rename to tests/config/split/unordered.tst/abc.yaml diff --git a/tests/config/split/unordered./xyz.yaml b/tests/config/split/unordered.tst/xyz.yaml similarity index 100% rename from tests/config/split/unordered./xyz.yaml rename to tests/config/split/unordered.tst/xyz.yaml diff --git a/tests/test_octodns_provider_yaml.py b/tests/test_octodns_provider_yaml.py index 15e90da..38dfc11 100644 --- a/tests/test_octodns_provider_yaml.py +++ b/tests/test_octodns_provider_yaml.py @@ -207,18 +207,20 @@ class TestSplitYamlProvider(TestCase): def test_zone_directory(self): source = SplitYamlProvider( - 'test', join(dirname(__file__), 'config/split')) + 'test', join(dirname(__file__), 'config/split'), + extension='.tst') zone = Zone('unit.tests.', []) self.assertEqual( - join(dirname(__file__), 'config/split/unit.tests.'), + join(dirname(__file__), 'config/split/unit.tests.tst'), source._zone_directory(zone)) def test_apply_handles_existing_zone_directory(self): with TemporaryDirectory() as td: - provider = SplitYamlProvider('test', join(td.dirname, 'config')) - makedirs(join(td.dirname, 'config', 'does.exist.')) + provider = SplitYamlProvider('test', join(td.dirname, 'config'), + extension='.tst') + makedirs(join(td.dirname, 'config', 'does.exist.tst')) zone = Zone('does.exist.', []) self.assertTrue(isdir(provider._zone_directory(zone))) @@ -227,7 +229,8 @@ class TestSplitYamlProvider(TestCase): def test_provider(self): source = SplitYamlProvider( - 'test', join(dirname(__file__), 'config/split')) + 'test', join(dirname(__file__), 'config/split'), + extension='.tst') zone = Zone('unit.tests.', []) dynamic_zone = Zone('dynamic.tests.', []) @@ -246,9 +249,10 @@ class TestSplitYamlProvider(TestCase): with TemporaryDirectory() as td: # Add some subdirs to make sure that it can create them directory = join(td.dirname, 'sub', 'dir') - zone_dir = join(directory, 'unit.tests.') - dynamic_zone_dir = join(directory, 'dynamic.tests.') - target = SplitYamlProvider('test', directory) + zone_dir = join(directory, 'unit.tests.tst') + dynamic_zone_dir = join(directory, 'dynamic.tests.tst') + target = SplitYamlProvider('test', directory, + extension='.tst') # We add everything plan = target.plan(zone) @@ -335,7 +339,8 @@ class TestSplitYamlProvider(TestCase): def test_empty(self): source = SplitYamlProvider( - 'test', join(dirname(__file__), 'config/split')) + 'test', join(dirname(__file__), 'config/split'), + extension='.tst') zone = Zone('empty.', []) @@ -345,7 +350,8 @@ class TestSplitYamlProvider(TestCase): def test_unsorted(self): source = SplitYamlProvider( - 'test', join(dirname(__file__), 'config/split')) + 'test', join(dirname(__file__), 'config/split'), + extension='.tst') zone = Zone('unordered.', []) @@ -356,14 +362,15 @@ class TestSplitYamlProvider(TestCase): source = SplitYamlProvider( 'test', join(dirname(__file__), 'config/split'), - enforce_order=False) + extension='.tst', enforce_order=False) # no exception source.populate(zone) self.assertEqual(2, len(zone.records)) def test_subzone_handling(self): source = SplitYamlProvider( - 'test', join(dirname(__file__), 'config/split')) + 'test', join(dirname(__file__), 'config/split'), + extension='.tst') # If we add `sub` as a sub-zone we'll reject `www.sub` zone = Zone('unit.tests.', ['sub']) diff --git a/tests/test_octodns_source_axfr.py b/tests/test_octodns_source_axfr.py index e0871ee..44e04d0 100644 --- a/tests/test_octodns_source_axfr.py +++ b/tests/test_octodns_source_axfr.py @@ -46,10 +46,10 @@ class TestAxfrSource(TestCase): class TestZoneFileSource(TestCase): - source = ZoneFileSource('test', './tests/zones', file_extension='tst') + source = ZoneFileSource('test', './tests/zones', file_extension='.tst') def test_zonefiles_with_extension(self): - source = ZoneFileSource('test', './tests/zones', 'extension') + source = ZoneFileSource('test', './tests/zones', '.extension') # Load zonefiles with a specified file extension valid = Zone('ext.unit.tests.', []) source.populate(valid) From a8505d66f13421f1c67242f4ecbf1fbeae85aef5 Mon Sep 17 00:00:00 2001 From: Robert Reichel Date: Fri, 29 Jan 2021 15:11:27 -0500 Subject: [PATCH 107/358] Improve checking and creating Azure DNS Zones This change improves the process for checking for AzureDNS zones by using the known set and not relying upon custom error handling. Since the provider already fetches the zones, octodns doesn't need to make a second call to check for the existence of the zone - _populate_zones already does that for us. --- octodns/provider/azuredns.py | 36 ++++++++++--------------- tests/test_octodns_provider_azuredns.py | 30 ++++++++++++++------- 2 files changed, 34 insertions(+), 32 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index a1ef6fe..0224f06 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -341,7 +341,8 @@ class AzureProvider(BaseProvider): self.log.debug('azure_zones: loading') list_zones = self._dns_client.zones.list_by_resource_group for zone in list_zones(self._resource_group): - self._azure_zones.add(zone.name) + if zone.name: + self._azure_zones.add(zone.name.rstrip('.')) def _check_zone(self, name, create=False): '''Checks whether a zone specified in a source exist in Azure server. @@ -356,29 +357,20 @@ class AzureProvider(BaseProvider): :type return: str or None ''' - self.log.debug('_check_zone: name=%s', name) - try: - if name in self._azure_zones: - return name - self._dns_client.zones.get(self._resource_group, name) + self.log.debug('_check_zone: name=%s create=%s', name, create) + # Check if the zone already exists in our set + if name in self._azure_zones: + return name + # If not, and its time to create, lets do it. + if create: + self.log.debug('_check_zone:no matching zone; creating %s', name) + create_zone = self._dns_client.zones.create_or_update + create_zone(self._resource_group, name, Zone(location='global')) self._azure_zones.add(name) return name - except CloudError as err: - msg = 'The Resource \'Microsoft.Network/dnszones/{}\''.format(name) - msg += ' under resource group \'{}\''.format(self._resource_group) - msg += ' was not found.' - if msg == err.message: - # Then the only error is that the zone doesn't currently exist - if create: - self.log.debug('_check_zone:no matching zone; creating %s', - name) - create_zone = self._dns_client.zones.create_or_update - create_zone(self._resource_group, name, - Zone(location='global')) - return name - else: - return - raise + else: + # Else return nothing (aka false) + return def populate(self, zone, target=False, lenient=False): '''Required function of manager.py to collect records from zone. diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index 2b2c1e7..3c93a27 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -498,32 +498,42 @@ class TestAzureDnsProvider(TestCase): record_list = provider._dns_client.record_sets.list_by_dns_zone record_list.return_value = rs + zone_list = provider._dns_client.zones.list_by_resource_group + zone_list.return_value = [zone] + exists = provider.populate(zone) - self.assertTrue(exists) + self.assertEquals(len(zone.records), 17) + self.assertTrue(exists) def test_populate_zone(self): provider = self._get_provider() zone_list = provider._dns_client.zones.list_by_resource_group - zone_list.return_value = [AzureZone(location='global'), - AzureZone(location='global')] + zone_1 = AzureZone(location='global') + # This is far from ideal but the zone constructor doesn't let me set it on creation + zone_1.name = "zone-1" + zone_2 = AzureZone(location='global') + # This is far from ideal but the zone constructor doesn't let me set it on creation + zone_2.name = "zone-2" + zone_list.return_value = [zone_1, + zone_2, + zone_1] provider._populate_zones() - self.assertEquals(len(provider._azure_zones), 1) + # This should be returning two zones since two zones are the same + self.assertEquals(len(provider._azure_zones), 2) def test_bad_zone_response(self): provider = self._get_provider() _get = provider._dns_client.zones.get _get.side_effect = CloudError(Mock(status=404), 'Azure Error') - trip = False - try: - provider._check_zone('unit.test', create=False) - except CloudError: - trip = True - self.assertEquals(trip, True) + self.assertEquals( + provider._check_zone('unit.test', create=False), + None + ) def test_apply(self): provider = self._get_provider() From 83cc7fbe1a5cca6d1f2ae2910378859ca39411a5 Mon Sep 17 00:00:00 2001 From: Robert Reichel Date: Fri, 29 Jan 2021 15:19:19 -0500 Subject: [PATCH 108/358] Appease the linter --- tests/test_octodns_provider_azuredns.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index 3c93a27..cb790ea 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -502,7 +502,7 @@ class TestAzureDnsProvider(TestCase): zone_list.return_value = [zone] exists = provider.populate(zone) - + self.assertEquals(len(zone.records), 17) self.assertTrue(exists) @@ -511,10 +511,12 @@ class TestAzureDnsProvider(TestCase): zone_list = provider._dns_client.zones.list_by_resource_group zone_1 = AzureZone(location='global') - # This is far from ideal but the zone constructor doesn't let me set it on creation + # This is far from ideal but the + # zone constructor doesn't let me set it on creation zone_1.name = "zone-1" zone_2 = AzureZone(location='global') - # This is far from ideal but the zone constructor doesn't let me set it on creation + # This is far from ideal but the + # zone constructor doesn't let me set it on creation zone_2.name = "zone-2" zone_list.return_value = [zone_1, zone_2, From f79ad89e82a04af07a5a15ec0c88932cbf32ddc5 Mon Sep 17 00:00:00 2001 From: Robert Reichel Date: Fri, 29 Jan 2021 15:22:13 -0500 Subject: [PATCH 109/358] Continue linter appeasement --- tests/test_octodns_provider_azuredns.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index cb790ea..b62762e 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -511,7 +511,7 @@ class TestAzureDnsProvider(TestCase): zone_list = provider._dns_client.zones.list_by_resource_group zone_1 = AzureZone(location='global') - # This is far from ideal but the + # This is far from ideal but the # zone constructor doesn't let me set it on creation zone_1.name = "zone-1" zone_2 = AzureZone(location='global') @@ -533,7 +533,7 @@ class TestAzureDnsProvider(TestCase): _get = provider._dns_client.zones.get _get.side_effect = CloudError(Mock(status=404), 'Azure Error') self.assertEquals( - provider._check_zone('unit.test', create=False), + provider._check_zone('unit.test', create=False), None ) From f92fdfce172bb3748e50e0cd2a5a01f2e41e0df1 Mon Sep 17 00:00:00 2001 From: Robert Reichel Date: Fri, 29 Jan 2021 15:24:42 -0500 Subject: [PATCH 110/358] Even more desperate attempts to appease linter --- octodns/provider/azuredns.py | 1 - 1 file changed, 1 deletion(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 0224f06..6a65354 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -7,7 +7,6 @@ from __future__ import absolute_import, division, print_function, \ from azure.common.credentials import ServicePrincipalCredentials from azure.mgmt.dns import DnsManagementClient -from msrestazure.azure_exceptions import CloudError from azure.mgmt.dns.models import ARecord, AaaaRecord, CaaRecord, \ CnameRecord, MxRecord, SrvRecord, NsRecord, PtrRecord, TxtRecord, Zone From 3ab532c5af1c71c5c35a2d6450ba0ab17a55cf31 Mon Sep 17 00:00:00 2001 From: Robert Reichel Date: Fri, 29 Jan 2021 15:30:17 -0500 Subject: [PATCH 111/358] Fix test coverage --- octodns/provider/azuredns.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 6a65354..30e9af5 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -340,8 +340,7 @@ class AzureProvider(BaseProvider): self.log.debug('azure_zones: loading') list_zones = self._dns_client.zones.list_by_resource_group for zone in list_zones(self._resource_group): - if zone.name: - self._azure_zones.add(zone.name.rstrip('.')) + self._azure_zones.add(zone.name.rstrip('.')) def _check_zone(self, name, create=False): '''Checks whether a zone specified in a source exist in Azure server. From 0b116a57b96c66e4e051e980d372820618937260 Mon Sep 17 00:00:00 2001 From: Robert Reichel Date: Tue, 2 Feb 2021 12:18:40 -0500 Subject: [PATCH 112/358] Modify Azure DNS Client logic to lazy load Lazy loading the Azure DNS client ensures that the necessary network calls only occur when it is time to actually do something with the client. --- octodns/provider/azuredns.py | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index a1ef6fe..888f5e3 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -323,6 +323,22 @@ class AzureProvider(BaseProvider): SUPPORTS = set(('A', 'AAAA', 'CAA', 'CNAME', 'MX', 'NS', 'PTR', 'SRV', 'TXT')) + def _get_dns_client(self): + if not self._dns_client_handle: + # Not initialized yet, we need to do that. + credentials = ServicePrincipalCredentials( + self._dns_client_client_id, + secret=self._dns_client_key, + tenant=self._dns_client_directory_id + ) + self._dns_client_handle = DnsManagementClient( + credentials, + self._dns_client_subscription_id + ) + + return self._dns_client_handle + + def __init__(self, id, client_id, key, directory_id, sub_id, resource_group, *args, **kwargs): self.log = logging.getLogger('AzureProvider[{}]'.format(id)) @@ -330,10 +346,13 @@ class AzureProvider(BaseProvider): 'key=***, directory_id:%s', id, client_id, directory_id) super(AzureProvider, self).__init__(id, *args, **kwargs) - credentials = ServicePrincipalCredentials( - client_id, secret=key, tenant=directory_id - ) - self._dns_client = DnsManagementClient(credentials, sub_id) + # Store necessary initialization params + self._dns_client_handle = None + self._dns_client_client_id = client_id + self._dns_client_key = key + self._dns_client_directory_id = directory_id + self._dns_client_subscription_id = sub_id + self._dns_client = property(_get_dns_client, _set_dns_client) self._resource_group = resource_group self._azure_zones = set() From 6146be8ec3ad861320f50f1d8542f3cd020a468b Mon Sep 17 00:00:00 2001 From: Robert Reichel Date: Tue, 2 Feb 2021 12:21:00 -0500 Subject: [PATCH 113/358] Remove unused set_dns_client property --- octodns/provider/azuredns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 888f5e3..9561385 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -352,7 +352,7 @@ class AzureProvider(BaseProvider): self._dns_client_key = key self._dns_client_directory_id = directory_id self._dns_client_subscription_id = sub_id - self._dns_client = property(_get_dns_client, _set_dns_client) + self._dns_client = property(_get_dns_client) self._resource_group = resource_group self._azure_zones = set() From 6fb77c08109c5a4d4537280fe291fb921ad7b28d Mon Sep 17 00:00:00 2001 From: Robert Reichel Date: Tue, 2 Feb 2021 12:21:48 -0500 Subject: [PATCH 114/358] Add set DNS client logic if needed for testing --- octodns/provider/azuredns.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 9561385..2431e7d 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -337,6 +337,8 @@ class AzureProvider(BaseProvider): ) return self._dns_client_handle + def _set_dns_client(self, client) + self.dns_client_handle = client def __init__(self, id, client_id, key, directory_id, sub_id, @@ -352,7 +354,7 @@ class AzureProvider(BaseProvider): self._dns_client_key = key self._dns_client_directory_id = directory_id self._dns_client_subscription_id = sub_id - self._dns_client = property(_get_dns_client) + self._dns_client = property(_get_dns_client, _set_dns_client) self._resource_group = resource_group self._azure_zones = set() From 975376d09dd58ef9f02aa522ef3842443cb595a4 Mon Sep 17 00:00:00 2001 From: Robert Reichel Date: Tue, 2 Feb 2021 12:26:04 -0500 Subject: [PATCH 115/358] Remove trailing whitespace --- octodns/provider/azuredns.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 2431e7d..0162e18 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -327,19 +327,19 @@ class AzureProvider(BaseProvider): if not self._dns_client_handle: # Not initialized yet, we need to do that. credentials = ServicePrincipalCredentials( - self._dns_client_client_id, - secret=self._dns_client_key, + self._dns_client_client_id, + secret=self._dns_client_key, tenant=self._dns_client_directory_id ) self._dns_client_handle = DnsManagementClient( - credentials, + credentials, self._dns_client_subscription_id ) - + return self._dns_client_handle + def _set_dns_client(self, client) self.dns_client_handle = client - def __init__(self, id, client_id, key, directory_id, sub_id, resource_group, *args, **kwargs): From 5e78d07a97c2634bf41cb67b2510240ca502726b Mon Sep 17 00:00:00 2001 From: Robert Reichel Date: Tue, 2 Feb 2021 12:31:04 -0500 Subject: [PATCH 116/358] Use @property in lieu of property() --- octodns/provider/azuredns.py | 35 ++++++++++++++++------------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 0162e18..53b5e48 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -323,24 +323,6 @@ class AzureProvider(BaseProvider): SUPPORTS = set(('A', 'AAAA', 'CAA', 'CNAME', 'MX', 'NS', 'PTR', 'SRV', 'TXT')) - def _get_dns_client(self): - if not self._dns_client_handle: - # Not initialized yet, we need to do that. - credentials = ServicePrincipalCredentials( - self._dns_client_client_id, - secret=self._dns_client_key, - tenant=self._dns_client_directory_id - ) - self._dns_client_handle = DnsManagementClient( - credentials, - self._dns_client_subscription_id - ) - - return self._dns_client_handle - - def _set_dns_client(self, client) - self.dns_client_handle = client - def __init__(self, id, client_id, key, directory_id, sub_id, resource_group, *args, **kwargs): self.log = logging.getLogger('AzureProvider[{}]'.format(id)) @@ -354,10 +336,25 @@ class AzureProvider(BaseProvider): self._dns_client_key = key self._dns_client_directory_id = directory_id self._dns_client_subscription_id = sub_id - self._dns_client = property(_get_dns_client, _set_dns_client) + self._dns_client = None + self._resource_group = resource_group self._azure_zones = set() + @property + def _dns_client(self) + if self._dns_client is None: + credentials = ServicePrincipalCredentials( + self._dns_client_client_id, + secret=self._dns_client_key, + tenant=self._dns_client_directory_id + ) + self._dns_client = DnsManagementClient( + credentials, + self._dns_client_subscription_id + ) + return self._dns_client + def _populate_zones(self): self.log.debug('azure_zones: loading') list_zones = self._dns_client.zones.list_by_resource_group From 831d1cc30b003f5e8f9f444c5a805777183f8007 Mon Sep 17 00:00:00 2001 From: Robert Reichel Date: Tue, 2 Feb 2021 12:34:44 -0500 Subject: [PATCH 117/358] Add missing colon --- octodns/provider/azuredns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 53b5e48..f41f892 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -342,7 +342,7 @@ class AzureProvider(BaseProvider): self._azure_zones = set() @property - def _dns_client(self) + def _dns_client(self): if self._dns_client is None: credentials = ServicePrincipalCredentials( self._dns_client_client_id, From a58371e3bbf896f8c5bd1edb91f0cfaccd7ae921 Mon Sep 17 00:00:00 2001 From: Robert Reichel Date: Tue, 2 Feb 2021 12:40:57 -0500 Subject: [PATCH 118/358] Apply suggestions from code review Co-authored-by: Ross McFarland --- octodns/provider/azuredns.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index f41f892..046ddfe 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -336,24 +336,24 @@ class AzureProvider(BaseProvider): self._dns_client_key = key self._dns_client_directory_id = directory_id self._dns_client_subscription_id = sub_id - self._dns_client = None + self.__dns_client = None self._resource_group = resource_group self._azure_zones = set() @property def _dns_client(self): - if self._dns_client is None: + if self.__dns_client is None: credentials = ServicePrincipalCredentials( self._dns_client_client_id, secret=self._dns_client_key, tenant=self._dns_client_directory_id ) - self._dns_client = DnsManagementClient( + self.__dns_client = DnsManagementClient( credentials, self._dns_client_subscription_id ) - return self._dns_client + return self.__dns_client def _populate_zones(self): self.log.debug('azure_zones: loading') From c2c05a761e6e6f83495f4bfda6f4ea5dadc47d48 Mon Sep 17 00:00:00 2001 From: Robert Reichel Date: Tue, 2 Feb 2021 13:14:29 -0500 Subject: [PATCH 119/358] Fix patching --- tests/test_octodns_provider_azuredns.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index 2b2c1e7..19cc826 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -376,8 +376,6 @@ class TestAzureDnsProvider(TestCase): def _provider(self): return self._get_provider('mock_spc', 'mock_dns_client') - @patch('octodns.provider.azuredns.DnsManagementClient') - @patch('octodns.provider.azuredns.ServicePrincipalCredentials') def _get_provider(self, mock_spc, mock_dns_client): '''Returns a mock AzureProvider object to use in testing. @@ -390,7 +388,9 @@ class TestAzureDnsProvider(TestCase): ''' return AzureProvider('mock_id', 'mock_client', 'mock_key', 'mock_directory', 'mock_sub', 'mock_rg') - + + @patch('octodns.provider.azuredns.DnsManagementClient') + @patch('octodns.provider.azuredns.ServicePrincipalCredentials') def test_populate_records(self): provider = self._get_provider() @@ -501,7 +501,9 @@ class TestAzureDnsProvider(TestCase): exists = provider.populate(zone) self.assertTrue(exists) self.assertEquals(len(zone.records), 17) - + + @patch('octodns.provider.azuredns.DnsManagementClient') + @patch('octodns.provider.azuredns.ServicePrincipalCredentials') def test_populate_zone(self): provider = self._get_provider() @@ -512,7 +514,9 @@ class TestAzureDnsProvider(TestCase): provider._populate_zones() self.assertEquals(len(provider._azure_zones), 1) - + + @patch('octodns.provider.azuredns.DnsManagementClient') + @patch('octodns.provider.azuredns.ServicePrincipalCredentials') def test_bad_zone_response(self): provider = self._get_provider() @@ -539,6 +543,8 @@ class TestAzureDnsProvider(TestCase): self.assertEquals(19, provider.apply(Plan(zone, zone, deletes, True))) + @patch('octodns.provider.azuredns.DnsManagementClient') + @patch('octodns.provider.azuredns.ServicePrincipalCredentials') def test_create_zone(self): provider = self._get_provider() @@ -555,6 +561,8 @@ class TestAzureDnsProvider(TestCase): self.assertEquals(19, provider.apply(Plan(None, desired, changes, True))) + @patch('octodns.provider.azuredns.DnsManagementClient') + @patch('octodns.provider.azuredns.ServicePrincipalCredentials') def test_check_zone_no_create(self): provider = self._get_provider() From 1982fbddac3533e87a3627e6fe2a3996dd0ab0b9 Mon Sep 17 00:00:00 2001 From: Robert Reichel Date: Tue, 2 Feb 2021 13:18:04 -0500 Subject: [PATCH 120/358] Update patching --- tests/test_octodns_provider_azuredns.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index 19cc826..030bf9f 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -375,7 +375,9 @@ class Test_CheckAzureAlias(TestCase): class TestAzureDnsProvider(TestCase): def _provider(self): return self._get_provider('mock_spc', 'mock_dns_client') - + + @patch('octodns.provider.azuredns.DnsManagementClient') + @patch('octodns.provider.azuredns.ServicePrincipalCredentials') def _get_provider(self, mock_spc, mock_dns_client): '''Returns a mock AzureProvider object to use in testing. @@ -386,8 +388,11 @@ class TestAzureDnsProvider(TestCase): :type return: AzureProvider ''' - return AzureProvider('mock_id', 'mock_client', 'mock_key', + provider = AzureProvider('mock_id', 'mock_client', 'mock_key', 'mock_directory', 'mock_sub', 'mock_rg') + # Fetch the client to force it to load the creds + client = provider._dns_client + return provider @patch('octodns.provider.azuredns.DnsManagementClient') @patch('octodns.provider.azuredns.ServicePrincipalCredentials') From 950090bb547aa58591d3535c2986a8b46b5ceba6 Mon Sep 17 00:00:00 2001 From: Robert Reichel Date: Tue, 2 Feb 2021 13:23:02 -0500 Subject: [PATCH 121/358] Update test_octodns_provider_azuredns.py --- tests/test_octodns_provider_azuredns.py | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index 030bf9f..8f840f0 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -375,7 +375,7 @@ class Test_CheckAzureAlias(TestCase): class TestAzureDnsProvider(TestCase): def _provider(self): return self._get_provider('mock_spc', 'mock_dns_client') - + @patch('octodns.provider.azuredns.DnsManagementClient') @patch('octodns.provider.azuredns.ServicePrincipalCredentials') def _get_provider(self, mock_spc, mock_dns_client): @@ -393,9 +393,7 @@ class TestAzureDnsProvider(TestCase): # Fetch the client to force it to load the creds client = provider._dns_client return provider - - @patch('octodns.provider.azuredns.DnsManagementClient') - @patch('octodns.provider.azuredns.ServicePrincipalCredentials') + def test_populate_records(self): provider = self._get_provider() @@ -506,9 +504,7 @@ class TestAzureDnsProvider(TestCase): exists = provider.populate(zone) self.assertTrue(exists) self.assertEquals(len(zone.records), 17) - - @patch('octodns.provider.azuredns.DnsManagementClient') - @patch('octodns.provider.azuredns.ServicePrincipalCredentials') + def test_populate_zone(self): provider = self._get_provider() @@ -519,9 +515,7 @@ class TestAzureDnsProvider(TestCase): provider._populate_zones() self.assertEquals(len(provider._azure_zones), 1) - - @patch('octodns.provider.azuredns.DnsManagementClient') - @patch('octodns.provider.azuredns.ServicePrincipalCredentials') + def test_bad_zone_response(self): provider = self._get_provider() @@ -548,8 +542,6 @@ class TestAzureDnsProvider(TestCase): self.assertEquals(19, provider.apply(Plan(zone, zone, deletes, True))) - @patch('octodns.provider.azuredns.DnsManagementClient') - @patch('octodns.provider.azuredns.ServicePrincipalCredentials') def test_create_zone(self): provider = self._get_provider() @@ -566,8 +558,6 @@ class TestAzureDnsProvider(TestCase): self.assertEquals(19, provider.apply(Plan(None, desired, changes, True))) - @patch('octodns.provider.azuredns.DnsManagementClient') - @patch('octodns.provider.azuredns.ServicePrincipalCredentials') def test_check_zone_no_create(self): provider = self._get_provider() From d94db03f5b32dc95d8f63cfe7c6a3a8eec8542fd Mon Sep 17 00:00:00 2001 From: Robert Reichel Date: Tue, 2 Feb 2021 13:26:28 -0500 Subject: [PATCH 122/358] Fix lint --- tests/test_octodns_provider_azuredns.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index 8f840f0..2b048c0 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -389,7 +389,8 @@ class TestAzureDnsProvider(TestCase): :type return: AzureProvider ''' provider = AzureProvider('mock_id', 'mock_client', 'mock_key', - 'mock_directory', 'mock_sub', 'mock_rg') + 'mock_directory', 'mock_sub', 'mock_rg' + ) # Fetch the client to force it to load the creds client = provider._dns_client return provider From 290a6303037a817fa7a3da80562946ef7b8c69b5 Mon Sep 17 00:00:00 2001 From: Robert Reichel Date: Tue, 2 Feb 2021 13:29:52 -0500 Subject: [PATCH 123/358] Update test_octodns_provider_azuredns.py --- tests/test_octodns_provider_azuredns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index 2d6f08d..b7721b7 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -390,7 +390,7 @@ class TestAzureDnsProvider(TestCase): ''' provider = AzureProvider('mock_id', 'mock_client', 'mock_key', 'mock_directory', 'mock_sub', 'mock_rg' - ) + ) # Fetch the client to force it to load the creds client = provider._dns_client return provider From ec0b309437369cefdda372aa9723e550201072bc Mon Sep 17 00:00:00 2001 From: Robert Reichel Date: Tue, 2 Feb 2021 13:34:15 -0500 Subject: [PATCH 124/358] Remove unused client ref --- tests/test_octodns_provider_azuredns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index b7721b7..9523b51 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -392,7 +392,7 @@ class TestAzureDnsProvider(TestCase): 'mock_directory', 'mock_sub', 'mock_rg' ) # Fetch the client to force it to load the creds - client = provider._dns_client + provider._dns_client return provider def test_populate_records(self): From cda56a3ca7ffed9b62a5f960682a9c25b9173c05 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Thu, 4 Feb 2021 10:48:45 -0800 Subject: [PATCH 125/358] Force the value passed to FQDN to be a str --- octodns/record/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/octodns/record/__init__.py b/octodns/record/__init__.py index f22eebf..7beb570 100644 --- a/octodns/record/__init__.py +++ b/octodns/record/__init__.py @@ -758,7 +758,9 @@ class _TargetValue(object): reasons.append('empty value') elif not data: reasons.append('missing value') - elif not FQDN(data, allow_underscores=True).is_valid: + # NOTE: FQDN complains if the data it receives isn't a str, it doesn't + # allow unicode... This is likely specific to 2.7 + elif not FQDN(str(data), allow_underscores=True).is_valid: reasons.append('{} value "{}" is not a valid FQDN' .format(_type, data)) elif not data.endswith('.'): From 4081c7b31b1ea31855b94c1572cc18e5a46410b7 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 5 Feb 2021 11:55:37 -0800 Subject: [PATCH 126/358] Add the number of changes and zone name to "making changes" --- octodns/provider/base.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/octodns/provider/base.py b/octodns/provider/base.py index ae87844..eb097a2 100644 --- a/octodns/provider/base.py +++ b/octodns/provider/base.py @@ -91,7 +91,10 @@ class BaseProvider(BaseSource): self.log.info('apply: disabled') return 0 - self.log.info('apply: making changes') + zone_name = plan.desired.name + num_changes = len(plan.changes) + self.log.info('apply: making %d changes to %s', num_changes, + zone_name) self._apply(plan) return len(plan.changes) From 858628a867218e566013cde9d5a5fa920ecce382 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 5 Feb 2021 12:06:46 -0800 Subject: [PATCH 127/358] Update yaml test path to work on windows --- tests/test_octodns_provider_yaml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_octodns_provider_yaml.py b/tests/test_octodns_provider_yaml.py index 38dfc11..f255238 100644 --- a/tests/test_octodns_provider_yaml.py +++ b/tests/test_octodns_provider_yaml.py @@ -213,7 +213,7 @@ class TestSplitYamlProvider(TestCase): zone = Zone('unit.tests.', []) self.assertEqual( - join(dirname(__file__), 'config/split/unit.tests.tst'), + join(dirname(__file__), 'config/split', 'unit.tests.tst'), source._zone_directory(zone)) def test_apply_handles_existing_zone_directory(self): From 2346ebc1ce72edca8d066218cafb29e46d1f7ee9 Mon Sep 17 00:00:00 2001 From: Patrick Connolly Date: Mon, 8 Feb 2021 21:24:41 -0500 Subject: [PATCH 128/358] Added list of projects & resources. --- README.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/README.md b/README.md index a65e28f..84dc033 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,7 @@ It is similar to [Netflix/denominator](https://github.com/Netflix/denominator). - [Dynamic sources](#dynamic-sources) - [Contributing](#contributing) - [Getting help](#getting-help) +- [Related Projects & Resources](#related-projects--resources) - [License](#license) - [Authors](#authors) @@ -225,6 +226,8 @@ Most of the things included in OctoDNS are providers, the obvious difference bei The `class` key in the providers config section can be used to point to arbitrary classes in the python path so internal or 3rd party providers can easily be included with no coordination beyond getting them into PYTHONPATH, most likely installed into the virtualenv with OctoDNS. +For examples of building third-party sources and providers, see [Related Projects & Resources](#related-projects--resources). + ## Other Uses ### Syncing between providers @@ -286,6 +289,29 @@ Please see our [contributing document](/CONTRIBUTING.md) if you would like to pa If you have a problem or suggestion, please [open an issue](https://github.com/octodns/octodns/issues/new) in this repository, and we will do our best to help. Please note that this project adheres to the [Contributor Covenant Code of Conduct](/CODE_OF_CONDUCT.md). +## Related Projects & Resources + +- **GitHub Action:** [OctoDNS-Sync](https://github.com/marketplace/actions/octodns-sync) +- **Sample Implementations.** See how others are using it + - [`hackclub/dns`](https://github.com/hackclub/dns) + - [`kubernetes/k8s.io:/dns`](https://github.com/kubernetes/k8s.io/tree/master/dns) + - [`g0v-network/domains`](https://github.com/g0v-network/domains) + - [`jekyll/dns`](https://github.com/jekyll/dns) + - [`parkr/dns`](https://github.com/parkr/dns) +- **Custom Sources & Providers.** + - [`octodns/octodns-ddns`](https://github.com/octodns/octodns-ddns): A simple Dynamic DNS source. + - [`doddo/octodns-lexicon`](https://github.com/doddo/octodns-lexicon): Use [Lexicon](https://github.com/AnalogJ/lexicon) providers as octoDNS providers. + - [`asyncon/octoblox`](https://github.com/asyncon/octoblox): [Infoblox](https://www.infoblox.com/) provider. + - [`sukiyaki/octodns-netbox`](https://github.com/sukiyaki/octodns-netbox): [NetBox](https://github.com/netbox-community/netbox) source. + - [`kompetenzbolzen/octodns-custom-provider`](https://github.com/kompetenzbolzen/octodns-custom-provider): zonefile provider & phpIPAM source. +- **Resources.** + - Article: [Visualising DNS records with Neo4j](https://medium.com/@costask/querying-and-visualising-octodns-records-with-neo4j-f4f72ab2d474) + code + - Video: [FOSDEM 2019 - DNS as code with octodns](https://archive.fosdem.org/2019/schedule/event/dns_octodns/) + - GitHub Blog: [Enabling DNS split authority with OctoDNS](https://github.blog/2017-04-27-enabling-split-authority-dns-with-octodns/) + - Tutorial: [How To Deploy and Manage Your DNS using OctoDNS on Ubuntu 18.04](https://www.digitalocean.com/community/tutorials/how-to-deploy-and-manage-your-dns-using-octodns-on-ubuntu-18-04) + +If you know of any other resources, please do let us know! + ## License OctoDNS is licensed under the [MIT license](LICENSE). From 9d4bd0aaec43764077a20ba23a2065cd1011c92a Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Sun, 29 Nov 2020 23:42:51 +0800 Subject: [PATCH 129/358] Add support for LOC records --- docs/records.md | 1 + octodns/provider/yaml.py | 2 +- octodns/record/__init__.py | 190 ++++++++ tests/config/unit.tests.yaml | 28 ++ tests/test_octodns_manager.py | 14 +- tests/test_octodns_provider_constellix.py | 2 +- tests/test_octodns_provider_digitalocean.py | 2 +- tests/test_octodns_provider_dnsimple.py | 4 +- tests/test_octodns_provider_dnsmadeeasy.py | 2 +- tests/test_octodns_provider_easydns.py | 2 +- tests/test_octodns_provider_gandi.py | 4 +- tests/test_octodns_provider_powerdns.py | 4 +- tests/test_octodns_provider_yaml.py | 9 +- tests/test_octodns_record.py | 488 +++++++++++++++++++- tests/zones/unit.tests.tst | 4 + 15 files changed, 730 insertions(+), 26 deletions(-) diff --git a/docs/records.md b/docs/records.md index 4cf1e4b..e39a85d 100644 --- a/docs/records.md +++ b/docs/records.md @@ -10,6 +10,7 @@ OctoDNS supports the following record types: * `CAA` * `CNAME` * `DNAME` +* `LOC` * `MX` * `NAPTR` * `NS` diff --git a/octodns/provider/yaml.py b/octodns/provider/yaml.py index 3deca01..8314f38 100644 --- a/octodns/provider/yaml.py +++ b/octodns/provider/yaml.py @@ -104,7 +104,7 @@ class YamlProvider(BaseProvider): ''' SUPPORTS_GEO = True SUPPORTS_DYNAMIC = True - SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CAA', 'CNAME', 'DNAME', 'MX', + SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CAA', 'CNAME', 'DNAME', 'LOC', 'MX', 'NAPTR', 'NS', 'PTR', 'SSHFP', 'SPF', 'SRV', 'TXT')) def __init__(self, id, directory, default_ttl=3600, enforce_order=True, diff --git a/octodns/record/__init__.py b/octodns/record/__init__.py index 7beb570..8ee2eaa 100644 --- a/octodns/record/__init__.py +++ b/octodns/record/__init__.py @@ -97,6 +97,7 @@ class Record(EqualityTupleMixin): 'CAA': CaaRecord, 'CNAME': CnameRecord, 'DNAME': DnameRecord, + 'LOC': LocRecord, 'MX': MxRecord, 'NAPTR': NaptrRecord, 'NS': NsRecord, @@ -879,6 +880,195 @@ class DnameRecord(_DynamicMixin, _ValueMixin, Record): _value_type = DnameValue +class LocValue(EqualityTupleMixin): + # TODO: work out how to do defaults per RFC + + @classmethod + def validate(cls, data, _type): + int_keys = [ + 'lat_degrees', + 'lat_minutes', + 'long_degrees', + 'long_minutes', + ] + + float_keys = [ + 'lat_seconds', + 'long_seconds', + 'altitude', + 'size', + 'precision_horz', + 'precision_vert', + ] + + direction_keys = [ + 'lat_direction', + 'long_direction', + ] + + if not isinstance(data, (list, tuple)): + data = (data,) + reasons = [] + for value in data: + for key in int_keys: + try: + int(value[key]) + if ( + ( + key == 'lat_degrees' and + not 0 <= int(value[key]) <= 90 + ) or ( + key == 'long_degrees' and + not 0 <= int(value[key]) <= 180 + ) or ( + key in ['lat_minutes', 'long_minutes'] and + not 0 <= int(value[key]) <= 59 + ) + ): + reasons.append('invalid value for {} "{}"' + .format(key, value[key])) + except KeyError: + reasons.append('missing {}'.format(key)) + except ValueError: + reasons.append('invalid {} "{}"' + .format(key, value[key])) + + for key in float_keys: + try: + float(value[key]) + if ( + ( + key in ['lat_seconds', 'long_seconds'] and + not 0 <= float(value[key]) <= 59.999 + ) or ( + key == 'altitude' and + not -100000.00 <= float(value[key]) <= 42849672.95 + ) or ( + key in ['size', + 'precision_horz', + 'precision_vert'] and + not 0 <= float(value[key]) <= 90000000.00 + ) + ): + reasons.append('invalid value for {} "{}"' + .format(key, value[key])) + except KeyError: + reasons.append('missing {}'.format(key)) + except ValueError: + reasons.append('invalid {} "{}"' + .format(key, value[key])) + + for key in direction_keys: + try: + str(value[key]) + if ( + key == 'lat_direction' and + value[key] not in ['N', 'S'] + ): + reasons.append('invalid direction for {} "{}"' + .format(key, value[key])) + if ( + key == 'long_direction' and + value[key] not in ['E', 'W'] + ): + reasons.append('invalid direction for {} "{}"' + .format(key, value[key])) + except KeyError: + reasons.append('missing {}'.format(key)) + return reasons + + @classmethod + def process(cls, values): + return [LocValue(v) for v in values] + + def __init__(self, value): + self.lat_degrees = int(value['lat_degrees']) + self.lat_minutes = int(value['lat_minutes']) + self.lat_seconds = float(value['lat_seconds']) + self.lat_direction = value['lat_direction'].upper() + self.long_degrees = int(value['long_degrees']) + self.long_minutes = int(value['long_minutes']) + self.long_seconds = float(value['long_seconds']) + self.long_direction = value['long_direction'].upper() + self.altitude = float(value['altitude']) + self.size = float(value['size']) + self.precision_horz = float(value['precision_horz']) + self.precision_vert = float(value['precision_vert']) + + @property + def data(self): + return { + 'lat_degrees': self.lat_degrees, + 'lat_minutes': self.lat_minutes, + 'lat_seconds': self.lat_seconds, + 'lat_direction': self.lat_direction, + 'long_degrees': self.long_degrees, + 'long_minutes': self.long_minutes, + 'long_seconds': self.long_seconds, + 'long_direction': self.long_direction, + 'altitude': self.altitude, + 'size': self.size, + 'precision_horz': self.precision_horz, + 'precision_vert': self.precision_vert, + } + + def __hash__(self): + return hash(( + self.lat_degrees, + self.lat_minutes, + self.lat_seconds, + self.lat_direction, + self.long_degrees, + self.long_minutes, + self.long_seconds, + self.long_direction, + self.altitude, + self.size, + self.precision_horz, + self.precision_vert, + )) + + def _equality_tuple(self): + return ( + self.lat_degrees, + self.lat_minutes, + self.lat_seconds, + self.lat_direction, + self.long_degrees, + self.long_minutes, + self.long_seconds, + self.long_direction, + self.altitude, + self.size, + self.precision_horz, + self.precision_vert, + ) + + def __repr__(self): + loc_format = "'{0} {1} {2:.3f} {3} " + \ + "{4} {5} {6:.3f} {7} " + \ + "{8:.2f}m {9:.2f}m {10:.2f}m {11:.2f}m'" + return loc_format.format( + self.lat_degrees, + self.lat_minutes, + self.lat_seconds, + self.lat_direction, + self.long_degrees, + self.long_minutes, + self.long_seconds, + self.long_direction, + self.altitude, + self.size, + self.precision_horz, + self.precision_vert, + ) + + +class LocRecord(_ValuesMixin, Record): + _type = 'LOC' + _value_type = LocValue + + class MxValue(EqualityTupleMixin): @classmethod diff --git a/tests/config/unit.tests.yaml b/tests/config/unit.tests.yaml index 7b84ac9..f5cf648 100644 --- a/tests/config/unit.tests.yaml +++ b/tests/config/unit.tests.yaml @@ -77,6 +77,34 @@ included: - test type: CNAME value: unit.tests. +loc: + ttl: 300 + type: LOC + values: + - altitude: 20 + lat_degrees: 31 + lat_direction: S + lat_minutes: 58 + lat_seconds: 52.1 + long_degrees: 115 + long_direction: E + long_minutes: 49 + long_seconds: 11.7 + precision_horz: 10 + precision_vert: 2 + size: 10 + - altitude: 20 + lat_degrees: 53 + lat_direction: N + lat_minutes: 13 + lat_seconds: 10 + long_degrees: 2 + long_direction: W + long_minutes: 18 + long_seconds: 26 + precision_horz: 1000 + precision_vert: 2 + size: 10 mx: ttl: 300 type: MX diff --git a/tests/test_octodns_manager.py b/tests/test_octodns_manager.py index f757466..3e0b122 100644 --- a/tests/test_octodns_manager.py +++ b/tests/test_octodns_manager.py @@ -118,12 +118,12 @@ class TestManager(TestCase): environ['YAML_TMP_DIR'] = tmpdir.dirname tc = Manager(get_config_filename('simple.yaml')) \ .sync(dry_run=False) - self.assertEquals(22, tc) + self.assertEquals(23, tc) # try with just one of the zones tc = Manager(get_config_filename('simple.yaml')) \ .sync(dry_run=False, eligible_zones=['unit.tests.']) - self.assertEquals(16, tc) + self.assertEquals(17, tc) # the subzone, with 2 targets tc = Manager(get_config_filename('simple.yaml')) \ @@ -138,18 +138,18 @@ class TestManager(TestCase): # Again with force tc = Manager(get_config_filename('simple.yaml')) \ .sync(dry_run=False, force=True) - self.assertEquals(22, tc) + self.assertEquals(23, tc) # Again with max_workers = 1 tc = Manager(get_config_filename('simple.yaml'), max_workers=1) \ .sync(dry_run=False, force=True) - self.assertEquals(22, tc) + self.assertEquals(23, tc) # Include meta tc = Manager(get_config_filename('simple.yaml'), max_workers=1, include_meta=True) \ .sync(dry_run=False, force=True) - self.assertEquals(26, tc) + self.assertEquals(27, tc) def test_eligible_sources(self): with TemporaryDirectory() as tmpdir: @@ -215,13 +215,13 @@ class TestManager(TestCase): fh.write('---\n{}') changes = manager.compare(['in'], ['dump'], 'unit.tests.') - self.assertEquals(16, len(changes)) + self.assertEquals(17, len(changes)) # Compound sources with varying support changes = manager.compare(['in', 'nosshfp'], ['dump'], 'unit.tests.') - self.assertEquals(15, len(changes)) + self.assertEquals(16, len(changes)) with self.assertRaises(ManagerException) as ctx: manager.compare(['nope'], ['dump'], 'unit.tests.') diff --git a/tests/test_octodns_provider_constellix.py b/tests/test_octodns_provider_constellix.py index bc17b50..7392271 100644 --- a/tests/test_octodns_provider_constellix.py +++ b/tests/test_octodns_provider_constellix.py @@ -132,7 +132,7 @@ class TestConstellixProvider(TestCase): plan = provider.plan(self.expected) # No root NS, no ignored, no excluded, no unsupported - n = len(self.expected.records) - 6 + n = len(self.expected.records) - 7 self.assertEquals(n, len(plan.changes)) self.assertEquals(n, provider.apply(plan)) diff --git a/tests/test_octodns_provider_digitalocean.py b/tests/test_octodns_provider_digitalocean.py index 0ad8f72..d1fa208 100644 --- a/tests/test_octodns_provider_digitalocean.py +++ b/tests/test_octodns_provider_digitalocean.py @@ -163,7 +163,7 @@ class TestDigitalOceanProvider(TestCase): plan = provider.plan(self.expected) # No root NS, no ignored, no excluded, no unsupported - n = len(self.expected.records) - 8 + n = len(self.expected.records) - 9 self.assertEquals(n, len(plan.changes)) self.assertEquals(n, provider.apply(plan)) self.assertFalse(plan.exists) diff --git a/tests/test_octodns_provider_dnsimple.py b/tests/test_octodns_provider_dnsimple.py index 92f32b1..e97751f 100644 --- a/tests/test_octodns_provider_dnsimple.py +++ b/tests/test_octodns_provider_dnsimple.py @@ -136,8 +136,8 @@ class TestDnsimpleProvider(TestCase): ] plan = provider.plan(self.expected) - # No root NS, no ignored, no excluded - n = len(self.expected.records) - 4 + # No root NS, no ignored, no excluded, no unsupported + n = len(self.expected.records) - 5 self.assertEquals(n, len(plan.changes)) self.assertEquals(n, provider.apply(plan)) self.assertFalse(plan.exists) diff --git a/tests/test_octodns_provider_dnsmadeeasy.py b/tests/test_octodns_provider_dnsmadeeasy.py index 0ad059d..3c709cf 100644 --- a/tests/test_octodns_provider_dnsmadeeasy.py +++ b/tests/test_octodns_provider_dnsmadeeasy.py @@ -134,7 +134,7 @@ class TestDnsMadeEasyProvider(TestCase): plan = provider.plan(self.expected) # No root NS, no ignored, no excluded, no unsupported - n = len(self.expected.records) - 6 + n = len(self.expected.records) - 7 self.assertEquals(n, len(plan.changes)) self.assertEquals(n, provider.apply(plan)) diff --git a/tests/test_octodns_provider_easydns.py b/tests/test_octodns_provider_easydns.py index 8df0e22..a0f03f9 100644 --- a/tests/test_octodns_provider_easydns.py +++ b/tests/test_octodns_provider_easydns.py @@ -374,7 +374,7 @@ class TestEasyDNSProvider(TestCase): plan = provider.plan(self.expected) # No root NS, no ignored, no excluded, no unsupported - n = len(self.expected.records) - 7 + n = len(self.expected.records) - 8 self.assertEquals(n, len(plan.changes)) self.assertEquals(n, provider.apply(plan)) self.assertFalse(plan.exists) diff --git a/tests/test_octodns_provider_gandi.py b/tests/test_octodns_provider_gandi.py index 7e1c866..41b0109 100644 --- a/tests/test_octodns_provider_gandi.py +++ b/tests/test_octodns_provider_gandi.py @@ -192,8 +192,8 @@ class TestGandiProvider(TestCase): ] plan = provider.plan(self.expected) - # No root NS, no ignored, no excluded - n = len(self.expected.records) - 4 + # No root NS, no ignored, no excluded, no LOC + n = len(self.expected.records) - 5 self.assertEquals(n, len(plan.changes)) self.assertEquals(n, provider.apply(plan)) self.assertFalse(plan.exists) diff --git a/tests/test_octodns_provider_powerdns.py b/tests/test_octodns_provider_powerdns.py index 33b5e44..5605c5b 100644 --- a/tests/test_octodns_provider_powerdns.py +++ b/tests/test_octodns_provider_powerdns.py @@ -185,7 +185,7 @@ class TestPowerDnsProvider(TestCase): expected = Zone('unit.tests.', []) source = YamlProvider('test', join(dirname(__file__), 'config')) source.populate(expected) - expected_n = len(expected.records) - 3 + expected_n = len(expected.records) - 4 self.assertEquals(16, expected_n) # No diffs == no changes @@ -291,7 +291,7 @@ class TestPowerDnsProvider(TestCase): expected = Zone('unit.tests.', []) source = YamlProvider('test', join(dirname(__file__), 'config')) source.populate(expected) - self.assertEquals(19, len(expected.records)) + self.assertEquals(20, len(expected.records)) # A small change to a single record with requests_mock() as mock: diff --git a/tests/test_octodns_provider_yaml.py b/tests/test_octodns_provider_yaml.py index f255238..08d1df0 100644 --- a/tests/test_octodns_provider_yaml.py +++ b/tests/test_octodns_provider_yaml.py @@ -35,7 +35,7 @@ class TestYamlProvider(TestCase): # without it we see everything source.populate(zone) - self.assertEquals(19, len(zone.records)) + self.assertEquals(20, len(zone.records)) source.populate(dynamic_zone) self.assertEquals(5, len(dynamic_zone.records)) @@ -58,12 +58,12 @@ class TestYamlProvider(TestCase): # We add everything plan = target.plan(zone) - self.assertEquals(16, len([c for c in plan.changes + self.assertEquals(17, len([c for c in plan.changes if isinstance(c, Create)])) self.assertFalse(isfile(yaml_file)) # Now actually do it - self.assertEquals(16, target.apply(plan)) + self.assertEquals(17, target.apply(plan)) self.assertTrue(isfile(yaml_file)) # Dynamic plan @@ -87,7 +87,7 @@ class TestYamlProvider(TestCase): # A 2nd sync should still create everything plan = target.plan(zone) - self.assertEquals(16, len([c for c in plan.changes + self.assertEquals(17, len([c for c in plan.changes if isinstance(c, Create)])) with open(yaml_file) as fh: @@ -106,6 +106,7 @@ class TestYamlProvider(TestCase): self.assertTrue('values' in data.pop('naptr')) self.assertTrue('values' in data.pop('sub')) self.assertTrue('values' in data.pop('txt')) + self.assertTrue('values' in data.pop('loc')) # these are stored as singular 'value' self.assertTrue('value' in data.pop('aaaa')) self.assertTrue('value' in data.pop('cname')) diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index d55b3b8..ce40b9b 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -9,10 +9,11 @@ from six import text_type from unittest import TestCase from octodns.record import ARecord, AaaaRecord, AliasRecord, CaaRecord, \ - CaaValue, CnameRecord, DnameRecord, Create, Delete, GeoValue, MxRecord, \ - MxValue, NaptrRecord, NaptrValue, NsRecord, PtrRecord, Record, \ - SshfpRecord, SshfpValue, SpfRecord, SrvRecord, SrvValue, TxtRecord, \ - Update, ValidationError, _Dynamic, _DynamicPool, _DynamicRule + CaaValue, CnameRecord, DnameRecord, Create, Delete, GeoValue, LocRecord, \ + LocValue, MxRecord, MxValue, NaptrRecord, NaptrValue, NsRecord, \ + PtrRecord, Record, SshfpRecord, SshfpValue, SpfRecord, SrvRecord, \ + SrvValue, TxtRecord, Update, ValidationError, _Dynamic, _DynamicPool, \ + _DynamicRule from octodns.zone import Zone from helpers import DynamicProvider, GeoProvider, SimpleProvider @@ -379,6 +380,98 @@ class TestRecord(TestCase): self.assertSingleValue(DnameRecord, 'target.foo.com.', 'other.foo.com.') + def test_loc(self): + a_values = [{ + 'lat_degrees': 31, + 'lat_minutes': 58, + 'lat_seconds': 52.1, + 'lat_direction': 'S', + 'long_degrees': 115, + 'long_minutes': 49, + 'long_seconds': 11.7, + 'long_direction': 'E', + 'altitude': 20, + 'size': 10, + 'precision_horz': 10, + 'precision_vert': 2, + }] + a_data = {'ttl': 30, 'values': a_values} + a = LocRecord(self.zone, 'a', a_data) + self.assertEquals('a', a.name) + self.assertEquals('a.unit.tests.', a.fqdn) + self.assertEquals(30, a.ttl) + self.assertEquals(a_values[0]['lat_degrees'], a.values[0].lat_degrees) + self.assertEquals(a_values[0]['lat_minutes'], a.values[0].lat_minutes) + self.assertEquals(a_values[0]['lat_seconds'], a.values[0].lat_seconds) + self.assertEquals(a_values[0]['lat_direction'], + a.values[0].lat_direction) + self.assertEquals(a_values[0]['long_degrees'], + a.values[0].long_degrees) + self.assertEquals(a_values[0]['long_minutes'], + a.values[0].long_minutes) + self.assertEquals(a_values[0]['long_seconds'], + a.values[0].long_seconds) + self.assertEquals(a_values[0]['long_direction'], + a.values[0].long_direction) + self.assertEquals(a_values[0]['altitude'], a.values[0].altitude) + self.assertEquals(a_values[0]['size'], a.values[0].size) + self.assertEquals(a_values[0]['precision_horz'], + a.values[0].precision_horz) + self.assertEquals(a_values[0]['precision_vert'], + a.values[0].precision_vert) + + b_value = { + 'lat_degrees': 32, + 'lat_minutes': 7, + 'lat_seconds': 19, + 'lat_direction': 'S', + 'long_degrees': 116, + 'long_minutes': 2, + 'long_seconds': 25, + 'long_direction': 'E', + 'altitude': 10, + 'size': 1, + 'precision_horz': 10000, + 'precision_vert': 10, + } + b_data = {'ttl': 30, 'value': b_value} + b = LocRecord(self.zone, 'b', b_data) + self.assertEquals(b_value['lat_degrees'], b.values[0].lat_degrees) + self.assertEquals(b_value['lat_minutes'], b.values[0].lat_minutes) + self.assertEquals(b_value['lat_seconds'], b.values[0].lat_seconds) + self.assertEquals(b_value['lat_direction'], b.values[0].lat_direction) + self.assertEquals(b_value['long_degrees'], b.values[0].long_degrees) + self.assertEquals(b_value['long_minutes'], b.values[0].long_minutes) + self.assertEquals(b_value['long_seconds'], b.values[0].long_seconds) + self.assertEquals(b_value['long_direction'], + b.values[0].long_direction) + self.assertEquals(b_value['altitude'], b.values[0].altitude) + self.assertEquals(b_value['size'], b.values[0].size) + self.assertEquals(b_value['precision_horz'], + b.values[0].precision_horz) + self.assertEquals(b_value['precision_vert'], + b.values[0].precision_vert) + self.assertEquals(b_data, b.data) + + target = SimpleProvider() + # No changes with self + self.assertFalse(a.changes(a, target)) + # Diff in lat_direction causes change + other = LocRecord(self.zone, 'a', {'ttl': 30, 'values': a_values}) + other.values[0].lat_direction = 'N' + change = a.changes(other, target) + self.assertEqual(change.existing, a) + self.assertEqual(change.new, other) + # Diff in altitude causes change + other.values[0].altitude = a.values[0].altitude + other.values[0].altitude = -10 + change = a.changes(other, target) + self.assertEqual(change.existing, a) + self.assertEqual(change.new, other) + + # __repr__ doesn't blow up + a.__repr__() + def test_mx(self): a_values = [{ 'preference': 10, @@ -1127,6 +1220,93 @@ class TestRecord(TestCase): self.assertTrue(d >= d) self.assertTrue(d <= d) + def test_loc_value(self): + a = LocValue({ + 'lat_degrees': 31, + 'lat_minutes': 58, + 'lat_seconds': 52.1, + 'lat_direction': 'S', + 'long_degrees': 115, + 'long_minutes': 49, + 'long_seconds': 11.7, + 'long_direction': 'E', + 'altitude': 20, + 'size': 10, + 'precision_horz': 10, + 'precision_vert': 2, + }) + b = LocValue({ + 'lat_degrees': 32, + 'lat_minutes': 7, + 'lat_seconds': 19, + 'lat_direction': 'S', + 'long_degrees': 116, + 'long_minutes': 2, + 'long_seconds': 25, + 'long_direction': 'E', + 'altitude': 10, + 'size': 1, + 'precision_horz': 10000, + 'precision_vert': 10, + }) + c = LocValue({ + 'lat_degrees': 53, + 'lat_minutes': 14, + 'lat_seconds': 10, + 'lat_direction': 'N', + 'long_degrees': 2, + 'long_minutes': 18, + 'long_seconds': 26, + 'long_direction': 'W', + 'altitude': 10, + 'size': 1, + 'precision_horz': 1000, + 'precision_vert': 10, + }) + + self.assertEqual(a, a) + self.assertEqual(b, b) + self.assertEqual(c, c) + + self.assertNotEqual(a, b) + self.assertNotEqual(a, c) + self.assertNotEqual(b, a) + self.assertNotEqual(b, c) + self.assertNotEqual(c, a) + self.assertNotEqual(c, b) + + self.assertTrue(a < b) + self.assertTrue(a < c) + + self.assertTrue(b > a) + self.assertTrue(b < c) + + self.assertTrue(c > a) + self.assertTrue(c > b) + + self.assertTrue(a <= b) + self.assertTrue(a <= c) + self.assertTrue(a <= a) + self.assertTrue(a >= a) + + self.assertTrue(b >= a) + self.assertTrue(b <= c) + self.assertTrue(b >= b) + self.assertTrue(b <= b) + + self.assertTrue(c >= a) + self.assertTrue(c >= b) + self.assertTrue(c >= c) + self.assertTrue(c <= c) + + # Hash + values = set() + values.add(a) + self.assertTrue(a in values) + self.assertFalse(b in values) + values.add(b) + self.assertTrue(b in values) + def test_mx_value(self): a = MxValue({'preference': 0, 'priority': 'a', 'exchange': 'v', 'value': '1'}) @@ -1960,6 +2140,306 @@ class TestRecordValidation(TestCase): self.assertEquals(['DNAME value "foo.bar.com" missing trailing .'], ctx.exception.reasons) + def test_LOC(self): + # doesn't blow up + Record.new(self.zone, '', { + 'type': 'LOC', + 'ttl': 600, + 'value': { + 'lat_degrees': 31, + 'lat_minutes': 58, + 'lat_seconds': 52.1, + 'lat_direction': 'S', + 'long_degrees': 115, + 'long_minutes': 49, + 'long_seconds': 11.7, + 'long_direction': 'E', + 'altitude': 20, + 'size': 10, + 'precision_horz': 10, + 'precision_vert': 2, + } + }) + + # missing int key + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'LOC', + 'ttl': 600, + 'value': { + 'lat_minutes': 58, + 'lat_seconds': 52.1, + 'lat_direction': 'S', + 'long_degrees': 115, + 'long_minutes': 49, + 'long_seconds': 11.7, + 'long_direction': 'E', + 'altitude': 20, + 'size': 10, + 'precision_horz': 10, + 'precision_vert': 2, + } + }) + + self.assertEquals(['missing lat_degrees'], ctx.exception.reasons) + + # missing float key + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'LOC', + 'ttl': 600, + 'value': { + 'lat_degrees': 31, + 'lat_minutes': 58, + 'lat_direction': 'S', + 'long_degrees': 115, + 'long_minutes': 49, + 'long_seconds': 11.7, + 'long_direction': 'E', + 'altitude': 20, + 'size': 10, + 'precision_horz': 10, + 'precision_vert': 2, + } + }) + + self.assertEquals(['missing lat_seconds'], ctx.exception.reasons) + + # missing text key + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'LOC', + 'ttl': 600, + 'value': { + 'lat_degrees': 31, + 'lat_minutes': 58, + 'lat_seconds': 52.1, + 'long_degrees': 115, + 'long_minutes': 49, + 'long_seconds': 11.7, + 'long_direction': 'E', + 'altitude': 20, + 'size': 10, + 'precision_horz': 10, + 'precision_vert': 2, + } + }) + + self.assertEquals(['missing lat_direction'], ctx.exception.reasons) + + # invalid direction + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'LOC', + 'ttl': 600, + 'value': { + 'lat_degrees': 31, + 'lat_minutes': 58, + 'lat_seconds': 52.1, + 'lat_direction': 'U', + 'long_degrees': 115, + 'long_minutes': 49, + 'long_seconds': 11.7, + 'long_direction': 'E', + 'altitude': 20, + 'size': 10, + 'precision_horz': 10, + 'precision_vert': 2, + } + }) + + self.assertEquals(['invalid direction for lat_direction "U"'], + ctx.exception.reasons) + + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'LOC', + 'ttl': 600, + 'value': { + 'lat_degrees': 31, + 'lat_minutes': 58, + 'lat_seconds': 52.1, + 'lat_direction': 'S', + 'long_degrees': 115, + 'long_minutes': 49, + 'long_seconds': 11.7, + 'long_direction': 'N', + 'altitude': 20, + 'size': 10, + 'precision_horz': 10, + 'precision_vert': 2, + } + }) + + self.assertEquals(['invalid direction for long_direction "N"'], + ctx.exception.reasons) + + # invalid degrees + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'LOC', + 'ttl': 600, + 'value': { + 'lat_degrees': 360, + 'lat_minutes': 58, + 'lat_seconds': 52.1, + 'lat_direction': 'S', + 'long_degrees': 115, + 'long_minutes': 49, + 'long_seconds': 11.7, + 'long_direction': 'E', + 'altitude': 20, + 'size': 10, + 'precision_horz': 10, + 'precision_vert': 2, + } + }) + + self.assertEquals(['invalid value for lat_degrees "360"'], + ctx.exception.reasons) + + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'LOC', + 'ttl': 600, + 'value': { + 'lat_degrees': 'nope', + 'lat_minutes': 58, + 'lat_seconds': 52.1, + 'lat_direction': 'S', + 'long_degrees': 115, + 'long_minutes': 49, + 'long_seconds': 11.7, + 'long_direction': 'E', + 'altitude': 20, + 'size': 10, + 'precision_horz': 10, + 'precision_vert': 2, + } + }) + + self.assertEquals(['invalid lat_degrees "nope"'], + ctx.exception.reasons) + + # invalid minutes + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'LOC', + 'ttl': 600, + 'value': { + 'lat_degrees': 31, + 'lat_minutes': 60, + 'lat_seconds': 52.1, + 'lat_direction': 'S', + 'long_degrees': 115, + 'long_minutes': 49, + 'long_seconds': 11.7, + 'long_direction': 'E', + 'altitude': 20, + 'size': 10, + 'precision_horz': 10, + 'precision_vert': 2, + } + }) + + self.assertEquals(['invalid value for lat_minutes "60"'], + ctx.exception.reasons) + + # invalid seconds + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'LOC', + 'ttl': 600, + 'value': { + 'lat_degrees': 31, + 'lat_minutes': 58, + 'lat_seconds': 60, + 'lat_direction': 'S', + 'long_degrees': 115, + 'long_minutes': 49, + 'long_seconds': 11.7, + 'long_direction': 'E', + 'altitude': 20, + 'size': 10, + 'precision_horz': 10, + 'precision_vert': 2, + } + }) + + self.assertEquals(['invalid value for lat_seconds "60"'], + ctx.exception.reasons) + + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'LOC', + 'ttl': 600, + 'value': { + 'lat_degrees': 31, + 'lat_minutes': 58, + 'lat_seconds': 'nope', + 'lat_direction': 'S', + 'long_degrees': 115, + 'long_minutes': 49, + 'long_seconds': 11.7, + 'long_direction': 'E', + 'altitude': 20, + 'size': 10, + 'precision_horz': 10, + 'precision_vert': 2, + } + }) + + self.assertEquals(['invalid lat_seconds "nope"'], + ctx.exception.reasons) + + # invalid altitude + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'LOC', + 'ttl': 600, + 'value': { + 'lat_degrees': 31, + 'lat_minutes': 58, + 'lat_seconds': 52.1, + 'lat_direction': 'S', + 'long_degrees': 115, + 'long_minutes': 49, + 'long_seconds': 11.7, + 'long_direction': 'E', + 'altitude': -666666, + 'size': 10, + 'precision_horz': 10, + 'precision_vert': 2, + } + }) + + self.assertEquals(['invalid value for altitude "-666666"'], + ctx.exception.reasons) + + # invalid size + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'LOC', + 'ttl': 600, + 'value': { + 'lat_degrees': 31, + 'lat_minutes': 58, + 'lat_seconds': 52.1, + 'lat_direction': 'S', + 'long_degrees': 115, + 'long_minutes': 49, + 'long_seconds': 11.7, + 'long_direction': 'E', + 'altitude': 20, + 'size': 99999999.99, + 'precision_horz': 10, + 'precision_vert': 2, + } + }) + + self.assertEquals(['invalid value for size "99999999.99"'], + ctx.exception.reasons) + def test_MX(self): # doesn't blow up Record.new(self.zone, '', { diff --git a/tests/zones/unit.tests.tst b/tests/zones/unit.tests.tst index 838de88..3a25415 100644 --- a/tests/zones/unit.tests.tst +++ b/tests/zones/unit.tests.tst @@ -32,6 +32,10 @@ mx 300 IN MX 20 smtp-2.unit.tests. mx 300 IN MX 30 smtp-3.unit.tests. mx 300 IN MX 40 smtp-1.unit.tests. +; LOC Records +loc 300 IN LOC 31 58 52.1 S 115 49 11.7 E 20m 10m 10m 2m +loc 300 IN LOC 53 14 10 N 2 18 26 W 20m 10m 1000m 2m + ; A Records @ 300 IN A 1.2.3.4 @ 300 IN A 1.2.3.5 From 3ac8d0fa1c3d6e1a97afd681d1872183ce5aeeae Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Mon, 30 Nov 2020 23:44:55 +0800 Subject: [PATCH 130/358] Add LOC record support to AXFR source --- README.md | 2 +- octodns/source/axfr.py | 31 +++++++++++++++++++++++++++++-- tests/test_octodns_source_axfr.py | 6 +++--- 3 files changed, 33 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index a65e28f..08933aa 100644 --- a/README.md +++ b/README.md @@ -205,7 +205,7 @@ The above command pulled the existing data out of Route53 and placed the results | [Selectel](/octodns/provider/selectel.py) | | A, AAAA, CNAME, MX, NS, SPF, SRV, TXT | No | | | [Transip](/octodns/provider/transip.py) | transip | A, AAAA, CNAME, MX, SRV, SPF, TXT, SSHFP, CAA | No | | | [UltraDns](/octodns/provider/ultra.py) | | A, AAAA, CAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | | -| [AxfrSource](/octodns/source/axfr.py) | | A, AAAA, CAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | read-only | +| [AxfrSource](/octodns/source/axfr.py) | | A, AAAA, CAA, CNAME, LOC, MX, NS, PTR, SPF, SRV, TXT | No | read-only | | [ZoneFileSource](/octodns/source/axfr.py) | | A, AAAA, CAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | read-only | | [TinyDnsFileSource](/octodns/source/tinydns.py) | | A, CNAME, MX, NS, PTR | No | read-only | | [YamlProvider](/octodns/provider/yaml.py) | | All | Yes | config | diff --git a/octodns/source/axfr.py b/octodns/source/axfr.py index e21f29f..7a45155 100644 --- a/octodns/source/axfr.py +++ b/octodns/source/axfr.py @@ -26,8 +26,8 @@ class AxfrBaseSource(BaseSource): SUPPORTS_GEO = False SUPPORTS_DYNAMIC = False - SUPPORTS = set(('A', 'AAAA', 'CAA', 'CNAME', 'MX', 'NS', 'PTR', 'SPF', - 'SRV', 'TXT')) + SUPPORTS = set(('A', 'AAAA', 'CAA', 'CNAME', 'LOC', 'MX', 'NS', 'PTR', + 'SPF', 'SRV', 'TXT')) def __init__(self, id): super(AxfrBaseSource, self).__init__(id) @@ -58,6 +58,33 @@ class AxfrBaseSource(BaseSource): 'values': values } + def _data_for_LOC(self, _type, records): + values = [] + for record in records: + lat_degrees, lat_minutes, lat_seconds, lat_direction, \ + long_degrees, long_minutes, long_seconds, long_direction, \ + altitude, size, precision_horz, precision_vert = \ + record['value'].replace('m', '').split(' ', 11) + values.append({ + 'lat_degrees': lat_degrees, + 'lat_minutes': lat_minutes, + 'lat_seconds': lat_seconds, + 'lat_direction': lat_direction, + 'long_degrees': long_degrees, + 'long_minutes': long_minutes, + 'long_seconds': long_seconds, + 'long_direction': long_direction, + 'altitude': altitude, + 'size': size, + 'precision_horz': precision_horz, + 'precision_vert': precision_vert, + }) + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': values + } + def _data_for_MX(self, _type, records): values = [] for record in records: diff --git a/tests/test_octodns_source_axfr.py b/tests/test_octodns_source_axfr.py index 44e04d0..8cf6929 100644 --- a/tests/test_octodns_source_axfr.py +++ b/tests/test_octodns_source_axfr.py @@ -36,7 +36,7 @@ class TestAxfrSource(TestCase): ] self.source.populate(got) - self.assertEquals(12, len(got.records)) + self.assertEquals(13, len(got.records)) with self.assertRaises(AxfrSourceZoneTransferFailed) as ctx: zone = Zone('unit.tests.', []) @@ -79,12 +79,12 @@ class TestZoneFileSource(TestCase): # Valid zone file in directory valid = Zone('unit.tests.', []) self.source.populate(valid) - self.assertEquals(12, len(valid.records)) + self.assertEquals(13, len(valid.records)) # 2nd populate does not read file again again = Zone('unit.tests.', []) self.source.populate(again) - self.assertEquals(12, len(again.records)) + self.assertEquals(13, len(again.records)) # bust the cache del self.source._zone_records[valid.name] From 8338e8db5855ff91a40847a5c6a4aaf6dc3945b8 Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Mon, 30 Nov 2020 23:56:04 +0800 Subject: [PATCH 131/358] Add LOC record support to Cloudflare provider --- README.md | 2 +- octodns/provider/cloudflare.py | 48 ++++++- .../cloudflare-dns_records-page-2.json | 56 +------- .../cloudflare-dns_records-page-3.json | 128 ++++++++++++++++++ tests/test_octodns_provider_cloudflare.py | 62 ++++++++- 5 files changed, 234 insertions(+), 62 deletions(-) create mode 100644 tests/fixtures/cloudflare-dns_records-page-3.json diff --git a/README.md b/README.md index 08933aa..5d57821 100644 --- a/README.md +++ b/README.md @@ -185,7 +185,7 @@ The above command pulled the existing data out of Route53 and placed the results |--|--|--|--|--| | [AzureProvider](/octodns/provider/azuredns.py) | azure-mgmt-dns | A, AAAA, CAA, CNAME, MX, NS, PTR, SRV, TXT | No | | | [Akamai](/octodns/provider/edgedns.py) | edgegrid-python | A, AAAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, SSHFP, TXT | No | | -| [CloudflareProvider](/octodns/provider/cloudflare.py) | | A, AAAA, ALIAS, CAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | CAA tags restricted | +| [CloudflareProvider](/octodns/provider/cloudflare.py) | | A, AAAA, ALIAS, CAA, CNAME, LOC, MX, NS, PTR, SPF, SRV, TXT | No | CAA tags restricted | | [ConstellixProvider](/octodns/provider/constellix.py) | | A, AAAA, ALIAS (ANAME), CAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | CAA tags restricted | | [DigitalOceanProvider](/octodns/provider/digitalocean.py) | | A, AAAA, CAA, CNAME, MX, NS, TXT, SRV | No | CAA tags restricted | | [DnsMadeEasyProvider](/octodns/provider/dnsmadeeasy.py) | | A, AAAA, ALIAS (ANAME), CAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | CAA tags restricted | diff --git a/octodns/provider/cloudflare.py b/octodns/provider/cloudflare.py index db937e5..05fef20 100644 --- a/octodns/provider/cloudflare.py +++ b/octodns/provider/cloudflare.py @@ -75,8 +75,8 @@ class CloudflareProvider(BaseProvider): ''' SUPPORTS_GEO = False SUPPORTS_DYNAMIC = False - SUPPORTS = set(('ALIAS', 'A', 'AAAA', 'CAA', 'CNAME', 'MX', 'NS', 'PTR', - 'SRV', 'SPF', 'TXT')) + SUPPORTS = set(('ALIAS', 'A', 'AAAA', 'CAA', 'CNAME', 'LOC', 'MX', 'NS', + 'PTR', 'SRV', 'SPF', 'TXT')) MIN_TTL = 120 TIMEOUT = 15 @@ -133,6 +133,7 @@ class CloudflareProvider(BaseProvider): timeout=self.TIMEOUT) self.log.debug('_request: status=%d', resp.status_code) if resp.status_code == 400: + self.log.debug('_request: data=%s', data) raise CloudflareError(resp.json()) if resp.status_code == 403: raise CloudflareAuthenticationError(resp.json()) @@ -216,6 +217,30 @@ class CloudflareProvider(BaseProvider): _data_for_ALIAS = _data_for_CNAME _data_for_PTR = _data_for_CNAME + def _data_for_LOC(self, _type, records): + values = [] + for record in records: + r = record['data'] + values.append({ + 'lat_degrees': int(r['lat_degrees']), + 'lat_minutes': int(r['lat_minutes']), + 'lat_seconds': float(r['lat_seconds']), + 'lat_direction': r['lat_direction'], + 'long_degrees': int(r['long_degrees']), + 'long_minutes': int(r['long_minutes']), + 'long_seconds': float(r['long_seconds']), + 'long_direction': r['long_direction'], + 'altitude': float(r['altitude']), + 'size': float(r['size']), + 'precision_horz': float(r['precision_horz']), + 'precision_vert': float(r['precision_vert']), + }) + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': values + } + def _data_for_MX(self, _type, records): values = [] for r in records: @@ -384,6 +409,25 @@ class CloudflareProvider(BaseProvider): _contents_for_PTR = _contents_for_CNAME + def _contents_for_LOC(self, record): + for value in record.values: + yield { + 'data': { + 'lat_degrees': value.lat_degrees, + 'lat_minutes': value.lat_minutes, + 'lat_seconds': value.lat_seconds, + 'lat_direction': value.lat_direction, + 'long_degrees': value.long_degrees, + 'long_minutes': value.long_minutes, + 'long_seconds': value.long_seconds, + 'long_direction': value.long_direction, + 'altitude': value.altitude, + 'size': value.size, + 'precision_horz': value.precision_horz, + 'precision_vert': value.precision_vert, + } + } + def _contents_for_MX(self, record): for value in record.values: yield { diff --git a/tests/fixtures/cloudflare-dns_records-page-2.json b/tests/fixtures/cloudflare-dns_records-page-2.json index b0bbaef..8075ba5 100644 --- a/tests/fixtures/cloudflare-dns_records-page-2.json +++ b/tests/fixtures/cloudflare-dns_records-page-2.json @@ -173,64 +173,14 @@ "meta": { "auto_added": false } - }, - { - "id": "fc12ab34cd5611334422ab3322997656", - "type": "SRV", - "name": "_srv._tcp.unit.tests", - "data": { - "service": "_srv", - "proto": "_tcp", - "name": "unit.tests", - "priority": 12, - "weight": 20, - "port": 30, - "target": "foo-2.unit.tests" - }, - "proxiable": true, - "proxied": false, - "ttl": 600, - "locked": false, - "zone_id": "ff12ab34cd5611334422ab3322997650", - "zone_name": "unit.tests", - "modified_on": "2017-03-11T18:01:43.940682Z", - "created_on": "2017-03-11T18:01:43.940682Z", - "meta": { - "auto_added": false - } - }, - { - "id": "fc12ab34cd5611334422ab3322997656", - "type": "SRV", - "name": "_srv._tcp.unit.tests", - "data": { - "service": "_srv", - "proto": "_tcp", - "name": "unit.tests", - "priority": 10, - "weight": 20, - "port": 30, - "target": "foo-1.unit.tests" - }, - "proxiable": true, - "proxied": false, - "ttl": 600, - "locked": false, - "zone_id": "ff12ab34cd5611334422ab3322997650", - "zone_name": "unit.tests", - "modified_on": "2017-03-11T18:01:43.940682Z", - "created_on": "2017-03-11T18:01:43.940682Z", - "meta": { - "auto_added": false - } } ], "result_info": { "page": 2, - "per_page": 11, - "total_pages": 2, + "per_page": 10, + "total_pages": 3, "count": 10, - "total_count": 20 + "total_count": 24 }, "success": true, "errors": [], diff --git a/tests/fixtures/cloudflare-dns_records-page-3.json b/tests/fixtures/cloudflare-dns_records-page-3.json new file mode 100644 index 0000000..0f06ab4 --- /dev/null +++ b/tests/fixtures/cloudflare-dns_records-page-3.json @@ -0,0 +1,128 @@ +{ + "result": [ + { + "id": "fc12ab34cd5611334422ab3322997656", + "type": "SRV", + "name": "_srv._tcp.unit.tests", + "data": { + "service": "_srv", + "proto": "_tcp", + "name": "unit.tests", + "priority": 12, + "weight": 20, + "port": 30, + "target": "foo-2.unit.tests" + }, + "proxiable": true, + "proxied": false, + "ttl": 600, + "locked": false, + "zone_id": "ff12ab34cd5611334422ab3322997650", + "zone_name": "unit.tests", + "modified_on": "2017-03-11T18:01:43.940682Z", + "created_on": "2017-03-11T18:01:43.940682Z", + "meta": { + "auto_added": false + } + }, + { + "id": "fc12ab34cd5611334422ab3322997656", + "type": "SRV", + "name": "_srv._tcp.unit.tests", + "data": { + "service": "_srv", + "proto": "_tcp", + "name": "unit.tests", + "priority": 10, + "weight": 20, + "port": 30, + "target": "foo-1.unit.tests" + }, + "proxiable": true, + "proxied": false, + "ttl": 600, + "locked": false, + "zone_id": "ff12ab34cd5611334422ab3322997650", + "zone_name": "unit.tests", + "modified_on": "2017-03-11T18:01:43.940682Z", + "created_on": "2017-03-11T18:01:43.940682Z", + "meta": { + "auto_added": false + } + }, + { + "id": "372e67954025e0ba6aaa6d586b9e0b59", + "type": "LOC", + "name": "loc.unit.tests", + "content": "IN LOC 31 58 52.1 S 115 49 11.7 E 20m 10m 10m 2m", + "proxiable": true, + "proxied": false, + "ttl": 300, + "locked": false, + "zone_id": "ff12ab34cd5611334422ab3322997650", + "zone_name": "unit.tests", + "created_on": "2020-01-28T05:20:00.12345Z", + "modified_on": "2020-01-28T05:20:00.12345Z", + "data": { + "lat_degrees": 31, + "lat_minutes": 58, + "lat_seconds": 52.1, + "lat_direction": "S", + "long_degrees": 115, + "long_minutes": 49, + "long_seconds": 11.7, + "long_direction": "E", + "altitude": 20, + "size": 10, + "precision_horz": 10, + "precision_vert": 2 + }, + "meta": { + "auto_added": true, + "source": "primary" + } + }, + { + "id": "372e67954025e0ba6aaa6d586b9e0b59", + "type": "LOC", + "name": "loc.unit.tests", + "content": "IN LOC 53 14 10 N 2 18 26 W 20m 10m 1000m 2m", + "proxiable": true, + "proxied": false, + "ttl": 300, + "locked": false, + "zone_id": "ff12ab34cd5611334422ab3322997650", + "zone_name": "unit.tests", + "created_on": "2020-01-28T05:20:00.12345Z", + "modified_on": "2020-01-28T05:20:00.12345Z", + "data": { + "lat_degrees": 53, + "lat_minutes": 13, + "lat_seconds": 10, + "lat_direction": "N", + "long_degrees": 2, + "long_minutes": 18, + "long_seconds": 26, + "long_direction": "W", + "altitude": 20, + "size": 10, + "precision_horz": 1000, + "precision_vert": 2 + }, + "meta": { + "auto_added": true, + "source": "primary" + } + } + ], + "result_info": { + "page": 3, + "per_page": 10, + "total_pages": 3, + "count": 4, + "total_count": 24 + }, + "success": true, + "errors": [], + "messages": [] +} diff --git a/tests/test_octodns_provider_cloudflare.py b/tests/test_octodns_provider_cloudflare.py index 735d95c..3727e20 100644 --- a/tests/test_octodns_provider_cloudflare.py +++ b/tests/test_octodns_provider_cloudflare.py @@ -177,10 +177,14 @@ class TestCloudflareProvider(TestCase): 'page-2.json') as fh: mock.get('{}?page=2'.format(base), status_code=200, text=fh.read()) + with open('tests/fixtures/cloudflare-dns_records-' + 'page-3.json') as fh: + mock.get('{}?page=3'.format(base), status_code=200, + text=fh.read()) zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals(13, len(zone.records)) + self.assertEquals(14, len(zone.records)) changes = self.expected.changes(zone, provider) @@ -189,7 +193,7 @@ class TestCloudflareProvider(TestCase): # re-populating the same zone/records comes out of cache, no calls again = Zone('unit.tests.', []) provider.populate(again) - self.assertEquals(13, len(again.records)) + self.assertEquals(14, len(again.records)) def test_apply(self): provider = CloudflareProvider('test', 'email', 'token', retry_period=0) @@ -203,12 +207,12 @@ class TestCloudflareProvider(TestCase): 'id': 42, } }, # zone create - ] + [None] * 22 # individual record creates + ] + [None] * 24 # individual record creates # non-existent zone, create everything plan = provider.plan(self.expected) - self.assertEquals(13, len(plan.changes)) - self.assertEquals(13, provider.apply(plan)) + self.assertEquals(14, len(plan.changes)) + self.assertEquals(14, provider.apply(plan)) self.assertFalse(plan.exists) provider._request.assert_has_calls([ @@ -234,7 +238,7 @@ class TestCloudflareProvider(TestCase): }), ], True) # expected number of total calls - self.assertEquals(23, provider._request.call_count) + self.assertEquals(25, provider._request.call_count) provider._request.reset_mock() @@ -566,6 +570,52 @@ class TestCloudflareProvider(TestCase): 'content': 'foo.bar.com.' }, list(ptr_record_contents)[0]) + def test_loc(self): + self.maxDiff = None + provider = CloudflareProvider('test', 'email', 'token') + + zone = Zone('unit.tests.', []) + # LOC record + loc_record = Record.new(zone, 'example', { + 'ttl': 300, + 'type': 'LOC', + 'value': { + 'lat_degrees': 31, + 'lat_minutes': 58, + 'lat_seconds': 52.1, + 'lat_direction': 'S', + 'long_degrees': 115, + 'long_minutes': 49, + 'long_seconds': 11.7, + 'long_direction': 'E', + 'altitude': 20, + 'size': 10, + 'precision_horz': 10, + 'precision_vert': 2, + } + }) + + loc_record_contents = provider._gen_data(loc_record) + self.assertEquals({ + 'name': 'example.unit.tests', + 'ttl': 300, + 'type': 'LOC', + 'data': { + 'lat_degrees': 31, + 'lat_minutes': 58, + 'lat_seconds': 52.1, + 'lat_direction': 'S', + 'long_degrees': 115, + 'long_minutes': 49, + 'long_seconds': 11.7, + 'long_direction': 'E', + 'altitude': 20, + 'size': 10, + 'precision_horz': 10, + 'precision_vert': 2, + } + }, list(loc_record_contents)[0]) + def test_srv(self): provider = CloudflareProvider('test', 'email', 'token') From 5963c8b89453e51e01d2b983d0421cb707fa1568 Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Tue, 1 Dec 2020 00:02:49 +0800 Subject: [PATCH 132/358] Force order of Delete() -> Create() -> Update() in Cloudflare provider Addresses issues with changing between A, AAAA and CNAME records in both directions with the Cloudflare API See also: github/octodns#507 See also: github/octodns#586 See also: github/octodns#587 --- octodns/provider/cloudflare.py | 10 ++++++++++ tests/test_octodns_provider_cloudflare.py | 10 +++++----- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/octodns/provider/cloudflare.py b/octodns/provider/cloudflare.py index 05fef20..f9d9fd0 100644 --- a/octodns/provider/cloudflare.py +++ b/octodns/provider/cloudflare.py @@ -143,6 +143,11 @@ class CloudflareProvider(BaseProvider): resp.raise_for_status() return resp.json() + def _change_keyer(self, change): + key = change.__class__.__name__ + order = {'Delete': 0, 'Create': 1, 'Update': 2} + return order[key] + @property def zones(self): if self._zones is None: @@ -660,6 +665,11 @@ class CloudflareProvider(BaseProvider): self.zones[name] = zone_id self._zone_records[name] = {} + # Force the operation order to be Delete() -> Create() -> Update() + # This will help avoid problems in updating a CNAME record into an + # A record and vice-versa + changes.sort(key=self._change_keyer) + for change in changes: class_name = change.__class__.__name__ getattr(self, '_apply_{}'.format(class_name))(change) diff --git a/tests/test_octodns_provider_cloudflare.py b/tests/test_octodns_provider_cloudflare.py index 3727e20..127480b 100644 --- a/tests/test_octodns_provider_cloudflare.py +++ b/tests/test_octodns_provider_cloudflare.py @@ -340,6 +340,10 @@ class TestCloudflareProvider(TestCase): self.assertTrue(plan.exists) # creates a the new value and then deletes all the old provider._request.assert_has_calls([ + call('DELETE', '/zones/ff12ab34cd5611334422ab3322997650/' + 'dns_records/fc12ab34cd5611334422ab3322997653'), + call('DELETE', '/zones/ff12ab34cd5611334422ab3322997650/' + 'dns_records/fc12ab34cd5611334422ab3322997654'), call('PUT', '/zones/42/dns_records/' 'fc12ab34cd5611334422ab3322997655', data={ 'content': '3.2.3.4', @@ -347,11 +351,7 @@ class TestCloudflareProvider(TestCase): 'name': 'ttl.unit.tests', 'proxied': False, 'ttl': 300 - }), - call('DELETE', '/zones/ff12ab34cd5611334422ab3322997650/' - 'dns_records/fc12ab34cd5611334422ab3322997653'), - call('DELETE', '/zones/ff12ab34cd5611334422ab3322997650/' - 'dns_records/fc12ab34cd5611334422ab3322997654') + }) ]) def test_update_add_swap(self): From 5852ae7a2ffdac74e3bff07b44d2a2c0db147024 Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Wed, 13 Jan 2021 21:46:23 +0800 Subject: [PATCH 133/358] Detect changes to LOC record correctly --- octodns/provider/cloudflare.py | 18 +++++++++++++++++- tests/test_octodns_provider_cloudflare.py | 17 +++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/octodns/provider/cloudflare.py b/octodns/provider/cloudflare.py index f9d9fd0..cc1169d 100644 --- a/octodns/provider/cloudflare.py +++ b/octodns/provider/cloudflare.py @@ -505,7 +505,7 @@ class CloudflareProvider(BaseProvider): # new records cleanly. In general when there are multiple records for a # name & type each will have a distinct/consistent `content` that can # serve as a unique identifier. - # BUT... there are exceptions. MX, CAA, and SRV don't have a simple + # BUT... there are exceptions. MX, CAA, LOC and SRV don't have a simple # content as things are currently implemented so we need to handle # those explicitly and create unique/hashable strings for them. _type = data['type'] @@ -517,6 +517,22 @@ class CloudflareProvider(BaseProvider): elif _type == 'SRV': data = data['data'] return '{port} {priority} {target} {weight}'.format(**data) + elif _type == 'LOC': + data = data['data'] + loc = ( + '{lat_degrees}', + '{lat_minutes}', + '{lat_seconds}', + '{lat_direction}', + '{long_degrees}', + '{long_minutes}', + '{long_seconds}', + '{long_direction}', + '{altitude}', + '{size}', + '{precision_horz}', + '{precision_vert}') + return ' '.join(loc).format(**data) return data['content'] def _apply_Create(self, change): diff --git a/tests/test_octodns_provider_cloudflare.py b/tests/test_octodns_provider_cloudflare.py index 127480b..d4fa74e 100644 --- a/tests/test_octodns_provider_cloudflare.py +++ b/tests/test_octodns_provider_cloudflare.py @@ -747,6 +747,23 @@ class TestCloudflareProvider(TestCase): }, 'type': 'SRV', }), + ('31 58 52.1 S 115 49 11.7 E 20 10 10 2', { + 'data': { + 'lat_degrees': 31, + 'lat_minutes': 58, + 'lat_seconds': 52.1, + 'lat_direction': 'S', + 'long_degrees': 115, + 'long_minutes': 49, + 'long_seconds': 11.7, + 'long_direction': 'E', + 'altitude': 20, + 'size': 10, + 'precision_horz': 10, + 'precision_vert': 2, + }, + 'type': 'LOC', + }), ): self.assertEqual(expected, provider._gen_key(data)) From f5c2f3a141ada969b9ef05e74c133abe877a8c10 Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Sun, 31 Jan 2021 22:45:48 +0800 Subject: [PATCH 134/358] Add LOC record support to PowerDNS provider --- octodns/provider/powerdns.py | 52 ++++++++++++++++++++++++- tests/fixtures/powerdns-full-data.json | 16 ++++++++ tests/test_octodns_provider_powerdns.py | 6 +-- 3 files changed, 69 insertions(+), 5 deletions(-) diff --git a/octodns/provider/powerdns.py b/octodns/provider/powerdns.py index de7743c..ee24eab 100644 --- a/octodns/provider/powerdns.py +++ b/octodns/provider/powerdns.py @@ -15,8 +15,8 @@ from .base import BaseProvider class PowerDnsBaseProvider(BaseProvider): SUPPORTS_GEO = False SUPPORTS_DYNAMIC = False - SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CAA', 'CNAME', 'MX', 'NAPTR', 'NS', - 'PTR', 'SPF', 'SSHFP', 'SRV', 'TXT')) + SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CAA', 'CNAME', 'LOC', 'MX', 'NAPTR', + 'NS', 'PTR', 'SPF', 'SSHFP', 'SRV', 'TXT')) TIMEOUT = 5 def __init__(self, id, host, api_key, port=8081, @@ -102,6 +102,33 @@ class PowerDnsBaseProvider(BaseProvider): _data_for_SPF = _data_for_quoted _data_for_TXT = _data_for_quoted + def _data_for_LOC(self, rrset): + values = [] + for record in rrset['records']: + lat_degrees, lat_minutes, lat_seconds, lat_direction, \ + long_degrees, long_minutes, long_seconds, long_direction, \ + altitude, size, precision_horz, precision_vert = \ + record['content'].replace('m', '').split(' ', 11) + values.append({ + 'lat_degrees': int(lat_degrees), + 'lat_minutes': int(lat_minutes), + 'lat_seconds': float(lat_seconds), + 'lat_direction': lat_direction, + 'long_degrees': int(long_degrees), + 'long_minutes': int(long_minutes), + 'long_seconds': float(long_seconds), + 'long_direction': long_direction, + 'altitude': float(altitude), + 'size': float(size), + 'precision_horz': float(precision_horz), + 'precision_vert': float(precision_vert), + }) + return { + 'ttl': rrset['ttl'], + 'type': rrset['type'], + 'values': values + } + def _data_for_MX(self, rrset): values = [] for record in rrset['records']: @@ -285,6 +312,27 @@ class PowerDnsBaseProvider(BaseProvider): _records_for_SPF = _records_for_quoted _records_for_TXT = _records_for_quoted + def _records_for_LOC(self, record): + return [{ + 'content': + '%d %d %0.3f %s %d %d %.3f %s %0.2fm %0.2fm %0.2fm %0.2fm' % + ( + int(v.lat_degrees), + int(v.lat_minutes), + float(v.lat_seconds), + v.lat_direction, + int(v.long_degrees), + int(v.long_minutes), + float(v.long_seconds), + v.long_direction, + float(v.altitude), + float(v.size), + float(v.precision_horz), + float(v.precision_vert) + ), + 'disabled': False + } for v in record.values] + def _records_for_MX(self, record): return [{ 'content': '{} {}'.format(v.preference, v.exchange), diff --git a/tests/fixtures/powerdns-full-data.json b/tests/fixtures/powerdns-full-data.json index 3d445d4..7da8232 100644 --- a/tests/fixtures/powerdns-full-data.json +++ b/tests/fixtures/powerdns-full-data.json @@ -32,6 +32,22 @@ "ttl": 300, "type": "MX" }, + { + "comments": [], + "name": "loc.unit.tests.", + "records": [ + { + "content": "31 58 52.100 S 115 49 11.700 E 20.00m 10.00m 10.00m 2.00m", + "disabled": false + }, + { + "content": "53 13 10.000 N 2 18 26.000 W 20.00m 10.00m 1000.00m 2.00m", + "disabled": false + } + ], + "ttl": 300, + "type": "LOC" + }, { "comments": [], "name": "sub.unit.tests.", diff --git a/tests/test_octodns_provider_powerdns.py b/tests/test_octodns_provider_powerdns.py index 5605c5b..f3f99e2 100644 --- a/tests/test_octodns_provider_powerdns.py +++ b/tests/test_octodns_provider_powerdns.py @@ -185,8 +185,8 @@ class TestPowerDnsProvider(TestCase): expected = Zone('unit.tests.', []) source = YamlProvider('test', join(dirname(__file__), 'config')) source.populate(expected) - expected_n = len(expected.records) - 4 - self.assertEquals(16, expected_n) + expected_n = len(expected.records) - 3 + self.assertEquals(17, expected_n) # No diffs == no changes with requests_mock() as mock: @@ -194,7 +194,7 @@ class TestPowerDnsProvider(TestCase): zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals(16, len(zone.records)) + self.assertEquals(17, len(zone.records)) changes = expected.changes(zone, provider) self.assertEquals(0, len(changes)) From 9e70caf92c8a44e0328117c3500c1af518e350aa Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Tue, 9 Feb 2021 20:47:03 +0800 Subject: [PATCH 135/358] Update test_octodns_source_axfr.py to catch up with upstream changes/tests --- tests/test_octodns_source_axfr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_octodns_source_axfr.py b/tests/test_octodns_source_axfr.py index 8cf6929..cd493cb 100644 --- a/tests/test_octodns_source_axfr.py +++ b/tests/test_octodns_source_axfr.py @@ -73,7 +73,7 @@ class TestZoneFileSource(TestCase): # Load zonefiles without a specified file extension valid = Zone('unit.tests.', []) source.populate(valid) - self.assertEquals(12, len(valid.records)) + self.assertEquals(13, len(valid.records)) def test_populate(self): # Valid zone file in directory From e991d8dc10d7693e8d4242f14e604fcd6d9df1c5 Mon Sep 17 00:00:00 2001 From: Patrick Connolly Date: Tue, 9 Feb 2021 16:02:52 -0500 Subject: [PATCH 136/358] Removed implementation example. --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 84dc033..9f608a0 100644 --- a/README.md +++ b/README.md @@ -297,7 +297,6 @@ If you have a problem or suggestion, please [open an issue](https://github.com/o - [`kubernetes/k8s.io:/dns`](https://github.com/kubernetes/k8s.io/tree/master/dns) - [`g0v-network/domains`](https://github.com/g0v-network/domains) - [`jekyll/dns`](https://github.com/jekyll/dns) - - [`parkr/dns`](https://github.com/parkr/dns) - **Custom Sources & Providers.** - [`octodns/octodns-ddns`](https://github.com/octodns/octodns-ddns): A simple Dynamic DNS source. - [`doddo/octodns-lexicon`](https://github.com/doddo/octodns-lexicon): Use [Lexicon](https://github.com/AnalogJ/lexicon) providers as octoDNS providers. From a8366aa02db6fac3d090f0d9a12edf3f7916ab4a Mon Sep 17 00:00:00 2001 From: Patrick Connolly Date: Tue, 9 Feb 2021 16:06:25 -0500 Subject: [PATCH 137/358] Stopped running CI for doc changes. --- .github/workflows/main.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0f62f96..337ebb0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,5 +1,8 @@ name: OctoDNS -on: [pull_request] +on: + pull_request: + paths-ignore: + - '**.md' jobs: ci: From 39d86f023e411c0af869a14ce7f12eda41f4d649 Mon Sep 17 00:00:00 2001 From: Steven Honson Date: Fri, 12 Feb 2021 00:29:48 +1100 Subject: [PATCH 138/358] powerdns: deletes before replaces --- octodns/provider/powerdns.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/octodns/provider/powerdns.py b/octodns/provider/powerdns.py index de7743c..a7f150b 100644 --- a/octodns/provider/powerdns.py +++ b/octodns/provider/powerdns.py @@ -7,6 +7,7 @@ from __future__ import absolute_import, division, print_function, \ from requests import HTTPError, Session import logging +from operator import itemgetter from ..record import Create, Record from .base import BaseProvider @@ -381,6 +382,12 @@ class PowerDnsBaseProvider(BaseProvider): for change in changes: class_name = change.__class__.__name__ mods.append(getattr(self, '_mod_{}'.format(class_name))(change)) + + # Ensure that any DELETE modifications always occur before any REPLACE + # modifications. This ensures that an A record can be replaced by a + # CNAME record and vice-versa. + mods = sorted(mods, key=itemgetter('changetype')) + self.log.debug('_apply: sending change request') try: From fdf74f9dd36365674473d8ee8d110da92f99b2c8 Mon Sep 17 00:00:00 2001 From: Steven Honson Date: Fri, 12 Feb 2021 00:48:42 +1100 Subject: [PATCH 139/358] powerdns: sort in place --- octodns/provider/powerdns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/powerdns.py b/octodns/provider/powerdns.py index a7f150b..ec30559 100644 --- a/octodns/provider/powerdns.py +++ b/octodns/provider/powerdns.py @@ -386,7 +386,7 @@ class PowerDnsBaseProvider(BaseProvider): # Ensure that any DELETE modifications always occur before any REPLACE # modifications. This ensures that an A record can be replaced by a # CNAME record and vice-versa. - mods = sorted(mods, key=itemgetter('changetype')) + mods.sort(key=itemgetter('changetype')) self.log.debug('_apply: sending change request') From 6034e8022f24a83ca80d1c32e1fc0420ecaca606 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Thu, 11 Feb 2021 17:05:42 -0800 Subject: [PATCH 140/358] Swap import order --- octodns/provider/powerdns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/powerdns.py b/octodns/provider/powerdns.py index ec30559..8ffff46 100644 --- a/octodns/provider/powerdns.py +++ b/octodns/provider/powerdns.py @@ -6,8 +6,8 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals from requests import HTTPError, Session -import logging from operator import itemgetter +import logging from ..record import Create, Record from .base import BaseProvider From 29c8f3253ed3d17cc8490bfc6e4b8e6f108b1411 Mon Sep 17 00:00:00 2001 From: Patrick Connolly Date: Thu, 11 Feb 2021 20:35:32 -0500 Subject: [PATCH 141/358] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 9f608a0..c784c0a 100644 --- a/README.md +++ b/README.md @@ -308,6 +308,7 @@ If you have a problem or suggestion, please [open an issue](https://github.com/o - Video: [FOSDEM 2019 - DNS as code with octodns](https://archive.fosdem.org/2019/schedule/event/dns_octodns/) - GitHub Blog: [Enabling DNS split authority with OctoDNS](https://github.blog/2017-04-27-enabling-split-authority-dns-with-octodns/) - Tutorial: [How To Deploy and Manage Your DNS using OctoDNS on Ubuntu 18.04](https://www.digitalocean.com/community/tutorials/how-to-deploy-and-manage-your-dns-using-octodns-on-ubuntu-18-04) + - Cloudflare Blog: [Improving the Resiliency of Our Infrastructure DNS Zone](https://blog.cloudflare.com/improving-the-resiliency-of-our-infrastructure-dns-zone/) If you know of any other resources, please do let us know! From 45d5da23cfc0940a536dc24102994820ab67f032 Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Wed, 9 Dec 2020 19:02:36 +0800 Subject: [PATCH 142/358] Add NULL SRV record examples to unit tests --- tests/config/unit.tests.yaml | 16 ++++++++++++++++ tests/zones/unit.tests.tst | 3 +++ 2 files changed, 19 insertions(+) diff --git a/tests/config/unit.tests.yaml b/tests/config/unit.tests.yaml index 7b84ac9..03f13a1 100644 --- a/tests/config/unit.tests.yaml +++ b/tests/config/unit.tests.yaml @@ -36,6 +36,22 @@ - flags: 0 tag: issue value: ca.unit.tests +_imap._tcp: + ttl: 600 + type: SRV + values: + - port: 0 + priority: 0 + target: . + weight: 0 +_pop3._tcp: + ttl: 600 + type: SRV + values: + - port: 0 + priority: 0 + target: . + weight: 0 _srv._tcp: ttl: 600 type: SRV diff --git a/tests/zones/unit.tests.tst b/tests/zones/unit.tests.tst index 838de88..b48b749 100644 --- a/tests/zones/unit.tests.tst +++ b/tests/zones/unit.tests.tst @@ -20,6 +20,9 @@ caa 1800 IN CAA 0 iodef "mailto:admin@unit.tests" ; SRV Records _srv._tcp 600 IN SRV 10 20 30 foo-1.unit.tests. _srv._tcp 600 IN SRV 10 20 30 foo-2.unit.tests. +; NULL SRV Records +_pop3._tcp 600 IN SRV 0 0 0 . +_imap._tcp 600 IN SRV 0 0 0 . ; TXT Records txt 600 IN TXT "Bah bah black sheep" From 403be8bb838bb8431700e61df6a4f150fe8b8a07 Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Wed, 9 Dec 2020 19:13:17 +0800 Subject: [PATCH 143/358] Fix handling of NULL SRV records in Cloudflare provider --- octodns/provider/cloudflare.py | 8 ++- .../cloudflare-dns_records-page-2.json | 50 +++++++++++++++++++ tests/test_octodns_provider_cloudflare.py | 12 ++--- 3 files changed, 62 insertions(+), 8 deletions(-) diff --git a/octodns/provider/cloudflare.py b/octodns/provider/cloudflare.py index db937e5..0f5c6cf 100644 --- a/octodns/provider/cloudflare.py +++ b/octodns/provider/cloudflare.py @@ -239,11 +239,13 @@ class CloudflareProvider(BaseProvider): def _data_for_SRV(self, _type, records): values = [] for r in records: + target = ('{}.'.format(r['data']['target']) + if r['data']['target'] != "." else ".") values.append({ 'priority': r['data']['priority'], 'weight': r['data']['weight'], 'port': r['data']['port'], - 'target': '{}.'.format(r['data']['target']), + 'target': target, }) return { 'type': _type, @@ -405,6 +407,8 @@ class CloudflareProvider(BaseProvider): name = subdomain for value in record.values: + target = value.target[:-1] if value.target != "." else "." + yield { 'data': { 'service': service, @@ -413,7 +417,7 @@ class CloudflareProvider(BaseProvider): 'priority': value.priority, 'weight': value.weight, 'port': value.port, - 'target': value.target[:-1], + 'target': target, } } diff --git a/tests/fixtures/cloudflare-dns_records-page-2.json b/tests/fixtures/cloudflare-dns_records-page-2.json index b0bbaef..860b6c3 100644 --- a/tests/fixtures/cloudflare-dns_records-page-2.json +++ b/tests/fixtures/cloudflare-dns_records-page-2.json @@ -174,6 +174,56 @@ "auto_added": false } }, + { + "id": "fc12ab34cd5611334422ab3322997656", + "type": "SRV", + "name": "_imap._tcp.unit.tests", + "data": { + "service": "_imap", + "proto": "_tcp", + "name": "unit.tests", + "priority": 0, + "weight": 0, + "port": 0, + "target": "." + }, + "proxiable": true, + "proxied": false, + "ttl": 600, + "locked": false, + "zone_id": "ff12ab34cd5611334422ab3322997650", + "zone_name": "unit.tests", + "modified_on": "2017-03-11T18:01:43.940682Z", + "created_on": "2017-03-11T18:01:43.940682Z", + "meta": { + "auto_added": false + } + }, + { + "id": "fc12ab34cd5611334422ab3322997656", + "type": "SRV", + "name": "_pop3._tcp.unit.tests", + "data": { + "service": "_imap", + "proto": "_pop3", + "name": "unit.tests", + "priority": 0, + "weight": 0, + "port": 0, + "target": "." + }, + "proxiable": true, + "proxied": false, + "ttl": 600, + "locked": false, + "zone_id": "ff12ab34cd5611334422ab3322997650", + "zone_name": "unit.tests", + "modified_on": "2017-03-11T18:01:43.940682Z", + "created_on": "2017-03-11T18:01:43.940682Z", + "meta": { + "auto_added": false + } + }, { "id": "fc12ab34cd5611334422ab3322997656", "type": "SRV", diff --git a/tests/test_octodns_provider_cloudflare.py b/tests/test_octodns_provider_cloudflare.py index 735d95c..94a37f4 100644 --- a/tests/test_octodns_provider_cloudflare.py +++ b/tests/test_octodns_provider_cloudflare.py @@ -180,7 +180,7 @@ class TestCloudflareProvider(TestCase): zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals(13, len(zone.records)) + self.assertEquals(15, len(zone.records)) changes = self.expected.changes(zone, provider) @@ -189,7 +189,7 @@ class TestCloudflareProvider(TestCase): # re-populating the same zone/records comes out of cache, no calls again = Zone('unit.tests.', []) provider.populate(again) - self.assertEquals(13, len(again.records)) + self.assertEquals(15, len(again.records)) def test_apply(self): provider = CloudflareProvider('test', 'email', 'token', retry_period=0) @@ -203,12 +203,12 @@ class TestCloudflareProvider(TestCase): 'id': 42, } }, # zone create - ] + [None] * 22 # individual record creates + ] + [None] * 24 # individual record creates # non-existent zone, create everything plan = provider.plan(self.expected) - self.assertEquals(13, len(plan.changes)) - self.assertEquals(13, provider.apply(plan)) + self.assertEquals(15, len(plan.changes)) + self.assertEquals(15, provider.apply(plan)) self.assertFalse(plan.exists) provider._request.assert_has_calls([ @@ -234,7 +234,7 @@ class TestCloudflareProvider(TestCase): }), ], True) # expected number of total calls - self.assertEquals(23, provider._request.call_count) + self.assertEquals(25, provider._request.call_count) provider._request.reset_mock() From 39412924da534adde67e1c9b11d0d93fd8e67db8 Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Wed, 9 Dec 2020 21:23:14 +0800 Subject: [PATCH 144/358] Fix handling of NULL SRV records in DigitalOcean provider --- octodns/provider/digitalocean.py | 6 +++++- tests/fixtures/digitalocean-page-2.json | 22 +++++++++++++++++++ tests/test_octodns_provider_digitalocean.py | 24 ++++++++++++++++++--- 3 files changed, 48 insertions(+), 4 deletions(-) diff --git a/octodns/provider/digitalocean.py b/octodns/provider/digitalocean.py index e192543..6ccee1d 100644 --- a/octodns/provider/digitalocean.py +++ b/octodns/provider/digitalocean.py @@ -186,10 +186,14 @@ class DigitalOceanProvider(BaseProvider): def _data_for_SRV(self, _type, records): values = [] for record in records: + target = ( + '{}.'.format(record['data']) + if record['data'] != "." else "." + ) values.append({ 'port': record['port'], 'priority': record['priority'], - 'target': '{}.'.format(record['data']), + 'target': target, 'weight': record['weight'] }) return { diff --git a/tests/fixtures/digitalocean-page-2.json b/tests/fixtures/digitalocean-page-2.json index 50f17f9..1405527 100644 --- a/tests/fixtures/digitalocean-page-2.json +++ b/tests/fixtures/digitalocean-page-2.json @@ -76,6 +76,28 @@ "weight": null, "flags": null, "tag": null + }, { + "id": 11189896, + "type": "SRV", + "name": "_imap._tcp", + "data": ".", + "priority": 0, + "port": 0, + "ttl": 600, + "weight": 0, + "flags": null, + "tag": null + }, { + "id": 11189897, + "type": "SRV", + "name": "_pop3._tcp", + "data": ".", + "priority": 0, + "port": 0, + "ttl": 600, + "weight": 0, + "flags": null, + "tag": null }], "links": { "pages": { diff --git a/tests/test_octodns_provider_digitalocean.py b/tests/test_octodns_provider_digitalocean.py index 0ad8f72..83fb5c3 100644 --- a/tests/test_octodns_provider_digitalocean.py +++ b/tests/test_octodns_provider_digitalocean.py @@ -83,14 +83,14 @@ class TestDigitalOceanProvider(TestCase): zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals(12, len(zone.records)) + self.assertEquals(14, len(zone.records)) changes = self.expected.changes(zone, provider) self.assertEquals(0, len(changes)) # 2nd populate makes no network calls/all from cache again = Zone('unit.tests.', []) provider.populate(again) - self.assertEquals(12, len(again.records)) + self.assertEquals(14, len(again.records)) # bust the cache del provider._zone_records[zone.name] @@ -190,6 +190,24 @@ class TestDigitalOceanProvider(TestCase): 'flags': 0, 'name': '@', 'tag': 'issue', 'ttl': 3600, 'type': 'CAA'}), + call('POST', '/domains/unit.tests/records', data={ + 'name': '_imap._tcp', + 'weight': 0, + 'data': '.', + 'priority': 0, + 'ttl': 600, + 'type': 'SRV', + 'port': 0 + }), + call('POST', '/domains/unit.tests/records', data={ + 'name': '_pop3._tcp', + 'weight': 0, + 'data': '.', + 'priority': 0, + 'ttl': 600, + 'type': 'SRV', + 'port': 0 + }), call('POST', '/domains/unit.tests/records', data={ 'name': '_srv._tcp', 'weight': 20, @@ -200,7 +218,7 @@ class TestDigitalOceanProvider(TestCase): 'port': 30 }), ]) - self.assertEquals(24, provider._client._request.call_count) + self.assertEquals(26, provider._client._request.call_count) provider._client._request.reset_mock() From 876a5b46f34693d7161b773babc95d1a38d02103 Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Sat, 19 Dec 2020 22:32:23 +0800 Subject: [PATCH 145/358] Update PowerDNS tests and fixtures for NULL SRV records --- tests/fixtures/powerdns-full-data.json | 24 ++++++++++++++++++++++++ tests/test_octodns_provider_powerdns.py | 6 +++--- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/tests/fixtures/powerdns-full-data.json b/tests/fixtures/powerdns-full-data.json index 3d445d4..a08f028 100644 --- a/tests/fixtures/powerdns-full-data.json +++ b/tests/fixtures/powerdns-full-data.json @@ -59,6 +59,30 @@ "ttl": 300, "type": "A" }, + { + "comments": [], + "name": "_imap._tcp.unit.tests.", + "records": [ + { + "content": "0 0 0 .", + "disabled": false + } + ], + "ttl": 600, + "type": "SRV" + }, + { + "comments": [], + "name": "_pop3._tcp.unit.tests.", + "records": [ + { + "content": "0 0 0 .", + "disabled": false + } + ], + "ttl": 600, + "type": "SRV" + }, { "comments": [], "name": "_srv._tcp.unit.tests.", diff --git a/tests/test_octodns_provider_powerdns.py b/tests/test_octodns_provider_powerdns.py index 33b5e44..7c418ff 100644 --- a/tests/test_octodns_provider_powerdns.py +++ b/tests/test_octodns_provider_powerdns.py @@ -186,7 +186,7 @@ class TestPowerDnsProvider(TestCase): source = YamlProvider('test', join(dirname(__file__), 'config')) source.populate(expected) expected_n = len(expected.records) - 3 - self.assertEquals(16, expected_n) + self.assertEquals(18, expected_n) # No diffs == no changes with requests_mock() as mock: @@ -194,7 +194,7 @@ class TestPowerDnsProvider(TestCase): zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals(16, len(zone.records)) + self.assertEquals(18, len(zone.records)) changes = expected.changes(zone, provider) self.assertEquals(0, len(changes)) @@ -291,7 +291,7 @@ class TestPowerDnsProvider(TestCase): expected = Zone('unit.tests.', []) source = YamlProvider('test', join(dirname(__file__), 'config')) source.populate(expected) - self.assertEquals(19, len(expected.records)) + self.assertEquals(21, len(expected.records)) # A small change to a single record with requests_mock() as mock: From 4105fb7ee798c83bf894ef39150335460536e888 Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Tue, 29 Dec 2020 17:06:50 +0800 Subject: [PATCH 146/358] Update Gandi tests and fixtures for NULL SRV records Thanks to @yzguy for assisting with API tests to confirm support --- tests/fixtures/gandi-no-changes.json | 18 ++++++++++++++++++ tests/test_octodns_provider_gandi.py | 20 ++++++++++++++++++-- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/tests/fixtures/gandi-no-changes.json b/tests/fixtures/gandi-no-changes.json index b018785..a67dc93 100644 --- a/tests/fixtures/gandi-no-changes.json +++ b/tests/fixtures/gandi-no-changes.json @@ -123,6 +123,24 @@ "2.2.3.6" ] }, + { + "rrset_type": "SRV", + "rrset_ttl": 600, + "rrset_name": "_imap._tcp", + "rrset_href": "https://api.gandi.net/v5/livedns/domains/unit.tests/records/_imap._tcp/SRV", + "rrset_values": [ + "0 0 0 ." + ] + }, + { + "rrset_type": "SRV", + "rrset_ttl": 600, + "rrset_name": "_pop3._tcp", + "rrset_href": "https://api.gandi.net/v5/livedns/domains/unit.tests/records/_pop3._tcp/SRV", + "rrset_values": [ + "0 0 0 ." + ] + }, { "rrset_type": "SRV", "rrset_ttl": 600, diff --git a/tests/test_octodns_provider_gandi.py b/tests/test_octodns_provider_gandi.py index 7e1c866..1b0443b 100644 --- a/tests/test_octodns_provider_gandi.py +++ b/tests/test_octodns_provider_gandi.py @@ -117,7 +117,7 @@ class TestGandiProvider(TestCase): zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals(14, len(zone.records)) + self.assertEquals(16, len(zone.records)) changes = self.expected.changes(zone, provider) self.assertEquals(0, len(changes)) @@ -284,6 +284,22 @@ class TestGandiProvider(TestCase): '12 20 30 foo-2.unit.tests.' ] }), + call('POST', '/livedns/domains/unit.tests/records', data={ + 'rrset_name': '_pop3._tcp', + 'rrset_ttl': 600, + 'rrset_type': 'SRV', + 'rrset_values': [ + '0 0 0 .', + ] + }), + call('POST', '/livedns/domains/unit.tests/records', data={ + 'rrset_name': '_imap._tcp', + 'rrset_ttl': 600, + 'rrset_type': 'SRV', + 'rrset_values': [ + '0 0 0 .', + ] + }), call('POST', '/livedns/domains/unit.tests/records', data={ 'rrset_name': '@', 'rrset_ttl': 3600, @@ -307,7 +323,7 @@ class TestGandiProvider(TestCase): }) ]) # expected number of total calls - self.assertEquals(17, provider._client._request.call_count) + self.assertEquals(19, provider._client._request.call_count) provider._client._request.reset_mock() From ae65311a96e77efea6719027f31b0a435ef7b9d8 Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Tue, 29 Dec 2020 17:11:22 +0800 Subject: [PATCH 147/358] Update Constellix tests and fixtures for NULL SRV records Thanks to @yzguy for assisting with API tests to confirm support --- tests/fixtures/constellix-records.json | 56 +++++++++++++++++++++++ tests/test_octodns_provider_constellix.py | 6 +-- 2 files changed, 59 insertions(+), 3 deletions(-) diff --git a/tests/fixtures/constellix-records.json b/tests/fixtures/constellix-records.json index 689fd53..282ca62 100644 --- a/tests/fixtures/constellix-records.json +++ b/tests/fixtures/constellix-records.json @@ -64,6 +64,62 @@ "roundRobinFailover": [], "pools": [], "poolsDetail": [] +}, { + "id": 1898527, + "type": "SRV", + "recordType": "srv", + "name": "_imap._tcp", + "recordOption": "roundRobin", + "noAnswer": false, + "note": "", + "ttl": 600, + "gtdRegion": 1, + "parentId": 123123, + "parent": "domain", + "source": "Domain", + "modifiedTs": 1565149714387, + "value": [{ + "value": ".", + "priority": 0, + "weight": 0, + "port": 0, + "disableFlag": false + }], + "roundRobin": [{ + "value": ".", + "priority": 0, + "weight": 0, + "port": 0, + "disableFlag": false + }] +}, { + "id": 1898528, + "type": "SRV", + "recordType": "srv", + "name": "_pop3._tcp", + "recordOption": "roundRobin", + "noAnswer": false, + "note": "", + "ttl": 600, + "gtdRegion": 1, + "parentId": 123123, + "parent": "domain", + "source": "Domain", + "modifiedTs": 1565149714387, + "value": [{ + "value": ".", + "priority": 0, + "weight": 0, + "port": 0, + "disableFlag": false + }], + "roundRobin": [{ + "value": ".", + "priority": 0, + "weight": 0, + "port": 0, + "disableFlag": false + }] }, { "id": 1808527, "type": "SRV", diff --git a/tests/test_octodns_provider_constellix.py b/tests/test_octodns_provider_constellix.py index bc17b50..8ba4860 100644 --- a/tests/test_octodns_provider_constellix.py +++ b/tests/test_octodns_provider_constellix.py @@ -101,14 +101,14 @@ class TestConstellixProvider(TestCase): zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals(14, len(zone.records)) + self.assertEquals(16, len(zone.records)) changes = self.expected.changes(zone, provider) self.assertEquals(0, len(changes)) # 2nd populate makes no network calls/all from cache again = Zone('unit.tests.', []) provider.populate(again) - self.assertEquals(14, len(again.records)) + self.assertEquals(16, len(again.records)) # bust the cache del provider._zone_records[zone.name] @@ -163,7 +163,7 @@ class TestConstellixProvider(TestCase): }), ]) - self.assertEquals(17, provider._client._request.call_count) + self.assertEquals(19, provider._client._request.call_count) provider._client._request.reset_mock() From 909c7ad7e88d54f4adeadbf9d3ad3d189de06360 Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Fri, 8 Jan 2021 16:51:36 +0800 Subject: [PATCH 148/358] Update EasyDNS tests and fixtures for NULL SRV records Thanks to @actazen for assisting with API tests to confirm support --- tests/fixtures/easydns-records.json | 26 ++++++++++++++++++++++++-- tests/test_octodns_provider_easydns.py | 6 +++--- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/tests/fixtures/easydns-records.json b/tests/fixtures/easydns-records.json index c3718b5..73ea953 100644 --- a/tests/fixtures/easydns-records.json +++ b/tests/fixtures/easydns-records.json @@ -264,10 +264,32 @@ "rdata": "v=DKIM1;k=rsa;s=email;h=sha256;p=A\/kinda+of\/long\/string+with+numb3rs", "geozone_id": "0", "last_mod": "2020-01-01 01:01:01" + }, + { + "id": "12340025", + "domain": "unit.tests", + "host": "_imap._tcp", + "ttl": "600", + "prio": "0", + "type": "SRV", + "rdata": "0 0 0 .", + "geozone_id": "0", + "last_mod": "2020-01-01 01:01:01" + }, + { + "id": "12340026", + "domain": "unit.tests", + "host": "_pop3._tcp", + "ttl": "600", + "prio": "0", + "type": "SRV", + "rdata": "0 0 0 .", + "geozone_id": "0", + "last_mod": "2020-01-01 01:01:01" } ], - "count": 24, - "total": 24, + "count": 26, + "total": 26, "start": 0, "max": 1000, "status": 200 diff --git a/tests/test_octodns_provider_easydns.py b/tests/test_octodns_provider_easydns.py index 8df0e22..2b137a6 100644 --- a/tests/test_octodns_provider_easydns.py +++ b/tests/test_octodns_provider_easydns.py @@ -80,14 +80,14 @@ class TestEasyDNSProvider(TestCase): text=fh.read()) provider.populate(zone) - self.assertEquals(13, len(zone.records)) + self.assertEquals(15, len(zone.records)) changes = self.expected.changes(zone, provider) self.assertEquals(0, len(changes)) # 2nd populate makes no network calls/all from cache again = Zone('unit.tests.', []) provider.populate(again) - self.assertEquals(13, len(again.records)) + self.assertEquals(15, len(again.records)) # bust the cache del provider._zone_records[zone.name] @@ -379,7 +379,7 @@ class TestEasyDNSProvider(TestCase): self.assertEquals(n, provider.apply(plan)) self.assertFalse(plan.exists) - self.assertEquals(23, provider._client._request.call_count) + self.assertEquals(25, provider._client._request.call_count) provider._client._request.reset_mock() From fb197b890e78d6a652fa276bb4148afd9174245b Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Mon, 11 Jan 2021 22:07:58 +0800 Subject: [PATCH 149/358] Update Mythic Beasts tests and fixtures for NULL SRV records Thanks to @pwaring for assisting with API tests to confirm support, and reaching out to Mythic Beasts to obtain a fix --- tests/fixtures/mythicbeasts-list.txt | 2 ++ tests/test_octodns_provider_mythicbeasts.py | 8 ++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/fixtures/mythicbeasts-list.txt b/tests/fixtures/mythicbeasts-list.txt index ed4ea4c..006a8ff 100644 --- a/tests/fixtures/mythicbeasts-list.txt +++ b/tests/fixtures/mythicbeasts-list.txt @@ -5,6 +5,8 @@ @ 3600 SSHFP 1 1 bf6b6825d2977c511a475bbefb88aad54a92ac73 @ 3600 SSHFP 1 1 7491973e5f8b39d5327cd4e08bc81b05f7710b49 @ 3600 CAA 0 issue ca.unit.tests +_imap._tcp 600 SRV 0 0 0 . +_pop3._tcp 600 SRV 0 0 0 . _srv._tcp 600 SRV 10 20 30 foo-1.unit.tests. _srv._tcp 600 SRV 12 20 30 foo-2.unit.tests. aaaa 600 AAAA 2601:644:500:e210:62f8:1dff:feb8:947a diff --git a/tests/test_octodns_provider_mythicbeasts.py b/tests/test_octodns_provider_mythicbeasts.py index f78cb0b..26af8c1 100644 --- a/tests/test_octodns_provider_mythicbeasts.py +++ b/tests/test_octodns_provider_mythicbeasts.py @@ -378,8 +378,8 @@ class TestMythicBeastsProvider(TestCase): zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals(15, len(zone.records)) - self.assertEquals(15, len(self.expected.records)) + self.assertEquals(17, len(zone.records)) + self.assertEquals(17, len(self.expected.records)) changes = self.expected.changes(zone, provider) self.assertEquals(0, len(changes)) @@ -445,7 +445,7 @@ class TestMythicBeastsProvider(TestCase): if isinstance(c, Update)])) self.assertEquals(1, len([c for c in plan.changes if isinstance(c, Delete)])) - self.assertEquals(14, len([c for c in plan.changes + self.assertEquals(16, len([c for c in plan.changes if isinstance(c, Create)])) - self.assertEquals(16, provider.apply(plan)) + self.assertEquals(18, provider.apply(plan)) self.assertTrue(plan.exists) From e0d79f826f752dc792ad03836e27daebe2c64691 Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Fri, 15 Jan 2021 18:32:11 +0800 Subject: [PATCH 150/358] Update Edge DNS tests and fixtures for NULL SRV records Thanks to @jdgri for assisting with API tests to confirm support --- tests/fixtures/edgedns-records.json | 20 ++++++++++++++++++-- tests/test_octodns_provider_edgedns.py | 10 +++++----- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/tests/fixtures/edgedns-records.json b/tests/fixtures/edgedns-records.json index 4693eb1..a5ce14f 100644 --- a/tests/fixtures/edgedns-records.json +++ b/tests/fixtures/edgedns-records.json @@ -9,6 +9,22 @@ "name": "_srv._tcp.unit.tests", "ttl": 600 }, + { + "rdata": [ + "0 0 0 ." + ], + "type": "SRV", + "name": "_imap._tcp.unit.tests", + "ttl": 600 + }, + { + "rdata": [ + "0 0 0 ." + ], + "type": "SRV", + "name": "_pop3._tcp.unit.tests", + "ttl": 600 + }, { "rdata": [ "2601:644:500:e210:62f8:1dff:feb8:947a" @@ -151,7 +167,7 @@ } ], "metadata": { - "totalElements": 16, + "totalElements": 18, "showAll": true } -} \ No newline at end of file +} diff --git a/tests/test_octodns_provider_edgedns.py b/tests/test_octodns_provider_edgedns.py index 20a9a07..694c762 100644 --- a/tests/test_octodns_provider_edgedns.py +++ b/tests/test_octodns_provider_edgedns.py @@ -77,14 +77,14 @@ class TestEdgeDnsProvider(TestCase): zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals(16, len(zone.records)) + self.assertEquals(18, len(zone.records)) changes = self.expected.changes(zone, provider) self.assertEquals(0, len(changes)) # 2nd populate makes no network calls/all from cache again = Zone('unit.tests.', []) provider.populate(again) - self.assertEquals(16, len(again.records)) + self.assertEquals(18, len(again.records)) # bust the cache del provider._zone_records[zone.name] @@ -105,7 +105,7 @@ class TestEdgeDnsProvider(TestCase): mock.delete(ANY, status_code=204) changes = provider.apply(plan) - self.assertEquals(29, changes) + self.assertEquals(31, changes) # Test against a zone that doesn't exist yet with requests_mock() as mock: @@ -118,7 +118,7 @@ class TestEdgeDnsProvider(TestCase): mock.delete(ANY, status_code=204) changes = provider.apply(plan) - self.assertEquals(14, changes) + self.assertEquals(16, changes) # Test against a zone that doesn't exist yet, but gid not provided with requests_mock() as mock: @@ -132,7 +132,7 @@ class TestEdgeDnsProvider(TestCase): mock.delete(ANY, status_code=204) changes = provider.apply(plan) - self.assertEquals(14, changes) + self.assertEquals(16, changes) # Test against a zone that doesn't exist, but cid not provided From 2cd5511dc68ccd0fad97dbc665e9cc20f20813d0 Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Sat, 30 Jan 2021 00:08:24 +0800 Subject: [PATCH 151/358] Warn that NULL SRV records are unsupported in DNSimple provider DNSimple does not handle NULL SRV records correctly (either via their web interface or API). Flag to end user if attempted. Issue noted with DNSimple support 2020-12-09 --- octodns/provider/dnsimple.py | 41 +++++++++++++++++++++++-- tests/test_octodns_provider_dnsimple.py | 2 +- 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/octodns/provider/dnsimple.py b/octodns/provider/dnsimple.py index f83098e..647b89c 100644 --- a/octodns/provider/dnsimple.py +++ b/octodns/provider/dnsimple.py @@ -218,12 +218,23 @@ class DnsimpleProvider(BaseProvider): try: weight, port, target = record['content'].split(' ', 2) except ValueError: - # see _data_for_NAPTR's continue + # their api/website will let you create invalid records, this + # essentially handles that by ignoring them for values + # purposes. That will cause updates to happen to delete them if + # they shouldn't exist or update them if they're wrong + self.log.warning( + '_data_for_SRV: unsupported %s record (%s)', + _type, + record['content'] + ) continue + + target = '{}.'.format(target) if target != "." else "." + values.append({ 'port': port, 'priority': record['priority'], - 'target': '{}.'.format(target), + 'target': target, 'weight': weight }) return { @@ -269,7 +280,12 @@ class DnsimpleProvider(BaseProvider): values = defaultdict(lambda: defaultdict(list)) for record in self.zone_records(zone): _type = record['type'] + data_for = getattr(self, '_data_for_{}'.format(_type), None) if _type not in self.SUPPORTS: + self.log.warning( + 'populate: skipping unsupported %s record', + _type + ) continue elif _type == 'TXT' and record['content'].startswith('ALIAS for'): # ALIAS has a "ride along" TXT record with 'ALIAS for XXXX', @@ -290,6 +306,27 @@ class DnsimpleProvider(BaseProvider): len(zone.records) - before, exists) return exists + def supports(self, record): + # DNSimple does not support empty/NULL SRV records + # + # Fails silently and leaves a corrupt record + # + # Skip the record and continue + if record._type == "SRV": + if 'value' in record.data: + targets = (record.data['value']['target'],) + else: + targets = [value['target'] for value in record.data['values']] + + if "." in targets: + self.log.warning( + 'supports: unsupported %s record with target (%s)', + record._type, targets + ) + return False + + return record._type in self.SUPPORTS + def _params_for_multiple(self, record): for value in record.values: yield { diff --git a/tests/test_octodns_provider_dnsimple.py b/tests/test_octodns_provider_dnsimple.py index 92f32b1..9f1dab3 100644 --- a/tests/test_octodns_provider_dnsimple.py +++ b/tests/test_octodns_provider_dnsimple.py @@ -137,7 +137,7 @@ class TestDnsimpleProvider(TestCase): plan = provider.plan(self.expected) # No root NS, no ignored, no excluded - n = len(self.expected.records) - 4 + n = len(self.expected.records) - 6 self.assertEquals(n, len(plan.changes)) self.assertEquals(n, provider.apply(plan)) self.assertFalse(plan.exists) From 84a0c089d4c7c28c20fa382d15113bac87f4caa6 Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Fri, 25 Dec 2020 19:21:41 +0800 Subject: [PATCH 152/358] Warn that NULL SRV records are unsupported in DNS Made Easy provider --- octodns/provider/dnsmadeeasy.py | 24 ++++++++++++++++++++++ tests/test_octodns_provider_dnsmadeeasy.py | 2 +- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/octodns/provider/dnsmadeeasy.py b/octodns/provider/dnsmadeeasy.py index 0bf05a0..7880280 100644 --- a/octodns/provider/dnsmadeeasy.py +++ b/octodns/provider/dnsmadeeasy.py @@ -284,6 +284,30 @@ class DnsMadeEasyProvider(BaseProvider): len(zone.records) - before, exists) return exists + def supports(self, record): + # DNS Made Easy does not support empty/NULL SRV records + # + # Attempting to sync such a record would generate the following error + # + # octodns.provider.dnsmadeeasy.DnsMadeEasyClientBadRequest: + # - Record value may not be a standalone dot. + # + # Skip the record and continue + if record._type == "SRV": + if 'value' in record.data: + targets = (record.data['value']['target'],) + else: + targets = [value['target'] for value in record.data['values']] + + if "." in targets: + self.log.warning( + 'supports: unsupported %s record with target (%s)', + record._type, targets + ) + return False + + return record._type in self.SUPPORTS + def _params_for_multiple(self, record): for value in record.values: yield { diff --git a/tests/test_octodns_provider_dnsmadeeasy.py b/tests/test_octodns_provider_dnsmadeeasy.py index 0ad059d..92aa547 100644 --- a/tests/test_octodns_provider_dnsmadeeasy.py +++ b/tests/test_octodns_provider_dnsmadeeasy.py @@ -134,7 +134,7 @@ class TestDnsMadeEasyProvider(TestCase): plan = provider.plan(self.expected) # No root NS, no ignored, no excluded, no unsupported - n = len(self.expected.records) - 6 + n = len(self.expected.records) - 8 self.assertEquals(n, len(plan.changes)) self.assertEquals(n, provider.apply(plan)) From 5d23977bbd26d4b12a96f7187ffc27fda8057bdd Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Sat, 30 Jan 2021 15:44:32 +0800 Subject: [PATCH 153/358] Adjust remaining unit tests due to extra records in test zone --- tests/test_octodns_manager.py | 14 +++++++------- tests/test_octodns_provider_transip.py | 4 ++-- tests/test_octodns_provider_ultra.py | 8 ++++---- tests/test_octodns_provider_yaml.py | 10 ++++++---- tests/test_octodns_source_axfr.py | 8 ++++---- 5 files changed, 23 insertions(+), 21 deletions(-) diff --git a/tests/test_octodns_manager.py b/tests/test_octodns_manager.py index f757466..1657f04 100644 --- a/tests/test_octodns_manager.py +++ b/tests/test_octodns_manager.py @@ -118,12 +118,12 @@ class TestManager(TestCase): environ['YAML_TMP_DIR'] = tmpdir.dirname tc = Manager(get_config_filename('simple.yaml')) \ .sync(dry_run=False) - self.assertEquals(22, tc) + self.assertEquals(24, tc) # try with just one of the zones tc = Manager(get_config_filename('simple.yaml')) \ .sync(dry_run=False, eligible_zones=['unit.tests.']) - self.assertEquals(16, tc) + self.assertEquals(18, tc) # the subzone, with 2 targets tc = Manager(get_config_filename('simple.yaml')) \ @@ -138,18 +138,18 @@ class TestManager(TestCase): # Again with force tc = Manager(get_config_filename('simple.yaml')) \ .sync(dry_run=False, force=True) - self.assertEquals(22, tc) + self.assertEquals(24, tc) # Again with max_workers = 1 tc = Manager(get_config_filename('simple.yaml'), max_workers=1) \ .sync(dry_run=False, force=True) - self.assertEquals(22, tc) + self.assertEquals(24, tc) # Include meta tc = Manager(get_config_filename('simple.yaml'), max_workers=1, include_meta=True) \ .sync(dry_run=False, force=True) - self.assertEquals(26, tc) + self.assertEquals(28, tc) def test_eligible_sources(self): with TemporaryDirectory() as tmpdir: @@ -215,13 +215,13 @@ class TestManager(TestCase): fh.write('---\n{}') changes = manager.compare(['in'], ['dump'], 'unit.tests.') - self.assertEquals(16, len(changes)) + self.assertEquals(18, len(changes)) # Compound sources with varying support changes = manager.compare(['in', 'nosshfp'], ['dump'], 'unit.tests.') - self.assertEquals(15, len(changes)) + self.assertEquals(17, len(changes)) with self.assertRaises(ManagerException) as ctx: manager.compare(['nope'], ['dump'], 'unit.tests.') diff --git a/tests/test_octodns_provider_transip.py b/tests/test_octodns_provider_transip.py index f792085..84cfebc 100644 --- a/tests/test_octodns_provider_transip.py +++ b/tests/test_octodns_provider_transip.py @@ -222,7 +222,7 @@ N4OiVz1I3rbZGYa396lpxO6ku8yCglisL1yrSP6DdEUp66ntpKVd provider._client = MockDomainService('unittest', self.bogus_key) plan = provider.plan(_expected) - self.assertEqual(12, plan.change_counts['Create']) + self.assertEqual(14, plan.change_counts['Create']) self.assertEqual(0, plan.change_counts['Update']) self.assertEqual(0, plan.change_counts['Delete']) @@ -235,7 +235,7 @@ N4OiVz1I3rbZGYa396lpxO6ku8yCglisL1yrSP6DdEUp66ntpKVd provider = TransipProvider('test', 'unittest', self.bogus_key) provider._client = MockDomainService('unittest', self.bogus_key) plan = provider.plan(_expected) - self.assertEqual(12, len(plan.changes)) + self.assertEqual(14, len(plan.changes)) changes = provider.apply(plan) self.assertEqual(changes, len(plan.changes)) diff --git a/tests/test_octodns_provider_ultra.py b/tests/test_octodns_provider_ultra.py index 43eac3c..b6d1017 100644 --- a/tests/test_octodns_provider_ultra.py +++ b/tests/test_octodns_provider_ultra.py @@ -285,12 +285,12 @@ class TestUltraProvider(TestCase): provider._request.side_effect = [ UltraNoZonesExistException('No Zones'), None, # zone create - ] + [None] * 13 # individual record creates + ] + [None] * 15 # individual record creates # non-existent zone, create everything plan = provider.plan(self.expected) - self.assertEquals(13, len(plan.changes)) - self.assertEquals(13, provider.apply(plan)) + self.assertEquals(15, len(plan.changes)) + self.assertEquals(15, provider.apply(plan)) self.assertFalse(plan.exists) provider._request.assert_has_calls([ @@ -320,7 +320,7 @@ class TestUltraProvider(TestCase): 'p=A/kinda+of/long/string+with+numb3rs']}), ], True) # expected number of total calls - self.assertEquals(15, provider._request.call_count) + self.assertEquals(17, provider._request.call_count) # Create sample rrset payload to attempt to alter page1 = json_load(open('tests/fixtures/ultra-records-page-1.json')) diff --git a/tests/test_octodns_provider_yaml.py b/tests/test_octodns_provider_yaml.py index f255238..d527fb3 100644 --- a/tests/test_octodns_provider_yaml.py +++ b/tests/test_octodns_provider_yaml.py @@ -35,7 +35,7 @@ class TestYamlProvider(TestCase): # without it we see everything source.populate(zone) - self.assertEquals(19, len(zone.records)) + self.assertEquals(21, len(zone.records)) source.populate(dynamic_zone) self.assertEquals(5, len(dynamic_zone.records)) @@ -58,12 +58,12 @@ class TestYamlProvider(TestCase): # We add everything plan = target.plan(zone) - self.assertEquals(16, len([c for c in plan.changes + self.assertEquals(18, len([c for c in plan.changes if isinstance(c, Create)])) self.assertFalse(isfile(yaml_file)) # Now actually do it - self.assertEquals(16, target.apply(plan)) + self.assertEquals(18, target.apply(plan)) self.assertTrue(isfile(yaml_file)) # Dynamic plan @@ -87,7 +87,7 @@ class TestYamlProvider(TestCase): # A 2nd sync should still create everything plan = target.plan(zone) - self.assertEquals(16, len([c for c in plan.changes + self.assertEquals(18, len([c for c in plan.changes if isinstance(c, Create)])) with open(yaml_file) as fh: @@ -107,6 +107,8 @@ class TestYamlProvider(TestCase): self.assertTrue('values' in data.pop('sub')) self.assertTrue('values' in data.pop('txt')) # these are stored as singular 'value' + self.assertTrue('value' in data.pop('_imap._tcp')) + self.assertTrue('value' in data.pop('_pop3._tcp')) self.assertTrue('value' in data.pop('aaaa')) self.assertTrue('value' in data.pop('cname')) self.assertTrue('value' in data.pop('dname')) diff --git a/tests/test_octodns_source_axfr.py b/tests/test_octodns_source_axfr.py index 44e04d0..f1a8109 100644 --- a/tests/test_octodns_source_axfr.py +++ b/tests/test_octodns_source_axfr.py @@ -36,7 +36,7 @@ class TestAxfrSource(TestCase): ] self.source.populate(got) - self.assertEquals(12, len(got.records)) + self.assertEquals(14, len(got.records)) with self.assertRaises(AxfrSourceZoneTransferFailed) as ctx: zone = Zone('unit.tests.', []) @@ -73,18 +73,18 @@ class TestZoneFileSource(TestCase): # Load zonefiles without a specified file extension valid = Zone('unit.tests.', []) source.populate(valid) - self.assertEquals(12, len(valid.records)) + self.assertEquals(14, len(valid.records)) def test_populate(self): # Valid zone file in directory valid = Zone('unit.tests.', []) self.source.populate(valid) - self.assertEquals(12, len(valid.records)) + self.assertEquals(14, len(valid.records)) # 2nd populate does not read file again again = Zone('unit.tests.', []) self.source.populate(again) - self.assertEquals(12, len(again.records)) + self.assertEquals(14, len(again.records)) # bust the cache del self.source._zone_records[valid.name] From cec53b2180fc02e9c47c3b2d11b62e00da9a6650 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Tue, 16 Feb 2021 07:11:14 -0800 Subject: [PATCH 154/358] Require different twines based on python version --- requirements-dev.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 146d673..85a93f5 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,4 +5,5 @@ pycodestyle==2.6.0 pyflakes==2.2.0 readme_renderer[md]==26.0 requests_mock -twine==3.2.0 +twine==1.15.0; python_version < '3.2' +twine==3.2.0; python_version >= '3.2' From f64ee57b14ba03363c69405ccdb08aca5bb880d0 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Tue, 16 Feb 2021 07:11:46 -0800 Subject: [PATCH 155/358] Actually only install twine on 3.x --- requirements-dev.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 85a93f5..522f112 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,5 +5,4 @@ pycodestyle==2.6.0 pyflakes==2.2.0 readme_renderer[md]==26.0 requests_mock -twine==1.15.0; python_version < '3.2' twine==3.2.0; python_version >= '3.2' From e516e7647b756c9c6d8d71a23ff5127e4d634d20 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Tue, 16 Feb 2021 07:39:52 -0800 Subject: [PATCH 156/358] Remove 2.7 form the version matix --- .github/workflows/main.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 337ebb0..b2f48dd 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -10,8 +10,7 @@ jobs: strategy: matrix: # Tested versions based on dates in https://devguide.python.org/devcycle/#end-of-life-branches, - # with the addition of 2.7 b/c it's still if pretty wide active use. - python-version: [2.7, 3.6, 3.7, 3.8, 3.9] + python-version: [3.6, 3.7, 3.8, 3.9] steps: - uses: actions/checkout@master - name: Setup python From 55af09d73cf512b00a03c58786a4f16971e8e915 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Thu, 18 Feb 2021 07:11:18 -0800 Subject: [PATCH 157/358] use super for supports base case, remove unused `data_for`. --- octodns/provider/dnsimple.py | 3 +-- octodns/provider/dnsmadeeasy.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/octodns/provider/dnsimple.py b/octodns/provider/dnsimple.py index 647b89c..599eacb 100644 --- a/octodns/provider/dnsimple.py +++ b/octodns/provider/dnsimple.py @@ -280,7 +280,6 @@ class DnsimpleProvider(BaseProvider): values = defaultdict(lambda: defaultdict(list)) for record in self.zone_records(zone): _type = record['type'] - data_for = getattr(self, '_data_for_{}'.format(_type), None) if _type not in self.SUPPORTS: self.log.warning( 'populate: skipping unsupported %s record', @@ -325,7 +324,7 @@ class DnsimpleProvider(BaseProvider): ) return False - return record._type in self.SUPPORTS + return super(DnsimpleProvider, self).supports(record) def _params_for_multiple(self, record): for value in record.values: diff --git a/octodns/provider/dnsmadeeasy.py b/octodns/provider/dnsmadeeasy.py index 7880280..b222b5c 100644 --- a/octodns/provider/dnsmadeeasy.py +++ b/octodns/provider/dnsmadeeasy.py @@ -306,7 +306,7 @@ class DnsMadeEasyProvider(BaseProvider): ) return False - return record._type in self.SUPPORTS + return super(DnsMadeEasyProvider, self).supports(record) def _params_for_multiple(self, record): for value in record.values: From d688c6123ad7c53f9b961f5eb73d44db34c8a120 Mon Sep 17 00:00:00 2001 From: "Yaroshevich, Denis" Date: Wed, 3 Mar 2021 18:54:19 +0300 Subject: [PATCH 158/358] Added G-Core DNS API v2 provider with support of A, AAAA records --- README.md | 1 + octodns/provider/gcore.py | 229 ++++++++++++++++++++++++ tests/fixtures/gcore-no-changes.json | 53 ++++++ tests/fixtures/gcore-records.json | 25 +++ tests/fixtures/gcore-zone.json | 27 +++ tests/test_octodns_provider_gcore.py | 251 +++++++++++++++++++++++++++ 6 files changed, 586 insertions(+) create mode 100644 octodns/provider/gcore.py create mode 100644 tests/fixtures/gcore-no-changes.json create mode 100644 tests/fixtures/gcore-records.json create mode 100644 tests/fixtures/gcore-zone.json create mode 100644 tests/test_octodns_provider_gcore.py diff --git a/README.md b/README.md index a65e28f..0be2427 100644 --- a/README.md +++ b/README.md @@ -195,6 +195,7 @@ The above command pulled the existing data out of Route53 and placed the results | [EtcHostsProvider](/octodns/provider/etc_hosts.py) | | A, AAAA, ALIAS, CNAME | No | | | [EnvVarSource](/octodns/source/envvar.py) | | TXT | No | read-only environment variable injection | | [GandiProvider](/octodns/provider/gandi.py) | | A, AAAA, ALIAS, CAA, CNAME, DNAME, MX, NS, PTR, SPF, SRV, SSHFP, TXT | No | | +| [GCoreProvider](/octodns/provider/gcore.py) | | A, AAAA | No | | | [GoogleCloudProvider](/octodns/provider/googlecloud.py) | google-cloud-dns | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | No | | | [MythicBeastsProvider](/octodns/provider/mythicbeasts.py) | Mythic Beasts | A, AAAA, ALIAS, CNAME, MX, NS, SRV, SSHFP, CAA, TXT | No | | | [Ns1Provider](/octodns/provider/ns1.py) | ns1-python | All | Yes | Missing `NA` geo target | diff --git a/octodns/provider/gcore.py b/octodns/provider/gcore.py new file mode 100644 index 0000000..42e2cb7 --- /dev/null +++ b/octodns/provider/gcore.py @@ -0,0 +1,229 @@ +# +# +# + +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + +from collections import defaultdict +from requests import Session +import logging + +from ..record import Record +from .base import BaseProvider + + +class GCoreClientException(Exception): + def __init__(self, r): + super(GCoreClientException, self).__init__(r.text) + + +class GCoreClientBadRequest(GCoreClientException): + def __init__(self, r): + super(GCoreClientBadRequest, self).__init__(r) + + +class GCoreClientNotFound(GCoreClientException): + def __init__(self, r): + super(GCoreClientNotFound, self).__init__(r) + + +class GCoreClient(object): + + ROOT_ZONES = "/zones" + + def __init__(self, base_url, token): + session = Session() + session.headers.update({"Authorization": "Bearer {}".format(token)}) + self._session = session + self._base_url = base_url + + def _request(self, method, path, params={}, data=None): + url = "{}{}".format(self._base_url, path) + r = self._session.request( + method, url, params=params, json=data, timeout=30.0 + ) + if r.status_code == 400: + raise GCoreClientBadRequest(r) + elif r.status_code == 404: + raise GCoreClientNotFound(r) + elif r.status_code == 500: + raise GCoreClientException(r) + r.raise_for_status() + return r + + def zone(self, zone_name): + return self._request( + "GET", "{}/{}".format(self.ROOT_ZONES, zone_name) + ).json() + + def zone_create(self, zone_name): + return self._request( + "POST", self.ROOT_ZONES, data={"name": zone_name} + ).json() + + def zone_records(self, zone_name): + rrsets = self._request( + "GET", "{}/{}/rrsets".format(self.ROOT_ZONES, zone_name) + ).json() + records = rrsets["rrsets"] + return records + + def record_create(self, zone_name, rrset_name, type_, data): + self._request( + "POST", self._rrset_url(zone_name, rrset_name, type_), data=data + ) + + def record_update(self, zone_name, rrset_name, type_, data): + self._request( + "PUT", self._rrset_url(zone_name, rrset_name, type_), data=data + ) + + def record_delete(self, zone_name, rrset_name, type_): + self._request("DELETE", self._rrset_url(zone_name, rrset_name, type_)) + + def _rrset_url(self, zone_name, rrset_name, type_): + return "{}/{}/{}/{}".format( + self.ROOT_ZONES, zone_name, rrset_name, type_ + ) + + +class GCoreProvider(BaseProvider): + """ + GCore provider using API v2. + + gcore: + class: octodns.provider.gcore.GCoreProvider + # Your API key (required) + token: XXXXXXXXXXXX + # url: https://dnsapi.gcorelabs.com/v2 + """ + + SUPPORTS_GEO = False + SUPPORTS_DYNAMIC = False + SUPPORTS = set(("A", "AAAA")) + + def __init__(self, id, token, *args, **kwargs): + base_url = kwargs.pop("url", "https://dnsapi.gcorelabs.com/v2") + self.log = logging.getLogger("GCoreProvider[{}]".format(id)) + self.log.debug("__init__: id=%s, token=***", id) + super(GCoreProvider, self).__init__(id, *args, **kwargs) + self._client = GCoreClient(base_url, token) + + def _data_for_single(self, _type, record): + return { + "ttl": record["ttl"], + "type": _type, + "values": record["resource_records"][0]["content"], + } + + _data_for_A = _data_for_single + _data_for_AAAA = _data_for_single + + def zone_records(self, zone): + try: + return self._client.zone_records(zone.name[:-1]), True + except GCoreClientNotFound: + return [], False + + def populate(self, zone, target=False, lenient=False): + self.log.debug( + "populate: name=%s, target=%s, lenient=%s", + zone.name, + target, + lenient, + ) + + values = defaultdict(defaultdict) + records, exists = self.zone_records(zone) + for record in records: + _type = record["type"] + if _type not in self.SUPPORTS: + continue + rr_name = record["name"].replace(zone.name, "") + if len(rr_name) > 0 and rr_name.endswith("."): + rr_name = rr_name[:-1] + values[rr_name][_type] = record + + before = len(zone.records) + for name, types in values.items(): + for _type, record in types.items(): + data_for = getattr(self, "_data_for_{}".format(_type)) + record = Record.new( + zone, + name, + data_for(_type, record), + source=self, + lenient=lenient, + ) + zone.add_record(record, lenient=lenient) + + self.log.info( + "populate: found %s records, exists=%s", + len(zone.records) - before, + exists, + ) + return exists + + def _params_for_single(self, record): + return { + "ttl": record.ttl, + "resource_records": [{"content": record.values}], + } + + _params_for_A = _params_for_single + _params_for_AAAA = _params_for_single + + def _apply_create(self, change): + new = change.new + rrset_name = self._build_rrset_name(new) + data = getattr(self, "_params_for_{}".format(new._type))(new) + self._client.record_create( + new.zone.name[:-1], rrset_name, new._type, data + ) + + def _apply_update(self, change): + new = change.new + rrset_name = self._build_rrset_name(new) + data = getattr(self, "_params_for_{}".format(new._type))(new) + self._client.record_update( + new.zone.name[:-1], rrset_name, new._type, data + ) + + def _apply_delete(self, change): + existing = change.existing + rrset_name = self._build_rrset_name(existing) + self._client.record_delete( + existing.zone.name[:-1], rrset_name, existing._type + ) + + def _apply(self, plan): + desired = plan.desired + changes = plan.changes + zone = desired.name[:-1] + self.log.debug( + "_apply: zone=%s, len(changes)=%d", desired.name, len(changes) + ) + + try: + self._client.zone(zone) + except GCoreClientNotFound: + self.log.info("_apply: no existing zone, trying to create it") + self._client.zone_create(zone) + self.log.info("_apply: zone has been successfully created") + + changes.reverse() + + for change in changes: + class_name = change.__class__.__name__ + getattr(self, "_apply_{}".format(class_name.lower()))(change) + + @staticmethod + def _build_rrset_name(record): + if len(record.name) > 0: + return "{}.{}".format(record.name, record.zone.name) + return record.zone.name diff --git a/tests/fixtures/gcore-no-changes.json b/tests/fixtures/gcore-no-changes.json new file mode 100644 index 0000000..ad62c94 --- /dev/null +++ b/tests/fixtures/gcore-no-changes.json @@ -0,0 +1,53 @@ +{ + "rrsets": [{ + "name": "unit.tests.", + "type": "A", + "ttl": 300, + "resource_records": [{ + "content": [ + "1.2.3.4", + "1.2.3.5" + ] + }] + }, { + "name": "aaaa.unit.tests.", + "type": "AAAA", + "ttl": 600, + "resource_records": [{ + "content": [ + "2601:644:500:e210:62f8:1dff:feb8:947a" + ] + }] + }, { + "name": "www.unit.tests.", + "type": "A", + "ttl": 300, + "resource_records": [{ + "content": [ + "2.2.3.6" + ] + }] + }, { + "name": "www.sub.unit.tests.", + "type": "A", + "ttl": 300, + "resource_records": [{ + "content": [ + "2.2.3.6" + ] + }] + }, { + "name": "unit.tests.", + "type": "ns", + "ttl": 300, + "resource_records": [{ + "content": [ + "ns2.gcdn.services" + ] + }, { + "content": [ + "ns1.gcorelabs.net" + ] + }] + }] +} diff --git a/tests/fixtures/gcore-records.json b/tests/fixtures/gcore-records.json new file mode 100644 index 0000000..4d4685e --- /dev/null +++ b/tests/fixtures/gcore-records.json @@ -0,0 +1,25 @@ +{ + "rrsets": [{ + "name": "unit.tests.", + "type": "A", + "ttl": 300, + "resource_records": [{ + "content": [ + "1.2.3.4" + ] + }] + }, { + "name": "unit.tests.", + "type": "ns", + "ttl": 300, + "resource_records": [{ + "content": [ + "ns2.gcdn.services" + ] + }, { + "content": [ + "ns1.gcorelabs.net" + ] + }] + }] +} diff --git a/tests/fixtures/gcore-zone.json b/tests/fixtures/gcore-zone.json new file mode 100644 index 0000000..7f70275 --- /dev/null +++ b/tests/fixtures/gcore-zone.json @@ -0,0 +1,27 @@ +{ + "id": 27757, + "name": "unit.test", + "nx_ttl": 300, + "retry": 5400, + "refresh": 0, + "expiry": 1209600, + "contact": "support@gcorelabs.com", + "serial": 1614752868, + "primary_server": "ns1.gcorelabs.net", + "records": [ + { + "id": 12419, + "name": "unit.test.", + "type": "ns", + "ttl": 300, + "short_answers": [ + "[ns2.gcdn.services]", + "[ns1.gcorelabs.net]" + ] + } + ], + "dns_servers": [ + "ns1.gcorelabs.net", + "ns2.gcdn.services" + ] +} \ No newline at end of file diff --git a/tests/test_octodns_provider_gcore.py b/tests/test_octodns_provider_gcore.py new file mode 100644 index 0000000..74762da --- /dev/null +++ b/tests/test_octodns_provider_gcore.py @@ -0,0 +1,251 @@ +# +# +# + +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + +from mock import Mock, call +from os.path import dirname, join +from requests_mock import ANY, mock as requests_mock +from six import text_type +from unittest import TestCase + +from octodns.record import Record, Update, Delete +from octodns.provider.gcore import ( + GCoreProvider, + GCoreClientBadRequest, + GCoreClientNotFound, + GCoreClientException, +) +from octodns.provider.yaml import YamlProvider +from octodns.zone import Zone + + +class TestGCoreProvider(TestCase): + expected = Zone("unit.tests.", []) + source = YamlProvider("test", join(dirname(__file__), "config")) + source.populate(expected) + + def test_populate(self): + + provider = GCoreProvider("test_id", "token") + + # 400 - Bad Request. + with requests_mock() as mock: + mock.get(ANY, status_code=400, text='{"error":"bad body"}') + + with self.assertRaises(GCoreClientBadRequest) as ctx: + zone = Zone("unit.tests.", []) + provider.populate(zone) + self.assertIn('"error":"bad body"', text_type(ctx.exception)) + + # 404 - Not Found. + with requests_mock() as mock: + mock.get( + ANY, status_code=404, text='{"error":"zone is not found"}' + ) + + with self.assertRaises(GCoreClientNotFound) as ctx: + zone = Zone("unit.tests.", []) + provider._client.zone(zone) + self.assertIn( + '"error":"zone is not found"', text_type(ctx.exception) + ) + + # General error + with requests_mock() as mock: + mock.get(ANY, status_code=500, text="Things caught fire") + + with self.assertRaises(GCoreClientException) as ctx: + zone = Zone("unit.tests.", []) + provider.populate(zone) + self.assertEquals("Things caught fire", text_type(ctx.exception)) + + # No diffs == no changes + with requests_mock() as mock: + base = "https://dnsapi.gcorelabs.com/v2/zones/unit.tests/rrsets" + with open("tests/fixtures/gcore-no-changes.json") as fh: + mock.get(base, text=fh.read()) + + zone = Zone("unit.tests.", []) + provider.populate(zone) + self.assertEquals(4, len(zone.records)) + changes = self.expected.changes(zone, provider) + self.assertEquals(0, len(changes)) + + # 3 removed + 1 modified + with requests_mock() as mock: + base = "https://dnsapi.gcorelabs.com/v2/zones/unit.tests/rrsets" + with open("tests/fixtures/gcore-records.json") as fh: + mock.get(base, text=fh.read()) + + zone = Zone("unit.tests.", []) + provider.populate(zone) + self.assertEquals(1, len(zone.records)) + changes = self.expected.changes(zone, provider) + self.assertEquals(4, len(changes)) + self.assertEquals( + 3, len([c for c in changes if isinstance(c, Delete)]) + ) + self.assertEquals( + 1, len([c for c in changes if isinstance(c, Update)]) + ) + + def test_apply(self): + provider = GCoreProvider("test_id", "token") + + # Zone does not exists but can be created. + with requests_mock() as mock: + mock.get( + ANY, status_code=404, text='{"error":"zone is not found"}' + ) + mock.post(ANY, status_code=200, text='{"id":1234}') + + plan = provider.plan(self.expected) + provider.apply(plan) + + # Zone does not exists and can't be created. + with requests_mock() as mock: + mock.get( + ANY, status_code=404, text='{"error":"zone is not found"}' + ) + mock.post( + ANY, + status_code=400, + text='{"error":"parent zone is already' + ' occupied by another client"}', + ) + + with self.assertRaises( + (GCoreClientNotFound, GCoreClientBadRequest) + ) as ctx: + plan = provider.plan(self.expected) + provider.apply(plan) + self.assertIn( + "parent zone is already occupied by another client", + text_type(ctx.exception), + ) + + resp = Mock() + resp.json = Mock() + provider._client._request = Mock(return_value=resp) + + with open("tests/fixtures/gcore-zone.json") as fh: + zone = fh.read() + + # non-existent domain + resp.json.side_effect = [ + GCoreClientNotFound(resp), # no zone in populate + GCoreClientNotFound(resp), # no domain during apply + zone, + ] + plan = provider.plan(self.expected) + + # create all + self.assertEquals(4, len(plan.changes)) + self.assertEquals(4, provider.apply(plan)) + self.assertFalse(plan.exists) + + provider._client._request.assert_has_calls( + [ + call("GET", "/zones/unit.tests/rrsets"), + call("GET", "/zones/unit.tests"), + call("POST", "/zones", data={"name": "unit.tests"}), + call( + "POST", + "/zones/unit.tests/www.sub.unit.tests./A", + data={ + "ttl": 300, + "resource_records": [{"content": ["2.2.3.6"]}], + }, + ), + call( + "POST", + "/zones/unit.tests/www.unit.tests./A", + data={ + "ttl": 300, + "resource_records": [{"content": ["2.2.3.6"]}], + }, + ), + call( + "POST", + "/zones/unit.tests/aaaa.unit.tests./AAAA", + data={ + "ttl": 600, + "resource_records": [ + { + "content": [ + "2601:644:500:e210:62f8:1dff:feb8:947a" + ] + } + ], + }, + ), + call( + "POST", + "/zones/unit.tests/unit.tests./A", + data={ + "ttl": 300, + "resource_records": [ + {"content": ["1.2.3.4", "1.2.3.5"]} + ], + }, + ), + ] + ) + # expected number of total calls + self.assertEquals(7, provider._client._request.call_count) + + provider._client._request.reset_mock() + + # delete 1 and update 1 + provider._client.zone_records = Mock( + return_value=[ + { + "name": "www", + "ttl": 300, + "type": "A", + "resource_records": [{"content": ["1.2.3.4"]}], + }, + { + "name": "ttl", + "ttl": 600, + "type": "A", + "resource_records": [{"content": ["3.2.3.4"]}], + }, + ] + ) + + # Domain exists, we don't care about return + resp.json.side_effect = ["{}"] + + wanted = Zone("unit.tests.", []) + wanted.add_record( + Record.new( + wanted, "ttl", {"ttl": 300, "type": "A", "value": "3.2.3.4"} + ) + ) + + plan = provider.plan(wanted) + self.assertTrue(plan.exists) + self.assertEquals(2, len(plan.changes)) + self.assertEquals(2, provider.apply(plan)) + + provider._client._request.assert_has_calls( + [ + call("DELETE", "/zones/unit.tests/www.unit.tests./A"), + call( + "PUT", + "/zones/unit.tests/ttl.unit.tests./A", + data={ + "ttl": 300, + "resource_records": [{"content": ["3.2.3.4"]}], + }, + ), + ] + ) From 88b003130e423896f90cf3767b2b263c4a13a6e7 Mon Sep 17 00:00:00 2001 From: "Yaroshevich, Denis" Date: Tue, 9 Mar 2021 11:28:09 +0300 Subject: [PATCH 159/358] fix API incorrect behaviour while returning records data: API always must return type in upper case, but at this moment it returns record type as it was provided on record creation --- octodns/provider/gcore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/gcore.py b/octodns/provider/gcore.py index 42e2cb7..5865431 100644 --- a/octodns/provider/gcore.py +++ b/octodns/provider/gcore.py @@ -141,7 +141,7 @@ class GCoreProvider(BaseProvider): values = defaultdict(defaultdict) records, exists = self.zone_records(zone) for record in records: - _type = record["type"] + _type = record["type"].upper() if _type not in self.SUPPORTS: continue rr_name = record["name"].replace(zone.name, "") From 2f90ce475652a32c67663e99f2737d31ca911dfd Mon Sep 17 00:00:00 2001 From: "Yaroshevich, Denis" Date: Tue, 9 Mar 2021 16:26:22 +0300 Subject: [PATCH 160/358] add more information in logs when RR create/update/delete fail --- octodns/provider/gcore.py | 46 ++++++++++++++++++++++++--------------- 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/octodns/provider/gcore.py b/octodns/provider/gcore.py index 5865431..6387771 100644 --- a/octodns/provider/gcore.py +++ b/octodns/provider/gcore.py @@ -179,27 +179,39 @@ class GCoreProvider(BaseProvider): _params_for_AAAA = _params_for_single def _apply_create(self, change): - new = change.new - rrset_name = self._build_rrset_name(new) - data = getattr(self, "_params_for_{}".format(new._type))(new) - self._client.record_create( - new.zone.name[:-1], rrset_name, new._type, data - ) + try: + new = change.new + rrset_name = self._build_rrset_name(new) + data = getattr(self, "_params_for_{}".format(new._type))(new) + self._client.record_create( + new.zone.name[:-1], rrset_name, new._type, data + ) + except: + self.log.exception("failed to create RR: %s", change) + rai def _apply_update(self, change): - new = change.new - rrset_name = self._build_rrset_name(new) - data = getattr(self, "_params_for_{}".format(new._type))(new) - self._client.record_update( - new.zone.name[:-1], rrset_name, new._type, data - ) + try: + new = change.new + rrset_name = self._build_rrset_name(new) + data = getattr(self, "_params_for_{}".format(new._type))(new) + self._client.record_update( + new.zone.name[:-1], rrset_name, new._type, data + ) + except: + self.log.exception("failed to update RR: %s", change) + raise def _apply_delete(self, change): - existing = change.existing - rrset_name = self._build_rrset_name(existing) - self._client.record_delete( - existing.zone.name[:-1], rrset_name, existing._type - ) + try: + existing = change.existing + rrset_name = self._build_rrset_name(existing) + self._client.record_delete( + existing.zone.name[:-1], rrset_name, existing._type + ) + except: + self.log.exception("failed to delete RR: %s", change) + raise def _apply(self, plan): desired = plan.desired From bec4d025dcd1297ea71335339a209b3dc1962443 Mon Sep 17 00:00:00 2001 From: Brian Surowiec Date: Wed, 10 Mar 2021 15:57:50 -0500 Subject: [PATCH 161/358] Update branch name --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5f3ef4c..9d267d7 100644 --- a/README.md +++ b/README.md @@ -294,7 +294,7 @@ If you have a problem or suggestion, please [open an issue](https://github.com/o - **GitHub Action:** [OctoDNS-Sync](https://github.com/marketplace/actions/octodns-sync) - **Sample Implementations.** See how others are using it - [`hackclub/dns`](https://github.com/hackclub/dns) - - [`kubernetes/k8s.io:/dns`](https://github.com/kubernetes/k8s.io/tree/master/dns) + - [`kubernetes/k8s.io:/dns`](https://github.com/kubernetes/k8s.io/tree/main/dns) - [`g0v-network/domains`](https://github.com/g0v-network/domains) - [`jekyll/dns`](https://github.com/jekyll/dns) - **Custom Sources & Providers.** From 04a9b2b9b869697e32e48472584feafde4cb2d6a Mon Sep 17 00:00:00 2001 From: "Yaroshevich, Denis" Date: Fri, 12 Mar 2021 15:00:24 +0300 Subject: [PATCH 163/358] add logging for all apply actions, instead of in case of error(s) --- octodns/provider/gcore.py | 49 ++++++++++++++++----------------------- 1 file changed, 20 insertions(+), 29 deletions(-) diff --git a/octodns/provider/gcore.py b/octodns/provider/gcore.py index 6387771..28c9fad 100644 --- a/octodns/provider/gcore.py +++ b/octodns/provider/gcore.py @@ -179,39 +179,30 @@ class GCoreProvider(BaseProvider): _params_for_AAAA = _params_for_single def _apply_create(self, change): - try: - new = change.new - rrset_name = self._build_rrset_name(new) - data = getattr(self, "_params_for_{}".format(new._type))(new) - self._client.record_create( - new.zone.name[:-1], rrset_name, new._type, data - ) - except: - self.log.exception("failed to create RR: %s", change) - rai + self.log.info("creating: %s", change) + new = change.new + rrset_name = self._build_rrset_name(new) + data = getattr(self, "_params_for_{}".format(new._type))(new) + self._client.record_create( + new.zone.name[:-1], rrset_name, new._type, data + ) def _apply_update(self, change): - try: - new = change.new - rrset_name = self._build_rrset_name(new) - data = getattr(self, "_params_for_{}".format(new._type))(new) - self._client.record_update( - new.zone.name[:-1], rrset_name, new._type, data - ) - except: - self.log.exception("failed to update RR: %s", change) - raise + self.log.info("updating: %s", change) + new = change.new + rrset_name = self._build_rrset_name(new) + data = getattr(self, "_params_for_{}".format(new._type))(new) + self._client.record_update( + new.zone.name[:-1], rrset_name, new._type, data + ) def _apply_delete(self, change): - try: - existing = change.existing - rrset_name = self._build_rrset_name(existing) - self._client.record_delete( - existing.zone.name[:-1], rrset_name, existing._type - ) - except: - self.log.exception("failed to delete RR: %s", change) - raise + self.log.info("deleting: %s", change) + existing = change.existing + rrset_name = self._build_rrset_name(existing) + self._client.record_delete( + existing.zone.name[:-1], rrset_name, existing._type + ) def _apply(self, plan): desired = plan.desired From 6c1cc47024ef1355abbc16f2daded10a8db0a1f9 Mon Sep 17 00:00:00 2001 From: Evgenii Tereshkov Date: Thu, 18 Mar 2021 02:31:10 +0700 Subject: [PATCH 164/358] Update README.md: fix typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9d267d7..0a26da9 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,7 @@ The `max_workers` key in the `manager` section of the config enables threading t In this example, `example.net` is an alias of zone `example.com`, which means they share the same sources and targets. They will therefore have identical records. -Now that we have something to tell OctoDNS about our providers & zones we need to tell it about or records. We'll keep it simple for now and just create a single `A` record at the top-level of the domain. +Now that we have something to tell OctoDNS about our providers & zones we need to tell it about our records. We'll keep it simple for now and just create a single `A` record at the top-level of the domain. `config/example.com.yaml` From 87450817461c912b26853d39a7ef5d7d50dd1b91 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Wed, 17 Mar 2021 17:04:10 -0700 Subject: [PATCH 165/358] Revert "Stopped running CI for doc changes." This reverts commit a8366aa02db6fac3d090f0d9a12edf3f7916ab4a. --- .github/workflows/main.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b2f48dd..b68df8d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,8 +1,5 @@ name: OctoDNS -on: - pull_request: - paths-ignore: - - '**.md' +on: [pull_request] jobs: ci: From ac56a36f432ab8a071d627828d1613b3e31a15a7 Mon Sep 17 00:00:00 2001 From: "Yaroshevich, Denis" Date: Mon, 22 Mar 2021 20:57:10 +0300 Subject: [PATCH 166/358] multiple resource records values should be sent as separate 'content'-s --- octodns/provider/gcore.py | 4 +++- tests/test_octodns_provider_gcore.py | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/octodns/provider/gcore.py b/octodns/provider/gcore.py index 28c9fad..246a77a 100644 --- a/octodns/provider/gcore.py +++ b/octodns/provider/gcore.py @@ -172,7 +172,9 @@ class GCoreProvider(BaseProvider): def _params_for_single(self, record): return { "ttl": record.ttl, - "resource_records": [{"content": record.values}], + "resource_records": [ + {"content": [value]} for value in record.values + ], } _params_for_A = _params_for_single diff --git a/tests/test_octodns_provider_gcore.py b/tests/test_octodns_provider_gcore.py index 74762da..5671e15 100644 --- a/tests/test_octodns_provider_gcore.py +++ b/tests/test_octodns_provider_gcore.py @@ -192,7 +192,8 @@ class TestGCoreProvider(TestCase): data={ "ttl": 300, "resource_records": [ - {"content": ["1.2.3.4", "1.2.3.5"]} + {"content": ["1.2.3.4"]}, + {"content": ["1.2.3.5"]}, ], }, ), From 70555ed546ce31d8cb72588ca2c66acedbdda43e Mon Sep 17 00:00:00 2001 From: "Yaroshevich, Denis" Date: Tue, 23 Mar 2021 11:35:12 +0300 Subject: [PATCH 167/358] add new API parameter has been added to fetch all resource records --- octodns/provider/gcore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/gcore.py b/octodns/provider/gcore.py index 246a77a..60e8bad 100644 --- a/octodns/provider/gcore.py +++ b/octodns/provider/gcore.py @@ -68,7 +68,7 @@ class GCoreClient(object): def zone_records(self, zone_name): rrsets = self._request( - "GET", "{}/{}/rrsets".format(self.ROOT_ZONES, zone_name) + "GET", "{}/{}/rrsets?all=true".format(self.ROOT_ZONES, zone_name) ).json() records = rrsets["rrsets"] return records From 4a1e11c3942ef526239b06673004293dbb8d6b3b Mon Sep 17 00:00:00 2001 From: "Yaroshevich, Denis" Date: Tue, 23 Mar 2021 11:48:23 +0300 Subject: [PATCH 168/358] commit missing changes for tests --- tests/test_octodns_provider_gcore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_octodns_provider_gcore.py b/tests/test_octodns_provider_gcore.py index 5671e15..785e5e9 100644 --- a/tests/test_octodns_provider_gcore.py +++ b/tests/test_octodns_provider_gcore.py @@ -153,7 +153,7 @@ class TestGCoreProvider(TestCase): provider._client._request.assert_has_calls( [ - call("GET", "/zones/unit.tests/rrsets"), + call("GET", "/zones/unit.tests/rrsets?all=true"), call("GET", "/zones/unit.tests"), call("POST", "/zones", data={"name": "unit.tests"}), call( From 1aefa1a496271b2fc1922d9ddd55cf06a3e29123 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 25 Mar 2021 21:56:29 +0000 Subject: [PATCH 169/358] Bump pyyaml from 5.3.1 to 5.4 Bumps [pyyaml](https://github.com/yaml/pyyaml) from 5.3.1 to 5.4. - [Release notes](https://github.com/yaml/pyyaml/releases) - [Changelog](https://github.com/yaml/pyyaml/blob/master/CHANGES) - [Commits](https://github.com/yaml/pyyaml/compare/5.3.1...5.4) Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8b9c052..933ac60 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -PyYaml==5.3.1 +PyYaml==5.4 azure-common==1.1.25 azure-mgmt-dns==3.0.0 boto3==1.15.9 From a99c71b3e566cb2619696c8a545028675127a6b5 Mon Sep 17 00:00:00 2001 From: Scott Little Date: Fri, 26 Mar 2021 19:48:38 +0100 Subject: [PATCH 170/358] Change the auth value to have a proper string not a binary rep --- octodns/provider/easydns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/easydns.py b/octodns/provider/easydns.py index 835fcb9..d7a75a4 100644 --- a/octodns/provider/easydns.py +++ b/octodns/provider/easydns.py @@ -59,7 +59,7 @@ class EasyDNSClient(object): self.base_path = self.SANDBOX if sandbox else self.LIVE sess = Session() sess.headers.update({'Authorization': 'Basic {}' - .format(self.auth_key)}) + .format(self.auth_key.decode('utf-8'))}) sess.headers.update({'accept': 'application/json'}) self._sess = sess From 374a5c62f25e93f155f8066717b5eeb71b2d1552 Mon Sep 17 00:00:00 2001 From: Maikel Poot Date: Wed, 31 Mar 2021 12:49:01 +0200 Subject: [PATCH 171/358] Enable NS support for non-root NS records --- octodns/provider/transip.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/transip.py b/octodns/provider/transip.py index 7458e36..ae2c5d5 100644 --- a/octodns/provider/transip.py +++ b/octodns/provider/transip.py @@ -50,7 +50,7 @@ class TransipProvider(BaseProvider): SUPPORTS_GEO = False SUPPORTS_DYNAMIC = False SUPPORTS = set( - ('A', 'AAAA', 'CNAME', 'MX', 'SRV', 'SPF', 'TXT', 'SSHFP', 'CAA')) + ('A', 'AAAA', 'CNAME', 'MX', 'SRV', 'SPF', 'TXT', 'SSHFP', 'CAA', 'NS')) # unsupported by OctoDNS: 'TLSA' MIN_TTL = 120 TIMEOUT = 15 From 87c31ca23d757a3de4606d830a9dc1e66fbe23f7 Mon Sep 17 00:00:00 2001 From: Maikel Poot Date: Wed, 31 Mar 2021 13:31:24 +0200 Subject: [PATCH 172/358] Update transip unittest --- tests/test_octodns_provider_transip.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/test_octodns_provider_transip.py b/tests/test_octodns_provider_transip.py index 84cfebc..234c95e 100644 --- a/tests/test_octodns_provider_transip.py +++ b/tests/test_octodns_provider_transip.py @@ -56,10 +56,11 @@ class MockDomainService(DomainService): _dns_entries.extend(entries_for(name, record)) - # NS is not supported as a DNS Entry, - # so it should cover the if statement + # Add a non-supported type + # so it triggers the "is supported" (transip.py:115) check and + # give 100% code coverage _dns_entries.append( - DnsEntry('@', '3600', 'NS', 'ns01.transip.nl.')) + DnsEntry('@', '3600', 'BOGUS', 'ns01.transip.nl.')) self.mockupEntries = _dns_entries @@ -222,7 +223,7 @@ N4OiVz1I3rbZGYa396lpxO6ku8yCglisL1yrSP6DdEUp66ntpKVd provider._client = MockDomainService('unittest', self.bogus_key) plan = provider.plan(_expected) - self.assertEqual(14, plan.change_counts['Create']) + self.assertEqual(15, plan.change_counts['Create']) self.assertEqual(0, plan.change_counts['Update']) self.assertEqual(0, plan.change_counts['Delete']) @@ -235,7 +236,7 @@ N4OiVz1I3rbZGYa396lpxO6ku8yCglisL1yrSP6DdEUp66ntpKVd provider = TransipProvider('test', 'unittest', self.bogus_key) provider._client = MockDomainService('unittest', self.bogus_key) plan = provider.plan(_expected) - self.assertEqual(14, len(plan.changes)) + self.assertEqual(15, len(plan.changes)) changes = provider.apply(plan) self.assertEqual(changes, len(plan.changes)) From 60e6f185cdbbf66544734bc2f6e21b65f4233f8c Mon Sep 17 00:00:00 2001 From: Maikel Poot Date: Wed, 31 Mar 2021 13:41:55 +0200 Subject: [PATCH 173/358] Update Documentation --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0a26da9..3186ed2 100644 --- a/README.md +++ b/README.md @@ -204,7 +204,7 @@ The above command pulled the existing data out of Route53 and placed the results | [Rackspace](/octodns/provider/rackspace.py) | | A, AAAA, ALIAS, CNAME, MX, NS, PTR, SPF, TXT | No | | | [Route53](/octodns/provider/route53.py) | boto3 | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | Both | CNAME health checks don't support a Host header | | [Selectel](/octodns/provider/selectel.py) | | A, AAAA, CNAME, MX, NS, SPF, SRV, TXT | No | | -| [Transip](/octodns/provider/transip.py) | transip | A, AAAA, CNAME, MX, SRV, SPF, TXT, SSHFP, CAA | No | | +| [Transip](/octodns/provider/transip.py) | transip | A, AAAA, CNAME, MX, NS, SRV, SPF, TXT, SSHFP, CAA | No | | | [UltraDns](/octodns/provider/ultra.py) | | A, AAAA, CAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | | | [AxfrSource](/octodns/source/axfr.py) | | A, AAAA, CAA, CNAME, LOC, MX, NS, PTR, SPF, SRV, TXT | No | read-only | | [ZoneFileSource](/octodns/source/axfr.py) | | A, AAAA, CAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | read-only | From 840cb03bf432e8c7cfba2df25aed45e838fedb19 Mon Sep 17 00:00:00 2001 From: Maikel Poot Date: Wed, 31 Mar 2021 13:44:37 +0200 Subject: [PATCH 174/358] Fix lint error --- octodns/provider/transip.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/octodns/provider/transip.py b/octodns/provider/transip.py index ae2c5d5..6ccbe22 100644 --- a/octodns/provider/transip.py +++ b/octodns/provider/transip.py @@ -49,8 +49,8 @@ class TransipProvider(BaseProvider): ''' SUPPORTS_GEO = False SUPPORTS_DYNAMIC = False - SUPPORTS = set( - ('A', 'AAAA', 'CNAME', 'MX', 'SRV', 'SPF', 'TXT', 'SSHFP', 'CAA', 'NS')) + SUPPORTS = set(('A', 'AAAA', 'CNAME', 'MX', 'NS', 'SRV', 'SPF', 'TXT', + 'SSHFP', 'CAA')) # unsupported by OctoDNS: 'TLSA' MIN_TTL = 120 TIMEOUT = 15 From 090dbe351554ca3e929f099fa9e127aceedccd79 Mon Sep 17 00:00:00 2001 From: Christian Funkhouser Date: Wed, 7 Apr 2021 16:00:05 -0400 Subject: [PATCH 175/358] sync accepts file handle for plan output --- octodns/manager.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/octodns/manager.py b/octodns/manager.py index 9ce10ff..f4f1491 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -9,6 +9,7 @@ from concurrent.futures import ThreadPoolExecutor from importlib import import_module from os import environ from six import text_type +from sys import stdout import logging from .provider.base import BaseProvider @@ -267,7 +268,7 @@ class Manager(object): return plans, zone def sync(self, eligible_zones=[], eligible_sources=[], eligible_targets=[], - dry_run=True, force=False): + dry_run=True, force=False, plan_output_fh=stdout): self.log.info('sync: eligible_zones=%s, eligible_targets=%s, ' 'dry_run=%s, force=%s', eligible_zones, eligible_targets, dry_run, force) @@ -276,7 +277,7 @@ class Manager(object): if eligible_zones: zones = [z for z in zones if z[0] in eligible_zones] - aliased_zones = {} + aliased_zones = {} futures = [] for zone_name, config in zones: self.log.info('sync: zone=%s', zone_name) @@ -402,7 +403,7 @@ class Manager(object): plans.sort(key=self._plan_keyer, reverse=True) for output in self.plan_outputs.values(): - output.run(plans=plans, log=self.log) + output.run(plans=plans, log=self.log, fh=plan_output_fh) if not force: self.log.debug('sync: checking safety') From 2075550f078942d642c873bd2ec62343eb1a0f16 Mon Sep 17 00:00:00 2001 From: Christian Funkhouser Date: Wed, 7 Apr 2021 18:21:34 -0400 Subject: [PATCH 176/358] Test that Manager passes fh to _PlanOutputs --- octodns/manager.py | 5 +++-- tests/config/plan-output-filehandle.yaml | 6 ++++++ tests/test_octodns_manager.py | 23 ++++++++++++++++++++++- 3 files changed, 31 insertions(+), 3 deletions(-) create mode 100644 tests/config/plan-output-filehandle.yaml diff --git a/octodns/manager.py b/octodns/manager.py index f4f1491..e94e41e 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -270,8 +270,9 @@ class Manager(object): def sync(self, eligible_zones=[], eligible_sources=[], eligible_targets=[], dry_run=True, force=False, plan_output_fh=stdout): self.log.info('sync: eligible_zones=%s, eligible_targets=%s, ' - 'dry_run=%s, force=%s', eligible_zones, eligible_targets, - dry_run, force) + 'dry_run=%s, force=%s, plan_output_fh=%s', + eligible_zones, eligible_targets, dry_run, force, + plan_output_fh) zones = self.config['zones'].items() if eligible_zones: diff --git a/tests/config/plan-output-filehandle.yaml b/tests/config/plan-output-filehandle.yaml new file mode 100644 index 0000000..9c9bb87 --- /dev/null +++ b/tests/config/plan-output-filehandle.yaml @@ -0,0 +1,6 @@ +manager: + plan_outputs: + "doesntexist": + class: octodns.provider.plan.DoesntExist +providers: {} +zones: {} diff --git a/tests/test_octodns_manager.py b/tests/test_octodns_manager.py index 442ed49..003d414 100644 --- a/tests/test_octodns_manager.py +++ b/tests/test_octodns_manager.py @@ -8,7 +8,6 @@ from __future__ import absolute_import, division, print_function, \ from os import environ from os.path import dirname, join from six import text_type -from unittest import TestCase from octodns.record import Record from octodns.manager import _AggregateTarget, MainThreadExecutor, Manager, \ @@ -16,6 +15,10 @@ from octodns.manager import _AggregateTarget, MainThreadExecutor, Manager, \ from octodns.yaml import safe_load from octodns.zone import Zone +from mock import MagicMock, patch +from unittest import TestCase + + from helpers import DynamicProvider, GeoProvider, NoSshFpProvider, \ SimpleProvider, TemporaryDirectory @@ -371,6 +374,24 @@ class TestManager(TestCase): with self.assertRaises(TypeError): manager._populate_and_plan('unit.tests.', [NoZone()], []) + @patch('octodns.manager.Manager._get_named_class') + def test_sync_passes_file_handle(self, mock): + plan_output_mock = MagicMock() + plan_output_class_mock = MagicMock() + plan_output_class_mock.return_value = plan_output_mock + mock.return_value = plan_output_class_mock + fh_mock = MagicMock() + + Manager(get_config_filename('plan-output-filehandle.yaml') + ).sync(plan_output_fh=fh_mock) + + # Since we only care about the fh kwarg, and different _PlanOutputs are + # are free to require arbitrary kwargs anyway, we concern ourselves + # with checking the value of fh only. + plan_output_mock.run.assert_called() + _, kwargs = plan_output_mock.run.call_args + self.assertEqual(fh_mock, kwargs.get('fh')) + class TestMainThreadExecutor(TestCase): From 55c194c2032fd65b1d61b0309d3d9c55f8e22489 Mon Sep 17 00:00:00 2001 From: Christian Funkhouser Date: Thu, 8 Apr 2021 09:40:04 -0400 Subject: [PATCH 177/358] Update tests/test_octodns_manager.py Co-authored-by: Ross McFarland --- tests/test_octodns_manager.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_octodns_manager.py b/tests/test_octodns_manager.py index 003d414..91ee374 100644 --- a/tests/test_octodns_manager.py +++ b/tests/test_octodns_manager.py @@ -18,7 +18,6 @@ from octodns.zone import Zone from mock import MagicMock, patch from unittest import TestCase - from helpers import DynamicProvider, GeoProvider, NoSshFpProvider, \ SimpleProvider, TemporaryDirectory From aa93e20f2e4f7fa597e61f11a5096a34fea32f20 Mon Sep 17 00:00:00 2001 From: Christian Funkhouser Date: Thu, 8 Apr 2021 11:03:30 -0400 Subject: [PATCH 178/358] Represent plan_output_fh less verbosely. Co-authored-by: Ross McFarland --- octodns/manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/manager.py b/octodns/manager.py index e94e41e..7326207 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -272,7 +272,7 @@ class Manager(object): self.log.info('sync: eligible_zones=%s, eligible_targets=%s, ' 'dry_run=%s, force=%s, plan_output_fh=%s', eligible_zones, eligible_targets, dry_run, force, - plan_output_fh) + getattr(plan_output_fh, 'name', plan_output_fh.__class__.__name__)) zones = self.config['zones'].items() if eligible_zones: From ada61f8d764cf4175f83376d989eaec3984e716a Mon Sep 17 00:00:00 2001 From: Christian Funkhouser Date: Thu, 8 Apr 2021 11:56:17 -0400 Subject: [PATCH 179/358] De-lint an aggressively-long log line --- octodns/manager.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/octodns/manager.py b/octodns/manager.py index 7326207..9b10196 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -269,10 +269,12 @@ class Manager(object): def sync(self, eligible_zones=[], eligible_sources=[], eligible_targets=[], dry_run=True, force=False, plan_output_fh=stdout): - self.log.info('sync: eligible_zones=%s, eligible_targets=%s, ' - 'dry_run=%s, force=%s, plan_output_fh=%s', - eligible_zones, eligible_targets, dry_run, force, - getattr(plan_output_fh, 'name', plan_output_fh.__class__.__name__)) + + self.log.info( + 'sync: eligible_zones=%s, eligible_targets=%s, dry_run=%s, ' + 'force=%s, plan_output_fh=%s', + eligible_zones, eligible_targets, dry_run, force, + getattr(plan_output_fh, 'name', plan_output_fh.__class__.__name__)) zones = self.config['zones'].items() if eligible_zones: From 988e8d27fb61ab86a7e8f565c53cf3ff9fae3de7 Mon Sep 17 00:00:00 2001 From: "Yaroshevich, Denis" Date: Fri, 9 Apr 2021 11:45:40 +0300 Subject: [PATCH 180/358] fix populate for multiple resource records in single rrset --- octodns/provider/gcore.py | 20 +++++++++++++++++--- tests/fixtures/gcore-no-changes.json | 5 ++++- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/octodns/provider/gcore.py b/octodns/provider/gcore.py index 60e8bad..5582dd8 100644 --- a/octodns/provider/gcore.py +++ b/octodns/provider/gcore.py @@ -36,9 +36,10 @@ class GCoreClient(object): ROOT_ZONES = "/zones" - def __init__(self, base_url, token): + def __init__(self, log, base_url, token): session = Session() session.headers.update({"Authorization": "Bearer {}".format(token)}) + self.log = log self._session = session self._base_url = base_url @@ -48,10 +49,19 @@ class GCoreClient(object): method, url, params=params, json=data, timeout=30.0 ) if r.status_code == 400: + self.log.error( + "bad request %r has been sent to %r: %s", data, url, r.text + ) raise GCoreClientBadRequest(r) elif r.status_code == 404: + self.log.error( + "resource %r not found: %s", url, r.text + ) raise GCoreClientNotFound(r) elif r.status_code == 500: + self.log.error( + "server error no %r to %r: %s", data, url, r.text + ) raise GCoreClientException(r) r.raise_for_status() return r @@ -112,13 +122,17 @@ class GCoreProvider(BaseProvider): self.log = logging.getLogger("GCoreProvider[{}]".format(id)) self.log.debug("__init__: id=%s, token=***", id) super(GCoreProvider, self).__init__(id, *args, **kwargs) - self._client = GCoreClient(base_url, token) + self._client = GCoreClient(self.log, base_url, token) def _data_for_single(self, _type, record): return { "ttl": record["ttl"], "type": _type, - "values": record["resource_records"][0]["content"], + "values": [ + rr_value + for resource_record in record["resource_records"] + for rr_value in resource_record["content"] + ], } _data_for_A = _data_for_single diff --git a/tests/fixtures/gcore-no-changes.json b/tests/fixtures/gcore-no-changes.json index ad62c94..a70fb82 100644 --- a/tests/fixtures/gcore-no-changes.json +++ b/tests/fixtures/gcore-no-changes.json @@ -5,7 +5,10 @@ "ttl": 300, "resource_records": [{ "content": [ - "1.2.3.4", + "1.2.3.4" + ] + }, { + "content": [ "1.2.3.5" ] }] From c71784b0e832850f1c535485994976d9d48b29da Mon Sep 17 00:00:00 2001 From: Ricard Bejarano Date: Mon, 12 Apr 2021 08:06:59 +0200 Subject: [PATCH 181/358] initial work on Hetzner provider - implemented HetznerClient API class - tested manually, lacking formal tests --- octodns/provider/hetzner.py | 91 +++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 octodns/provider/hetzner.py diff --git a/octodns/provider/hetzner.py b/octodns/provider/hetzner.py new file mode 100644 index 0000000..17dc046 --- /dev/null +++ b/octodns/provider/hetzner.py @@ -0,0 +1,91 @@ +# +# +# + +from requests import Session + + +class HetznerClientException(Exception): + pass + + +class HetznerClientNotFound(HetznerClientException): + + def __init__(self): + super(HetznerClientNotFound, self).__init__('Not Found') + + +class HetznerClient(object): + BASE_URL = 'https://dns.hetzner.com/api/v1' + + def __init__(self, token): + session = Session() + session.headers.update({'Auth-API-Token': token}) + self._session = session + + def _do(self, method, path, params=None, data=None): + url = self.BASE_URL + path + response = self._session.request(method, url, params=params, json=data) + if response.status_code == 404: + raise HetznerClientNotFound() + response.raise_for_status() + return response.json() + + def _do_with_pagination(self, method, path, key, params=None, data=None, + per_page=100): + pagination_params = {'page': 1, 'per_page': per_page} + if params is not None: + params = {**params, **pagination_params} + else: + params = pagination_params + + items = [] + while True: + response = self._do(method, path, params, data) + items += response[key] + if response['meta']['pagination']['page'] >= \ + response['meta']['pagination']['last_page']: + break + params['page'] += 1 + return items + + def zones_get(self, name=None, search_name=None): + params = {'name': name, 'search_name': search_name} + return self._do_with_pagination('GET', '/zones', 'zones', + params=params) + + def zone_get(self, zone_id): + return self._do('GET', '/zones/' + zone_id)['zone'] + + def zone_create(self, name, ttl=None): + data = {'name': name, 'ttl': ttl} + return self._do('POST', '/zones', data=data)['zone'] + + def zone_update(self, zone_id, name, ttl=None): + data = {'name': name, 'ttl': ttl} + return self._do('PUT', '/zones/' + zone_id, data=data)['zone'] + + def zone_delete(self, zone_id): + return self._do('DELETE', '/zones/' + zone_id) + + def zone_records_get(self, zone_id): + params = {'zone_id': zone_id} + # No need to handle pagination as it returns all records by default. + return self._do('GET', '/records', params=params)['records'] + + def zone_record_get(self, record_id): + return self._do('GET', '/records/' + record_id)['record'] + + def zone_record_create(self, zone_id, name, _type, value, ttl=None): + data = {'name': name, 'ttl': ttl, 'type': _type, 'value': value, + 'zone_id': zone_id} + return self._do('POST', '/records', data=data)['record'] + + def zone_record_update(self, zone_id, record_id, name, _type, value, + ttl=None): + data = {'name': name, 'ttl': ttl, 'type': _type, 'value': value, + 'zone_id': zone_id} + return self._do('PUT', '/records/' + record_id, data=data)['record'] + + def zone_record_delete(self, zone_id, record_id): + return self._do('DELETE', '/records/' + record_id) From c8e91c1e11b86106645ac67055b34eec4b6724f4 Mon Sep 17 00:00:00 2001 From: Ricard Bejarano Date: Mon, 12 Apr 2021 21:08:53 +0200 Subject: [PATCH 182/358] added HetznerClient docstring --- octodns/provider/hetzner.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/octodns/provider/hetzner.py b/octodns/provider/hetzner.py index 17dc046..1002e7d 100644 --- a/octodns/provider/hetzner.py +++ b/octodns/provider/hetzner.py @@ -16,6 +16,15 @@ class HetznerClientNotFound(HetznerClientException): class HetznerClient(object): + ''' + Hetzner DNS Public API v1 client class. + + Zone and Record resources are (almost) fully supported, even if unnecessary + to future-proof this client. Bulk Record create/update is not supported. + + No support for Primary Servers. + ''' + BASE_URL = 'https://dns.hetzner.com/api/v1' def __init__(self, token): From 8a9743b36ec56b81429b0bf9481c0f674a9834a9 Mon Sep 17 00:00:00 2001 From: Ricard Bejarano Date: Tue, 13 Apr 2021 08:08:48 +0200 Subject: [PATCH 183/358] added HetznerClient._replace_at to address the "@"/"" record name problem --- octodns/provider/hetzner.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/octodns/provider/hetzner.py b/octodns/provider/hetzner.py index 1002e7d..2e069d0 100644 --- a/octodns/provider/hetzner.py +++ b/octodns/provider/hetzner.py @@ -77,24 +77,34 @@ class HetznerClient(object): def zone_delete(self, zone_id): return self._do('DELETE', '/zones/' + zone_id) + def _replace_at(self, record): + record['name'] = '' if record['name'] == '@' else record['name'] + return record + def zone_records_get(self, zone_id): params = {'zone_id': zone_id} # No need to handle pagination as it returns all records by default. - return self._do('GET', '/records', params=params)['records'] + return [ + self._replace_at(record) + for record in self._do('GET', '/records', params=params)['records'] + ] def zone_record_get(self, record_id): - return self._do('GET', '/records/' + record_id)['record'] + record = self._do('GET', '/records/' + record_id)['record'] + return self._replace_at(record) def zone_record_create(self, zone_id, name, _type, value, ttl=None): - data = {'name': name, 'ttl': ttl, 'type': _type, 'value': value, + data = {'name': name or '@', 'ttl': ttl, 'type': _type, 'value': value, 'zone_id': zone_id} - return self._do('POST', '/records', data=data)['record'] + record = self._do('POST', '/records', data=data)['record'] + return self._replace_at(record) def zone_record_update(self, zone_id, record_id, name, _type, value, ttl=None): - data = {'name': name, 'ttl': ttl, 'type': _type, 'value': value, + data = {'name': name or '@', 'ttl': ttl, 'type': _type, 'value': value, 'zone_id': zone_id} - return self._do('PUT', '/records/' + record_id, data=data)['record'] + record = self._do('PUT', '/records/' + record_id, data=data)['record'] + return self._replace_at(record) def zone_record_delete(self, zone_id, record_id): return self._do('DELETE', '/records/' + record_id) From f507349ce506af5554de338a0eb56977b8632a60 Mon Sep 17 00:00:00 2001 From: Ricard Bejarano Date: Wed, 14 Apr 2021 07:57:23 +0200 Subject: [PATCH 184/358] implemented HetznerProvider --- octodns/provider/hetzner.py | 254 ++++++++++++++++++++++++++++++++++++ 1 file changed, 254 insertions(+) diff --git a/octodns/provider/hetzner.py b/octodns/provider/hetzner.py index 2e069d0..570cefe 100644 --- a/octodns/provider/hetzner.py +++ b/octodns/provider/hetzner.py @@ -2,8 +2,17 @@ # # +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +import logging +from collections import defaultdict + from requests import Session +from ..record import Record +from .base import BaseProvider + class HetznerClientException(Exception): pass @@ -108,3 +117,248 @@ class HetznerClient(object): def zone_record_delete(self, zone_id, record_id): return self._do('DELETE', '/records/' + record_id) + + +class HetznerProvider(BaseProvider): + ''' + Hetzner DNS provider using API v1 + + hetzner: + class: octodns.provider.hetzner.HetznerProvider + # Your Hetzner API token (required) + token: foo + ''' + SUPPORTS_GEO = False + SUPPORTS_DYNAMIC = False + SUPPORTS = set(('A', 'AAAA', 'CAA', 'CNAME', 'MX', 'NS', 'SRV', 'TXT')) + + def __init__(self, id, token, *args, **kwargs): + self.log = logging.getLogger('HetznerProvider[{}]'.format(id)) + self.log.debug('__init__: id=%s, token=***', id) + super(HetznerProvider, self).__init__(id, *args, **kwargs) + self._client = HetznerClient(token) + + self._zone_records = {} + + def _append_dot(self, value): + return value if value[-1] == '.' else '{}.'.format(value) + + def _data_for_multiple(self, _type, records): + values = [record['value'].replace(';', '\\;') for record in records] + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': values + } + + _data_for_A = _data_for_multiple + _data_for_AAAA = _data_for_multiple + + def _data_for_CAA(self, _type, records): + values = [] + for record in records: + value_without_spaces = record['value'].replace(' ', '') + flags = value_without_spaces[0] + tag = value_without_spaces[1:].split('"')[0] + value = record['value'].split('"')[1] + values.append({ + 'flags': int(flags), + 'tag': tag, + 'value': value, + }) + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': values + } + + def _data_for_CNAME(self, _type, records): + record = records[0] + return { + 'ttl': record['ttl'], + 'type': _type, + 'value': self._append_dot(record['value']) + } + + def _data_for_MX(self, _type, records): + values = [] + for record in records: + value_stripped_split = record['value'].strip().split(' ') + preference = value_stripped_split[0] + exchange = value_stripped_split[-1] + values.append({ + 'preference': int(preference), + 'exchange': self._append_dot(exchange) + }) + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': values + } + + def _data_for_NS(self, _type, records): + values = [] + for record in records: + values.append(self._append_dot(record['value'])) + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': values, + } + + def _data_for_SRV(self, _type, records): + values = [] + for record in records: + value_stripped = record['value'].strip() + priority = value_stripped.split(' ')[0] + weight = value_stripped[len(priority):].strip().split(' ')[0] + target = value_stripped.split(' ')[-1] + port = value_stripped[:-len(target)].strip().split(' ')[-1] + values.append({ + 'port': int(port), + 'priority': int(priority), + 'target': self._append_dot(target), + 'weight': int(weight) + }) + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': values + } + + _data_for_TXT = _data_for_multiple + + def zone_records(self, zone): + if zone.name not in self._zone_records: + try: + zone_id = self._client.zones_get(name=zone.name[:-1])[0]['id'] + self._zone_records[zone.name] = \ + self._client.zone_records_get(zone_id) + except HetznerClientNotFound: + return [] + + return self._zone_records[zone.name] + + def populate(self, zone, target=False, lenient=False): + self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, + target, lenient) + + values = defaultdict(lambda: defaultdict(list)) + for record in self.zone_records(zone): + _type = record['type'] + if _type not in self.SUPPORTS: + self.log.warning('populate: skipping unsupported %s record', + _type) + continue + values[record['name']][record['type']].append(record) + + before = len(zone.records) + for name, types in values.items(): + for _type, records in types.items(): + data_for = getattr(self, '_data_for_{}'.format(_type)) + record = Record.new(zone, name, data_for(_type, records), + source=self, lenient=lenient) + zone.add_record(record, lenient=lenient) + + exists = zone.name in self._zone_records + self.log.info('populate: found %s records, exists=%s', + len(zone.records) - before, exists) + return exists + + def _params_for_multiple(self, record): + for value in record.values: + yield { + 'value': value.replace('\\;', ';'), + 'name': record.name, + 'ttl': record.ttl, + 'type': record._type + } + + _params_for_A = _params_for_multiple + _params_for_AAAA = _params_for_multiple + + def _params_for_CAA(self, record): + for value in record.values: + data = '{} {} "{}"'.format(value.flags, value.tag, value.value) + yield { + 'value': data, + 'name': record.name, + 'ttl': record.ttl, + 'type': record._type + } + + def _params_for_single(self, record): + yield { + 'value': record.value, + 'name': record.name, + 'ttl': record.ttl, + 'type': record._type + } + + _params_for_CNAME = _params_for_single + + def _params_for_MX(self, record): + for value in record.values: + data = '{} {}'.format(value.preference, value.exchange) + yield { + 'value': data, + 'name': record.name, + 'ttl': record.ttl, + 'type': record._type + } + + _params_for_NS = _params_for_multiple + + def _params_for_SRV(self, record): + for value in record.values: + data = '{} {} {} {}'.format(value.priority, value.weight, + value.port, value.target) + yield { + 'value': data, + 'name': record.name, + 'ttl': record.ttl, + 'type': record._type + } + + _params_for_TXT = _params_for_multiple + + def _apply_Create(self, zone_id, change): + new = change.new + params_for = getattr(self, '_params_for_{}'.format(new._type)) + for params in params_for(new): + self._client.zone_record_create(zone_id, params['name'], + params['type'], params['value'], + params['ttl']) + + def _apply_Update(self, zone_id, change): + # It's way simpler to delete-then-recreate than to update + self._apply_Delete(zone_id, change) + self._apply_Create(zone_id, change) + + def _apply_Delete(self, zone_id, change): + existing = change.existing + zone = existing.zone + for record in self.zone_records(zone): + if existing.name == record['name'] and \ + existing._type == record['type']: + self._client.zone_record_delete(zone_id, record['id']) + + def _apply(self, plan): + desired = plan.desired + changes = plan.changes + self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name, + len(changes)) + + zone_name = desired.name[:-1] + try: + zone_id = self._client.zones_get(name=zone_name)[0]['id'] + except HetznerClientNotFound: + self.log.debug('_apply: no matching zone, creating domain') + zone_id = self._client.zone_create(zone_name)['id'] + + for change in changes: + class_name = change.__class__.__name__ + getattr(self, '_apply_{}'.format(class_name))(zone_id, change) + + # Clear out the cache if any + self._zone_records.pop(desired.name, None) From 8d1dd926ea6a8241928140e62258221f8636b782 Mon Sep 17 00:00:00 2001 From: Ricard Bejarano Date: Wed, 14 Apr 2021 07:59:00 +0200 Subject: [PATCH 185/358] added Hetzner provider to README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 0a26da9..42c91a0 100644 --- a/README.md +++ b/README.md @@ -197,6 +197,7 @@ The above command pulled the existing data out of Route53 and placed the results | [EnvVarSource](/octodns/source/envvar.py) | | TXT | No | read-only environment variable injection | | [GandiProvider](/octodns/provider/gandi.py) | | A, AAAA, ALIAS, CAA, CNAME, DNAME, MX, NS, PTR, SPF, SRV, SSHFP, TXT | No | | | [GoogleCloudProvider](/octodns/provider/googlecloud.py) | google-cloud-dns | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | No | | +| [HetznerProvider](/octodns/provider/hetzner.py) | | A, AAAA, CAA, CNAME, MX, NS, SRV, TXT | No | | | [MythicBeastsProvider](/octodns/provider/mythicbeasts.py) | Mythic Beasts | A, AAAA, ALIAS, CNAME, MX, NS, SRV, SSHFP, CAA, TXT | No | | | [Ns1Provider](/octodns/provider/ns1.py) | ns1-python | All | Yes | Missing `NA` geo target | | [OVH](/octodns/provider/ovh.py) | ovh | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, SSHFP, TXT, DKIM | No | | From 0ff933244a4bb2d4777a74fd3af39c00a6b19125 Mon Sep 17 00:00:00 2001 From: "Yaroshevich, Denis" Date: Tue, 20 Apr 2021 14:33:42 +0300 Subject: [PATCH 186/358] allow to pass login/password which will be used to acquire token for further usage --- octodns/provider/gcore.py | 106 ++++++++++++++++++++------- tests/test_octodns_provider_gcore.py | 72 +++++++++++++++--- 2 files changed, 138 insertions(+), 40 deletions(-) diff --git a/octodns/provider/gcore.py b/octodns/provider/gcore.py index 5582dd8..a52439f 100644 --- a/octodns/provider/gcore.py +++ b/octodns/provider/gcore.py @@ -11,7 +11,9 @@ from __future__ import ( from collections import defaultdict from requests import Session +import http import logging +import urllib.parse from ..record import Record from .base import BaseProvider @@ -34,51 +36,82 @@ class GCoreClientNotFound(GCoreClientException): class GCoreClient(object): - ROOT_ZONES = "/zones" - - def __init__(self, log, base_url, token): - session = Session() - session.headers.update({"Authorization": "Bearer {}".format(token)}) + ROOT_ZONES = "zones" + + def __init__( + self, + log, + api_url, + auth_url, + token=None, + login=None, + password=None, + ): self.log = log - self._session = session - self._base_url = base_url + self._session = Session() + self._api_url = api_url + if token is not None: + self._session.headers.update( + {"Authorization": "APIKey {}".format(token)} + ) + elif login is not None and password is not None: + token = self._auth(auth_url, login, password) + self._session.headers.update( + {"Authorization": "Bearer {}".format(token)} + ) + else: + raise ValueError("either token or login & password must be set") + + def _auth(self, url, login, password): + # well, can't use _request, since API returns 400 if credentials + # invalid which will be logged, but we don't want do this + r = self._session.request( + "POST", + self._build_url(url, "auth", "jwt", "login"), + json={"username": login, "password": password}, + ) + r.raise_for_status() + return r.json()["access"] - def _request(self, method, path, params={}, data=None): - url = "{}{}".format(self._base_url, path) + def _request(self, method, url, params=None, data=None): r = self._session.request( method, url, params=params, json=data, timeout=30.0 ) - if r.status_code == 400: + if r.status_code == http.HTTPStatus.BAD_REQUEST: self.log.error( "bad request %r has been sent to %r: %s", data, url, r.text ) raise GCoreClientBadRequest(r) - elif r.status_code == 404: - self.log.error( - "resource %r not found: %s", url, r.text - ) + elif r.status_code == http.HTTPStatus.NOT_FOUND: + self.log.error("resource %r not found: %s", url, r.text) raise GCoreClientNotFound(r) - elif r.status_code == 500: - self.log.error( - "server error no %r to %r: %s", data, url, r.text - ) + elif r.status_code == http.HTTPStatus.INTERNAL_SERVER_ERROR: + self.log.error("server error no %r to %r: %s", data, url, r.text) raise GCoreClientException(r) r.raise_for_status() return r def zone(self, zone_name): return self._request( - "GET", "{}/{}".format(self.ROOT_ZONES, zone_name) + "GET", self._build_url(self._api_url, self.ROOT_ZONES, zone_name) ).json() def zone_create(self, zone_name): return self._request( - "POST", self.ROOT_ZONES, data={"name": zone_name} + "POST", + self._build_url(self._api_url, self.ROOT_ZONES), + data={"name": zone_name}, ).json() def zone_records(self, zone_name): rrsets = self._request( - "GET", "{}/{}/rrsets?all=true".format(self.ROOT_ZONES, zone_name) + "GET", + "{}".format( + self._build_url( + self._api_url, self.ROOT_ZONES, zone_name, "rrsets" + ) + ), + params={"all": "true"}, ).json() records = rrsets["rrsets"] return records @@ -97,10 +130,17 @@ class GCoreClient(object): self._request("DELETE", self._rrset_url(zone_name, rrset_name, type_)) def _rrset_url(self, zone_name, rrset_name, type_): - return "{}/{}/{}/{}".format( - self.ROOT_ZONES, zone_name, rrset_name, type_ + return self._build_url( + self._api_url, self.ROOT_ZONES, zone_name, rrset_name, type_ ) + @staticmethod + def _build_url(base, *items): + for i in items: + base = base.strip("/") + "/" + base = urllib.parse.urljoin(base, i) + return base + class GCoreProvider(BaseProvider): """ @@ -108,8 +148,12 @@ class GCoreProvider(BaseProvider): gcore: class: octodns.provider.gcore.GCoreProvider - # Your API key (required) + # Your API key token: XXXXXXXXXXXX + # or login + password + login: XXXXXXXXXXXX + password: XXXXXXXXXXXX + # auth_url: https://api.gcdn.co # url: https://dnsapi.gcorelabs.com/v2 """ @@ -117,12 +161,18 @@ class GCoreProvider(BaseProvider): SUPPORTS_DYNAMIC = False SUPPORTS = set(("A", "AAAA")) - def __init__(self, id, token, *args, **kwargs): - base_url = kwargs.pop("url", "https://dnsapi.gcorelabs.com/v2") + def __init__(self, id, *args, **kwargs): + token = kwargs.pop("token", None) + login = kwargs.pop("login", None) + password = kwargs.pop("password", None) + api_url = kwargs.pop("url", "https://dnsapi.gcorelabs.com/v2") + auth_url = kwargs.pop("auth_url", "https://api.gcdn.co") self.log = logging.getLogger("GCoreProvider[{}]".format(id)) - self.log.debug("__init__: id=%s, token=***", id) + self.log.debug("__init__: id=%s", id) super(GCoreProvider, self).__init__(id, *args, **kwargs) - self._client = GCoreClient(self.log, base_url, token) + self._client = GCoreClient( + self.log, api_url, auth_url, token, login, password + ) def _data_for_single(self, _type, record): return { diff --git a/tests/test_octodns_provider_gcore.py b/tests/test_octodns_provider_gcore.py index 785e5e9..aa536f1 100644 --- a/tests/test_octodns_provider_gcore.py +++ b/tests/test_octodns_provider_gcore.py @@ -33,7 +33,7 @@ class TestGCoreProvider(TestCase): def test_populate(self): - provider = GCoreProvider("test_id", "token") + provider = GCoreProvider("test_id", token="token") # 400 - Bad Request. with requests_mock() as mock: @@ -52,7 +52,7 @@ class TestGCoreProvider(TestCase): with self.assertRaises(GCoreClientNotFound) as ctx: zone = Zone("unit.tests.", []) - provider._client.zone(zone) + provider._client.zone(zone.name) self.assertIn( '"error":"zone is not found"', text_type(ctx.exception) ) @@ -66,6 +66,48 @@ class TestGCoreProvider(TestCase): provider.populate(zone) self.assertEquals("Things caught fire", text_type(ctx.exception)) + # No credentials or token error + with requests_mock() as mock: + with self.assertRaises(ValueError) as ctx: + GCoreProvider("test_id") + self.assertEquals( + "either token or login & password must be set", + text_type(ctx.exception), + ) + + # Auth with login password + with requests_mock() as mock: + + def match_body(request): + return {"username": "foo", "password": "bar"} == request.json() + + auth_url = "http://api/auth/jwt/login" + mock.post( + auth_url, + additional_matcher=match_body, + status_code=200, + json={"access": "access"}, + ) + + providerPassword = GCoreProvider( + "test_id", + url="http://dns", + auth_url="http://api", + login="foo", + password="bar", + ) + assert mock.called + + # make sure token passed in header + zone_rrset_url = "http://dns/zones/unit.tests/rrsets?all=true" + mock.get( + zone_rrset_url, + request_headers={"Authorization": "Bearer access"}, + status_code=404, + ) + zone = Zone("unit.tests.", []) + assert not providerPassword.populate(zone) + # No diffs == no changes with requests_mock() as mock: base = "https://dnsapi.gcorelabs.com/v2/zones/unit.tests/rrsets" @@ -97,7 +139,7 @@ class TestGCoreProvider(TestCase): ) def test_apply(self): - provider = GCoreProvider("test_id", "token") + provider = GCoreProvider("test_id", url="http://api", token="token") # Zone does not exists but can be created. with requests_mock() as mock: @@ -153,12 +195,16 @@ class TestGCoreProvider(TestCase): provider._client._request.assert_has_calls( [ - call("GET", "/zones/unit.tests/rrsets?all=true"), - call("GET", "/zones/unit.tests"), - call("POST", "/zones", data={"name": "unit.tests"}), + call( + "GET", + "http://api/zones/unit.tests/rrsets", + params={"all": "true"}, + ), + call("GET", "http://api/zones/unit.tests"), + call("POST", "http://api/zones", data={"name": "unit.tests"}), call( "POST", - "/zones/unit.tests/www.sub.unit.tests./A", + "http://api/zones/unit.tests/www.sub.unit.tests./A", data={ "ttl": 300, "resource_records": [{"content": ["2.2.3.6"]}], @@ -166,7 +212,7 @@ class TestGCoreProvider(TestCase): ), call( "POST", - "/zones/unit.tests/www.unit.tests./A", + "http://api/zones/unit.tests/www.unit.tests./A", data={ "ttl": 300, "resource_records": [{"content": ["2.2.3.6"]}], @@ -174,7 +220,7 @@ class TestGCoreProvider(TestCase): ), call( "POST", - "/zones/unit.tests/aaaa.unit.tests./AAAA", + "http://api/zones/unit.tests/aaaa.unit.tests./AAAA", data={ "ttl": 600, "resource_records": [ @@ -188,7 +234,7 @@ class TestGCoreProvider(TestCase): ), call( "POST", - "/zones/unit.tests/unit.tests./A", + "http://api/zones/unit.tests/unit.tests./A", data={ "ttl": 300, "resource_records": [ @@ -239,10 +285,12 @@ class TestGCoreProvider(TestCase): provider._client._request.assert_has_calls( [ - call("DELETE", "/zones/unit.tests/www.unit.tests./A"), + call( + "DELETE", "http://api/zones/unit.tests/www.unit.tests./A" + ), call( "PUT", - "/zones/unit.tests/ttl.unit.tests./A", + "http://api/zones/unit.tests/ttl.unit.tests./A", data={ "ttl": 300, "resource_records": [{"content": ["3.2.3.4"]}], From 04d87fdf35ed4cc9e57c462f8721acf3f14d7910 Mon Sep 17 00:00:00 2001 From: "Yaroshevich, Denis" Date: Tue, 20 Apr 2021 16:37:09 +0300 Subject: [PATCH 187/358] allow to customize token_type, since there are two types of tokens: permanent (APIKey) and JWT (Bearer) --- octodns/provider/gcore.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/octodns/provider/gcore.py b/octodns/provider/gcore.py index a52439f..041e19f 100644 --- a/octodns/provider/gcore.py +++ b/octodns/provider/gcore.py @@ -44,15 +44,16 @@ class GCoreClient(object): api_url, auth_url, token=None, + token_type=None, login=None, password=None, ): self.log = log self._session = Session() self._api_url = api_url - if token is not None: + if token is not None and token_type is not None: self._session.headers.update( - {"Authorization": "APIKey {}".format(token)} + {"Authorization": "{} {}".format(token_type, token)} ) elif login is not None and password is not None: token = self._auth(auth_url, login, password) @@ -150,6 +151,7 @@ class GCoreProvider(BaseProvider): class: octodns.provider.gcore.GCoreProvider # Your API key token: XXXXXXXXXXXX + # token_type: APIKey # or login + password login: XXXXXXXXXXXX password: XXXXXXXXXXXX @@ -163,6 +165,7 @@ class GCoreProvider(BaseProvider): def __init__(self, id, *args, **kwargs): token = kwargs.pop("token", None) + token_type = kwargs.pop("token_type", "APIKey") login = kwargs.pop("login", None) password = kwargs.pop("password", None) api_url = kwargs.pop("url", "https://dnsapi.gcorelabs.com/v2") @@ -171,7 +174,13 @@ class GCoreProvider(BaseProvider): self.log.debug("__init__: id=%s", id) super(GCoreProvider, self).__init__(id, *args, **kwargs) self._client = GCoreClient( - self.log, api_url, auth_url, token, login, password + self.log, + api_url, + auth_url, + token=token, + token_type=token_type, + login=login, + password=password, ) def _data_for_single(self, _type, record): From 1a2cb50c6326b22a870694c51374bd2bb05556fa Mon Sep 17 00:00:00 2001 From: Ricard Bejarano Date: Wed, 21 Apr 2021 07:52:38 +0200 Subject: [PATCH 188/358] fixed potential KeyError when record ttl field is missing --- octodns/provider/hetzner.py | 38 ++++++++++++++++++++++++++----------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/octodns/provider/hetzner.py b/octodns/provider/hetzner.py index 570cefe..17b1141 100644 --- a/octodns/provider/hetzner.py +++ b/octodns/provider/hetzner.py @@ -5,10 +5,9 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -import logging from collections import defaultdict - from requests import Session +import logging from ..record import Record from .base import BaseProvider @@ -24,6 +23,12 @@ class HetznerClientNotFound(HetznerClientException): super(HetznerClientNotFound, self).__init__('Not Found') +class HetznerClientUnauthorized(HetznerClientException): + + def __init__(self): + super(HetznerClientUnauthorized, self).__init__('Unauthorized') + + class HetznerClient(object): ''' Hetzner DNS Public API v1 client class. @@ -44,6 +49,8 @@ class HetznerClient(object): def _do(self, method, path, params=None, data=None): url = self.BASE_URL + path response = self._session.request(method, url, params=params, json=data) + if response.status_code == 401: + raise HetznerClientUnauthorized() if response.status_code == 404: raise HetznerClientNotFound() response.raise_for_status() @@ -139,14 +146,22 @@ class HetznerProvider(BaseProvider): self._client = HetznerClient(token) self._zone_records = {} + self._zone_metadata = {} def _append_dot(self, value): - return value if value[-1] == '.' else '{}.'.format(value) + if value == '@' or value[-1] == '.': + return value + return '{}.'.format(value) + + def _record_ttl(self, record): + if 'ttl' in record: + return record['ttl'] + return self._zone_metadata[record['zone_id']]['ttl'] def _data_for_multiple(self, _type, records): values = [record['value'].replace(';', '\\;') for record in records] return { - 'ttl': records[0]['ttl'], + 'ttl': self._record_ttl(records[0]), 'type': _type, 'values': values } @@ -167,7 +182,7 @@ class HetznerProvider(BaseProvider): 'value': value, }) return { - 'ttl': records[0]['ttl'], + 'ttl': self._record_ttl(records[0]), 'type': _type, 'values': values } @@ -175,7 +190,7 @@ class HetznerProvider(BaseProvider): def _data_for_CNAME(self, _type, records): record = records[0] return { - 'ttl': record['ttl'], + 'ttl': self._record_ttl(record), 'type': _type, 'value': self._append_dot(record['value']) } @@ -191,7 +206,7 @@ class HetznerProvider(BaseProvider): 'exchange': self._append_dot(exchange) }) return { - 'ttl': records[0]['ttl'], + 'ttl': self._record_ttl(records[0]), 'type': _type, 'values': values } @@ -201,7 +216,7 @@ class HetznerProvider(BaseProvider): for record in records: values.append(self._append_dot(record['value'])) return { - 'ttl': records[0]['ttl'], + 'ttl': self._record_ttl(records[0]), 'type': _type, 'values': values, } @@ -221,7 +236,7 @@ class HetznerProvider(BaseProvider): 'weight': int(weight) }) return { - 'ttl': records[0]['ttl'], + 'ttl': self._record_ttl(records[0]), 'type': _type, 'values': values } @@ -231,9 +246,10 @@ class HetznerProvider(BaseProvider): def zone_records(self, zone): if zone.name not in self._zone_records: try: - zone_id = self._client.zones_get(name=zone.name[:-1])[0]['id'] + zone_metadata = self._client.zones_get(name=zone.name[:-1])[0] + self._zone_metadata[zone_metadata['id']] = zone_metadata self._zone_records[zone.name] = \ - self._client.zone_records_get(zone_id) + self._client.zone_records_get(zone_metadata['id']) except HetznerClientNotFound: return [] From ab436af92d42643eac3796ee58a8109be737ace9 Mon Sep 17 00:00:00 2001 From: Ricard Bejarano Date: Wed, 21 Apr 2021 07:55:18 +0200 Subject: [PATCH 189/358] added populate() tests --- tests/fixtures/hetzner-records.json | 223 +++++++++++++++++++++++++ tests/fixtures/hetzner-zones.json | 43 +++++ tests/test_octodns_provider_hetzner.py | 84 ++++++++++ 3 files changed, 350 insertions(+) create mode 100644 tests/fixtures/hetzner-records.json create mode 100644 tests/fixtures/hetzner-zones.json create mode 100644 tests/test_octodns_provider_hetzner.py diff --git a/tests/fixtures/hetzner-records.json b/tests/fixtures/hetzner-records.json new file mode 100644 index 0000000..bbafdcb --- /dev/null +++ b/tests/fixtures/hetzner-records.json @@ -0,0 +1,223 @@ +{ + "records": [ + { + "id": "SOA", + "type": "SOA", + "name": "@", + "value": "hydrogen.ns.hetzner.com. dns.hetzner.com. 1 86400 10800 3600000 3600", + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "NS:sub:0", + "type": "NS", + "name": "sub", + "value": "6.2.3.4", + "ttl": 3600, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "NS:sub:1", + "type": "NS", + "name": "sub", + "value": "7.2.3.4", + "ttl": 3600, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "SRV:_srv._tcp:0", + "type": "SRV", + "name": "_srv._tcp", + "value": "10 20 30 foo-1.unit.tests", + "ttl": 600, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "SRV:_srv._tcp:1", + "type": "SRV", + "name": "_srv._tcp", + "value": "12 20 30 foo-2.unit.tests", + "ttl": 600, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "TXT:txt:0", + "type": "TXT", + "name": "txt", + "value": "\"Bah bah black sheep\"", + "ttl": 600, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "TXT:txt:1", + "type": "TXT", + "name": "txt", + "value": "\"have you any wool.\"", + "ttl": 600, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "A:@:0", + "type": "A", + "name": "@", + "value": "1.2.3.4", + "ttl": 300, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "A:@:1", + "type": "A", + "name": "@", + "value": "1.2.3.5", + "ttl": 300, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "A:www:0", + "type": "A", + "name": "www", + "value": "2.2.3.6", + "ttl": 300, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "MX:mx:0", + "type": "MX", + "name": "mx", + "value": "10 smtp-4.unit.tests", + "ttl": 300, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "MX:mx:1", + "type": "MX", + "name": "mx", + "value": "20 smtp-2.unit.tests", + "ttl": 300, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "MX:mx:2", + "type": "MX", + "name": "mx", + "value": "30 smtp-3.unit.tests", + "ttl": 300, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "MX:mx:3", + "type": "MX", + "name": "mx", + "value": "40 smtp-1.unit.tests", + "ttl": 300, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "AAAA:aaaa:0", + "type": "AAAA", + "name": "aaaa", + "value": "2601:644:500:e210:62f8:1dff:feb8:947a", + "ttl": 600, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "CNAME:cname:0", + "type": "CNAME", + "name": "cname", + "value": "unit.tests", + "ttl": 300, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "A:www.sub:0", + "type": "A", + "name": "www.sub", + "value": "2.2.3.6", + "ttl": 300, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "TXT:txt:2", + "type": "TXT", + "name": "txt", + "value": "v=DKIM1;k=rsa;s=email;h=sha256;p=A/kinda+of/long/string+with+numb3rs", + "ttl": 600, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "CAA:@:0", + "type": "CAA", + "name": "@", + "value": "0 issue \"ca.unit.tests\"", + "ttl": 3600, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "CNAME:included:0", + "type": "CNAME", + "name": "included", + "value": "unit.tests", + "ttl": 3600, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "SRV:_imap._tcp:0", + "type": "SRV", + "name": "_imap._tcp", + "value": "0 0 0 .", + "ttl": 600, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + }, + { + "id": "SRV:_pop3._tcp:0", + "type": "SRV", + "name": "_pop3._tcp", + "value": "0 0 0 .", + "ttl": 600, + "zone_id": "unit.tests", + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "modified": "0000-00-00 00:00:00.000 +0000 UTC" + } + ] +} diff --git a/tests/fixtures/hetzner-zones.json b/tests/fixtures/hetzner-zones.json new file mode 100644 index 0000000..4d9b897 --- /dev/null +++ b/tests/fixtures/hetzner-zones.json @@ -0,0 +1,43 @@ +{ + "zones": [ + { + "id": "unit.tests", + "name": "unit.tests", + "ttl": 3600, + "registrar": "", + "legacy_dns_host": "", + "legacy_ns": [], + "ns": [], + "created": "0000-00-00 00:00:00.000 +0000 UTC", + "verified": "", + "modified": "0000-00-00 00:00:00.000 +0000 UTC", + "project": "", + "owner": "", + "permission": "", + "zone_type": { + "id": "", + "name": "", + "description": "", + "prices": null + }, + "status": "verified", + "paused": false, + "is_secondary_dns": false, + "txt_verification": { + "name": "", + "token": "" + }, + "records_count": null + } + ], + "meta": { + "pagination": { + "page": 1, + "per_page": 100, + "previous_page": 1, + "next_page": 1, + "last_page": 1, + "total_entries": 1 + } + } +} diff --git a/tests/test_octodns_provider_hetzner.py b/tests/test_octodns_provider_hetzner.py new file mode 100644 index 0000000..bf619fb --- /dev/null +++ b/tests/test_octodns_provider_hetzner.py @@ -0,0 +1,84 @@ +# +# +# + + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from os.path import dirname, join +from requests import HTTPError +from requests_mock import ANY, mock as requests_mock +from six import text_type +from unittest import TestCase + +from octodns.provider.hetzner import HetznerProvider +from octodns.provider.yaml import YamlProvider +from octodns.zone import Zone + + +class TestdHetznerProvider(TestCase): + expected = Zone('unit.tests.', []) + source = YamlProvider('test', join(dirname(__file__), 'config')) + source.populate(expected) + + def test_populate(self): + provider = HetznerProvider('test', 'token') + + # Bad auth + with requests_mock() as mock: + mock.get(ANY, status_code=401, + text='{"message":"Invalid authentication credentials"}') + + with self.assertRaises(Exception) as ctx: + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals('Unauthorized', text_type(ctx.exception)) + + # General error + with requests_mock() as mock: + mock.get(ANY, status_code=502, text='Things caught fire') + + with self.assertRaises(HTTPError) as ctx: + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals(502, ctx.exception.response.status_code) + + # Non-existent zone doesn't populate anything + with requests_mock() as mock: + mock.get(ANY, status_code=404, + text='{"zone":{"id":"","name":"","ttl":0,"registrar":"",' + '"legacy_dns_host":"","legacy_ns":null,"ns":null,' + '"created":"","verified":"","modified":"","project":"",' + '"owner":"","permission":"","zone_type":{"id":"",' + '"name":"","description":"","prices":null},"status":"",' + '"paused":false,"is_secondary_dns":false,' + '"txt_verification":{"name":"","token":""},' + '"records_count":0},"error":{' + '"message":"zone not found","code":404}}') + + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals(set(), zone.records) + + # No diffs == no changes + with requests_mock() as mock: + base = 'https://dns.hetzner.com/api/v1' + with open('tests/fixtures/hetzner-zones.json') as fh: + mock.get('{}/zones'.format(base), text=fh.read()) + with open('tests/fixtures/hetzner-records.json') as fh: + mock.get('{}/records'.format(base), text=fh.read()) + + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals(13, len(zone.records)) + changes = self.expected.changes(zone, provider) + self.assertEquals(0, len(changes)) + + # 2nd populate makes no network calls/all from cache + again = Zone('unit.tests.', []) + provider.populate(again) + self.assertEquals(13, len(again.records)) + + # bust the cache + del provider._zone_records[zone.name] From 3451c79c6ac012a3dda29fdca36234a5bfe0966f Mon Sep 17 00:00:00 2001 From: Cadu Ribeiro Date: Wed, 21 Apr 2021 12:41:11 -0300 Subject: [PATCH 190/358] Change DNSimple's Provider to query zones instead domains A quick summary of a problem with have with DNSimple provider: Let's suppose that I have the following config: zones: 30.114.195.in-addr.arpa.: sources: - config targets: - dnsimple Even if a customer has this Reverse zone configured in DNSimple, this fails with: 400 Bad Request for url: https://api.sandbox.dnsimple.com/v2/x/domains because it is trying to create a domain because the zone wasn't found. octodns.provider.dnsimple.DnsimpleClientNotFound: Not found This happens because the GET /domains endpoint at DNSimple does not bring Reverse Zones. To make this work nice, we should use /zones/ instead making it return properly the reverse zones. --- octodns/provider/dnsimple.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/octodns/provider/dnsimple.py b/octodns/provider/dnsimple.py index 599eacb..a4b9350 100644 --- a/octodns/provider/dnsimple.py +++ b/octodns/provider/dnsimple.py @@ -51,8 +51,8 @@ class DnsimpleClient(object): resp.raise_for_status() return resp - def domain(self, name): - path = '/domains/{}'.format(name) + def zone(self, name): + path = '/zones/{}'.format(name) return self._request('GET', path).json() def domain_create(self, name): @@ -442,7 +442,7 @@ class DnsimpleProvider(BaseProvider): domain_name = desired.name[:-1] try: - self._client.domain(domain_name) + self._client.zone(domain_name) except DnsimpleClientNotFound: self.log.debug('_apply: no matching zone, creating domain') self._client.domain_create(domain_name) From e90aeb5d34fe0b1d423daffb2cdd2b831fbf1775 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Thu, 22 Apr 2021 18:45:14 -0700 Subject: [PATCH 191/358] pools used as fallbacks should count as seen --- octodns/record/__init__.py | 12 ++++++++---- tests/config/dynamic.tests.yaml | 23 +++++++++++++++++++++++ tests/test_octodns_provider_yaml.py | 14 +++++++++----- 3 files changed, 40 insertions(+), 9 deletions(-) diff --git a/octodns/record/__init__.py b/octodns/record/__init__.py index 8ee2eaa..8c8760a 100644 --- a/octodns/record/__init__.py +++ b/octodns/record/__init__.py @@ -531,6 +531,7 @@ class _DynamicMixin(object): pools_exist = set() pools_seen = set() + pools_seen_as_fallback = set() if not isinstance(pools, dict): reasons.append('pools must be a dict') elif not pools: @@ -573,9 +574,12 @@ class _DynamicMixin(object): 'value {}'.format(_id, value_num)) fallback = pool.get('fallback', None) - if fallback is not None and fallback not in pools: - reasons.append('undefined fallback "{}" for pool "{}"' - .format(fallback, _id)) + if fallback is not None: + if fallback in pools: + pools_seen_as_fallback.add(fallback) + else: + reasons.append('undefined fallback "{}" for pool "{}"' + .format(fallback, _id)) # Check for loops fallback = pools[_id].get('fallback', None) @@ -644,7 +648,7 @@ class _DynamicMixin(object): reasons.extend(GeoCodes.validate(geo, 'rule {} ' .format(rule_num))) - unused = pools_exist - pools_seen + unused = pools_exist - pools_seen - pools_seen_as_fallback if unused: unused = '", "'.join(sorted(unused)) reasons.append('unused pools: "{}"'.format(unused)) diff --git a/tests/config/dynamic.tests.yaml b/tests/config/dynamic.tests.yaml index 4bd97a7..d25f63a 100644 --- a/tests/config/dynamic.tests.yaml +++ b/tests/config/dynamic.tests.yaml @@ -109,6 +109,29 @@ cname: - pool: iad type: CNAME value: target.unit.tests. +pool-only-in-fallback: + dynamic: + pools: + one: + fallback: two + values: + - value: 1.1.1.1 + three: + values: + - value: 3.3.3.3 + two: + values: + - value: 2.2.2.2 + rules: + - geos: + - NA-US + pool: one + - geos: + - AS-SG + pool: three + ttl: 300 + type: A + values: [4.4.4.4] real-ish-a: dynamic: pools: diff --git a/tests/test_octodns_provider_yaml.py b/tests/test_octodns_provider_yaml.py index 2ce9b4a..872fcca 100644 --- a/tests/test_octodns_provider_yaml.py +++ b/tests/test_octodns_provider_yaml.py @@ -38,7 +38,7 @@ class TestYamlProvider(TestCase): self.assertEquals(22, len(zone.records)) source.populate(dynamic_zone) - self.assertEquals(5, len(dynamic_zone.records)) + self.assertEquals(6, len(dynamic_zone.records)) # Assumption here is that a clean round-trip means that everything # worked as expected, data that went in came back out and could be @@ -68,11 +68,11 @@ class TestYamlProvider(TestCase): # Dynamic plan plan = target.plan(dynamic_zone) - self.assertEquals(5, len([c for c in plan.changes + self.assertEquals(6, len([c for c in plan.changes if isinstance(c, Create)])) self.assertFalse(isfile(dynamic_yaml_file)) # Apply it - self.assertEquals(5, target.apply(plan)) + self.assertEquals(6, target.apply(plan)) self.assertTrue(isfile(dynamic_yaml_file)) # There should be no changes after the round trip @@ -148,6 +148,10 @@ class TestYamlProvider(TestCase): self.assertTrue('value' in dyna) # self.assertTrue('dynamic' in dyna) + dyna = data.pop('pool-only-in-fallback') + self.assertTrue('value' in dyna) + # self.assertTrue('dynamic' in dyna) + # make sure nothing is left self.assertEquals([], list(data.keys())) @@ -397,7 +401,7 @@ class TestOverridingYamlProvider(TestCase): # Load the base, should see the 5 records base.populate(zone) got = {r.name: r for r in zone.records} - self.assertEquals(5, len(got)) + self.assertEquals(6, len(got)) # We get the "dynamic" A from the base config self.assertTrue('dynamic' in got['a'].data) # No added @@ -406,7 +410,7 @@ class TestOverridingYamlProvider(TestCase): # Load the overrides, should replace one and add 1 override.populate(zone) got = {r.name: r for r in zone.records} - self.assertEquals(6, len(got)) + self.assertEquals(7, len(got)) # 'a' was replaced with a generic record self.assertEquals({ 'ttl': 3600, From b55a647e6e915ea5395711152ce3f2745a376062 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 26 Apr 2021 06:36:05 -0700 Subject: [PATCH 192/358] Remove redudant pools_seen.add(pool) call --- octodns/record/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/octodns/record/__init__.py b/octodns/record/__init__.py index 8c8760a..135baf8 100644 --- a/octodns/record/__init__.py +++ b/octodns/record/__init__.py @@ -628,7 +628,6 @@ class _DynamicMixin(object): if pool not in pools: reasons.append('rule {} undefined pool "{}"' .format(rule_num, pool)) - pools_seen.add(pool) elif pool in pools_seen and geos: reasons.append('rule {} invalid, target pool "{}" ' 'reused'.format(rule_num, pool)) From 078576520d67e7b6360f5f1ef1166ee6b3f0f2d2 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 26 Apr 2021 08:34:02 -0700 Subject: [PATCH 193/358] Rework NS1 pool handling to support fallback-only pools --- octodns/provider/ns1.py | 63 +++++++++++++++++++----------- tests/test_octodns_provider_ns1.py | 15 +++++-- 2 files changed, 51 insertions(+), 27 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 6cea185..7724fcb 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -406,7 +406,7 @@ class Ns1Provider(BaseProvider): for piece in note.split(' '): try: k, v = piece.split(':', 1) - data[k] = v + data[k] = v if v != '' else None except ValueError: pass return data @@ -479,31 +479,45 @@ class Ns1Provider(BaseProvider): # region. pools = defaultdict(lambda: {'fallback': None, 'values': []}) for answer in record['answers']: - # region (group name in the UI) is the pool name - pool_name = answer['region'] - # Get the actual pool name by removing the type - pool_name = self._parse_dynamic_pool_name(pool_name) - pool = pools[pool_name] - meta = answer['meta'] + notes = self._parse_notes(meta.get('note', '')) + value = text_type(answer['answer'][0]) - if meta['priority'] == 1: - # priority 1 means this answer is part of the pools own values - value_dict = { - 'value': value, - 'weight': int(meta.get('weight', 1)), - } - # If we have the original pool name and the catchall pool name - # in the answers, they point at the same pool. Add values only - # once - if value_dict not in pool['values']: - pool['values'].append(value_dict) + if notes.get('from', False) == '--default--': + # It's a final/default value, record it and move on + default.add(value) + continue + + # NS1 pool names can be found in notes > v0.9.11, in order to allow + # us to find fallback-only pools/values. Before that we used + # `region` (group name in the UI) and only paid attention to + # priority=1 (first level) + notes_pool_name = notes.get('pool', None) + if notes_pool_name is None: + # < v0.9.11 + if meta['priority'] != 1: + # Ignore all but priority 1 + continue + # And use region's pool name as the pool name + pool_name = self._parse_dynamic_pool_name(answer['region']) else: - # It's a fallback, we only care about it if it's a - # final/default - notes = self._parse_notes(meta.get('note', '')) - if notes.get('from', False) == '--default--': - default.add(value) + # > v0.9.11, use the notes-based name and consider all values + pool_name = notes_pool_name + + pool = pools[pool_name] + value_dict = { + 'value': value, + 'weight': int(meta.get('weight', 1)), + } + if value_dict not in pool['values']: + # If we haven't seen this value before add it to the pool + pool['values'].append(value_dict) + + # If there's a fallback recorded in the value for its pool go ahead + # and use it, another v0.9.11 thing + fallback = notes.get('fallback', None) + if fallback is not None: + pool['fallback'] = fallback # The regions objects map to rules, but it's a bit fuzzy since they're # tied to pools on the NS1 side, e.g. we can only have 1 rule per pool, @@ -978,12 +992,15 @@ class Ns1Provider(BaseProvider): seen.add(current_pool_name) pool = pools[current_pool_name] for answer in pool_answers[current_pool_name]: + fallback = pool.data['fallback'] answer = { 'answer': answer['answer'], 'meta': { 'priority': priority, 'note': self._encode_notes({ 'from': pool_label, + 'pool': current_pool_name, + 'fallback': fallback or '', }), 'up': { 'feed': answer['feed_id'], diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index 00b068b..b7270fb 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -1117,14 +1117,21 @@ class TestNs1ProviderDynamic(TestCase): # finally has a catchall. Those are examples of the two ways pools get # expanded. # - # lhr splits in two, with a region and country. + # lhr splits in two, with a region and country and includes a fallback + # + # All values now include their own `pool:` name # # well as both lhr georegion (for contients) and country. The first is # an example of a repeated target pool in a rule (only allowed when the # 2nd is a catchall.) - self.assertEquals(['from:--default--', 'from:iad__catchall', - 'from:iad__country', 'from:iad__georegion', - 'from:lhr__country', 'from:lhr__georegion'], + self.assertEquals(['fallback: from:iad__catchall pool:iad', + 'fallback: from:iad__country pool:iad', + 'fallback: from:iad__georegion pool:iad', + 'fallback: from:lhr__country pool:iad', + 'fallback: from:lhr__georegion pool:iad', + 'fallback:iad from:lhr__country pool:lhr', + 'fallback:iad from:lhr__georegion pool:lhr', + 'from:--default--'], sorted(notes.keys())) # All the iad's should match (after meta and region were removed) From fe74c462138a951577ce4693e8eafa5e32764557 Mon Sep 17 00:00:00 2001 From: Ricard Bejarano Date: Mon, 26 Apr 2021 19:02:44 +0200 Subject: [PATCH 194/358] made hetzner.HetznerProvider._do Mock-able for testing purposes making it a wrapper for _do_raw --- octodns/provider/hetzner.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/octodns/provider/hetzner.py b/octodns/provider/hetzner.py index 17b1141..78be756 100644 --- a/octodns/provider/hetzner.py +++ b/octodns/provider/hetzner.py @@ -46,7 +46,7 @@ class HetznerClient(object): session.headers.update({'Auth-API-Token': token}) self._session = session - def _do(self, method, path, params=None, data=None): + def _do_raw(self, method, path, params=None, data=None): url = self.BASE_URL + path response = self._session.request(method, url, params=params, json=data) if response.status_code == 401: @@ -54,7 +54,10 @@ class HetznerClient(object): if response.status_code == 404: raise HetznerClientNotFound() response.raise_for_status() - return response.json() + return response + + def _do(self, method, path, params=None, data=None): + return self._do_raw(method, path, params, data).json() def _do_with_pagination(self, method, path, key, params=None, data=None, per_page=100): From 612738b327b26b417a683ec5dd0c3c55291b6bd4 Mon Sep 17 00:00:00 2001 From: Ricard Bejarano Date: Mon, 26 Apr 2021 19:03:25 +0200 Subject: [PATCH 195/358] renamed TestdHetznerProvider -> TestHetznerProvider (missing "d") --- tests/test_octodns_provider_hetzner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_octodns_provider_hetzner.py b/tests/test_octodns_provider_hetzner.py index bf619fb..a8a67fb 100644 --- a/tests/test_octodns_provider_hetzner.py +++ b/tests/test_octodns_provider_hetzner.py @@ -17,7 +17,7 @@ from octodns.provider.yaml import YamlProvider from octodns.zone import Zone -class TestdHetznerProvider(TestCase): +class TestHetznerProvider(TestCase): expected = Zone('unit.tests.', []) source = YamlProvider('test', join(dirname(__file__), 'config')) source.populate(expected) From 3f3d500152baa55e9d88c27ced0e83101699472c Mon Sep 17 00:00:00 2001 From: Ricard Bejarano Date: Mon, 26 Apr 2021 19:06:29 +0200 Subject: [PATCH 196/358] fixed missing whitespace in plan() debug logging When debug logging is enabled, the plan() function logs lines like: Plan: DEBUG: __init__: Creates=13, Updates=0, Deletes=0Existing=0 Where a space between "Deletes=0" and "Existing=0" is missing. This adds it. --- octodns/provider/plan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/plan.py b/octodns/provider/plan.py index af6863a..69bd2b2 100644 --- a/octodns/provider/plan.py +++ b/octodns/provider/plan.py @@ -50,7 +50,7 @@ class Plan(object): except AttributeError: existing_n = 0 - self.log.debug('__init__: Creates=%d, Updates=%d, Deletes=%d' + self.log.debug('__init__: Creates=%d, Updates=%d, Deletes=%d ' 'Existing=%d', self.change_counts['Create'], self.change_counts['Update'], From fbd838990364dec250d3aa58e3ed1b441f853ef4 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 26 Apr 2021 17:10:22 -0700 Subject: [PATCH 197/358] Tests for new-style ns1 data_for_dynamic_A fallback only pools --- octodns/provider/ns1.py | 2 +- tests/test_octodns_provider_ns1.py | 109 +++++++++++++++++++++++++++++ 2 files changed, 110 insertions(+), 1 deletion(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 7724fcb..52fb3d2 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -542,7 +542,7 @@ class Ns1Provider(BaseProvider): rules[rule_order] = rule # The group notes field in the UI is a `note` on the region here, - # that's where we can find our pool's fallback. + # that's where we can find our pool's fallback in < v0.9.11 anyway if 'fallback' in notes: # set the fallback pool name pools[pool_name]['fallback'] = notes['fallback'] diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index b7270fb..db713eb 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -1478,6 +1478,115 @@ class TestNs1ProviderDynamic(TestCase): self.assertTrue( 'OC-{}'.format(c) in data4['dynamic']['rules'][0]['geos']) + # Test out fallback only pools and new-style notes + ns1_record = { + 'answers': [{ + 'answer': ['1.1.1.1'], + 'meta': { + 'priority': 1, + 'note': 'from:one__country pool:one fallback:two', + }, + 'region': 'one_country', + }, { + 'answer': ['2.2.2.2'], + 'meta': { + 'priority': 2, + 'note': 'from:one__country pool:two fallback:three', + }, + 'region': 'one_country', + }, { + 'answer': ['3.3.3.3'], + 'meta': { + 'priority': 3, + 'note': 'from:one__country pool:three fallback:', + }, + 'region': 'one_country', + }, { + 'answer': ['5.5.5.5'], + 'meta': { + 'priority': 4, + 'note': 'from:--default--', + }, + 'region': 'one_country', + }, { + 'answer': ['4.4.4.4'], + 'meta': { + 'priority': 1, + 'note': 'from:four__country pool:four fallback:', + }, + 'region': 'four_country', + }, { + 'answer': ['5.5.5.5'], + 'meta': { + 'priority': 2, + 'note': 'from:--default--', + }, + 'region': 'four_country', + }], + 'domain': 'unit.tests', + 'filters': filters, + 'regions': { + 'one__country': { + 'meta': { + 'note': 'rule-order:1 fallback:two', + 'country': ['CA'], + 'us_state': ['OR'], + }, + }, + 'four__country': { + 'meta': { + 'note': 'rule-order:2', + 'country': ['CA'], + 'us_state': ['OR'], + }, + }, + catchall_pool_name: { + 'meta': { + 'note': 'rule-order:3', + }, + } + }, + 'tier': 3, + 'ttl': 42, + } + data = provider._data_for_dynamic_A('A', ns1_record) + self.assertEquals({ + 'dynamic': { + 'pools': { + 'four': { + 'fallback': None, + 'values': [{'value': '4.4.4.4', 'weight': 1}] + }, + 'one': { + 'fallback': 'two', + 'values': [{'value': '1.1.1.1', 'weight': 1}] + }, + 'three': { + 'fallback': None, + 'values': [{'value': '3.3.3.3', 'weight': 1}] + }, + 'two': { + 'fallback': 'three', + 'values': [{'value': '2.2.2.2', 'weight': 1}] + }, + }, + 'rules': [{ + '_order': '1', + 'geos': ['NA-CA', 'NA-US-OR'], + 'pool': 'one' + }, { + '_order': '2', + 'geos': ['NA-CA', 'NA-US-OR'], + 'pool': 'four' + }, { + '_order': '3', 'pool': 'iad'} + ] + }, + 'ttl': 42, + 'type': 'A', + 'values': ['5.5.5.5'] + }, data) + @patch('ns1.rest.records.Records.retrieve') @patch('ns1.rest.zones.Zones.retrieve') @patch('octodns.provider.ns1.Ns1Provider._monitors_for') From 7832fb880910e50c7cb65d80b3a7659d9dac941e Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 26 Apr 2021 20:08:42 -0700 Subject: [PATCH 198/358] Better name for _create_zone --- octodns/processors/__init__.py | 2 +- octodns/processors/filters.py | 4 ++-- octodns/processors/ownership.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/octodns/processors/__init__.py b/octodns/processors/__init__.py index 6b999e2..327b1c2 100644 --- a/octodns/processors/__init__.py +++ b/octodns/processors/__init__.py @@ -13,7 +13,7 @@ class BaseProcessor(object): def __init__(self, name): self.name = name - def _create_zone(self, zone): + def _clone_zone(self, zone): return Zone(zone.name, sub_zones=zone.sub_zones) def process_source_zone(self, zone, sources): diff --git a/octodns/processors/filters.py b/octodns/processors/filters.py index 413964f..2c5f87f 100644 --- a/octodns/processors/filters.py +++ b/octodns/processors/filters.py @@ -15,7 +15,7 @@ class TypeAllowlistFilter(BaseProcessor): self.allowlist = allowlist def _process(self, zone, *args, **kwargs): - ret = self._create_zone(zone) + ret = self._clone_zone(zone) for record in zone.records: if record._type in self.allowlist: ret.add_record(record) @@ -33,7 +33,7 @@ class TypeRejectlistFilter(BaseProcessor): self.rejectlist = rejectlist def _process(self, zone, *args, **kwargs): - ret = self._create_zone(zone) + ret = self._clone_zone(zone) for record in zone.records: if record._type not in self.rejectlist: ret.add_record(record) diff --git a/octodns/processors/ownership.py b/octodns/processors/ownership.py index 374e380..67bbb9a 100644 --- a/octodns/processors/ownership.py +++ b/octodns/processors/ownership.py @@ -26,7 +26,7 @@ class OwnershipProcessor(BaseProcessor): self._txt_values = [txt_value] def process_source_zone(self, zone, *args, **kwargs): - ret = self._create_zone(zone) + ret = self._clone_zone(zone) for record in zone.records: # Always copy over the source records ret.add_record(record) From 0de9efd03268ae0d447997fe1960dc314880348d Mon Sep 17 00:00:00 2001 From: Ricard Bejarano Date: Tue, 27 Apr 2021 07:31:05 +0200 Subject: [PATCH 199/358] removed unused HetznerClient methods to fix imparial coverage --- octodns/provider/hetzner.py | 70 +++++++++---------------------------- 1 file changed, 17 insertions(+), 53 deletions(-) diff --git a/octodns/provider/hetzner.py b/octodns/provider/hetzner.py index 78be756..0be01ea 100644 --- a/octodns/provider/hetzner.py +++ b/octodns/provider/hetzner.py @@ -32,11 +32,6 @@ class HetznerClientUnauthorized(HetznerClientException): class HetznerClient(object): ''' Hetzner DNS Public API v1 client class. - - Zone and Record resources are (almost) fully supported, even if unnecessary - to future-proof this client. Bulk Record create/update is not supported. - - No support for Primary Servers. ''' BASE_URL = 'https://dns.hetzner.com/api/v1' @@ -46,8 +41,8 @@ class HetznerClient(object): session.headers.update({'Auth-API-Token': token}) self._session = session - def _do_raw(self, method, path, params=None, data=None): - url = self.BASE_URL + path + def _do(self, method, path, params=None, data=None): + url = '{}{}'.format(self.BASE_URL, path) response = self._session.request(method, url, params=params, json=data) if response.status_code == 401: raise HetznerClientUnauthorized() @@ -56,20 +51,15 @@ class HetznerClient(object): response.raise_for_status() return response - def _do(self, method, path, params=None, data=None): - return self._do_raw(method, path, params, data).json() - - def _do_with_pagination(self, method, path, key, params=None, data=None, - per_page=100): - pagination_params = {'page': 1, 'per_page': per_page} - if params is not None: - params = {**params, **pagination_params} - else: - params = pagination_params + def _do_json(self, method, path, params=None, data=None): + return self._do(method, path, params, data).json() + def _do_json_paginate(self, method, path, key, params=None, data=None, + per_page=100): items = [] + params = {**{'page': 1, 'per_page': per_page}, **params} while True: - response = self._do(method, path, params, data) + response = self._do_json(method, path, params, data) items += response[key] if response['meta']['pagination']['page'] >= \ response['meta']['pagination']['last_page']: @@ -79,54 +69,28 @@ class HetznerClient(object): def zones_get(self, name=None, search_name=None): params = {'name': name, 'search_name': search_name} - return self._do_with_pagination('GET', '/zones', 'zones', - params=params) - - def zone_get(self, zone_id): - return self._do('GET', '/zones/' + zone_id)['zone'] + return self._do_json_paginate('GET', '/zones', 'zones', params=params) def zone_create(self, name, ttl=None): data = {'name': name, 'ttl': ttl} - return self._do('POST', '/zones', data=data)['zone'] - - def zone_update(self, zone_id, name, ttl=None): - data = {'name': name, 'ttl': ttl} - return self._do('PUT', '/zones/' + zone_id, data=data)['zone'] - - def zone_delete(self, zone_id): - return self._do('DELETE', '/zones/' + zone_id) - - def _replace_at(self, record): - record['name'] = '' if record['name'] == '@' else record['name'] - return record + return self._do_json('POST', '/zones', data=data)['zone'] def zone_records_get(self, zone_id): params = {'zone_id': zone_id} # No need to handle pagination as it returns all records by default. - return [ - self._replace_at(record) - for record in self._do('GET', '/records', params=params)['records'] - ] - - def zone_record_get(self, record_id): - record = self._do('GET', '/records/' + record_id)['record'] - return self._replace_at(record) + records = self._do_json('GET', '/records', params=params)['records'] + for record in records: + if record['name'] == '@': + record['name'] = '' + return records def zone_record_create(self, zone_id, name, _type, value, ttl=None): data = {'name': name or '@', 'ttl': ttl, 'type': _type, 'value': value, 'zone_id': zone_id} - record = self._do('POST', '/records', data=data)['record'] - return self._replace_at(record) - - def zone_record_update(self, zone_id, record_id, name, _type, value, - ttl=None): - data = {'name': name or '@', 'ttl': ttl, 'type': _type, 'value': value, - 'zone_id': zone_id} - record = self._do('PUT', '/records/' + record_id, data=data)['record'] - return self._replace_at(record) + self._do('POST', '/records', data=data) def zone_record_delete(self, zone_id, record_id): - return self._do('DELETE', '/records/' + record_id) + self._do('DELETE', '/records/{}'.format(record_id)) class HetznerProvider(BaseProvider): From 192231109181dac8f61c1b41be156ba4c53cbac9 Mon Sep 17 00:00:00 2001 From: Ricard Bejarano Date: Tue, 27 Apr 2021 07:53:17 +0200 Subject: [PATCH 200/358] WIP added TestHetznerProvider.test_apply --- tests/test_octodns_provider_hetzner.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/tests/test_octodns_provider_hetzner.py b/tests/test_octodns_provider_hetzner.py index a8a67fb..0c18731 100644 --- a/tests/test_octodns_provider_hetzner.py +++ b/tests/test_octodns_provider_hetzner.py @@ -6,13 +6,15 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals +from mock import Mock from os.path import dirname, join from requests import HTTPError from requests_mock import ANY, mock as requests_mock from six import text_type from unittest import TestCase -from octodns.provider.hetzner import HetznerProvider +from octodns.provider.hetzner import HetznerClientNotFound, \ + HetznerProvider from octodns.provider.yaml import YamlProvider from octodns.zone import Zone @@ -82,3 +84,24 @@ class TestHetznerProvider(TestCase): # bust the cache del provider._zone_records[zone.name] + + def test_apply(self): + provider = HetznerProvider('test', 'token') + + resp = Mock() + resp.json = Mock() + provider._client._do = Mock(return_value=resp) + + # non-existent domain, create everything + resp.json.side_effect = [ + HetznerClientNotFound, # no zone in populate + HetznerClientNotFound, # no zone during apply + {"zone": {"id": "string"}} + ] + plan = provider.plan(self.expected) + + # No root NS, no ignored, no excluded, no unsupported + n = len(self.expected.records) - 9 + self.assertEquals(n, len(plan.changes)) + self.assertEquals(n, provider.apply(plan)) + self.assertFalse(plan.exists) From 716d06819611b917d0d6c81c5138eb4f5b5d4809 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Tue, 27 Apr 2021 06:44:09 -0700 Subject: [PATCH 201/358] Backwards compat for plan overrides, 100% manager coverage, singular processor module name --- octodns/manager.py | 14 +- octodns/processor/__init__.py | 6 + .../__init__.py => processor/base.py} | 0 octodns/{processors => processor}/filters.py | 0 .../{processors => processor}/ownership.py | 0 tests/config/processors-missing-class.yaml | 23 ++++ tests/config/processors-wants-config.yaml | 25 ++++ tests/config/processors.yaml | 33 +++++ tests/helpers.py | 30 ++++ tests/test_octodns_manager.py | 129 +++++++++++++++++- tests/test_octodns_provider_base.py | 67 ++++++++- 11 files changed, 314 insertions(+), 13 deletions(-) create mode 100644 octodns/processor/__init__.py rename octodns/{processors/__init__.py => processor/base.py} (100%) rename octodns/{processors => processor}/filters.py (100%) rename octodns/{processors => processor}/ownership.py (100%) create mode 100644 tests/config/processors-missing-class.yaml create mode 100644 tests/config/processors-wants-config.yaml create mode 100644 tests/config/processors.yaml diff --git a/octodns/manager.py b/octodns/manager.py index f2a2da6..c7173c6 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -263,7 +263,7 @@ class Manager(object): except TypeError as e: if "keyword argument 'lenient'" not in text_type(e): raise - self.log.warn(': provider %s does not accept lenient ' + self.log.warn('provider %s does not accept lenient ' 'param', source.__class__.__name__) source.populate(zone) @@ -281,9 +281,15 @@ class Manager(object): 'value': 'provider={}'.format(target.id) }) zone.add_record(meta, replace=True) - # TODO: if someone has overrriden plan already this will be a - # breaking change so we probably need to try both ways - plan = target.plan(zone, processors=processors) + try: + plan = target.plan(zone, processors=processors) + except TypeError as e: + if "keyword argument 'processors'" not in text_type(e): + raise + self.log.warn('provider.plan %s does not accept processors ' + 'param', target.__class__.__name__) + plan = target.plan(zone) + for processor in processors: plan = processor.process_plan(plan, sources=sources, target=target) diff --git a/octodns/processor/__init__.py b/octodns/processor/__init__.py new file mode 100644 index 0000000..14ccf18 --- /dev/null +++ b/octodns/processor/__init__.py @@ -0,0 +1,6 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals diff --git a/octodns/processors/__init__.py b/octodns/processor/base.py similarity index 100% rename from octodns/processors/__init__.py rename to octodns/processor/base.py diff --git a/octodns/processors/filters.py b/octodns/processor/filters.py similarity index 100% rename from octodns/processors/filters.py rename to octodns/processor/filters.py diff --git a/octodns/processors/ownership.py b/octodns/processor/ownership.py similarity index 100% rename from octodns/processors/ownership.py rename to octodns/processor/ownership.py diff --git a/tests/config/processors-missing-class.yaml b/tests/config/processors-missing-class.yaml new file mode 100644 index 0000000..4594307 --- /dev/null +++ b/tests/config/processors-missing-class.yaml @@ -0,0 +1,23 @@ +providers: + config: + class: octodns.provider.yaml.YamlProvider + directory: tests/config + dump: + class: octodns.provider.yaml.YamlProvider + directory: env/YAML_TMP_DIR + geo: + class: helpers.GeoProvider + nosshfp: + class: helpers.NoSshFpProvider + +processors: + no-class: {} + +zones: + unit.tests.: + processors: + - noop + sources: + - in + targets: + - dump diff --git a/tests/config/processors-wants-config.yaml b/tests/config/processors-wants-config.yaml new file mode 100644 index 0000000..53fc397 --- /dev/null +++ b/tests/config/processors-wants-config.yaml @@ -0,0 +1,25 @@ +providers: + config: + class: octodns.provider.yaml.YamlProvider + directory: tests/config + dump: + class: octodns.provider.yaml.YamlProvider + directory: env/YAML_TMP_DIR + geo: + class: helpers.GeoProvider + nosshfp: + class: helpers.NoSshFpProvider + +processors: + # valid class, but it wants a param and we're not passing it + wants-config: + class: helpers.WantsConfigProcessor + +zones: + unit.tests.: + processors: + - noop + sources: + - in + targets: + - dump diff --git a/tests/config/processors.yaml b/tests/config/processors.yaml new file mode 100644 index 0000000..097024b --- /dev/null +++ b/tests/config/processors.yaml @@ -0,0 +1,33 @@ +providers: + config: + class: octodns.provider.yaml.YamlProvider + directory: tests/config + dump: + class: octodns.provider.yaml.YamlProvider + directory: env/YAML_TMP_DIR + geo: + class: helpers.GeoProvider + nosshfp: + class: helpers.NoSshFpProvider + +processors: + # Just testing config so any processor will do + noop: + class: octodns.processor.base.BaseProcessor + +zones: + unit.tests.: + processors: + - noop + sources: + - config + targets: + - dump + + bad.unit.tests.: + processors: + - doesnt-exist + sources: + - in + targets: + - dump diff --git a/tests/helpers.py b/tests/helpers.py index ff7f7cc..eedfd8b 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -7,6 +7,10 @@ from __future__ import absolute_import, division, print_function, \ from shutil import rmtree from tempfile import mkdtemp +from logging import getLogger + +from octodns.processor.base import BaseProcessor +from octodns.provider.base import BaseProvider class SimpleSource(object): @@ -90,3 +94,29 @@ class TemporaryDirectory(object): rmtree(self.dirname) else: raise Exception(self.dirname) + + +class WantsConfigProcessor(BaseProcessor): + + def __init__(self, name, some_config): + super(WantsConfigProcessor, self).__init__(name) + + +class PlannableProvider(BaseProvider): + log = getLogger('PlannableProvider') + + SUPPORTS_GEO = False + SUPPORTS_DYNAMIC = False + SUPPORTS = set(('A',)) + + def __init__(self, *args, **kwargs): + super(PlannableProvider, self).__init__(*args, **kwargs) + + def populate(self, zone, source=False, target=False, lenient=False): + pass + + def supports(self, record): + return True + + def __repr__(self): + return self.__class__.__name__ diff --git a/tests/test_octodns_manager.py b/tests/test_octodns_manager.py index e4b5211..069bc0b 100644 --- a/tests/test_octodns_manager.py +++ b/tests/test_octodns_manager.py @@ -9,9 +9,10 @@ from os import environ from os.path import dirname, join from six import text_type -from octodns.record import Record from octodns.manager import _AggregateTarget, MainThreadExecutor, Manager, \ ManagerException +from octodns.processor.base import BaseProcessor +from octodns.record import Create, Delete, Record from octodns.yaml import safe_load from octodns.zone import Zone @@ -19,7 +20,7 @@ from mock import MagicMock, patch from unittest import TestCase from helpers import DynamicProvider, GeoProvider, NoSshFpProvider, \ - SimpleProvider, TemporaryDirectory + PlannableProvider, SimpleProvider, TemporaryDirectory config_dir = join(dirname(__file__), 'config') @@ -358,20 +359,48 @@ class TestManager(TestCase): class NoLenient(SimpleProvider): - def populate(self, zone, source=False): + def populate(self, zone): pass # This should be ok, we'll fall back to not passing it manager._populate_and_plan('unit.tests.', [], [NoLenient()], []) - class NoZone(SimpleProvider): + class OtherType(SimpleProvider): - def populate(self, lenient=False): + def populate(self, zone, lenient=False): + raise TypeError('something else') + + # This will blow up, we don't fallback for source + with self.assertRaises(TypeError) as ctx: + manager._populate_and_plan('unit.tests.', [], [OtherType()], + []) + self.assertEquals('something else', text_type(ctx.exception)) + + def test_plan_processors_fallback(self): + with TemporaryDirectory() as tmpdir: + environ['YAML_TMP_DIR'] = tmpdir.dirname + # Only allow a target that doesn't exist + manager = Manager(get_config_filename('simple.yaml')) + + class NoProcessors(SimpleProvider): + + def plan(self, zone): pass + # This should be ok, we'll fall back to not passing it + manager._populate_and_plan('unit.tests.', [], [], + [NoProcessors()]) + + class OtherType(SimpleProvider): + + def plan(self, zone, processors): + raise TypeError('something else') + # This will blow up, we don't fallback for source - with self.assertRaises(TypeError): - manager._populate_and_plan('unit.tests.', [NoZone()], []) + with self.assertRaises(TypeError) as ctx: + manager._populate_and_plan('unit.tests.', [], [], + [OtherType()]) + self.assertEquals('something else', text_type(ctx.exception)) @patch('octodns.manager.Manager._get_named_class') def test_sync_passes_file_handle(self, mock): @@ -391,6 +420,92 @@ class TestManager(TestCase): _, kwargs = plan_output_mock.run.call_args self.assertEqual(fh_mock, kwargs.get('fh')) + def test_processor_config(self): + # Smoke test loading a valid config + manager = Manager(get_config_filename('processors.yaml')) + self.assertEquals(['noop'], list(manager.processors.keys())) + # This zone specifies a valid processor + manager.sync(['unit.tests.']) + + with self.assertRaises(ManagerException) as ctx: + # This zone specifies a non-existant processor + manager.sync(['bad.unit.tests.']) + self.assertTrue('Zone bad.unit.tests., unknown processor: ' + 'doesnt-exist' in text_type(ctx.exception)) + + with self.assertRaises(ManagerException) as ctx: + Manager(get_config_filename('processors-missing-class.yaml')) + self.assertTrue('Processor no-class is missing class' in + text_type(ctx.exception)) + + with self.assertRaises(ManagerException) as ctx: + Manager(get_config_filename('processors-wants-config.yaml')) + self.assertTrue('Incorrect processor config for wants-config' in + text_type(ctx.exception)) + + def test_processors(self): + manager = Manager(get_config_filename('simple.yaml')) + + targets = [PlannableProvider('prov')] + + zone = Zone('unit.tests.', []) + record = Record.new(zone, 'a', { + 'ttl': 30, + 'type': 'A', + 'value': '1.2.3.4', + }) + + # muck with sources + class MockProcessor(BaseProcessor): + + def process_source_zone(self, zone, sources): + zone = self._clone_zone(zone) + zone.add_record(record) + return zone + + mock = MockProcessor('mock') + plans, zone = manager._populate_and_plan('unit.tests.', [mock], [], + targets) + # Our mock was called and added the record + self.assertEquals(record, list(zone.records)[0]) + # We got a create for the thing added to the expected state (source) + self.assertIsInstance(plans[0][1].changes[0], Create) + + # muck with targets + class MockProcessor(BaseProcessor): + + def process_target_zone(self, zone, target): + zone = self._clone_zone(zone) + zone.add_record(record) + return zone + + mock = MockProcessor('mock') + plans, zone = manager._populate_and_plan('unit.tests.', [mock], [], + targets) + # No record added since it's target this time + self.assertFalse(zone.records) + # We got a delete for the thing added to the existing state (target) + self.assertIsInstance(plans[0][1].changes[0], Delete) + + # muck with plans + class MockProcessor(BaseProcessor): + + def process_target_zone(self, zone, target): + zone = self._clone_zone(zone) + zone.add_record(record) + return zone + + def process_plan(self, plans, sources, target): + # get rid of the change + plans.changes.pop(0) + + mock = MockProcessor('mock') + plans, zone = manager._populate_and_plan('unit.tests.', [mock], [], + targets) + # We planned a delete again, but this time removed it from the plan, so + # no plans + self.assertFalse(plans) + class TestMainThreadExecutor(TestCase): diff --git a/tests/test_octodns_provider_base.py b/tests/test_octodns_provider_base.py index f33db0f..4dfce48 100644 --- a/tests/test_octodns_provider_base.py +++ b/tests/test_octodns_provider_base.py @@ -9,9 +9,10 @@ from logging import getLogger from six import text_type from unittest import TestCase -from octodns.record import Create, Delete, Record, Update +from octodns.processor.base import BaseProcessor from octodns.provider.base import BaseProvider from octodns.provider.plan import Plan, UnsafePlan +from octodns.record import Create, Delete, Record, Update from octodns.zone import Zone @@ -21,7 +22,7 @@ class HelperProvider(BaseProvider): SUPPORTS = set(('A',)) id = 'test' - def __init__(self, extra_changes, apply_disabled=False, + def __init__(self, extra_changes=[], apply_disabled=False, include_change_callback=None): self.__extra_changes = extra_changes self.apply_disabled = apply_disabled @@ -43,6 +44,29 @@ class HelperProvider(BaseProvider): pass +class TrickyProcessor(BaseProcessor): + + def __init__(self, name, add_during_process_target_zone): + super(TrickyProcessor, self).__init__(name) + self.add_during_process_target_zone = add_during_process_target_zone + self.reset() + + def reset(self): + self.existing = None + self.target = None + + def process_target_zone(self, existing, target): + self.existing = existing + self.target = target + + new = self._clone_zone(existing) + for record in existing.records: + new.add_record(record) + for record in self.add_during_process_target_zone: + new.add_record(record) + return new + + class TestBaseProvider(TestCase): def test_base_provider(self): @@ -138,6 +162,45 @@ class TestBaseProvider(TestCase): self.assertTrue(plan) self.assertEquals(1, len(plan.changes)) + def test_plan_with_processors(self): + zone = Zone('unit.tests.', []) + + record = Record.new(zone, 'a', { + 'ttl': 30, + 'type': 'A', + 'value': '1.2.3.4', + }) + provider = HelperProvider() + # Processor that adds a record to the zone, which planning will then + # delete since it won't know anything about it + tricky = TrickyProcessor('tricky', [record]) + plan = provider.plan(zone, processors=[tricky]) + self.assertTrue(plan) + self.assertEquals(1, len(plan.changes)) + self.assertIsInstance(plan.changes[0], Delete) + # Called processor stored its params + self.assertTrue(tricky.existing) + self.assertEquals(zone.name, tricky.existing.name) + + # Chain of processors happen one after the other + other = Record.new(zone, 'b', { + 'ttl': 30, + 'type': 'A', + 'value': '5.6.7.8', + }) + # Another processor will add its record, thus 2 deletes + another = TrickyProcessor('tricky', [other]) + plan = provider.plan(zone, processors=[tricky, another]) + self.assertTrue(plan) + self.assertEquals(2, len(plan.changes)) + self.assertIsInstance(plan.changes[0], Delete) + self.assertIsInstance(plan.changes[1], Delete) + # 2nd processor stored its params, and we'll see the record the + # first one added + self.assertTrue(another.existing) + self.assertEquals(zone.name, another.existing.name) + self.assertEquals(1, len(another.existing.records)) + def test_apply(self): ignored = Zone('unit.tests.', []) From f387a61561e3841d77f2040c6302b04b1d391248 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Tue, 27 Apr 2021 07:52:41 -0700 Subject: [PATCH 202/358] Allow/reject use sets and now have tests --- octodns/processor/{filters.py => filter.py} | 6 +- tests/test_octodns_processor_filter.py | 92 +++++++++++++++++++++ 2 files changed, 95 insertions(+), 3 deletions(-) rename octodns/processor/{filters.py => filter.py} (89%) create mode 100644 tests/test_octodns_processor_filter.py diff --git a/octodns/processor/filters.py b/octodns/processor/filter.py similarity index 89% rename from octodns/processor/filters.py rename to octodns/processor/filter.py index 2c5f87f..369e987 100644 --- a/octodns/processor/filters.py +++ b/octodns/processor/filter.py @@ -5,14 +5,14 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -from . import BaseProcessor +from .base import BaseProcessor class TypeAllowlistFilter(BaseProcessor): def __init__(self, name, allowlist): super(TypeAllowlistFilter, self).__init__(name) - self.allowlist = allowlist + self.allowlist = set(allowlist) def _process(self, zone, *args, **kwargs): ret = self._clone_zone(zone) @@ -30,7 +30,7 @@ class TypeRejectlistFilter(BaseProcessor): def __init__(self, name, rejectlist): super(TypeRejectlistFilter, self).__init__(name) - self.rejectlist = rejectlist + self.rejectlist = set(rejectlist) def _process(self, zone, *args, **kwargs): ret = self._clone_zone(zone) diff --git a/tests/test_octodns_processor_filter.py b/tests/test_octodns_processor_filter.py new file mode 100644 index 0000000..1febfbd --- /dev/null +++ b/tests/test_octodns_processor_filter.py @@ -0,0 +1,92 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from logging import getLogger +from six import StringIO, text_type +from unittest import TestCase + +from octodns.processor.filter import TypeAllowlistFilter, TypeRejectlistFilter +from octodns.record import Record +from octodns.zone import Zone + +zone = Zone('unit.tests.', []) +for record in [ + Record.new(zone, 'a', { + 'ttl': 30, + 'type': 'A', + 'value': '1.2.3.4', + }), + Record.new(zone, 'aaaa', { + 'ttl': 30, + 'type': 'AAAA', + 'value': '::1', + }), + Record.new(zone, 'txt', { + 'ttl': 30, + 'type': 'TXT', + 'value': 'Hello World!', + }), + Record.new(zone, 'a2', { + 'ttl': 30, + 'type': 'A', + 'value': '2.3.4.5', + }), + Record.new(zone, 'txt2', { + 'ttl': 30, + 'type': 'TXT', + 'value': 'That will do', + }), +]: + zone.add_record(record) + + +class TestTypeAllowListFilter(TestCase): + + def test_basics(self): + filter_a = TypeAllowlistFilter('only-a', set(('A'))) + + got = filter_a.process_source_zone(zone) + self.assertEquals(['a', 'a2'], sorted([r.name for r in got.records])) + + filter_aaaa = TypeAllowlistFilter('only-aaaa', ('AAAA',)) + got = filter_aaaa.process_source_zone(zone) + self.assertEquals(['aaaa'], sorted([r.name for r in got.records])) + + filter_txt = TypeAllowlistFilter('only-txt', ['TXT']) + got = filter_txt.process_source_zone(zone) + self.assertEquals(['txt', 'txt2'], + sorted([r.name for r in got.records])) + + filter_a_aaaa = TypeAllowlistFilter('only-aaaa', set(('A', 'AAAA'))) + got = filter_a_aaaa.process_source_zone(zone) + self.assertEquals(['a', 'a2', 'aaaa'], + sorted([r.name for r in got.records])) + + +class TestTypeRejectListFilter(TestCase): + + def test_basics(self): + filter_a = TypeRejectlistFilter('not-a', set(('A'))) + + got = filter_a.process_source_zone(zone) + self.assertEquals(['aaaa', 'txt', 'txt2'], + sorted([r.name for r in got.records])) + + filter_aaaa = TypeRejectlistFilter('not-aaaa', ('AAAA',)) + got = filter_aaaa.process_source_zone(zone) + self.assertEquals(['a', 'a2', 'txt', 'txt2'], + sorted([r.name for r in got.records])) + + filter_txt = TypeRejectlistFilter('not-txt', ['TXT']) + got = filter_txt.process_source_zone(zone) + self.assertEquals(['a', 'a2', 'aaaa'], + sorted([r.name for r in got.records])) + + filter_a_aaaa = TypeRejectlistFilter('not-a-aaaa', set(('A', 'AAAA'))) + got = filter_a_aaaa.process_source_zone(zone) + self.assertEquals(['txt', 'txt2'], + sorted([r.name for r in got.records])) From 540fb27263a3b16524c125340c580efd59ed61ec Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Tue, 27 Apr 2021 09:25:24 -0700 Subject: [PATCH 203/358] Clean up and test OwnershipProcessor --- octodns/processor/ownership.py | 14 +------------- tests/test_octodns_processor_filter.py | 10 ++++------ 2 files changed, 5 insertions(+), 19 deletions(-) diff --git a/octodns/processor/ownership.py b/octodns/processor/ownership.py index 67bbb9a..172f349 100644 --- a/octodns/processor/ownership.py +++ b/octodns/processor/ownership.py @@ -6,12 +6,11 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals from collections import defaultdict -from pprint import pprint from ..provider.plan import Plan from ..record import Record -from . import BaseProcessor +from .base import BaseProcessor # Mark anything octoDNS is managing that way it can know it's safe to modify or @@ -71,8 +70,6 @@ class OwnershipProcessor(BaseProcessor): name = '' owned[name][_type.upper()] = True - pprint(dict(owned)) - # Cases: # - Configured in source # - We'll fully CRU/manage it adding ownership TXT, @@ -83,17 +80,10 @@ class OwnershipProcessor(BaseProcessor): # - Special records like octodns-meta # - Should be left alone and should not have ownerthis TXTs - pprint(plan.changes) - filtered_changes = [] for change in plan.changes: record = change.record - pprint([change, - not self._is_ownership(record), - record._type not in owned[record.name], - record.name != 'octodns-meta']) - if not self._is_ownership(record) and \ record._type not in owned[record.name] and \ record.name != 'octodns-meta': @@ -105,8 +95,6 @@ class OwnershipProcessor(BaseProcessor): # change is we should do filtered_changes.append(change) - pprint(filtered_changes) - if plan.changes != filtered_changes: return Plan(plan.existing, plan.desired, filtered_changes, plan.exists, plan.update_pcent_threshold, diff --git a/tests/test_octodns_processor_filter.py b/tests/test_octodns_processor_filter.py index 1febfbd..f421676 100644 --- a/tests/test_octodns_processor_filter.py +++ b/tests/test_octodns_processor_filter.py @@ -5,8 +5,6 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -from logging import getLogger -from six import StringIO, text_type from unittest import TestCase from octodns.processor.filter import TypeAllowlistFilter, TypeRejectlistFilter @@ -57,12 +55,12 @@ class TestTypeAllowListFilter(TestCase): self.assertEquals(['aaaa'], sorted([r.name for r in got.records])) filter_txt = TypeAllowlistFilter('only-txt', ['TXT']) - got = filter_txt.process_source_zone(zone) + got = filter_txt.process_target_zone(zone) self.assertEquals(['txt', 'txt2'], sorted([r.name for r in got.records])) filter_a_aaaa = TypeAllowlistFilter('only-aaaa', set(('A', 'AAAA'))) - got = filter_a_aaaa.process_source_zone(zone) + got = filter_a_aaaa.process_target_zone(zone) self.assertEquals(['a', 'a2', 'aaaa'], sorted([r.name for r in got.records])) @@ -82,11 +80,11 @@ class TestTypeRejectListFilter(TestCase): sorted([r.name for r in got.records])) filter_txt = TypeRejectlistFilter('not-txt', ['TXT']) - got = filter_txt.process_source_zone(zone) + got = filter_txt.process_target_zone(zone) self.assertEquals(['a', 'a2', 'aaaa'], sorted([r.name for r in got.records])) filter_a_aaaa = TypeRejectlistFilter('not-a-aaaa', set(('A', 'AAAA'))) - got = filter_a_aaaa.process_source_zone(zone) + got = filter_a_aaaa.process_target_zone(zone) self.assertEquals(['txt', 'txt2'], sorted([r.name for r in got.records])) From f8659bec0451d7b61d4c6135d3a6a87610026fff Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Tue, 27 Apr 2021 09:30:34 -0700 Subject: [PATCH 204/358] Helps if you add the file --- tests/test_octodns_processor_ownership.py | 146 ++++++++++++++++++++++ 1 file changed, 146 insertions(+) create mode 100644 tests/test_octodns_processor_ownership.py diff --git a/tests/test_octodns_processor_ownership.py b/tests/test_octodns_processor_ownership.py new file mode 100644 index 0000000..959f4c2 --- /dev/null +++ b/tests/test_octodns_processor_ownership.py @@ -0,0 +1,146 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from unittest import TestCase + +from octodns.processor.ownership import OwnershipProcessor +from octodns.record import Delete, Record +from octodns.zone import Zone + +from helpers import PlannableProvider + + +zone = Zone('unit.tests.', []) +records = {} +for record in [ + Record.new(zone, '', { + 'ttl': 30, + 'type': 'A', + 'values': [ + '1.2.3.4', + '5.6.7.8', + ], + }), + Record.new(zone, 'the-a', { + 'ttl': 30, + 'type': 'A', + 'value': '1.2.3.4', + }), + Record.new(zone, 'the-aaaa', { + 'ttl': 30, + 'type': 'AAAA', + 'value': '::1', + }), + Record.new(zone, 'the-txt', { + 'ttl': 30, + 'type': 'TXT', + 'value': 'Hello World!', + }), + Record.new(zone, '*', { + 'ttl': 30, + 'type': 'A', + 'value': '4.3.2.1', + }), +]: + records[record.name] = record + zone.add_record(record) + + +class TestOwnershipProcessor(TestCase): + + def test_process_source_zone(self): + ownership = OwnershipProcessor('ownership') + + got = ownership.process_source_zone(zone) + self.assertEquals([ + '', + '*', + '_owner.a', + '_owner.a._wildcard', + '_owner.a.the-a', + '_owner.aaaa.the-aaaa', + '_owner.txt.the-txt', + 'the-a', + 'the-aaaa', + 'the-txt', + ], sorted([r.name for r in got.records])) + + found = False + for record in got.records: + if record.name.startswith(ownership.txt_name): + self.assertEquals([ownership.txt_value], record.values) + # test _is_ownership while we're in here + self.assertTrue(ownership._is_ownership(record)) + found = True + else: + self.assertFalse(ownership._is_ownership(record)) + self.assertTrue(found) + + def test_process_plan(self): + ownership = OwnershipProcessor('ownership') + provider = PlannableProvider('helper') + + # No plan, is a quick noop + self.assertFalse(ownership.process_plan(None)) + + # Nothing exists create both records and ownership + ownership_added = ownership.process_source_zone(zone) + plan = provider.plan(ownership_added) + self.assertTrue(plan) + # Double the number of records + self.assertEquals(len(records) * 2, len(plan.changes)) + # Now process the plan, shouldn't make any changes, we're creating + # everything + got = ownership.process_plan(plan) + self.assertTrue(got) + self.assertEquals(len(records) * 2, len(got.changes)) + + # Something extra exists and doesn't have ownership TXT, leave it + # alone, we don't own it. + extra_a = Record.new(zone, 'extra-a', { + 'ttl': 30, + 'type': 'A', + 'value': '4.4.4.4', + }) + plan.existing.add_record(extra_a) + # If we'd done a "real" plan we'd have a delete for the extra thing. + plan.changes.append(Delete(extra_a)) + # Process the plan, shouldn't make any changes since the extra bit is + # something we don't own + got = ownership.process_plan(plan) + self.assertTrue(got) + self.assertEquals(len(records) * 2, len(got.changes)) + + # Something extra exists and does have an ownership record so we will + # delete it... + copy = Zone('unit.tests.', []) + for record in records.values(): + if record.name != 'the-a': + copy.add_record(record) + # New ownership, without the `the-a` + ownership_added = ownership.process_source_zone(copy) + self.assertEquals(len(records) * 2 - 2, len(ownership_added.records)) + plan = provider.plan(ownership_added) + # Fake the extra existing by adding the record, its ownership, and the + # two delete changes. + the_a = records['the-a'] + plan.existing.add_record(the_a) + name = '{}.a.the-a'.format(ownership.txt_name) + the_a_ownership = Record.new(zone, name, { + 'ttl': 30, + 'type': 'TXT', + 'value': ownership.txt_value, + }) + plan.existing.add_record(the_a_ownership) + plan.changes.append(Delete(the_a)) + plan.changes.append(Delete(the_a_ownership)) + # Finally process the plan, should be a noop and we should get the same + # plan out, meaning the planned deletes were allowed to happen. + got = ownership.process_plan(plan) + self.assertTrue(got) + self.assertEquals(plan, got) + self.assertEquals(len(plan.changes), len(got.changes)) From a0c4e9ecd76af5f2a830b8cb430468b147e070f3 Mon Sep 17 00:00:00 2001 From: Ricard Bejarano Date: Tue, 27 Apr 2021 19:56:39 +0200 Subject: [PATCH 205/358] simplified HetznerClient by removing unused pagination handling --- octodns/provider/hetzner.py | 24 +++--------------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/octodns/provider/hetzner.py b/octodns/provider/hetzner.py index 0be01ea..d396e18 100644 --- a/octodns/provider/hetzner.py +++ b/octodns/provider/hetzner.py @@ -30,10 +30,6 @@ class HetznerClientUnauthorized(HetznerClientException): class HetznerClient(object): - ''' - Hetzner DNS Public API v1 client class. - ''' - BASE_URL = 'https://dns.hetzner.com/api/v1' def __init__(self, token): @@ -54,22 +50,9 @@ class HetznerClient(object): def _do_json(self, method, path, params=None, data=None): return self._do(method, path, params, data).json() - def _do_json_paginate(self, method, path, key, params=None, data=None, - per_page=100): - items = [] - params = {**{'page': 1, 'per_page': per_page}, **params} - while True: - response = self._do_json(method, path, params, data) - items += response[key] - if response['meta']['pagination']['page'] >= \ - response['meta']['pagination']['last_page']: - break - params['page'] += 1 - return items - - def zones_get(self, name=None, search_name=None): - params = {'name': name, 'search_name': search_name} - return self._do_json_paginate('GET', '/zones', 'zones', params=params) + def zone_get(self, name): + params = {'name': name} + return self._do_json('GET', '/zones', params)['zones'][0] def zone_create(self, name, ttl=None): data = {'name': name, 'ttl': ttl} @@ -77,7 +60,6 @@ class HetznerClient(object): def zone_records_get(self, zone_id): params = {'zone_id': zone_id} - # No need to handle pagination as it returns all records by default. records = self._do_json('GET', '/records', params=params)['records'] for record in records: if record['name'] == '@': From d4c6836d0bb8b0d8bd6b3fb58ba9abb2378f706a Mon Sep 17 00:00:00 2001 From: Ricard Bejarano Date: Tue, 27 Apr 2021 19:58:07 +0200 Subject: [PATCH 206/358] implemented HetznerProvider.zone_metadata as the equivalent to zone_records for zone metadata --- octodns/provider/hetzner.py | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/octodns/provider/hetzner.py b/octodns/provider/hetzner.py index d396e18..53914f5 100644 --- a/octodns/provider/hetzner.py +++ b/octodns/provider/hetzner.py @@ -96,16 +96,28 @@ class HetznerProvider(BaseProvider): self._zone_records = {} self._zone_metadata = {} + self._zone_name_to_id = {} def _append_dot(self, value): if value == '@' or value[-1] == '.': return value return '{}.'.format(value) + def zone_metadata(self, zone_id=None, zone_name=None): + if zone_name is not None: + if zone_name in self._zone_name_to_id: + zone_id = self._zone_name_to_id[zone_name] + else: + zone = self._client.zone_get(name=zone_name[:-1]) + zone_id = zone['id'] + self._zone_name_to_id[zone_name] = zone_id + self._zone_metadata[zone_id] = zone + + return self._zone_metadata[zone_id] + def _record_ttl(self, record): - if 'ttl' in record: - return record['ttl'] - return self._zone_metadata[record['zone_id']]['ttl'] + default_ttl = self.zone_metadata(zone_id=record['zone_id'])['ttl'] + return record['ttl'] if 'ttl' in record else default_ttl def _data_for_multiple(self, _type, records): values = [record['value'].replace(';', '\\;') for record in records] @@ -195,10 +207,9 @@ class HetznerProvider(BaseProvider): def zone_records(self, zone): if zone.name not in self._zone_records: try: - zone_metadata = self._client.zones_get(name=zone.name[:-1])[0] - self._zone_metadata[zone_metadata['id']] = zone_metadata + zone_id = self.zone_metadata(zone_name=zone.name)['id'] self._zone_records[zone.name] = \ - self._client.zone_records_get(zone_metadata['id']) + self._client.zone_records_get(zone_id) except HetznerClientNotFound: return [] @@ -314,12 +325,11 @@ class HetznerProvider(BaseProvider): self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name, len(changes)) - zone_name = desired.name[:-1] try: - zone_id = self._client.zones_get(name=zone_name)[0]['id'] + zone_id = self.zone_metadata(zone_name=desired.name)['id'] except HetznerClientNotFound: self.log.debug('_apply: no matching zone, creating domain') - zone_id = self._client.zone_create(zone_name)['id'] + zone_id = self._client.zone_create(desired.name[:-1])['id'] for change in changes: class_name = change.__class__.__name__ From 6bf24d867894c8df992b8491762a9217e563d7d5 Mon Sep 17 00:00:00 2001 From: Ricard Bejarano Date: Tue, 27 Apr 2021 19:59:00 +0200 Subject: [PATCH 207/358] implemented TestHetznerProvider.test_apply --- tests/test_octodns_provider_hetzner.py | 230 ++++++++++++++++++++++++- 1 file changed, 229 insertions(+), 1 deletion(-) diff --git a/tests/test_octodns_provider_hetzner.py b/tests/test_octodns_provider_hetzner.py index 0c18731..02da32e 100644 --- a/tests/test_octodns_provider_hetzner.py +++ b/tests/test_octodns_provider_hetzner.py @@ -6,13 +6,14 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -from mock import Mock +from mock import Mock, call from os.path import dirname, join from requests import HTTPError from requests_mock import ANY, mock as requests_mock from six import text_type from unittest import TestCase +from octodns.record import Record from octodns.provider.hetzner import HetznerClientNotFound, \ HetznerProvider from octodns.provider.yaml import YamlProvider @@ -105,3 +106,230 @@ class TestHetznerProvider(TestCase): self.assertEquals(n, len(plan.changes)) self.assertEquals(n, provider.apply(plan)) self.assertFalse(plan.exists) + + provider._client._do.assert_has_calls([ + # created the zone + call('POST', '/zones', None, { + 'name': 'unit.tests', + 'ttl': None, + }), + # created all the records with their expected data + call('POST', '/records', data={ + 'name': '@', + 'ttl': 300, + 'type': 'A', + 'value': '1.2.3.4', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': '@', + 'ttl': 300, + 'type': 'A', + 'value': '1.2.3.5', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': '@', + 'ttl': 3600, + 'type': 'CAA', + 'value': '0 issue "ca.unit.tests"', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': '_imap._tcp', + 'ttl': 600, + 'type': 'SRV', + 'value': '0 0 0 .', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': '_pop3._tcp', + 'ttl': 600, + 'type': 'SRV', + 'value': '0 0 0 .', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': '_srv._tcp', + 'ttl': 600, + 'type': 'SRV', + 'value': '10 20 30 foo-1.unit.tests.', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': '_srv._tcp', + 'ttl': 600, + 'type': 'SRV', + 'value': '12 20 30 foo-2.unit.tests.', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': 'aaaa', + 'ttl': 600, + 'type': 'AAAA', + 'value': '2601:644:500:e210:62f8:1dff:feb8:947a', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': 'cname', + 'ttl': 300, + 'type': 'CNAME', + 'value': 'unit.tests.', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': 'included', + 'ttl': 3600, + 'type': 'CNAME', + 'value': 'unit.tests.', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': 'mx', + 'ttl': 300, + 'type': 'MX', + 'value': '10 smtp-4.unit.tests.', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': 'mx', + 'ttl': 300, + 'type': 'MX', + 'value': '20 smtp-2.unit.tests.', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': 'mx', + 'ttl': 300, + 'type': 'MX', + 'value': '30 smtp-3.unit.tests.', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': 'mx', + 'ttl': 300, + 'type': 'MX', + 'value': '40 smtp-1.unit.tests.', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': 'sub', + 'ttl': 3600, + 'type': 'NS', + 'value': '6.2.3.4.', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': 'sub', + 'ttl': 3600, + 'type': 'NS', + 'value': '7.2.3.4.', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': 'txt', + 'ttl': 600, + 'type': 'TXT', + 'value': 'Bah bah black sheep', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': 'txt', + 'ttl': 600, + 'type': 'TXT', + 'value': 'have you any wool.', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': 'txt', + 'ttl': 600, + 'type': 'TXT', + 'value': 'v=DKIM1;k=rsa;s=email;h=sha256;' + 'p=A/kinda+of/long/string+with+numb3rs', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': 'www', + 'ttl': 300, + 'type': 'A', + 'value': '2.2.3.6', + 'zone_id': 'unit.tests', + }), + call('POST', '/records', data={ + 'name': 'www.sub', + 'ttl': 300, + 'type': 'A', + 'value': '2.2.3.6', + 'zone_id': 'unit.tests', + }), + ]) + self.assertEquals(24, provider._client._do.call_count) + + provider._client._do.reset_mock() + + # delete 1 and update 1 + provider._client.zone_get = Mock(return_value={ + 'id': 'unit.tests', + 'name': 'unit.tests', + 'ttl': 3600, + }) + provider._client.zone_records_get = Mock(return_value=[ + { + 'type': 'A', + 'id': 'one', + 'created': '0000-00-00T00:00:00Z', + 'modified': '0000-00-00T00:00:00Z', + 'zone_id': 'unit.tests', + 'name': 'www', + 'value': '1.2.3.4', + 'ttl': 300, + }, + { + 'type': 'A', + 'id': 'two', + 'created': '0000-00-00T00:00:00Z', + 'modified': '0000-00-00T00:00:00Z', + 'zone_id': 'unit.tests', + 'name': 'www', + 'value': '2.2.3.4', + 'ttl': 300, + }, + { + 'type': 'A', + 'id': 'three', + 'created': '0000-00-00T00:00:00Z', + 'modified': '0000-00-00T00:00:00Z', + 'zone_id': 'unit.tests', + 'name': 'ttl', + 'value': '3.2.3.4', + 'ttl': 600, + }, + ]) + + # Domain exists, we don't care about return + resp.json.side_effect = ['{}'] + + wanted = Zone('unit.tests.', []) + wanted.add_record(Record.new(wanted, 'ttl', { + 'ttl': 300, + 'type': 'A', + 'value': '3.2.3.4', + })) + + plan = provider.plan(wanted) + self.assertTrue(plan.exists) + self.assertEquals(2, len(plan.changes)) + self.assertEquals(2, provider.apply(plan)) + # recreate for update, and delete for the 2 parts of the other + provider._client._do.assert_has_calls([ + call('POST', '/records', data={ + 'name': 'ttl', + 'ttl': 300, + 'type': 'A', + 'value': '3.2.3.4', + 'zone_id': 'unit.tests', + }), + call('DELETE', '/records/one'), + call('DELETE', '/records/two'), + call('DELETE', '/records/three'), + ], any_order=True) From b3c394a5e0056e5c0f4382e89e7854b4e02c02e4 Mon Sep 17 00:00:00 2001 From: Ricard Bejarano Date: Tue, 27 Apr 2021 19:59:26 +0200 Subject: [PATCH 208/358] minor correctness tweaks --- tests/test_octodns_provider_hetzner.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/test_octodns_provider_hetzner.py b/tests/test_octodns_provider_hetzner.py index 02da32e..4167944 100644 --- a/tests/test_octodns_provider_hetzner.py +++ b/tests/test_octodns_provider_hetzner.py @@ -66,7 +66,7 @@ class TestHetznerProvider(TestCase): # No diffs == no changes with requests_mock() as mock: - base = 'https://dns.hetzner.com/api/v1' + base = provider._client.BASE_URL with open('tests/fixtures/hetzner-zones.json') as fh: mock.get('{}/zones'.format(base), text=fh.read()) with open('tests/fixtures/hetzner-records.json') as fh: @@ -93,11 +93,17 @@ class TestHetznerProvider(TestCase): resp.json = Mock() provider._client._do = Mock(return_value=resp) + domain_after_creation = {'zone': { + 'id': 'unit.tests', + 'name': 'unit.tests', + 'ttl': 3600, + }} + # non-existent domain, create everything resp.json.side_effect = [ HetznerClientNotFound, # no zone in populate HetznerClientNotFound, # no zone during apply - {"zone": {"id": "string"}} + domain_after_creation, ] plan = provider.plan(self.expected) From a564270ef5cf8030e55e3a2434ad2e7b51c9c370 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Thu, 29 Apr 2021 09:12:35 -0700 Subject: [PATCH 209/358] Add a blurb on pip installing a sha --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index cd9b884..5eaca41 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,14 @@ $ pip install octodns $ mkdir config ``` +#### Installing a specific commit SHA + +If you'd like to install a version that has not yet been released in a repetable/safe manner you can do the following. In general octoDNS is fairly stable inbetween releases thanks to the plan and apply process, but care should be taken regardless. + +```shell +$ pip install -e git+https://git@github.com/github/octodns.git@#egg=octodns +``` + ### Config We start by creating a config file to tell OctoDNS about our providers and the zone(s) we want it to manage. Below we're setting up a `YamlProvider` to source records from our config files and both a `Route53Provider` and `DynProvider` to serve as the targets for those records. You can have any number of zones set up and any number of sources of data and targets for records for each. You can also have multiple config files, that make use of separate accounts and each manage a distinct set of zones. A good example of this this might be `./config/staging.yaml` & `./config/production.yaml`. We'll focus on a `config/production.yaml`. From ad37b997739fc76c370aba1983f156bd8e32c03a Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Thu, 29 Apr 2021 09:12:38 -0700 Subject: [PATCH 210/358] Add shell code block types --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 5eaca41..4ab3c6b 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ It is similar to [Netflix/denominator](https://github.com/Netflix/denominator). Running through the following commands will install the latest release of OctoDNS and set up a place for your config files to live. To determine if provider specific requirements are necessary see the [Supported providers table](#supported-providers) below. -``` +```shell $ mkdir dns $ cd dns $ virtualenv env @@ -121,7 +121,7 @@ Further information can be found in [Records Documentation](/docs/records.md). We're ready to do a dry-run with our new setup to see what changes it would make. Since we're pretending here we'll act like there are no existing records for `example.com.` in our accounts on either provider. -``` +```shell $ octodns-sync --config-file=./config/production.yaml ... ******************************************************************************** @@ -145,7 +145,7 @@ There will be other logging information presented on the screen, but successful Now it's time to tell OctoDNS to make things happen. We'll invoke it again with the same options and add a `--doit` on the end to tell it this time we actually want it to try and make the specified changes. -``` +```shell $ octodns-sync --config-file=./config/production.yaml --doit ... ``` @@ -176,7 +176,7 @@ If that goes smoothly, you again see the expected changes, and verify them with Very few situations will involve starting with a blank slate which is why there's tooling built in to pull existing data out of providers into a matching config file. -``` +```shell $ octodns-dump --config-file=config/production.yaml --output-dir=tmp/ example.com. route53 2017-03-15T13:33:34 INFO Manager __init__: config_file=tmp/production.yaml 2017-03-15T13:33:34 INFO Manager dump: zone=example.com., sources=('route53',) From 58c7f431e86ce441767ab1f5a7f6dee7837465e8 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 30 Apr 2021 14:43:23 -0700 Subject: [PATCH 211/358] v0.9.12 version bump and CHANGELOG update --- CHANGELOG.md | 25 ++++++++++++++++++++++++- octodns/__init__.py | 2 +- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fb68e25..1d16544 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,29 @@ +## v0.9.12 - 2021-04-30 - Enough time has passed + +#### Noteworthy changes + +* Formal Python 2.7 support removed, deps and tooling were becoming + unmaintainable +* octodns/octodns move, from github/octodns, more to come + +#### Stuff + +* ZoneFileSource supports specifying an extension & no files end in . to better + support Windows +* LOC record type support added +* Support for pre-release versions of PowerDNS +* PowerDNS delete before create which allows A <-> CNAME etc. +* Improved validation of fqdn's in ALIAS, CNAME, etc. +* Transip support for NS records +* Support for sending plan output to a file +* DNSimple uses zone api rather than domain to support non-registered stuff, + e.g. reverse zones. +* Support for fallback-only dynamic pools and related fixes to NS1 provider +* Initial Hetzner provider + ## v0.9.11 - 2020-11-05 - We still don't know edition -#### Noteworthy changtes +#### Noteworthy changes * ALIAS records only allowed at the root of zones - see `leient` in record docs for work-arounds if you really need them. diff --git a/octodns/__init__.py b/octodns/__init__.py index 3fcdaa1..1885d42 100644 --- a/octodns/__init__.py +++ b/octodns/__init__.py @@ -3,4 +3,4 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -__VERSION__ = '0.9.11' +__VERSION__ = '0.9.12' From 40569945d2bc0fb89735fd97e3cdbbd00b1379c1 Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Fri, 30 Apr 2021 16:53:37 -0700 Subject: [PATCH 212/358] Add support for dynamic CNAME records in NS1 --- octodns/provider/ns1.py | 24 +++++- tests/test_octodns_provider_ns1.py | 118 +++++++++++++++++++++++++++++ 2 files changed, 138 insertions(+), 4 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 6cea185..4da0d33 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -588,16 +588,22 @@ class Ns1Provider(BaseProvider): rules = list(rules.values()) rules.sort(key=lambda r: (r['_order'], r['pool'])) - return { + data = { 'dynamic': { 'pools': pools, 'rules': rules, }, 'ttl': record['ttl'], 'type': _type, - 'values': sorted(default), } + if _type == 'CNAME': + data['value'] = default[0] + else: + data['values'] = default + + return data + def _data_for_A(self, _type, record): if record.get('tier', 1) > 1: # Advanced record, see if it's first answer has a note @@ -646,6 +652,10 @@ class Ns1Provider(BaseProvider): } def _data_for_CNAME(self, _type, record): + if record.get('tier', 1) > 1: + # Advanced dynamic record + return self._data_for_dynamic_A(_type, record) + try: value = record['short_answers'][0] except IndexError: @@ -1114,10 +1124,14 @@ class Ns1Provider(BaseProvider): 'feed_id': feed_id, }) + if record._type == 'CNAME': + default_values = [record.value] + else: + default_values = record.values default_answers = [{ 'answer': [v], 'weight': 1, - } for v in record.values] + } for v in default_values] # Build our list of answers # The regions dictionary built above already has the required pool @@ -1171,8 +1185,10 @@ class Ns1Provider(BaseProvider): values = [(v.flags, v.tag, v.value) for v in record.values] return {'answers': values, 'ttl': record.ttl}, None - # TODO: dynamic CNAME support def _params_for_CNAME(self, record): + if getattr(record, 'dynamic', False): + return self._params_for_dynamic_A(record) + return {'answers': [record.value], 'ttl': record.ttl}, None _params_for_ALIAS = _params_for_CNAME diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index 00b068b..c16a573 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -1270,6 +1270,63 @@ class TestNs1ProviderDynamic(TestCase): params, _ = provider._params_for_geo_A(record) self.assertEquals([], params['filters']) + @patch('octodns.provider.ns1.Ns1Provider._monitor_sync') + @patch('octodns.provider.ns1.Ns1Provider._monitors_for') + def test_params_for_dynamic_CNAME(self, monitors_for_mock, + monitor_sync_mock): + provider = Ns1Provider('test', 'api-key') + + # pre-fill caches to avoid extranious calls (things we're testing + # elsewhere) + provider._client._datasource_id = 'foo' + provider._client._feeds_for_monitors = { + 'mon-id': 'feed-id', + } + + # provider._params_for_A() calls provider._monitors_for() and + # provider._monitor_sync(). Mock their return values so that we don't + # make NS1 API calls during tests + monitors_for_mock.reset_mock() + monitor_sync_mock.reset_mock() + monitors_for_mock.side_effect = [{ + 'iad.unit.tests.': 'mid-1', + }] + monitor_sync_mock.side_effect = [ + ('mid-1', 'fid-1'), + ] + + record = Record.new(self.zone, 'foo', { + 'dynamic': { + 'pools': { + 'iad': { + 'values': [{ + 'value': 'iad.unit.tests.', + }], + }, + }, + 'rules': [{ + 'pool': 'iad', + }], + }, + 'octodns': { + 'healthcheck': { + 'host': 'send.me', + 'path': '/_ping', + 'port': 80, + 'protocol': 'HTTP', + } + }, + 'ttl': 32, + 'type': 'CNAME', + 'value': 'value.unit.tests.', + 'meta': {}, + }) + ret, _ = provider._params_for_CNAME(record) + + # Check if the default value was correctly read and populated + # All other dynamic record test cases are covered by dynamic_A tests + self.assertEquals(ret['answers'][1]['answer'][0], 'value.unit.tests.') + def test_data_for_dynamic_A(self): provider = Ns1Provider('test', 'api-key') @@ -1471,6 +1528,67 @@ class TestNs1ProviderDynamic(TestCase): self.assertTrue( 'OC-{}'.format(c) in data4['dynamic']['rules'][0]['geos']) + def test_data_for_dynamic_CNAME(self): + provider = Ns1Provider('test', 'api-key') + + # Test out a small setup that just covers default value validation + # Everything else is same as dynamic A whose tests will cover all + # other options and test cases + # Not testing for geo/region specific cases + filters = provider._get_updated_filter_chain(False, False) + catchall_pool_name = 'iad__catchall' + ns1_record = { + 'answers': [{ + 'answer': ['2.3.4.5.unit.tests.'], + 'meta': { + 'priority': 1, + 'weight': 12, + 'note': 'from:{}'.format(catchall_pool_name), + }, + 'region': catchall_pool_name, + }, { + 'answer': ['1.2.3.4.unit.tests.'], + 'meta': { + 'priority': 2, + 'note': 'from:--default--', + }, + 'region': catchall_pool_name, + }], + 'domain': 'foo.unit.tests', + 'filters': filters, + 'regions': { + catchall_pool_name: { + 'meta': { + 'note': 'rule-order:1', + }, + } + }, + 'tier': 3, + 'ttl': 42, + 'type': 'CNAME', + } + data = provider._data_for_CNAME('CNAME', ns1_record) + self.assertEquals({ + 'dynamic': { + 'pools': { + 'iad': { + 'fallback': None, + 'values': [{ + 'value': '2.3.4.5.unit.tests.', + 'weight': 12, + }], + }, + }, + 'rules': [{ + '_order': '1', + 'pool': 'iad', + }], + }, + 'ttl': 42, + 'type': 'CNAME', + 'value': '1.2.3.4.unit.tests.', + }, data) + @patch('ns1.rest.records.Records.retrieve') @patch('ns1.rest.zones.Zones.retrieve') @patch('octodns.provider.ns1.Ns1Provider._monitors_for') From 15eb23eeb672f743cb21724eddb2fa394c931aea Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Fri, 30 Apr 2021 20:26:11 -0700 Subject: [PATCH 213/358] Trim trailing dot from CNAME answers for NS1 monitors --- octodns/provider/ns1.py | 11 +++- tests/test_octodns_provider_ns1.py | 87 +++++++++++++++++++----------- 2 files changed, 64 insertions(+), 34 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 4da0d33..6ade3b9 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -832,6 +832,10 @@ class Ns1Provider(BaseProvider): # This monitor does not belong to this record config = monitor['config'] value = config['host'] + if record._type == 'CNAME': + # Append a trailing dot for CNAME records so that + # lookup by a CNAME answer works + value = value + '.' monitors[value] = monitor return monitors @@ -882,6 +886,10 @@ class Ns1Provider(BaseProvider): host = record.fqdn[:-1] _type = record._type + if _type == 'CNAME': + # NS1 does not accept a host value with a trailing dot + value = value[:-1] + ret = { 'active': True, 'config': { @@ -1266,8 +1274,7 @@ class Ns1Provider(BaseProvider): extra.append(Update(record, record)) continue - for have in self._monitors_for(record).values(): - value = have['config']['host'] + for value, have in self._monitors_for(record).items(): expected = self._monitor_gen(record, value) # TODO: find values which have missing monitors if not self._monitor_is_match(expected, have): diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index c16a573..a5c41a6 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -578,6 +578,34 @@ class TestNs1ProviderDynamic(TestCase): 'meta': {}, }) + def cname_record(self): + return Record.new(self.zone, 'foo', { + 'dynamic': { + 'pools': { + 'iad': { + 'values': [{ + 'value': 'iad.unit.tests.', + }], + }, + }, + 'rules': [{ + 'pool': 'iad', + }], + }, + 'octodns': { + 'healthcheck': { + 'host': 'send.me', + 'path': '/_ping', + 'port': 80, + 'protocol': 'HTTP', + } + }, + 'ttl': 33, + 'type': 'CNAME', + 'value': 'value.unit.tests.', + 'meta': {}, + }) + def test_notes(self): provider = Ns1Provider('test', 'api-key') @@ -609,6 +637,12 @@ class TestNs1ProviderDynamic(TestCase): }, 'notes': 'host:unit.tests type:A', } + monitor_five = { + 'config': { + 'host': 'iad.unit.tests', + }, + 'notes': 'host:foo.unit.tests type:CNAME', + } provider._client._monitors_cache = { 'one': monitor_one, 'two': { @@ -624,6 +658,7 @@ class TestNs1ProviderDynamic(TestCase): 'notes': 'host:other.unit.tests type:A', }, 'four': monitor_four, + 'five': monitor_five, } # Would match, but won't get there b/c it's not dynamic @@ -641,6 +676,11 @@ class TestNs1ProviderDynamic(TestCase): '2.3.4.5': monitor_four, }, provider._monitors_for(self.record())) + # Check match for CNAME values + self.assertEquals({ + 'iad.unit.tests.': monitor_five, + }, provider._monitors_for(self.cname_record())) + def test_uuid(self): # Just a smoke test/for coverage provider = Ns1Provider('test', 'api-key') @@ -728,6 +768,14 @@ class TestNs1ProviderDynamic(TestCase): # No http response expected self.assertFalse('rules' in monitor) + def test_monitor_gen_CNAME(self): + provider = Ns1Provider('test', 'api-key') + + value = 'iad.unit.tests.' + record = self.cname_record() + monitor = provider._monitor_gen(record, value) + self.assertEquals(value[:-1], monitor['config']['host']) + def test_monitor_is_match(self): provider = Ns1Provider('test', 'api-key') @@ -1295,32 +1343,7 @@ class TestNs1ProviderDynamic(TestCase): ('mid-1', 'fid-1'), ] - record = Record.new(self.zone, 'foo', { - 'dynamic': { - 'pools': { - 'iad': { - 'values': [{ - 'value': 'iad.unit.tests.', - }], - }, - }, - 'rules': [{ - 'pool': 'iad', - }], - }, - 'octodns': { - 'healthcheck': { - 'host': 'send.me', - 'path': '/_ping', - 'port': 80, - 'protocol': 'HTTP', - } - }, - 'ttl': 32, - 'type': 'CNAME', - 'value': 'value.unit.tests.', - 'meta': {}, - }) + record = self.cname_record() ret, _ = provider._params_for_CNAME(record) # Check if the default value was correctly read and populated @@ -1539,7 +1562,7 @@ class TestNs1ProviderDynamic(TestCase): catchall_pool_name = 'iad__catchall' ns1_record = { 'answers': [{ - 'answer': ['2.3.4.5.unit.tests.'], + 'answer': ['iad.unit.tests.'], 'meta': { 'priority': 1, 'weight': 12, @@ -1547,7 +1570,7 @@ class TestNs1ProviderDynamic(TestCase): }, 'region': catchall_pool_name, }, { - 'answer': ['1.2.3.4.unit.tests.'], + 'answer': ['value.unit.tests.'], 'meta': { 'priority': 2, 'note': 'from:--default--', @@ -1564,7 +1587,7 @@ class TestNs1ProviderDynamic(TestCase): } }, 'tier': 3, - 'ttl': 42, + 'ttl': 43, 'type': 'CNAME', } data = provider._data_for_CNAME('CNAME', ns1_record) @@ -1574,7 +1597,7 @@ class TestNs1ProviderDynamic(TestCase): 'iad': { 'fallback': None, 'values': [{ - 'value': '2.3.4.5.unit.tests.', + 'value': 'iad.unit.tests.', 'weight': 12, }], }, @@ -1584,9 +1607,9 @@ class TestNs1ProviderDynamic(TestCase): 'pool': 'iad', }], }, - 'ttl': 42, + 'ttl': 43, 'type': 'CNAME', - 'value': '1.2.3.4.unit.tests.', + 'value': 'value.unit.tests.', }, data) @patch('ns1.rest.records.Records.retrieve') From 3de5cd27404f5fae35feb14e7e4f01addace2528 Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Fri, 30 Apr 2021 20:38:08 -0700 Subject: [PATCH 214/358] More future proof index lookup --- tests/test_octodns_provider_ns1.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index a5c41a6..c14173b 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -1348,7 +1348,7 @@ class TestNs1ProviderDynamic(TestCase): # Check if the default value was correctly read and populated # All other dynamic record test cases are covered by dynamic_A tests - self.assertEquals(ret['answers'][1]['answer'][0], 'value.unit.tests.') + self.assertEquals(ret['answers'][-1]['answer'][0], 'value.unit.tests.') def test_data_for_dynamic_A(self): provider = Ns1Provider('test', 'api-key') From 6d7cab43e881247201ea77cd5ef281fb925674a6 Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Mon, 3 May 2021 10:59:12 -0700 Subject: [PATCH 215/358] Rename data/params for dynamic methods --- octodns/provider/ns1.py | 14 +++++++------- tests/test_octodns_provider_ns1.py | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 6ade3b9..38f9da3 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -464,10 +464,10 @@ class Ns1Provider(BaseProvider): pass return pool_name - def _data_for_dynamic_A(self, _type, record): + def _data_for_dynamic(self, _type, record): # First make sure we have the expected filters config if not self._valid_filter_config(record['filters'], record['domain']): - self.log.error('_data_for_dynamic_A: %s %s has unsupported ' + self.log.error('_data_for_dynamic: %s %s has unsupported ' 'filters', record['domain'], _type) raise Ns1Exception('Unrecognized advanced record') @@ -613,7 +613,7 @@ class Ns1Provider(BaseProvider): first_answer_note = '' # If that note includes a `from` (pool name) it's a dynamic record if 'from:' in first_answer_note: - return self._data_for_dynamic_A(_type, record) + return self._data_for_dynamic(_type, record) # If not it's an old geo record return self._data_for_geo_A(_type, record) @@ -654,7 +654,7 @@ class Ns1Provider(BaseProvider): def _data_for_CNAME(self, _type, record): if record.get('tier', 1) > 1: # Advanced dynamic record - return self._data_for_dynamic_A(_type, record) + return self._data_for_dynamic(_type, record) try: value = record['short_answers'][0] @@ -1031,7 +1031,7 @@ class Ns1Provider(BaseProvider): } answers.append(answer) - def _params_for_dynamic_A(self, record): + def _params_for_dynamic(self, record): pools = record.dynamic.pools # Convert rules to regions @@ -1168,7 +1168,7 @@ class Ns1Provider(BaseProvider): def _params_for_A(self, record): if getattr(record, 'dynamic', False): - return self._params_for_dynamic_A(record) + return self._params_for_dynamic(record) elif hasattr(record, 'geo'): return self._params_for_geo_A(record) @@ -1195,7 +1195,7 @@ class Ns1Provider(BaseProvider): def _params_for_CNAME(self, record): if getattr(record, 'dynamic', False): - return self._params_for_dynamic_A(record) + return self._params_for_dynamic(record) return {'answers': [record.value], 'ttl': record.ttl}, None diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index c14173b..4d6f02a 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -1290,7 +1290,7 @@ class TestNs1ProviderDynamic(TestCase): ('mid-2', 'fid-2'), ('mid-3', 'fid-3'), ] - # This indirectly calls into _params_for_dynamic_A and tests the + # This indirectly calls into _params_for_dynamic and tests the # handling to get there record = self.record() ret, _ = provider._params_for_A(record) @@ -1350,7 +1350,7 @@ class TestNs1ProviderDynamic(TestCase): # All other dynamic record test cases are covered by dynamic_A tests self.assertEquals(ret['answers'][-1]['answer'][0], 'value.unit.tests.') - def test_data_for_dynamic_A(self): + def test_data_for_dynamic(self): provider = Ns1Provider('test', 'api-key') # Unexpected filters throws an error @@ -1359,7 +1359,7 @@ class TestNs1ProviderDynamic(TestCase): 'filters': [], } with self.assertRaises(Ns1Exception) as ctx: - provider._data_for_dynamic_A('A', ns1_record) + provider._data_for_dynamic('A', ns1_record) self.assertEquals('Unrecognized advanced record', text_type(ctx.exception)) @@ -1371,7 +1371,7 @@ class TestNs1ProviderDynamic(TestCase): 'regions': {}, 'ttl': 42, } - data = provider._data_for_dynamic_A('A', ns1_record) + data = provider._data_for_dynamic('A', ns1_record) self.assertEquals({ 'dynamic': { 'pools': {}, @@ -1476,7 +1476,7 @@ class TestNs1ProviderDynamic(TestCase): 'tier': 3, 'ttl': 42, } - data = provider._data_for_dynamic_A('A', ns1_record) + data = provider._data_for_dynamic('A', ns1_record) self.assertEquals({ 'dynamic': { 'pools': { @@ -1520,7 +1520,7 @@ class TestNs1ProviderDynamic(TestCase): }, data) # Same answer if we go through _data_for_A which out sources the job to - # _data_for_dynamic_A + # _data_for_dynamic data2 = provider._data_for_A('A', ns1_record) self.assertEquals(data, data2) @@ -1531,7 +1531,7 @@ class TestNs1ProviderDynamic(TestCase): ns1_record['regions'][old_style_catchall_pool_name] = \ ns1_record['regions'][catchall_pool_name] del ns1_record['regions'][catchall_pool_name] - data3 = provider._data_for_dynamic_A('A', ns1_record) + data3 = provider._data_for_dynamic('A', ns1_record) self.assertEquals(data, data2) # Oceania test cases From ebfb9355b136dfce46db0905bd6c4e714d6563b7 Mon Sep 17 00:00:00 2001 From: omar Date: Mon, 10 May 2021 19:32:38 -0700 Subject: [PATCH 216/358] Update the AzureProvider to support azure-mgmt-dns 8.0.0 and azure-identity. --- README.md | 2 +- octodns/provider/azuredns.py | 22 +++++++++++----------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 4ab3c6b..0a04e27 100644 --- a/README.md +++ b/README.md @@ -192,7 +192,7 @@ The above command pulled the existing data out of Route53 and placed the results | Provider | Requirements | Record Support | Dynamic | Notes | |--|--|--|--|--| -| [AzureProvider](/octodns/provider/azuredns.py) | azure-mgmt-dns | A, AAAA, CAA, CNAME, MX, NS, PTR, SRV, TXT | No | | +| [AzureProvider](/octodns/provider/azuredns.py) | azure-identity, azure-mgmt-dns | A, AAAA, CAA, CNAME, MX, NS, PTR, SRV, TXT | No | | | [Akamai](/octodns/provider/edgedns.py) | edgegrid-python | A, AAAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, SSHFP, TXT | No | | | [CloudflareProvider](/octodns/provider/cloudflare.py) | | A, AAAA, ALIAS, CAA, CNAME, LOC, MX, NS, PTR, SPF, SRV, TXT | No | CAA tags restricted | | [ConstellixProvider](/octodns/provider/constellix.py) | | A, AAAA, ALIAS (ANAME), CAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | CAA tags restricted | diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 2fca5af..282077f 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -5,7 +5,7 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -from azure.common.credentials import ServicePrincipalCredentials +from azure.identity import ClientSecretCredential from azure.mgmt.dns import DnsManagementClient from azure.mgmt.dns.models import ARecord, AaaaRecord, CaaRecord, \ @@ -71,10 +71,10 @@ class _AzureRecord(object): '''Constructor for _AzureRecord. Notes on Azure records: An Azure record set has the form - RecordSet(name=<...>, type=<...>, arecords=[...], aaaa_records, ..) + RecordSet(name=<...>, type=<...>, a_records=[...], aaaa_records, ..) When constructing an azure record as done in self._apply_Create, the argument parameters for an A record would be - parameters={'ttl': , 'arecords': [ARecord(),]}. + parameters={'ttl': , 'a_records': [ARecord(),]}. As another example for CNAME record: parameters={'ttl': , 'cname_record': CnameRecord()}. @@ -263,7 +263,7 @@ def _parse_azure_type(string): def _check_for_alias(azrecord): - if (azrecord.target_resource.id and not azrecord.arecords and not + if (azrecord.target_resource.id and not azrecord.a_records and not azrecord.cname_record): return True return False @@ -343,14 +343,14 @@ class AzureProvider(BaseProvider): @property def _dns_client(self): if self.__dns_client is None: - credentials = ServicePrincipalCredentials( - self._dns_client_client_id, - secret=self._dns_client_key, - tenant=self._dns_client_directory_id + credential = ClientSecretCredential( + client_id=self._dns_client_client_id, + client_secret=self._dns_client_key, + tenant_id=self._dns_client_directory_id ) self.__dns_client = DnsManagementClient( - credentials, - self._dns_client_subscription_id + credential=credential, + subscription_id=self._dns_client_subscription_id ) return self.__dns_client @@ -452,7 +452,7 @@ class AzureProvider(BaseProvider): return exists def _data_for_A(self, azrecord): - return {'values': [ar.ipv4_address for ar in azrecord.arecords]} + return {'values': [ar.ipv4_address for ar in azrecord.a_records]} def _data_for_AAAA(self, azrecord): return {'values': [ar.ipv6_address for ar in azrecord.aaaa_records]} From 93de918e01b0c97a89ba8ff74f926534c966863b Mon Sep 17 00:00:00 2001 From: omar Date: Mon, 10 May 2021 20:38:30 -0700 Subject: [PATCH 217/358] Fix lint, requirements.txt, and all the tests but one. --- octodns/provider/azuredns.py | 3 ++- requirements.txt | 5 +++-- tests/test_octodns_provider_azuredns.py | 28 ++++++++++++------------- 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 282077f..7a1883f 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -71,7 +71,8 @@ class _AzureRecord(object): '''Constructor for _AzureRecord. Notes on Azure records: An Azure record set has the form - RecordSet(name=<...>, type=<...>, a_records=[...], aaaa_records, ..) + RecordSet(name=<...>, type=<...>, a_records=[...], + aaaa_records=[...], ...) When constructing an azure record as done in self._apply_Create, the argument parameters for an A record would be parameters={'ttl': , 'a_records': [ARecord(),]}. diff --git a/requirements.txt b/requirements.txt index 933ac60..54bd2a3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ PyYaml==5.4 -azure-common==1.1.25 -azure-mgmt-dns==3.0.0 +azure-common==1.1.27 +azure-identity==1.5.0 +azure-mgmt-dns==8.0.0 boto3==1.15.9 botocore==1.18.9 dnspython==1.16.0 diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index 9523b51..d9b2b5a 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -152,8 +152,8 @@ _base0.zone_name = 'unit.tests' _base0.relative_record_set_name = '@' _base0.record_type = 'A' _base0.params['ttl'] = 0 -_base0.params['arecords'] = [ARecord(ipv4_address='1.2.3.4'), - ARecord(ipv4_address='10.10.10.10')] +_base0.params['a_records'] = [ARecord(ipv4_address='1.2.3.4'), + ARecord(ipv4_address='10.10.10.10')] azure_records.append(_base0) _base1 = _AzureRecord('TestAzure', octo_records[1]) @@ -161,8 +161,8 @@ _base1.zone_name = 'unit.tests' _base1.relative_record_set_name = 'a' _base1.record_type = 'A' _base1.params['ttl'] = 1 -_base1.params['arecords'] = [ARecord(ipv4_address='1.2.3.4'), - ARecord(ipv4_address='1.1.1.1')] +_base1.params['a_records'] = [ARecord(ipv4_address='1.2.3.4'), + ARecord(ipv4_address='1.1.1.1')] azure_records.append(_base1) _base2 = _AzureRecord('TestAzure', octo_records[2]) @@ -170,7 +170,7 @@ _base2.zone_name = 'unit.tests' _base2.relative_record_set_name = 'aa' _base2.record_type = 'A' _base2.params['ttl'] = 9001 -_base2.params['arecords'] = ARecord(ipv4_address='1.2.4.3') +_base2.params['a_records'] = ARecord(ipv4_address='1.2.4.3') azure_records.append(_base2) _base3 = _AzureRecord('TestAzure', octo_records[3]) @@ -178,7 +178,7 @@ _base3.zone_name = 'unit.tests' _base3.relative_record_set_name = 'aaa' _base3.record_type = 'A' _base3.params['ttl'] = 2 -_base3.params['arecords'] = ARecord(ipv4_address='1.1.1.3') +_base3.params['a_records'] = ARecord(ipv4_address='1.1.1.3') azure_records.append(_base3) _base4 = _AzureRecord('TestAzure', octo_records[4]) @@ -366,7 +366,7 @@ class Test_CheckAzureAlias(TestCase): alias_record = type('C', (object,), {}) alias_record.target_resource = type('C', (object,), {}) alias_record.target_resource.id = "/subscriptions/x/resourceGroups/y/z" - alias_record.arecords = None + alias_record.a_records = None alias_record.cname_record = None self.assertEquals(_check_for_alias(alias_record), True) @@ -377,7 +377,7 @@ class TestAzureDnsProvider(TestCase): return self._get_provider('mock_spc', 'mock_dns_client') @patch('octodns.provider.azuredns.DnsManagementClient') - @patch('octodns.provider.azuredns.ServicePrincipalCredentials') + @patch('octodns.provider.azuredns.ClientSecretCredential') def _get_provider(self, mock_spc, mock_dns_client): '''Returns a mock AzureProvider object to use in testing. @@ -399,12 +399,12 @@ class TestAzureDnsProvider(TestCase): provider = self._get_provider() rs = [] - recordSet = RecordSet(arecords=[ARecord(ipv4_address='1.1.1.1')]) + recordSet = RecordSet(a_records=[ARecord(ipv4_address='1.1.1.1')]) recordSet.name, recordSet.ttl, recordSet.type = 'a1', 0, 'A' recordSet.target_resource = SubResource() rs.append(recordSet) - recordSet = RecordSet(arecords=[ARecord(ipv4_address='1.1.1.1'), - ARecord(ipv4_address='2.2.2.2')]) + recordSet = RecordSet(a_records=[ARecord(ipv4_address='1.1.1.1'), + ARecord(ipv4_address='2.2.2.2')]) recordSet.name, recordSet.ttl, recordSet.type = 'a2', 1, 'A' recordSet.target_resource = SubResource() rs.append(recordSet) @@ -575,11 +575,11 @@ class TestAzureDnsProvider(TestCase): provider = self._get_provider() rs = [] - recordSet = RecordSet(arecords=[ARecord(ipv4_address='1.1.1.1')]) + recordSet = RecordSet(a_records=[ARecord(ipv4_address='1.1.1.1')]) recordSet.name, recordSet.ttl, recordSet.type = 'a1', 0, 'A' rs.append(recordSet) - recordSet = RecordSet(arecords=[ARecord(ipv4_address='1.1.1.1'), - ARecord(ipv4_address='2.2.2.2')]) + recordSet = RecordSet(a_records=[ARecord(ipv4_address='1.1.1.1'), + ARecord(ipv4_address='2.2.2.2')]) recordSet.name, recordSet.ttl, recordSet.type = 'a2', 1, 'A' rs.append(recordSet) From 758c7fab61209084dc96cc845b08fa154835943c Mon Sep 17 00:00:00 2001 From: omar Date: Mon, 10 May 2021 20:49:28 -0700 Subject: [PATCH 218/358] Fix the last test. --- octodns/provider/azuredns.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 7a1883f..83cfeb0 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -102,8 +102,7 @@ class _AzureRecord(object): return # Refer to function docstring for key_name and class_name. - format_u_s = '' if record._type == 'A' else '_' - key_name = '{}{}records'.format(self.record_type, format_u_s).lower() + key_name = '{}_records'.format(self.record_type).lower() if record._type == 'CNAME': key_name = key_name[:len(key_name) - 1] azure_class = self.TYPE_MAP[self.record_type] From d619025040dc1e8087290e70554f9ccc9c028b78 Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Tue, 11 May 2021 15:39:00 -0700 Subject: [PATCH 219/358] Support dynamic records in Azure DNS --- README.md | 2 +- octodns/provider/azuredns.py | 632 +++++++++++++++-- requirements.txt | 1 + tests/test_octodns_provider_azuredns.py | 887 +++++++++++++++++++++++- 4 files changed, 1440 insertions(+), 82 deletions(-) diff --git a/README.md b/README.md index 0a04e27..8d59cd0 100644 --- a/README.md +++ b/README.md @@ -192,7 +192,7 @@ The above command pulled the existing data out of Route53 and placed the results | Provider | Requirements | Record Support | Dynamic | Notes | |--|--|--|--|--| -| [AzureProvider](/octodns/provider/azuredns.py) | azure-identity, azure-mgmt-dns | A, AAAA, CAA, CNAME, MX, NS, PTR, SRV, TXT | No | | +| [AzureProvider](/octodns/provider/azuredns.py) | azure-identity, azure-mgmt-dns, azure-mgmt-trafficmanager | A, AAAA, CAA, CNAME, MX, NS, PTR, SRV, TXT | Yes (CNAMEs only) | | | [Akamai](/octodns/provider/edgedns.py) | edgegrid-python | A, AAAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, SSHFP, TXT | No | | | [CloudflareProvider](/octodns/provider/cloudflare.py) | | A, AAAA, ALIAS, CAA, CNAME, LOC, MX, NS, PTR, SPF, SRV, TXT | No | CAA tags restricted | | [ConstellixProvider](/octodns/provider/constellix.py) | | A, AAAA, ALIAS (ANAME), CAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | CAA tags restricted | diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 83cfeb0..4d3dcc5 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -5,18 +5,28 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals +from collections import defaultdict + from azure.identity import ClientSecretCredential +from azure.common.credentials import ServicePrincipalCredentials from azure.mgmt.dns import DnsManagementClient +from azure.mgmt.trafficmanager import TrafficManagerManagementClient from azure.mgmt.dns.models import ARecord, AaaaRecord, CaaRecord, \ CnameRecord, MxRecord, SrvRecord, NsRecord, PtrRecord, TxtRecord, Zone +from azure.mgmt.trafficmanager.models import Profile, DnsConfig, \ + MonitorConfig, Endpoint, MonitorConfigCustomHeadersItem import logging from functools import reduce -from ..record import Record +from ..record import Record, Update, GeoCodes from .base import BaseProvider +class AzureException(Exception): + pass + + def escape_semicolon(s): assert s return s.replace(';', '\\;') @@ -67,7 +77,8 @@ class _AzureRecord(object): 'TXT': TxtRecord } - def __init__(self, resource_group, record, delete=False): + def __init__(self, resource_group, record, delete=False, + traffic_manager=None): '''Constructor for _AzureRecord. Notes on Azure records: An Azure record set has the form @@ -94,9 +105,11 @@ class _AzureRecord(object): self.log = logging.getLogger('AzureRecord') self.resource_group = resource_group - self.zone_name = record.zone.name[:len(record.zone.name) - 1] + self.zone_name = record.zone.name[:-1] self.relative_record_set_name = record.name or '@' self.record_type = record._type + self._record = record + self.traffic_manager = traffic_manager if delete: return @@ -104,11 +117,11 @@ class _AzureRecord(object): # Refer to function docstring for key_name and class_name. key_name = '{}_records'.format(self.record_type).lower() if record._type == 'CNAME': - key_name = key_name[:len(key_name) - 1] + key_name = key_name[:-1] azure_class = self.TYPE_MAP[self.record_type] - self.params = getattr(self, '_params_for_{}'.format(record._type)) - self.params = self.params(record.data, key_name, azure_class) + params_for = getattr(self, '_params_for_{}'.format(record._type)) + self.params = params_for(record.data, key_name, azure_class) self.params['ttl'] = record.ttl def _params_for_A(self, data, key_name, azure_class): @@ -139,6 +152,9 @@ class _AzureRecord(object): return {key_name: params} def _params_for_CNAME(self, data, key_name, azure_class): + if self._record.dynamic and self.traffic_manager: + return {'target_resource': self.traffic_manager} + return {key_name: azure_class(cname=data['value'])} def _params_for_MX(self, data, key_name, azure_class): @@ -227,25 +243,6 @@ class _AzureRecord(object): (parse_dict(self.params) == parse_dict(b.params)) & \ (self.relative_record_set_name == b.relative_record_set_name) - def __str__(self): - '''String representation of an _AzureRecord. - :type return: str - ''' - string = 'Zone: {}; '.format(self.zone_name) - string += 'Name: {}; '.format(self.relative_record_set_name) - string += 'Type: {}; '.format(self.record_type) - if not hasattr(self, 'params'): - return string - string += 'Ttl: {}; '.format(self.params['ttl']) - for char in self.params: - if char != 'ttl': - try: - for rec in self.params[char]: - string += 'Record: {}; '.format(rec.__dict__) - except: - string += 'Record: {}; '.format(self.params[char].__dict__) - return string - def _check_endswith_dot(string): return string if string.endswith('.') else string + '.' @@ -259,14 +256,88 @@ def _parse_azure_type(string): :type return: str ''' - return string.split('/')[len(string.split('/')) - 1] - + return string.split('/')[-1] + + +def _traffic_manager_suffix(record): + return record.fqdn[:-1].replace('.', '-') + + +def _get_monitor(record): + monitor = MonitorConfig( + protocol=record.healthcheck_protocol, + port=record.healthcheck_port, + path=record.healthcheck_path, + ) + host = record.healthcheck_host + if host: + monitor.custom_headers = [MonitorConfigCustomHeadersItem( + name='Host', value=host + )] + return monitor + + +def _profile_is_match(have, desired): + if have is None or desired is None: + return False + + # compare basic attributes + if have.name != desired.name or \ + have.traffic_routing_method != desired.traffic_routing_method or \ + have.dns_config.ttl != desired.dns_config.ttl or \ + len(have.endpoints) != len(desired.endpoints): + return False + + # compare monitoring configuration + monitor_have = have.monitor_config + monitor_desired = desired.monitor_config + if monitor_have.protocol != monitor_desired.protocol or \ + monitor_have.port != monitor_desired.port or \ + monitor_have.path != monitor_desired.path or \ + monitor_have.custom_headers != monitor_desired.custom_headers: + return False + + # compare endpoints + method = have.traffic_routing_method + if method == 'Priority': + have_endpoints = sorted(have.endpoints, key=lambda e: e.priority) + desired_endpoints = sorted(desired.endpoints, + key=lambda e: e.priority) + elif method == 'Weighted': + have_endpoints = sorted(have.endpoints, key=lambda e: e.target) + desired_endpoints = sorted(desired.endpoints, key=lambda e: e.target) + else: + have_endpoints = have.endpoints + desired_endpoints = desired.endpoints + endpoints = zip(have_endpoints, desired_endpoints) + for have_endpoint, desired_endpoint in endpoints: + if have_endpoint.name != desired_endpoint.name or \ + have_endpoint.type != desired_endpoint.type: + return False + target_type = have_endpoint.type.split('/')[-1] + if target_type == 'externalEndpoints': + # compare value, weight, priority + if have_endpoint.target != desired_endpoint.target: + return False + if method == 'Weighted' and \ + have_endpoint.weight != desired_endpoint.weight: + return False + elif target_type == 'nestedEndpoints': + # compare targets + if have_endpoint.target_resource_id != \ + desired_endpoint.target_resource_id: + return False + # compare geos + if method == 'Geographic': + have_geos = sorted(have_endpoint.geo_mapping) + desired_geos = sorted(desired_endpoint.geo_mapping) + if have_geos != desired_geos: + return False + else: + # unexpected, give up + return False -def _check_for_alias(azrecord): - if (azrecord.target_resource.id and not azrecord.a_records and not - azrecord.cname_record): - return True - return False + return True class AzureProvider(BaseProvider): @@ -318,7 +389,7 @@ class AzureProvider(BaseProvider): possible to also hard-code into the config file: eg, resource_group. ''' SUPPORTS_GEO = False - SUPPORTS_DYNAMIC = False + SUPPORTS_DYNAMIC = True SUPPORTS = set(('A', 'AAAA', 'CAA', 'CNAME', 'MX', 'NS', 'PTR', 'SRV', 'TXT')) @@ -336,24 +407,45 @@ class AzureProvider(BaseProvider): self._dns_client_directory_id = directory_id self._dns_client_subscription_id = sub_id self.__dns_client = None + self.__tm_client = None self._resource_group = resource_group self._azure_zones = set() + self._traffic_managers = dict() @property def _dns_client(self): if self.__dns_client is None: - credential = ClientSecretCredential( - client_id=self._dns_client_client_id, - client_secret=self._dns_client_key, - tenant_id=self._dns_client_directory_id - ) + # Azure's logger spits out a lot of debug messages at 'INFO' + # level, override it by re-assigning `info` method to `debug` + # (ugly hack until I find a better way) + logger_name = 'azure.core.pipeline.policies.http_logging_policy' + logger = logging.getLogger(logger_name) + logger.info = logger.debug self.__dns_client = DnsManagementClient( - credential=credential, - subscription_id=self._dns_client_subscription_id + credential=ClientSecretCredential( + client_id=self._dns_client_client_id, + client_secret=self._dns_client_key, + tenant_id=self._dns_client_directory_id, + logger=logger, + ), + subscription_id=self._dns_client_subscription_id, ) return self.__dns_client + @property + def _tm_client(self): + if self.__tm_client is None: + self.__tm_client = TrafficManagerManagementClient( + ServicePrincipalCredentials( + self._dns_client_client_id, + secret=self._dns_client_key, + tenant=self._dns_client_directory_id, + ), + self._dns_client_subscription_id, + ) + return self.__tm_client + def _populate_zones(self): self.log.debug('azure_zones: loading') list_zones = self._dns_client.zones.list_by_resource_group @@ -388,6 +480,42 @@ class AzureProvider(BaseProvider): # Else return nothing (aka false) return + def _populate_traffic_managers(self): + self.log.debug('traffic managers: loading') + list_profiles = self._tm_client.profiles.list_by_resource_group + for profile in list_profiles(self._resource_group): + self._traffic_managers[profile.id] = profile + # link nested profiles in advance for convenience + for _, profile in self._traffic_managers.items(): + self._populate_nested_profiles(profile) + + def _populate_nested_profiles(self, profile): + for ep in profile.endpoints: + target_id = ep.target_resource_id + if target_id and target_id in self._traffic_managers: + target = self._traffic_managers[target_id] + ep.target_resource = self._populate_nested_profiles(target) + return profile + + def _get_tm_profile_by_id(self, resource_id): + if not self._traffic_managers: + self._populate_traffic_managers() + return self._traffic_managers.get(resource_id) + + def _profile_name_to_id(self, name): + return '/subscriptions/' + self._dns_client_subscription_id + \ + '/resourceGroups/' + self._resource_group + \ + '/providers/Microsoft.Network/trafficManagerProfiles/' + \ + name + + def _get_tm_profile_by_name(self, name): + profile_id = self._profile_name_to_id(name) + return self._get_tm_profile_by_id(profile_id) + + def _get_tm_for_dynamic_record(self, record): + name = _traffic_manager_suffix(record) + return self._get_tm_profile_by_name(name) + def populate(self, zone, target=False, lenient=False): '''Required function of manager.py to collect records from zone. @@ -417,40 +545,35 @@ class AzureProvider(BaseProvider): exists = False before = len(zone.records) - zone_name = zone.name[:len(zone.name) - 1] + zone_name = zone.name[:-1] self._populate_zones() - self._check_zone(zone_name) - _records = [] records = self._dns_client.record_sets.list_by_dns_zone if self._check_zone(zone_name): exists = True for azrecord in records(self._resource_group, zone_name): - if _parse_azure_type(azrecord.type) in self.SUPPORTS: - _records.append(azrecord) - for azrecord in _records: - record_name = azrecord.name if azrecord.name != '@' else '' typ = _parse_azure_type(azrecord.type) + if typ not in self.SUPPORTS: + continue - if typ in ['A', 'CNAME']: - if _check_for_alias(azrecord): - self.log.debug( - 'Skipping - ALIAS. zone=%s record=%s, type=%s', - zone_name, record_name, typ) # pragma: no cover - continue # pragma: no cover - - data = getattr(self, '_data_for_{}'.format(typ)) - data = data(azrecord) - data['type'] = typ - data['ttl'] = azrecord.ttl - record = Record.new(zone, record_name, data, source=self) - + record = self._populate_record(zone, azrecord, lenient) zone.add_record(record, lenient=lenient) self.log.info('populate: found %s records, exists=%s', len(zone.records) - before, exists) return exists + def _populate_record(self, zone, azrecord, lenient=False): + record_name = azrecord.name if azrecord.name != '@' else '' + typ = _parse_azure_type(azrecord.type) + + data_for = getattr(self, '_data_for_{}'.format(typ)) + data = data_for(azrecord) + data['type'] = typ + data['ttl'] = azrecord.ttl + return Record.new(zone, record_name, data, source=self, + lenient=lenient) + def _data_for_A(self, azrecord): return {'values': [ar.ipv4_address for ar in azrecord.a_records]} @@ -470,6 +593,9 @@ class AzureProvider(BaseProvider): :type return: dict ''' + if azrecord.cname_record is None and azrecord.target_resource.id: + return self._data_for_dynamic(azrecord) + return {'value': _check_endswith_dot(azrecord.cname_record.cname)} def _data_for_MX(self, azrecord): @@ -495,6 +621,322 @@ class AzureProvider(BaseProvider): ar.value)) for ar in azrecord.txt_records]} + def _data_for_dynamic(self, azrecord): + default = set() + pools = defaultdict(lambda: {'fallback': None, 'values': []}) + rules = [] + + # top level geo profile + geo_profile = self._get_tm_profile_by_id(azrecord.target_resource.id) + for geo_ep in geo_profile.endpoints: + rule = {} + + # resolve list of regions + geo_map = list(geo_ep.geo_mapping) + if geo_map != ['WORLD']: + if 'GEO-ME' in geo_map: + # Azure treats Middle East as a separate group, but + # its part of Asia in octoDNS, so we need to remove GEO-ME + # if GEO-AS is also in the list + # Throw exception otherwise, it should not happen if the + # profile was generated by octoDNS + if 'GEO-AS' not in geo_map: + msg = '_data_for_dynamic: Profile={}: '.format( + geo_profile.name) + msg += 'Middle East (GEO-ME) is not supported by ' + \ + 'octoDNS. It needs to be either paired ' + \ + 'with Asia (GEO-AS) or expanded into ' + \ + 'individual list of countries.' + raise AzureException(msg) + geo_map.remove('GEO-ME') + geos = rule.setdefault('geos', []) + for code in geo_map: + if code.startswith('GEO-'): + geos.append(code[len('GEO-'):]) + elif '-' in code: + country, province = code.split('-', 1) + country = GeoCodes.country_to_code(country) + geos.append('{}-{}'.format(country, province)) + else: + geos.append(GeoCodes.country_to_code(code)) + + # second level priority profile + pool = None + rule_endpoints = geo_ep.target_resource.endpoints + rule_endpoints = sorted(rule_endpoints, key=lambda e: e.priority) + for rule_ep in rule_endpoints: + pool_name = rule_ep.name + + # third (and last) level weighted RR profile + # these should be leaf node profiles with no further nesting + pool_profile = rule_ep.target_resource + + # last/default pool + if pool_name == '--default--': + for pool_ep in pool_profile.endpoints: + default.add(pool_ep.target) + # this should be the last one, so let's break here + break + + # set first priority endpoint as the rule's primary pool + if 'pool' not in rule: + rule['pool'] = pool_name + + if pool: + # set current pool as fallback of the previous pool + pool['fallback'] = pool_name + + pool = pools[pool_name] + for pool_ep in pool_profile.endpoints: + val = pool_ep.target + value_dict = { + 'value': _check_endswith_dot(val), + 'weight': pool_ep.weight, + } + if value_dict not in pool['values']: + pool['values'].append(value_dict) + + if 'pool' not in rule or not default: + # this will happen if the priority profile does not have + # enough endpoints + msg = 'Expected at least 2 endpoints in {}, got {}'.format( + geo_ep.target_resource.name, len(rule_endpoints) + ) + raise AzureException(msg) + rules.append(rule) + + # Order and convert to a list + default = sorted(default) + + data = { + 'dynamic': { + 'pools': pools, + 'rules': rules, + }, + 'value': _check_endswith_dot(default[0]), + } + + return data + + def _extra_changes(self, existing, desired, changes): + changed = set() + + # Abort if there are non-CNAME dynamic records + for change in changes: + record = change.record + changed.add(record) + typ = record._type + dynamic = getattr(record, 'dynamic', False) + if dynamic and typ != 'CNAME': + msg = '{}: Dynamic records in Azure must be of type CNAME' + msg = msg.format(record.fqdn) + raise AzureException(msg) + + log = self.log.info + extra = [] + for record in desired.records: + if not getattr(record, 'dynamic', False): + # Already changed, or not dynamic, no need to check it + continue + + # let's walk through and show what will be changed even if + # the record is already be in list of changes + added = (record in changed) + + active = set() + profiles = self._generate_traffic_managers(record) + for profile in profiles: + name = profile.name + active.add(name) + existing_profile = self._get_tm_profile_by_name(name) + if not _profile_is_match(existing_profile, profile): + log('_extra_changes: Profile name=%s will be synced', + name) + if not added: + extra.append(Update(record, record)) + added = True + + existing_profiles = self._find_traffic_managers(record) + for name in existing_profiles - active: + log('_extra_changes: Profile name=%s will be destroyed', name) + if not added: + extra.append(Update(record, record)) + added = True + + return extra + + def _generate_tm_profile(self, name, routing, endpoints, record): + # set appropriate endpoint types + endpoint_type_prefix = 'Microsoft.Network/trafficManagerProfiles/' + for ep in endpoints: + if ep.target_resource_id: + ep.type = endpoint_type_prefix + 'nestedEndpoints' + elif ep.target: + ep.type = endpoint_type_prefix + 'externalEndpoints' + else: + msg = ('_generate_tm_profile: Invalid endpoint {} ' + + 'in profile {}, needs to have either target or ' + + 'target_resource_id').format(ep.name, name) + raise AzureException(msg) + + # build and return + return Profile( + id=self._profile_name_to_id(name), + name=name, + traffic_routing_method=routing, + dns_config=DnsConfig( + relative_name=name, + ttl=record.ttl, + ), + monitor_config=_get_monitor(record), + endpoints=endpoints, + location='global', + ) + + def _generate_traffic_managers(self, record): + traffic_managers = [] + pools = record.dynamic.pools + + tm_suffix = _traffic_manager_suffix(record) + profile = self._generate_tm_profile + + # construct the default pool that will be used at the end of + # all rules + target = record.value[:-1] + default_endpoints = [Endpoint( + name=target, + target=target, + weight=1, + )] + default_profile_name = 'default--{}'.format(tm_suffix) + default_profile = profile(default_profile_name, 'Weighted', + default_endpoints, record) + traffic_managers.append(default_profile) + + geo_endpoints = [] + + for rule in record.dynamic.rules: + pool_name = rule.data['pool'] + rule_endpoints = [] + priority = 1 + + while pool_name: + # iterate until we reach end of fallback chain + pool = pools[pool_name].data + profile_name = 'pool-{}--{}'.format(pool_name, tm_suffix) + endpoints = [] + for val in pool['values']: + target = val['value'] + # strip trailing dot from CNAME value + target = target[:-1] + endpoints.append(Endpoint( + name=target, + target=target, + weight=val.get('weight', 1), + )) + pool_profile = profile(profile_name, 'Weighted', endpoints, + record) + traffic_managers.append(pool_profile) + + # append pool to endpoint list of fallback rule profile + rule_endpoints.append(Endpoint( + name=pool_name, + target_resource_id=pool_profile.id, + priority=priority, + )) + + priority += 1 + pool_name = pool.get('fallback') + + # append default profile to the end + rule_endpoints.append(Endpoint( + name='--default--', + target_resource_id=default_profile.id, + priority=priority, + )) + # create rule profile with fallback chain + rule_profile_name = 'rule-{}--{}'.format(rule.data['pool'], + tm_suffix) + rule_profile = profile(rule_profile_name, 'Priority', + rule_endpoints, record) + traffic_managers.append(rule_profile) + + # append rule profile to top-level geo profile + rule_geos = rule.data.get('geos', []) + geos = [] + if len(rule_geos) > 0: + for geo in rule_geos: + if '-' in geo: + geos.append(geo.split('-', 1)[-1]) + else: + geos.append('GEO-{}'.format(geo)) + if geo == 'AS': + # Middle East is part of Asia in octoDNS, but + # Azure treats it as a separate "group", so let's + # add it in the list of geo mappings. We will drop + # it when we later parse the list of regions. + geos.append('GEO-ME') + else: + geos.append('WORLD') + geo_endpoints.append(Endpoint( + name='rule-{}'.format(rule.data['pool']), + target_resource_id=rule_profile.id, + geo_mapping=geos, + )) + + geo_profile = profile(tm_suffix, 'Geographic', geo_endpoints, record) + traffic_managers.append(geo_profile) + + return traffic_managers + + def _sync_traffic_managers(self, record): + desired_profiles = self._generate_traffic_managers(record) + seen = set() + + tm_sync = self._tm_client.profiles.create_or_update + populate = self._populate_nested_profiles + + for desired in desired_profiles: + name = desired.name + if name in seen: + continue + + existing = self._get_tm_profile_by_name(name) + if not _profile_is_match(existing, desired): + self.log.info( + '_sync_traffic_managers: Syncing profile=%s', name) + profile = tm_sync(self._resource_group, name, desired) + self._traffic_managers[profile.id] = populate(profile) + else: + self.log.debug( + '_sync_traffic_managers: Skipping profile=%s: up to date', + name) + seen.add(name) + + return seen + + def _find_traffic_managers(self, record): + tm_suffix = _traffic_manager_suffix(record) + + profiles = set() + for profile_id in self._traffic_managers: + # match existing profiles with record's suffix + name = profile_id.split('/')[-1] + if name == tm_suffix or \ + name.endswith('--{}'.format(tm_suffix)): + profiles.add(name) + + return profiles + + def _traffic_managers_gc(self, record, active_profiles): + existing_profiles = self._find_traffic_managers(record) + + # delete unused profiles + for profile_name in existing_profiles - active_profiles: + self.log.info('_traffic_managers_gc: Deleting profile=%s', + profile_name) + self._tm_client.profiles.delete(self._resource_group, profile_name) + def _apply_Create(self, change): '''A record from change must be created. @@ -503,7 +945,15 @@ class AzureProvider(BaseProvider): :type return: void ''' - ar = _AzureRecord(self._resource_group, change.new) + record = change.new + + dynamic = getattr(record, 'dynamic', False) + if dynamic: + self._sync_traffic_managers(record) + + profile = self._get_tm_for_dynamic_record(record) + ar = _AzureRecord(self._resource_group, record, + traffic_manager=profile) create = self._dns_client.record_sets.create_or_update create(resource_group_name=ar.resource_group, @@ -512,17 +962,71 @@ class AzureProvider(BaseProvider): record_type=ar.record_type, parameters=ar.params) - self.log.debug('* Success Create/Update: {}'.format(ar)) + self.log.debug('* Success Create: {}'.format(record)) - _apply_Update = _apply_Create + def _apply_Update(self, change): + '''A record from change must be created. + + :param change: a change object + :type change: octodns.record.Change + + :type return: void + ''' + existing = change.existing + new = change.new + existing_is_dynamic = getattr(existing, 'dynamic', False) + new_is_dynamic = getattr(new, 'dynamic', False) + + update_record = True + + if new_is_dynamic: + active = self._sync_traffic_managers(new) + # only TTL is configured in record, everything else goes inside + # traffic managers, so no need to update if TTL is unchanged + # and existing record is already aliased to its traffic manager + if existing.ttl == new.ttl and existing_is_dynamic: + update_record = False + + if update_record: + profile = self._get_tm_for_dynamic_record(new) + ar = _AzureRecord(self._resource_group, new, + traffic_manager=profile) + update = self._dns_client.record_sets.create_or_update + + update(resource_group_name=ar.resource_group, + zone_name=ar.zone_name, + relative_record_set_name=ar.relative_record_set_name, + record_type=ar.record_type, + parameters=ar.params) + + if new_is_dynamic: + # let's cleanup unused traffic managers + self._traffic_managers_gc(new, active) + elif existing_is_dynamic: + # cleanup traffic managers when a dynamic record gets + # changed to a simple record + self._traffic_managers_gc(existing, set()) + + self.log.debug('* Success Update: {}'.format(new)) def _apply_Delete(self, change): - ar = _AzureRecord(self._resource_group, change.existing, delete=True) + '''A record from change must be deleted. + + :param change: a change object + :type change: octodns.record.Change + + :type return: void + ''' + record = change.record + ar = _AzureRecord(self._resource_group, record, delete=True) delete = self._dns_client.record_sets.delete delete(self._resource_group, ar.zone_name, ar.relative_record_set_name, ar.record_type) + if getattr(record, 'dynamic', False): + self._traffic_managers_gc(record, set()) + self.log.debug('* Success Delete: {}'.format(ar)) def _apply(self, plan): diff --git a/requirements.txt b/requirements.txt index 54bd2a3..143ba67 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,7 @@ PyYaml==5.4 azure-common==1.1.27 azure-identity==1.5.0 azure-mgmt-dns==8.0.0 +azure-mgmt-trafficmanager==0.51.0 boto3==1.15.9 botocore==1.18.9 dnspython==1.16.0 diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index d9b2b5a..5655719 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -5,19 +5,23 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -from octodns.record import Create, Delete, Record +from octodns.record import Create, Update, Delete, Record from octodns.provider.azuredns import _AzureRecord, AzureProvider, \ - _check_endswith_dot, _parse_azure_type, _check_for_alias + _check_endswith_dot, _parse_azure_type, _traffic_manager_suffix, \ + _get_monitor, _profile_is_match, AzureException from octodns.zone import Zone from octodns.provider.base import Plan from azure.mgmt.dns.models import ARecord, AaaaRecord, CaaRecord, \ CnameRecord, MxRecord, SrvRecord, NsRecord, PtrRecord, TxtRecord, \ RecordSet, SoaRecord, SubResource, Zone as AzureZone +from azure.mgmt.trafficmanager.models import Profile, DnsConfig, \ + MonitorConfig, Endpoint, MonitorConfigCustomHeadersItem from msrestazure.azure_exceptions import CloudError +from six import text_type from unittest import TestCase -from mock import Mock, patch +from mock import Mock, patch, call zone = Zone(name='unit.tests.', sub_zones=[]) @@ -343,6 +347,43 @@ class Test_AzureRecord(TestCase): assert(azure_records[i]._equals(octo)) +class Test_DynamicAzureRecord(TestCase): + def test_azure_record(self): + tm_profile = Profile() + data = { + 'ttl': 60, + 'type': 'CNAME', + 'value': 'default.unit.tests.', + 'dynamic': { + 'pools': { + 'one': { + 'values': [ + {'value': 'one.unit.tests.', 'weight': 1} + ], + 'fallback': 'two', + }, + 'two': { + 'values': [ + {'value': 'two.unit.tests.', 'weight': 1} + ], + }, + }, + 'rules': [ + {'geos': ['AF'], 'pool': 'one'}, + {'pool': 'two'}, + ], + } + } + octo_record = Record.new(zone, 'foo', data) + azure_record = _AzureRecord('TestAzure', octo_record, + traffic_manager=tm_profile) + self.assertEqual(azure_record.zone_name, zone.name[:-1]) + self.assertEqual(azure_record.relative_record_set_name, 'foo') + self.assertEqual(azure_record.record_type, 'CNAME') + self.assertEqual(azure_record.params['ttl'], 60) + self.assertEqual(azure_record.params['target_resource'], tm_profile) + + class Test_ParseAzureType(TestCase): def test_parse_azure_type(self): for expected, test in [['A', 'Microsoft.Network/dnszones/A'], @@ -361,40 +402,369 @@ class Test_CheckEndswithDot(TestCase): self.assertEquals(expected, _check_endswith_dot(test)) -class Test_CheckAzureAlias(TestCase): - def test_check_for_alias(self): - alias_record = type('C', (object,), {}) - alias_record.target_resource = type('C', (object,), {}) - alias_record.target_resource.id = "/subscriptions/x/resourceGroups/y/z" - alias_record.a_records = None - alias_record.cname_record = None +class Test_TrafficManagerSuffix(TestCase): + def test_traffic_manager_suffix(self): + test = Record.new(zone, 'foo', data={ + 'ttl': 60, 'type': 'CNAME', 'value': 'default.unit.tests.', + }) + self.assertEqual(_traffic_manager_suffix(test), 'foo-unit-tests') + + +class Test_GetMonitor(TestCase): + def test_get_monitor(self): + record = Record.new(zone, 'foo', data={ + 'type': 'CNAME', 'ttl': 60, 'value': 'default.unit.tests.', + 'octodns': { + 'healthcheck': { + 'path': '/_ping', + 'port': 4443, + 'protocol': 'HTTPS', + } + }, + }) + + monitor = _get_monitor(record) + self.assertEqual(monitor.protocol, 'HTTPS') + self.assertEqual(monitor.port, 4443) + self.assertEqual(monitor.path, '/_ping') + headers = monitor.custom_headers + self.assertIsInstance(headers, list) + self.assertEquals(len(headers), 1) + headers = headers[0] + self.assertEqual(headers.name, 'Host') + self.assertEqual(headers.value, record.healthcheck_host) + + # test TCP monitor + record._octodns['healthcheck']['protocol'] = 'TCP' + monitor = _get_monitor(record) + self.assertEqual(monitor.protocol, 'TCP') + self.assertIsNone(monitor.custom_headers) + + +class Test_ProfileIsMatch(TestCase): + def test_profile_is_match(self): + is_match = _profile_is_match + + self.assertFalse(is_match(None, Profile())) + + # Profile object builder with default property values that can be + # overridden for testing below + def profile( + name = 'foo-unit-tests', + ttl = 60, + method = 'Geographic', + monitor_proto = 'HTTPS', + monitor_port = 4443, + monitor_path = '/_ping', + endpoints = 1, + endpoint_name = 'name', + endpoint_type = 'profile/nestedEndpoints', + target = 'target.unit.tests', + target_id = 'resource/id', + geos = ['GEO-AF'], + weight = 1, + priority = 1, + ): + dns = DnsConfig(ttl=ttl) + return Profile( + name=name, traffic_routing_method=method, dns_config=dns, + monitor_config=MonitorConfig( + protocol=monitor_proto, + port=monitor_port, + path=monitor_path, + ), + endpoints=[Endpoint( + name=endpoint_name, + type=endpoint_type, + target=target, + target_resource_id=target_id, + geo_mapping=geos, + weight=weight, + priority=priority, + )] + [Endpoint()] * (endpoints - 1), + ) + + self.assertTrue(is_match(profile(), profile())) + + self.assertFalse(is_match(profile(), profile(name='two'))) + self.assertFalse(is_match(profile(), profile(endpoints=2))) + self.assertFalse(is_match(profile(), profile(monitor_proto='HTTP'))) + self.assertFalse(is_match(profile(), profile(endpoint_name='a'))) + self.assertFalse(is_match(profile(), profile(endpoint_type='b'))) + self.assertFalse( + is_match(profile(endpoint_type='b'), profile(endpoint_type='b')) + ) + self.assertFalse(is_match(profile(), profile(target_id='rsrc/id2'))) + self.assertFalse(is_match(profile(), profile(geos=['IN']))) + + def wprofile(**kwargs): + kwargs['method'] = 'Weighted' + kwargs['endpoint_type'] = 'profile/externalEndpoints' + return profile(**kwargs) - self.assertEquals(_check_for_alias(alias_record), True) + self.assertFalse(is_match(wprofile(), wprofile(target='bar.unit'))) + self.assertFalse(is_match(wprofile(), wprofile(weight=3))) class TestAzureDnsProvider(TestCase): def _provider(self): return self._get_provider('mock_spc', 'mock_dns_client') + @patch('octodns.provider.azuredns.TrafficManagerManagementClient') @patch('octodns.provider.azuredns.DnsManagementClient') @patch('octodns.provider.azuredns.ClientSecretCredential') - def _get_provider(self, mock_spc, mock_dns_client): + @patch('octodns.provider.azuredns.ServicePrincipalCredentials') + def _get_provider(self, mock_spc, mock_css, mock_dns_client, + mock_tm_client): '''Returns a mock AzureProvider object to use in testing. :param mock_spc: placeholder :type mock_spc: str :param mock_dns_client: placeholder :type mock_dns_client: str + :param mock_tm_client: placeholder + :type mock_tm_client: str :type return: AzureProvider ''' provider = AzureProvider('mock_id', 'mock_client', 'mock_key', 'mock_directory', 'mock_sub', 'mock_rg' ) + # Fetch the client to force it to load the creds provider._dns_client + + # set critical functions to return properly + tm_list = provider._tm_client.profiles.list_by_resource_group + tm_list.return_value = [] + tm_sync = provider._tm_client.profiles.create_or_update + + def side_effect(rg, name, profile): + return profile + + tm_sync.side_effect = side_effect + return provider + def _get_dynamic_record(self, zone): + return Record.new(zone, 'foo', data={ + 'type': 'CNAME', + 'ttl': 60, + 'value': 'default.unit.tests.', + 'dynamic': { + 'pools': { + 'one': { + 'values': [ + {'value': 'one.unit.tests.', 'weight': 11}, + ], + 'fallback': 'two', + }, + 'two': { + 'values': [ + {'value': 'two1.unit.tests.', 'weight': 3}, + {'value': 'two2.unit.tests.', 'weight': 4}, + ], + 'fallback': 'three', + }, + 'three': { + 'values': [ + {'value': 'three.unit.tests.', 'weight': 13}, + ], + }, + }, + 'rules': [ + {'geos': ['AF', 'EU-DE', 'NA-US-CA'], 'pool': 'one'}, + {'pool': 'three'}, + ], + }, + 'octodns': { + 'healthcheck': { + 'path': '/_ping', + 'port': 4443, + 'protocol': 'HTTPS', + } + }, + }) + + def _get_tm_profiles(self, provider): + sub = provider._dns_client_subscription_id + rg = provider._resource_group + base_id = '/subscriptions/' + sub + \ + '/resourceGroups/' + rg + \ + '/providers/Microsoft.Network/trafficManagerProfiles/' + suffix = 'foo-unit-tests' + id_format = base_id + '{}--' + suffix + name_format = '{}--' + suffix + + dns = DnsConfig(ttl=60) + header = MonitorConfigCustomHeadersItem(name='Host', + value='foo.unit.tests') + monitor = MonitorConfig(protocol='HTTPS', port=4443, path='/_ping', + custom_headers=[header]) + external = 'Microsoft.Network/trafficManagerProfiles/externalEndpoints' + nested = 'Microsoft.Network/trafficManagerProfiles/nestedEndpoints' + + return [ + Profile( + id=id_format.format('default'), + name=name_format.format('default'), + traffic_routing_method='Weighted', + dns_config=dns, + monitor_config=monitor, + endpoints=[ + Endpoint( + name='default.unit.tests', + type=external, + target='default.unit.tests', + weight=1, + ), + ], + ), + Profile( + id=id_format.format('pool-one'), + name=name_format.format('pool-one'), + traffic_routing_method='Weighted', + dns_config=dns, + monitor_config=monitor, + endpoints=[ + Endpoint( + name='one.unit.tests', + type=external, + target='one.unit.tests', + weight=11, + ), + ], + ), + Profile( + id=id_format.format('pool-two'), + name=name_format.format('pool-two'), + traffic_routing_method='Weighted', + dns_config=dns, + monitor_config=monitor, + endpoints=[ + Endpoint( + name='two1.unit.tests', + type=external, + target='two1.unit.tests', + weight=3, + ), + Endpoint( + name='two2.unit.tests', + type=external, + target='two2.unit.tests', + weight=4, + ), + ], + ), + Profile( + id=id_format.format('pool-three'), + name=name_format.format('pool-three'), + traffic_routing_method='Weighted', + dns_config=dns, + monitor_config=monitor, + endpoints=[ + Endpoint( + name='three.unit.tests', + type=external, + target='three.unit.tests', + weight=13, + ), + ], + ), + Profile( + id=id_format.format('rule-one'), + name=name_format.format('rule-one'), + traffic_routing_method='Priority', + dns_config=dns, + monitor_config=monitor, + endpoints=[ + Endpoint( + name='one', + type=nested, + target_resource_id=id_format.format('pool-one'), + priority=1, + ), + Endpoint( + name='two', + type=nested, + target_resource_id=id_format.format('pool-two'), + priority=2, + ), + Endpoint( + name='three', + type=nested, + target_resource_id=id_format.format('pool-three'), + priority=3, + ), + Endpoint( + name='--default--', + type=nested, + target_resource_id=id_format.format('default'), + priority=4, + ), + ], + ), + Profile( + id=id_format.format('rule-three'), + name=name_format.format('rule-three'), + traffic_routing_method='Priority', + dns_config=dns, + monitor_config=monitor, + endpoints=[ + Endpoint( + name='three', + type=nested, + target_resource_id=id_format.format('pool-three'), + priority=1, + ), + Endpoint( + name='--default--', + type=nested, + target_resource_id=id_format.format('default'), + priority=2, + ), + ], + ), + Profile( + id=base_id + suffix, + name=suffix, + traffic_routing_method='Geographic', + dns_config=dns, + monitor_config=monitor, + endpoints=[ + Endpoint( + geo_mapping=['GEO-AF', 'DE', 'US-CA'], + name='rule-one', + type=nested, + target_resource_id=id_format.format('rule-one'), + ), + Endpoint( + geo_mapping=['WORLD'], + name='rule-three', + type=nested, + target_resource_id=id_format.format('rule-three'), + ), + ], + ), + ] + + def _get_dynamic_package(self): + '''Convenience function to setup a sample dynamic record. + ''' + provider = self._get_provider() + + # setup traffic manager profiles + tm_list = provider._tm_client.profiles.list_by_resource_group + tm_list.return_value = self._get_tm_profiles(provider) + + # setup zone with dynamic record + zone = Zone(name='unit.tests.', sub_zones=[]) + record = self._get_dynamic_record(zone) + zone.add_record(record) + + # return everything + return provider, zone, record + def test_populate_records(self): provider = self._get_provider() @@ -510,6 +880,121 @@ class TestAzureDnsProvider(TestCase): self.assertEquals(len(zone.records), 17) self.assertTrue(exists) + def test_populate_dynamic(self): + # Middle east without Asia raises exception + provider, zone, record = self._get_dynamic_package() + tm_suffix = _traffic_manager_suffix(record) + tm_id = provider._profile_name_to_id + tm_list = provider._tm_client.profiles.list_by_resource_group + rule_name = 'rule-one--{}'.format(tm_suffix) + nested = 'Microsoft.Network/trafficManagerProfiles/nestedEndpoints' + tm_list.return_value = [ + Profile( + id=tm_id(tm_suffix), + name=tm_suffix, + traffic_routing_method='Geographic', + endpoints=[ + Endpoint( + geo_mapping=['GEO-ME'], + ), + ], + ), + ] + azrecord = RecordSet( + ttl=60, + target_resource=SubResource(id=tm_id(tm_suffix)), + ) + azrecord.name = record.name or '@' + azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) + with self.assertRaises(AzureException) as ctx: + provider._populate_record(zone, azrecord) + self.assertTrue(text_type(ctx).startswith( + 'Middle East (GEO-ME) is not supported' + )) + + # empty priority profile raises exception + provider, zone, record = self._get_dynamic_package() + tm_list = provider._tm_client.profiles.list_by_resource_group + rule_name = 'rule-one--{}'.format(tm_suffix) + nested = 'Microsoft.Network/trafficManagerProfiles/nestedEndpoints' + tm_list.return_value = [ + Profile( + id=tm_id(rule_name), + name=rule_name, + traffic_routing_method='Priority', + endpoints=[], + ), + Profile( + id=tm_id(tm_suffix), + name=tm_suffix, + traffic_routing_method='Geographic', + endpoints=[ + Endpoint( + geo_mapping=['WORLD'], + name='rule-one', + type=nested, + target_resource_id=tm_id(rule_name), + ), + ], + ), + ] + with self.assertRaises(AzureException) as ctx: + provider._populate_record(zone, azrecord) + self.assertTrue(text_type(ctx).startswith( + 'Expected at least 2 endpoints' + )) + + # valid set of profiles produce expected dynamic record + provider, zone, record = self._get_dynamic_package() + root_profile_id = provider._profile_name_to_id( + _traffic_manager_suffix(record) + ) + azrecord = RecordSet( + ttl=60, + target_resource=SubResource(id=root_profile_id), + ) + azrecord.name = record.name or '@' + azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) + + record = provider._populate_record(zone, azrecord) + self.assertEqual(record.name, 'foo') + self.assertEqual(record.ttl, 60) + self.assertEqual(record.value, 'default.unit.tests.') + self.assertEqual(record.dynamic._data(), { + 'pools': { + 'one': { + 'values': [ + {'value': 'one.unit.tests.', 'weight': 11}, + ], + 'fallback': 'two', + }, + 'two': { + 'values': [ + {'value': 'two1.unit.tests.', 'weight': 3}, + {'value': 'two2.unit.tests.', 'weight': 4}, + ], + 'fallback': 'three', + }, + 'three': { + 'values': [ + {'value': 'three.unit.tests.', 'weight': 13}, + ], + 'fallback': None, + }, + }, + 'rules': [ + {'geos': ['AF', 'EU-DE', 'NA-US-CA'], 'pool': 'one'}, + {'pool': 'three'}, + ], + }) + + # valid profiles with Middle East test case + geo_profile = provider._get_tm_for_dynamic_record(record) + geo_profile.endpoints[0].geo_mapping.extend(['GEO-ME', 'GEO-AS']) + record = provider._populate_record(zone, azrecord) + self.assertIn('AS', record.dynamic.rules[0].data['geos']) + self.assertNotIn('ME', record.dynamic.rules[0].data['geos']) + def test_populate_zone(self): provider = self._get_provider() @@ -541,20 +1026,388 @@ class TestAzureDnsProvider(TestCase): None ) + def test_extra_changes(self): + provider, existing, record = self._get_dynamic_package() + + # test simple records produce no extra changes + desired = Zone(name=existing.name, sub_zones=[]) + desired.add_record(Record.new(desired, 'simple', data={ + 'type': record._type, + 'ttl': record.ttl, + 'value': record.value, + })) + extra = provider._extra_changes(desired, desired, []) + self.assertEqual(len(extra), 0) + + # test an unchanged dynamic record produces no extra changes + desired.add_record(record) + extra = provider._extra_changes(existing, desired, []) + self.assertEqual(len(extra), 0) + + # test unused TM produces the extra change for clean up + sample_profile = self._get_tm_profiles(provider)[0] + tm_id = provider._profile_name_to_id + root_profile_name = _traffic_manager_suffix(record) + extra_profile = Profile( + id=tm_id('random--{}'.format(root_profile_name)), + name='random--{}'.format(root_profile_name), + traffic_routing_method='Weighted', + dns_config=sample_profile.dns_config, + monitor_config=sample_profile.monitor_config, + endpoints=sample_profile.endpoints, + ) + tm_list = provider._tm_client.profiles.list_by_resource_group + tm_list.return_value.append(extra_profile) + provider._populate_traffic_managers() + extra = provider._extra_changes(existing, desired, []) + self.assertEqual(len(extra), 1) + extra = extra[0] + self.assertIsInstance(extra, Update) + self.assertEqual(extra.new, record) + desired._remove_record(record) + tm_list.return_value.pop() + + # test new dynamic record does not produce an extra change for it + new_dynamic = Record.new(desired, record.name + '2', data={ + 'type': record._type, + 'ttl': record.ttl, + 'value': record.value, + 'dynamic': record.dynamic._data(), + 'octodns': record._octodns, + }) + # test change in healthcheck by using a different port number + update_dynamic = Record.new(desired, record.name, data={ + 'type': record._type, + 'ttl': record.ttl, + 'value': record.value, + 'dynamic': record.dynamic._data(), + 'octodns': { + 'healthcheck': { + 'path': '/_ping', + 'port': 443, + 'protocol': 'HTTPS', + }, + }, + }) + desired.add_record(new_dynamic) + desired.add_record(update_dynamic) + changes = [Create(new_dynamic)] + extra = provider._extra_changes(existing, desired, changes) + # implicitly asserts that new_dynamic was not added to extra changes + # as it was already in the `changes` list + self.assertEqual(len(extra), 1) + extra = extra[0] + self.assertIsInstance(extra, Update) + self.assertEqual(extra.new, update_dynamic) + + # test non-CNAME dynamic record throws exception + a_dynamic = Record.new(desired, record.name + '3', data={ + 'type': 'A', + 'ttl': record.ttl, + 'values': ['1.1.1.1'], + 'dynamic': { + 'pools': { + 'one': {'values': [{'value': '2.2.2.2'}]}, + }, + 'rules': [ + {'pool': 'one'}, + ], + }, + }) + desired.add_record(a_dynamic) + changes.append(Create(a_dynamic)) + with self.assertRaises(AzureException): + provider._extra_changes(existing, desired, changes) + + def test_generate_tm_profile(self): + provider, zone, record = self._get_dynamic_package() + profile_gen = provider._generate_tm_profile + + name = 'foobar' + routing = 'Priority' + endpoints = [ + Endpoint(target='one.unit.tests'), + Endpoint(target_resource_id='/s/1/rg/foo/tm/foobar2'), + Endpoint(name='invalid'), + ] + + # invalid endpoint raises exception + with self.assertRaises(AzureException): + profile_gen(name, routing, endpoints, record) + + # regular test + endpoints.pop() + profile = profile_gen(name, routing, endpoints, record) + + # implicitly tests _profile_name_to_id + sub = provider._dns_client_subscription_id + rg = provider._resource_group + expected_id = '/subscriptions/' + sub + \ + '/resourceGroups/' + rg + \ + '/providers/Microsoft.Network/trafficManagerProfiles/' + name + self.assertEqual(profile.id, expected_id) + self.assertEqual(profile.name, name) + self.assertEqual(profile.traffic_routing_method, routing) + self.assertEqual(profile.dns_config.ttl, record.ttl) + self.assertEqual(len(profile.endpoints), len(endpoints)) + + self.assertEqual( + profile.endpoints[0].type, + 'Microsoft.Network/trafficManagerProfiles/externalEndpoints' + ) + self.assertEqual( + profile.endpoints[1].type, + 'Microsoft.Network/trafficManagerProfiles/nestedEndpoints' + ) + + def test_generate_traffic_managers(self): + provider, zone, record = self._get_dynamic_package() + profiles = provider._generate_traffic_managers(record) + deduped = [] + seen = set() + for profile in profiles: + if profile.name not in seen: + deduped.append(profile) + seen.add(profile.name) + + # check that every profile is a match with what we expect + expected_profiles = self._get_tm_profiles(provider) + self.assertEqual(len(expected_profiles), len(deduped)) + for have, expected in zip(deduped, expected_profiles): + self.assertTrue(_profile_is_match(have, expected)) + + # check Asia/Middle East test case + record.dynamic._data()['rules'][0]['geos'].append('AS') + profiles = provider._generate_traffic_managers(record) + geo_profile_name = _traffic_manager_suffix(record) + geo_profile = next( + profile + for profile in profiles + if profile.name == geo_profile_name + ) + self.assertIn('GEO-ME', geo_profile.endpoints[0].geo_mapping) + self.assertIn('GEO-AS', geo_profile.endpoints[0].geo_mapping) + + def test_sync_traffic_managers(self): + provider, zone, record = self._get_dynamic_package() + provider._populate_traffic_managers() + + tm_sync = provider._tm_client.profiles.create_or_update + + suffix = 'foo-unit-tests' + expected_seen = { + suffix, 'default--{}'.format(suffix), + 'rule-one--{}'.format(suffix), 'rule-three--{}'.format(suffix), + 'pool-one--{}'.format(suffix), 'pool-two--{}'.format(suffix), + 'pool-three--{}'.format(suffix), + } + + # test no change + seen = provider._sync_traffic_managers(record) + self.assertEqual(seen, expected_seen) + tm_sync.assert_not_called() + + # test that changing weight causes update API call + dynamic = record.dynamic._data() + dynamic['pools']['one']['values'][0]['weight'] = 14 + data = { + 'type': 'CNAME', + 'ttl': record.ttl, + 'value': record.value, + 'dynamic': dynamic, + 'octodns': record._octodns, + } + new_record = Record.new(zone, record.name, data) + tm_sync.reset_mock() + seen2 = provider._sync_traffic_managers(new_record) + self.assertEqual(seen2, expected_seen) + tm_sync.assert_called_once() + + # test that new profile was successfully inserted in cache + new_profile = provider._get_tm_profile_by_name( + 'pool-one--{}'.format(suffix) + ) + self.assertEqual(new_profile.endpoints[0].weight, 14) + + def test_find_traffic_managers(self): + provider, zone, record = self._get_dynamic_package() + + # insert a non-matching profile + sample_profile = self._get_tm_profiles(provider)[0] + # dummy record for generating suffix + record2 = Record.new(zone, record.name + '2', data={ + 'type': record._type, + 'ttl': record.ttl, + 'value': record.value, + }) + suffix2 = _traffic_manager_suffix(record2) + tm_id = provider._profile_name_to_id + extra_profile = Profile( + id=tm_id('random--{}'.format(suffix2)), + name='random--{}'.format(suffix2), + traffic_routing_method='Weighted', + dns_config=sample_profile.dns_config, + monitor_config=sample_profile.monitor_config, + endpoints=sample_profile.endpoints, + ) + tm_list = provider._tm_client.profiles.list_by_resource_group + tm_list.return_value.append(extra_profile) + provider._populate_traffic_managers() + + # implicitly asserts that non-matching profile is not included + suffix = _traffic_manager_suffix(record) + self.assertEqual(provider._find_traffic_managers(record), { + suffix, 'default--{}'.format(suffix), + 'rule-one--{}'.format(suffix), 'rule-three--{}'.format(suffix), + 'pool-one--{}'.format(suffix), 'pool-two--{}'.format(suffix), + 'pool-three--{}'.format(suffix), + }) + + def test_traffic_manager_gc(self): + provider, zone, record = self._get_dynamic_package() + provider._populate_traffic_managers() + + profiles = provider._find_traffic_managers(record) + profile_delete_mock = provider._tm_client.profiles.delete + + provider._traffic_managers_gc(record, profiles) + profile_delete_mock.assert_not_called() + + profile_delete_mock.reset_mock() + remove = list(profiles)[3] + profiles.discard(remove) + + provider._traffic_managers_gc(record, profiles) + profile_delete_mock.assert_has_calls( + [call(provider._resource_group, remove)] + ) + def test_apply(self): provider = self._get_provider() - changes = [] - deletes = [] - for i in octo_records: - changes.append(Create(i)) - deletes.append(Delete(i)) + half = int(len(octo_records) / 2) + changes = [Create(r) for r in octo_records[:half]] + \ + [Update(r, r) for r in octo_records[half:]] + deletes = [Delete(r) for r in octo_records] self.assertEquals(19, provider.apply(Plan(None, zone, changes, True))) self.assertEquals(19, provider.apply(Plan(zone, zone, deletes, True))) + def test_apply_create_dynamic(self): + provider = self._get_provider() + + tm_list = provider._tm_client.profiles.list_by_resource_group + tm_list.return_value = [] + + tm_sync = provider._tm_client.profiles.create_or_update + + zone = Zone(name='unit.tests.', sub_zones=[]) + record = self._get_dynamic_record(zone) + + profiles = self._get_tm_profiles(provider) + + provider._apply_Create(Create(record)) + # create was called as many times as number of profiles required for + # the dynamic record + self.assertEqual(tm_sync.call_count, len(profiles)) + + create = provider._dns_client.record_sets.create_or_update + create.assert_called_once() + + def test_apply_update_dynamic(self): + # existing is simple, new is dynamic + provider = self._get_provider() + tm_list = provider._tm_client.profiles.list_by_resource_group + tm_list.return_value = [] + profiles = self._get_tm_profiles(provider) + dynamic_record = self._get_dynamic_record(zone) + simple_record = Record.new(zone, dynamic_record.name, data={ + 'type': 'CNAME', + 'ttl': 3600, + 'value': 'cname.unit.tests.', + }) + change = Update(simple_record, dynamic_record) + provider._apply_Update(change) + tm_sync, dns_update, tm_delete = ( + provider._tm_client.profiles.create_or_update, + provider._dns_client.record_sets.create_or_update, + provider._tm_client.profiles.delete + ) + self.assertEqual(tm_sync.call_count, len(profiles)) + dns_update.assert_called_once() + tm_delete.assert_not_called() + + # existing is dynamic, new is simple + provider, existing, dynamic_record = self._get_dynamic_package() + profiles = self._get_tm_profiles(provider) + change = Update(dynamic_record, simple_record) + provider._apply_Update(change) + tm_sync, dns_update, tm_delete = ( + provider._tm_client.profiles.create_or_update, + provider._dns_client.record_sets.create_or_update, + provider._tm_client.profiles.delete + ) + tm_sync.assert_not_called() + dns_update.assert_called_once() + self.assertEqual(tm_delete.call_count, len(profiles)) + + # both are dynamic, healthcheck port is changed + provider, existing, dynamic_record = self._get_dynamic_package() + profiles = self._get_tm_profiles(provider) + dynamic_record2 = self._get_dynamic_record(existing) + dynamic_record2._octodns['healthcheck']['port'] += 1 + change = Update(dynamic_record, dynamic_record2) + provider._apply_Update(change) + tm_sync, dns_update, tm_delete = ( + provider._tm_client.profiles.create_or_update, + provider._dns_client.record_sets.create_or_update, + provider._tm_client.profiles.delete + ) + self.assertEqual(tm_sync.call_count, len(profiles)) + dns_update.assert_not_called() + tm_delete.assert_not_called() + + # both are dynamic, extra profile should be deleted + provider, existing, dynamic_record = self._get_dynamic_package() + sample_profile = self._get_tm_profiles(provider)[0] + tm_id = provider._profile_name_to_id + root_profile_name = _traffic_manager_suffix(dynamic_record) + extra_profile = Profile( + id=tm_id('random--{}'.format(root_profile_name)), + name='random--{}'.format(root_profile_name), + traffic_routing_method='Weighted', + dns_config=sample_profile.dns_config, + monitor_config=sample_profile.monitor_config, + endpoints=sample_profile.endpoints, + ) + tm_list = provider._tm_client.profiles.list_by_resource_group + tm_list.return_value.append(extra_profile) + change = Update(dynamic_record, dynamic_record) + provider._apply_Update(change) + tm_sync, dns_update, tm_delete = ( + provider._tm_client.profiles.create_or_update, + provider._dns_client.record_sets.create_or_update, + provider._tm_client.profiles.delete + ) + tm_sync.assert_not_called() + dns_update.assert_not_called() + tm_delete.assert_called_once() + + def test_apply_delete_dynamic(self): + provider, existing, record = self._get_dynamic_package() + provider._populate_traffic_managers() + profiles = self._get_tm_profiles(provider) + change = Delete(record) + provider._apply_Delete(change) + dns_delete, tm_delete = ( + provider._dns_client.record_sets.delete, + provider._tm_client.profiles.delete + ) + dns_delete.assert_called_once() + self.assertEqual(tm_delete.call_count, len(profiles)) + def test_create_zone(self): provider = self._get_provider() From 72b0438b7fb120e1155bf9934ddde34e9f28e267 Mon Sep 17 00:00:00 2001 From: Steven Hollingsworth Date: Wed, 12 May 2021 09:40:06 -0700 Subject: [PATCH 220/358] Updated doc to reflect config file location for zone wide lenient flag --- docs/records.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/records.md b/docs/records.md index e39a85d..c6b2a77 100644 --- a/docs/records.md +++ b/docs/records.md @@ -114,8 +114,7 @@ If you'd like to enable lenience for a whole zone you can do so with the followi ```yaml non-compliant-zone.com.: - octodns: - lenient: true + lenient: true sources: - route53 targets: From 9b5c8be01ece38e740fb7ee77aefec783812c852 Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Wed, 12 May 2021 10:02:04 -0700 Subject: [PATCH 221/358] optimize by not creating traffic manager for single-value pools If single-value pools have a weight defined, it will be lost by this optimization. Next time octodns-sync is run, it will show an update for setting the weight on remote. To overcome this, this commit includes a change to Record object that ignores the weight in single-value pools. --- octodns/provider/azuredns.py | 82 +++++++++--------- octodns/record/__init__.py | 4 + tests/test_octodns_provider_azuredns.py | 105 +++++++----------------- tests/test_octodns_record.py | 1 + 4 files changed, 78 insertions(+), 114 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 4d3dcc5..a792b9b 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -667,14 +667,9 @@ class AzureProvider(BaseProvider): for rule_ep in rule_endpoints: pool_name = rule_ep.name - # third (and last) level weighted RR profile - # these should be leaf node profiles with no further nesting - pool_profile = rule_ep.target_resource - # last/default pool if pool_name == '--default--': - for pool_ep in pool_profile.endpoints: - default.add(pool_ep.target) + default.add(rule_ep.target) # this should be the last one, so let's break here break @@ -687,11 +682,20 @@ class AzureProvider(BaseProvider): pool['fallback'] = pool_name pool = pools[pool_name] - for pool_ep in pool_profile.endpoints: + endpoints = [] + # these should be leaf node entries with no further nesting + if rule_ep.target_resource_id: + # third (and last) level weighted RR profile + endpoints = rule_ep.target_resource.endpoints + else: + # single-value pool + endpoints = [rule_ep] + + for pool_ep in endpoints: val = pool_ep.target value_dict = { 'value': _check_endswith_dot(val), - 'weight': pool_ep.weight, + 'weight': pool_ep.weight or 1, } if value_dict not in pool['values']: pool['values'].append(value_dict) @@ -703,6 +707,7 @@ class AzureProvider(BaseProvider): geo_ep.target_resource.name, len(rule_endpoints) ) raise AzureException(msg) + rules.append(rule) # Order and convert to a list @@ -800,19 +805,6 @@ class AzureProvider(BaseProvider): tm_suffix = _traffic_manager_suffix(record) profile = self._generate_tm_profile - # construct the default pool that will be used at the end of - # all rules - target = record.value[:-1] - default_endpoints = [Endpoint( - name=target, - target=target, - weight=1, - )] - default_profile_name = 'default--{}'.format(tm_suffix) - default_profile = profile(default_profile_name, 'Weighted', - default_endpoints, record) - traffic_managers.append(default_profile) - geo_endpoints = [] for rule in record.dynamic.rules: @@ -824,26 +816,36 @@ class AzureProvider(BaseProvider): # iterate until we reach end of fallback chain pool = pools[pool_name].data profile_name = 'pool-{}--{}'.format(pool_name, tm_suffix) - endpoints = [] - for val in pool['values']: - target = val['value'] - # strip trailing dot from CNAME value - target = target[:-1] - endpoints.append(Endpoint( - name=target, + if len(pool['values']) > 1: + # create Weighted profile for multi-value pool + endpoints = [] + for val in pool['values']: + target = val['value'] + # strip trailing dot from CNAME value + target = target[:-1] + endpoints.append(Endpoint( + name=target, + target=target, + weight=val.get('weight', 1), + )) + pool_profile = profile(profile_name, 'Weighted', endpoints, + record) + traffic_managers.append(pool_profile) + + # append pool to endpoint list of fallback rule profile + rule_endpoints.append(Endpoint( + name=pool_name, + target_resource_id=pool_profile.id, + priority=priority, + )) + else: + # add single-value pool as an external endpoint + target = pool['values'][0]['value'][:-1] + rule_endpoints.append(Endpoint( + name=pool_name, target=target, - weight=val.get('weight', 1), + priority=priority, )) - pool_profile = profile(profile_name, 'Weighted', endpoints, - record) - traffic_managers.append(pool_profile) - - # append pool to endpoint list of fallback rule profile - rule_endpoints.append(Endpoint( - name=pool_name, - target_resource_id=pool_profile.id, - priority=priority, - )) priority += 1 pool_name = pool.get('fallback') @@ -851,7 +853,7 @@ class AzureProvider(BaseProvider): # append default profile to the end rule_endpoints.append(Endpoint( name='--default--', - target_resource_id=default_profile.id, + target=record.value[:-1], priority=priority, )) # create rule profile with fallback chain diff --git a/octodns/record/__init__.py b/octodns/record/__init__.py index 135baf8..cc503d6 100644 --- a/octodns/record/__init__.py +++ b/octodns/record/__init__.py @@ -429,6 +429,10 @@ class _DynamicPool(object): ] values.sort(key=lambda d: d['value']) + # normalize weight of a single-value pool + if len(values) == 1: + values[0]['weight'] = 1 + fallback = data.get('fallback', None) self.data = { 'fallback': fallback if fallback != 'default' else None, diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index 5655719..fe867fe 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -574,7 +574,7 @@ class TestAzureDnsProvider(TestCase): }, 'rules': [ {'geos': ['AF', 'EU-DE', 'NA-US-CA'], 'pool': 'one'}, - {'pool': 'three'}, + {'pool': 'two'}, ], }, 'octodns': { @@ -605,36 +605,6 @@ class TestAzureDnsProvider(TestCase): nested = 'Microsoft.Network/trafficManagerProfiles/nestedEndpoints' return [ - Profile( - id=id_format.format('default'), - name=name_format.format('default'), - traffic_routing_method='Weighted', - dns_config=dns, - monitor_config=monitor, - endpoints=[ - Endpoint( - name='default.unit.tests', - type=external, - target='default.unit.tests', - weight=1, - ), - ], - ), - Profile( - id=id_format.format('pool-one'), - name=name_format.format('pool-one'), - traffic_routing_method='Weighted', - dns_config=dns, - monitor_config=monitor, - endpoints=[ - Endpoint( - name='one.unit.tests', - type=external, - target='one.unit.tests', - weight=11, - ), - ], - ), Profile( id=id_format.format('pool-two'), name=name_format.format('pool-two'), @@ -656,21 +626,6 @@ class TestAzureDnsProvider(TestCase): ), ], ), - Profile( - id=id_format.format('pool-three'), - name=name_format.format('pool-three'), - traffic_routing_method='Weighted', - dns_config=dns, - monitor_config=monitor, - endpoints=[ - Endpoint( - name='three.unit.tests', - type=external, - target='three.unit.tests', - weight=13, - ), - ], - ), Profile( id=id_format.format('rule-one'), name=name_format.format('rule-one'), @@ -680,8 +635,8 @@ class TestAzureDnsProvider(TestCase): endpoints=[ Endpoint( name='one', - type=nested, - target_resource_id=id_format.format('pool-one'), + type=external, + target='one.unit.tests', priority=1, ), Endpoint( @@ -692,37 +647,43 @@ class TestAzureDnsProvider(TestCase): ), Endpoint( name='three', - type=nested, - target_resource_id=id_format.format('pool-three'), + type=external, + target='three.unit.tests', priority=3, ), Endpoint( name='--default--', - type=nested, - target_resource_id=id_format.format('default'), + type=external, + target='default.unit.tests', priority=4, ), ], ), Profile( - id=id_format.format('rule-three'), - name=name_format.format('rule-three'), + id=id_format.format('rule-two'), + name=name_format.format('rule-two'), traffic_routing_method='Priority', dns_config=dns, monitor_config=monitor, endpoints=[ Endpoint( - name='three', + name='two', type=nested, - target_resource_id=id_format.format('pool-three'), + target_resource_id=id_format.format('pool-two'), priority=1, ), Endpoint( - name='--default--', - type=nested, - target_resource_id=id_format.format('default'), + name='three', + type=external, + target='three.unit.tests', priority=2, ), + Endpoint( + name='--default--', + type=external, + target='default.unit.tests', + priority=3, + ), ], ), Profile( @@ -740,9 +701,9 @@ class TestAzureDnsProvider(TestCase): ), Endpoint( geo_mapping=['WORLD'], - name='rule-three', + name='rule-two', type=nested, - target_resource_id=id_format.format('rule-three'), + target_resource_id=id_format.format('rule-two'), ), ], ), @@ -964,7 +925,7 @@ class TestAzureDnsProvider(TestCase): 'pools': { 'one': { 'values': [ - {'value': 'one.unit.tests.', 'weight': 11}, + {'value': 'one.unit.tests.', 'weight': 1}, ], 'fallback': 'two', }, @@ -977,14 +938,14 @@ class TestAzureDnsProvider(TestCase): }, 'three': { 'values': [ - {'value': 'three.unit.tests.', 'weight': 13}, + {'value': 'three.unit.tests.', 'weight': 1}, ], 'fallback': None, }, }, 'rules': [ {'geos': ['AF', 'EU-DE', 'NA-US-CA'], 'pool': 'one'}, - {'pool': 'three'}, + {'pool': 'two'}, ], }) @@ -1196,10 +1157,8 @@ class TestAzureDnsProvider(TestCase): suffix = 'foo-unit-tests' expected_seen = { - suffix, 'default--{}'.format(suffix), - 'rule-one--{}'.format(suffix), 'rule-three--{}'.format(suffix), - 'pool-one--{}'.format(suffix), 'pool-two--{}'.format(suffix), - 'pool-three--{}'.format(suffix), + suffix, 'pool-two--{}'.format(suffix), + 'rule-one--{}'.format(suffix), 'rule-two--{}'.format(suffix), } # test no change @@ -1209,7 +1168,7 @@ class TestAzureDnsProvider(TestCase): # test that changing weight causes update API call dynamic = record.dynamic._data() - dynamic['pools']['one']['values'][0]['weight'] = 14 + dynamic['pools']['two']['values'][0]['weight'] = 14 data = { 'type': 'CNAME', 'ttl': record.ttl, @@ -1225,7 +1184,7 @@ class TestAzureDnsProvider(TestCase): # test that new profile was successfully inserted in cache new_profile = provider._get_tm_profile_by_name( - 'pool-one--{}'.format(suffix) + 'pool-two--{}'.format(suffix) ) self.assertEqual(new_profile.endpoints[0].weight, 14) @@ -1257,10 +1216,8 @@ class TestAzureDnsProvider(TestCase): # implicitly asserts that non-matching profile is not included suffix = _traffic_manager_suffix(record) self.assertEqual(provider._find_traffic_managers(record), { - suffix, 'default--{}'.format(suffix), - 'rule-one--{}'.format(suffix), 'rule-three--{}'.format(suffix), - 'pool-one--{}'.format(suffix), 'pool-two--{}'.format(suffix), - 'pool-three--{}'.format(suffix), + suffix, 'pool-two--{}'.format(suffix), + 'rule-one--{}'.format(suffix), 'rule-two--{}'.format(suffix), }) def test_traffic_manager_gc(self): diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index ce40b9b..3bb5d24 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -3013,6 +3013,7 @@ class TestDynamicRecords(TestCase): 'pools': { 'one': { 'values': [{ + 'weight': 10, 'value': '3.3.3.3', }], }, From 5df2077ed02124487b7a69be024ddb2a0e6a9386 Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Fri, 14 May 2021 23:21:26 -0700 Subject: [PATCH 222/358] clarify the limitations and caveats of azure dynamic records --- README.md | 2 +- octodns/provider/azuredns.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 8d59cd0..2ce9445 100644 --- a/README.md +++ b/README.md @@ -192,7 +192,7 @@ The above command pulled the existing data out of Route53 and placed the results | Provider | Requirements | Record Support | Dynamic | Notes | |--|--|--|--|--| -| [AzureProvider](/octodns/provider/azuredns.py) | azure-identity, azure-mgmt-dns, azure-mgmt-trafficmanager | A, AAAA, CAA, CNAME, MX, NS, PTR, SRV, TXT | Yes (CNAMEs only) | | +| [AzureProvider](/octodns/provider/azuredns.py) | azure-identity, azure-mgmt-dns, azure-mgmt-trafficmanager | A, AAAA, CAA, CNAME, MX, NS, PTR, SRV, TXT | Alpha (CNAMEs only) | | | [Akamai](/octodns/provider/edgedns.py) | edgegrid-python | A, AAAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, SSHFP, TXT | No | | | [CloudflareProvider](/octodns/provider/cloudflare.py) | | A, AAAA, ALIAS, CAA, CNAME, LOC, MX, NS, PTR, SPF, SRV, TXT | No | CAA tags restricted | | [ConstellixProvider](/octodns/provider/constellix.py) | | A, AAAA, ALIAS (ANAME), CAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | CAA tags restricted | diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index a792b9b..8474014 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -387,6 +387,9 @@ class AzureProvider(BaseProvider): The first four variables above can be hidden in environment variables and octoDNS will automatically search for them in the shell. It is possible to also hard-code into the config file: eg, resource_group. + + Please read https://github.com/octodns/octodns/pull/706 for an overview + of how dynamic records are designed and caveats of using them. ''' SUPPORTS_GEO = False SUPPORTS_DYNAMIC = True From 1b5bf75c585a3a79c72ee655cbb383b8b962df72 Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Mon, 17 May 2021 13:16:00 -0700 Subject: [PATCH 223/358] drop method names from exceptions --- octodns/provider/azuredns.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 8474014..5c2d23c 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -644,8 +644,8 @@ class AzureProvider(BaseProvider): # Throw exception otherwise, it should not happen if the # profile was generated by octoDNS if 'GEO-AS' not in geo_map: - msg = '_data_for_dynamic: Profile={}: '.format( - geo_profile.name) + msg = 'Profile={} for record {}: '.format( + geo_profile.name, azrecord.fqdn) msg += 'Middle East (GEO-ME) is not supported by ' + \ 'octoDNS. It needs to be either paired ' + \ 'with Asia (GEO-AS) or expanded into ' + \ @@ -782,9 +782,9 @@ class AzureProvider(BaseProvider): elif ep.target: ep.type = endpoint_type_prefix + 'externalEndpoints' else: - msg = ('_generate_tm_profile: Invalid endpoint {} ' + - 'in profile {}, needs to have either target or ' + - 'target_resource_id').format(ep.name, name) + msg = ('Invalid endpoint {} in profile {}, needs to have' + + 'either target or target_resource_id').format( + ep.name, name) raise AzureException(msg) # build and return @@ -1032,7 +1032,7 @@ class AzureProvider(BaseProvider): if getattr(record, 'dynamic', False): self._traffic_managers_gc(record, set()) - self.log.debug('* Success Delete: {}'.format(ar)) + self.log.debug('* Success Delete: {}'.format(record)) def _apply(self, plan): '''Required function of manager.py to actually apply a record change. From cca288faa6463e2abfd572612ff1209e8d1053c5 Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Mon, 17 May 2021 13:19:43 -0700 Subject: [PATCH 224/358] log warning when non-1 weight is set for single-value pools --- octodns/record/__init__.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/octodns/record/__init__.py b/octodns/record/__init__.py index cc503d6..45cea51 100644 --- a/octodns/record/__init__.py +++ b/octodns/record/__init__.py @@ -417,6 +417,7 @@ class _ValueMixin(object): class _DynamicPool(object): + log = getLogger('_DynamicPool') def __init__(self, _id, data): self._id = _id @@ -431,7 +432,12 @@ class _DynamicPool(object): # normalize weight of a single-value pool if len(values) == 1: - values[0]['weight'] = 1 + weight = data['values'][0].get('weight', 1) + if weight != 1: + self.log.warn( + 'Using weight=1 instead of %s for single-value pool %s', + weight, _id) + values[0]['weight'] = 1 fallback = data.get('fallback', None) self.data = { From e1d262a3013942e2a614622ce6d819deab3909c9 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 17 May 2021 17:06:40 -0700 Subject: [PATCH 225/358] Add a validation requiring single value weight=1 --- octodns/record/__init__.py | 4 ++++ tests/test_octodns_record.py | 29 ++++++++++++++++++++++++++++- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/octodns/record/__init__.py b/octodns/record/__init__.py index 135baf8..f960998 100644 --- a/octodns/record/__init__.py +++ b/octodns/record/__init__.py @@ -573,6 +573,10 @@ class _DynamicMixin(object): reasons.append('missing value in pool "{}" ' 'value {}'.format(_id, value_num)) + if len(values) == 1 and values[0].get('weight', 1) != 1: + reasons.append('pool "{}" has single value with ' + 'weight!=1'.format(_id)) + fallback = pool.get('fallback', None) if fallback is not None: if fallback in pools: diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index ce40b9b..5a4704a 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -3412,7 +3412,7 @@ class TestDynamicRecords(TestCase): self.assertEquals(['pool "one" is missing values'], ctx.exception.reasons) - # pool valu not a dict + # pool value not a dict a_data = { 'dynamic': { 'pools': { @@ -3596,6 +3596,33 @@ class TestDynamicRecords(TestCase): self.assertEquals(['invalid weight "foo" in pool "three" value 2'], ctx.exception.reasons) + # single value with weight!=1 + a_data = { + 'dynamic': { + 'pools': { + 'one': { + 'values': [{ + 'weight': 12, + 'value': '6.6.6.6', + }], + }, + }, + 'rules': [{ + 'pool': 'one', + }], + }, + 'ttl': 60, + 'type': 'A', + 'values': [ + '1.1.1.1', + '2.2.2.2', + ], + } + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, 'bad', a_data) + self.assertEquals(['pool "one" has single value with weight!=1'], + ctx.exception.reasons) + # invalid fallback a_data = { 'dynamic': { From 753a337ecc604d837f018ac7f29313fd2e083b36 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 17 May 2021 17:49:31 -0700 Subject: [PATCH 226/358] Fix weights on new Azure dynamic record tests --- tests/test_octodns_provider_azuredns.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index fe867fe..609fc5c 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -555,7 +555,7 @@ class TestAzureDnsProvider(TestCase): 'pools': { 'one': { 'values': [ - {'value': 'one.unit.tests.', 'weight': 11}, + {'value': 'one.unit.tests.', 'weight': 1}, ], 'fallback': 'two', }, @@ -568,7 +568,7 @@ class TestAzureDnsProvider(TestCase): }, 'three': { 'values': [ - {'value': 'three.unit.tests.', 'weight': 13}, + {'value': 'three.unit.tests.', 'weight': 1}, ], }, }, From b02d5d0a2de8ff61271458da9d570283e325aff3 Mon Sep 17 00:00:00 2001 From: Tom Kaminski Date: Mon, 17 May 2021 19:29:10 -0500 Subject: [PATCH 227/358] Do not trigger change for health checks on cname dynamic records --- octodns/provider/route53.py | 11 +- tests/test_octodns_provider_route53.py | 157 +++++++++++++++++++++++++ 2 files changed, 167 insertions(+), 1 deletion(-) diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index 0d5bab9..a3b3a57 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -1253,7 +1253,16 @@ class Route53Provider(BaseProvider): return self._gen_mods('DELETE', existing_records, existing_rrsets) def _extra_changes_update_needed(self, record, rrset): - healthcheck_host = record.healthcheck_host + value = rrset['ResourceRecords'][0]['Value'] + + try: + ip_address(text_type(value)) + # We're working with an IP + healthcheck_host = record.healthcheck_host + except (AddressValueError, ValueError): + # This isn't an IP, host is the value + healthcheck_host = value + healthcheck_path = record.healthcheck_path healthcheck_protocol = record.healthcheck_protocol healthcheck_port = record.healthcheck_port diff --git a/tests/test_octodns_provider_route53.py b/tests/test_octodns_provider_route53.py index a2b61e7..1e7210d 100644 --- a/tests/test_octodns_provider_route53.py +++ b/tests/test_octodns_provider_route53.py @@ -1962,6 +1962,163 @@ class TestRoute53Provider(TestCase): self.assertEquals(1, len(extra)) stubber.assert_no_pending_responses() + def test_extra_change_dynamic_has_health_check_cname(self): + provider, stubber = self._get_stubbed_provider() + + list_hosted_zones_resp = { + 'HostedZones': [{ + 'Name': 'unit.tests.', + 'Id': 'z42', + 'CallerReference': 'abc', + }], + 'Marker': 'm', + 'IsTruncated': False, + 'MaxItems': '100', + } + stubber.add_response('list_hosted_zones', list_hosted_zones_resp, {}) + + # record with geo and no health check returns change + desired = Zone('unit.tests.', []) + record = Record.new(desired, 'cname', { + 'ttl': 30, + 'type': 'CNAME', + 'value': 'cname.unit.tests.', + 'dynamic': { + 'pools': { + 'one': { + 'values': [{ + 'value': 'one.cname.unit.tests.', + }], + }, + }, + 'rules': [{ + 'pool': 'one', + }], + }, + }) + desired.add_record(record) + list_resource_record_sets_resp = { + 'ResourceRecordSets': [{ + # Not dynamic value and other name + 'Name': 'unit.tests.', + 'Type': 'CNAME', + 'GeoLocation': { + 'CountryCode': '*', + }, + 'ResourceRecords': [{ + 'Value': 'cname.unit.tests.', + }], + 'TTL': 61, + # All the non-matches have a different Id so we'll fail if they + # match + 'HealthCheckId': '33', + }, { + # Not dynamic value, matching name, other type + 'Name': 'cname.unit.tests.', + 'Type': 'AAAA', + 'ResourceRecords': [{ + 'Value': '2001:0db8:3c4d:0015:0000:0000:1a2f:1a4b' + }], + 'TTL': 61, + 'HealthCheckId': '33', + }, { + # default value pool + 'Name': '_octodns-default-value.cname.unit.tests.', + 'Type': 'CNAME', + 'GeoLocation': { + 'CountryCode': '*', + }, + 'ResourceRecords': [{ + 'Value': 'cname.unit.tests.', + }], + 'TTL': 61, + 'HealthCheckId': '33', + }, { + # different record + 'Name': '_octodns-two-value.other.unit.tests.', + 'Type': 'CNAME', + 'GeoLocation': { + 'CountryCode': '*', + }, + 'ResourceRecords': [{ + 'Value': 'cname.unit.tests.', + }], + 'TTL': 61, + 'HealthCheckId': '33', + }, { + # same everything, but different type + 'Name': '_octodns-one-value.cname.unit.tests.', + 'Type': 'AAAA', + 'ResourceRecords': [{ + 'Value': '2001:0db8:3c4d:0015:0000:0000:1a2f:1a4b' + }], + 'TTL': 61, + 'HealthCheckId': '33', + }, { + # same everything, sub + 'Name': '_octodns-one-value.sub.cname.unit.tests.', + 'Type': 'CNAME', + 'ResourceRecords': [{ + 'Value': 'cname.unit.tests.', + }], + 'TTL': 61, + 'HealthCheckId': '33', + }, { + # match + 'Name': '_octodns-one-value.cname.unit.tests.', + 'Type': 'CNAME', + 'ResourceRecords': [{ + 'Value': 'one.cname.unit.tests.', + }], + 'TTL': 61, + 'HealthCheckId': '42', + }], + 'IsTruncated': False, + 'MaxItems': '100', + } + stubber.add_response('list_resource_record_sets', + list_resource_record_sets_resp, + {'HostedZoneId': 'z42'}) + + stubber.add_response('list_health_checks', { + 'HealthChecks': [{ + 'Id': '42', + 'CallerReference': self.caller_ref, + 'HealthCheckConfig': { + 'Type': 'HTTPS', + 'FullyQualifiedDomainName': 'one.cname.unit.tests.', + 'ResourcePath': '/_dns', + 'Type': 'HTTPS', + 'Port': 443, + 'MeasureLatency': True, + 'RequestInterval': 10, + }, + 'HealthCheckVersion': 2, + }], + 'IsTruncated': False, + 'MaxItems': '100', + 'Marker': '', + }) + extra = provider._extra_changes(desired=desired, changes=[]) + self.assertEquals(0, len(extra)) + stubber.assert_no_pending_responses() + + # change b/c of healthcheck path + record._octodns['healthcheck'] = { + 'path': '/_ready' + } + extra = provider._extra_changes(desired=desired, changes=[]) + self.assertEquals(1, len(extra)) + stubber.assert_no_pending_responses() + + # no change b/c healthcheck host ignored for dynamic cname + record._octodns['healthcheck'] = { + 'host': 'foo.bar.io' + } + extra = provider._extra_changes(desired=desired, changes=[]) + self.assertEquals(0, len(extra)) + stubber.assert_no_pending_responses() + def _get_test_plan(self, max_changes): provider = Route53Provider('test', 'abc', '123', max_changes) From 5f57b52d07f5a75777e36837a76828c3af9c48a3 Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Tue, 18 May 2021 10:12:59 -0700 Subject: [PATCH 228/358] map Oceania to Australia/Pacific in Azure --- octodns/provider/azuredns.py | 21 +++++++++++++++++++-- tests/test_octodns_provider_azuredns.py | 6 +++--- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 5c2d23c..8eeba02 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -655,12 +655,22 @@ class AzureProvider(BaseProvider): geos = rule.setdefault('geos', []) for code in geo_map: if code.startswith('GEO-'): - geos.append(code[len('GEO-'):]) + # continent + if code == 'GEO-AP': + # Azure uses Australia/Pacific (AP) instead of + # Oceania https://docs.microsoft.com/en-us/azure/ + # traffic-manager/ + # traffic-manager-geographic-regions + geos.append('OC') + else: + geos.append(code[len('GEO-'):]) elif '-' in code: + # state country, province = code.split('-', 1) country = GeoCodes.country_to_code(country) geos.append('{}-{}'.format(country, province)) else: + # country geos.append(GeoCodes.country_to_code(code)) # second level priority profile @@ -872,15 +882,22 @@ class AzureProvider(BaseProvider): if len(rule_geos) > 0: for geo in rule_geos: if '-' in geo: + # country or state geos.append(geo.split('-', 1)[-1]) else: - geos.append('GEO-{}'.format(geo)) + # continent if geo == 'AS': # Middle East is part of Asia in octoDNS, but # Azure treats it as a separate "group", so let's # add it in the list of geo mappings. We will drop # it when we later parse the list of regions. geos.append('GEO-ME') + elif geo == 'OC': + # Azure uses Australia/Pacific (AP) instead of + # Oceania + geo = 'AP' + + geos.append('GEO-{}'.format(geo)) else: geos.append('WORLD') geo_endpoints.append(Endpoint( diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index 609fc5c..e8e5281 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -573,7 +573,7 @@ class TestAzureDnsProvider(TestCase): }, }, 'rules': [ - {'geos': ['AF', 'EU-DE', 'NA-US-CA'], 'pool': 'one'}, + {'geos': ['AF', 'EU-DE', 'NA-US-CA', 'OC'], 'pool': 'one'}, {'pool': 'two'}, ], }, @@ -694,7 +694,7 @@ class TestAzureDnsProvider(TestCase): monitor_config=monitor, endpoints=[ Endpoint( - geo_mapping=['GEO-AF', 'DE', 'US-CA'], + geo_mapping=['GEO-AF', 'DE', 'US-CA', 'GEO-AP'], name='rule-one', type=nested, target_resource_id=id_format.format('rule-one'), @@ -944,7 +944,7 @@ class TestAzureDnsProvider(TestCase): }, }, 'rules': [ - {'geos': ['AF', 'EU-DE', 'NA-US-CA'], 'pool': 'one'}, + {'geos': ['AF', 'EU-DE', 'NA-US-CA', 'OC'], 'pool': 'one'}, {'pool': 'two'}, ], }) From 62122e442931cf30f7b98cd91280419a3c58fac7 Mon Sep 17 00:00:00 2001 From: Tom Kaminski Date: Thu, 20 May 2021 15:24:34 -0500 Subject: [PATCH 229/358] More explicit check for CNAME records when checking for extra updates --- octodns/provider/route53.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index a3b3a57..ad59eb0 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -1253,15 +1253,11 @@ class Route53Provider(BaseProvider): return self._gen_mods('DELETE', existing_records, existing_rrsets) def _extra_changes_update_needed(self, record, rrset): - value = rrset['ResourceRecords'][0]['Value'] - - try: - ip_address(text_type(value)) - # We're working with an IP + if record._type == 'CNAME': + # For CNAME, healthcheck host by default points to the CNAME value + healthcheck_host = rrset['ResourceRecords'][0]['Value'] + else: healthcheck_host = record.healthcheck_host - except (AddressValueError, ValueError): - # This isn't an IP, host is the value - healthcheck_host = value healthcheck_path = record.healthcheck_path healthcheck_protocol = record.healthcheck_protocol From 6ac368c488f41857b5fe4ac1a686d4de4db8fe5f Mon Sep 17 00:00:00 2001 From: "Yaroshevich, Denis" Date: Mon, 24 May 2021 15:38:31 +0300 Subject: [PATCH 230/358] rrset names may include ending dot, may not since new API update --- octodns/provider/gcore.py | 19 ++++--------------- tests/fixtures/gcore-no-changes.json | 6 +++--- tests/fixtures/gcore-records.json | 4 ++-- tests/fixtures/gcore-zone.json | 2 +- tests/test_octodns_provider_gcore.py | 3 +++ 5 files changed, 13 insertions(+), 21 deletions(-) diff --git a/octodns/provider/gcore.py b/octodns/provider/gcore.py index 041e19f..d5c7fa0 100644 --- a/octodns/provider/gcore.py +++ b/octodns/provider/gcore.py @@ -217,9 +217,7 @@ class GCoreProvider(BaseProvider): _type = record["type"].upper() if _type not in self.SUPPORTS: continue - rr_name = record["name"].replace(zone.name, "") - if len(rr_name) > 0 and rr_name.endswith("."): - rr_name = rr_name[:-1] + rr_name = zone.hostname_from_fqdn(record["name"]) values[rr_name][_type] = record before = len(zone.records) @@ -256,27 +254,24 @@ class GCoreProvider(BaseProvider): def _apply_create(self, change): self.log.info("creating: %s", change) new = change.new - rrset_name = self._build_rrset_name(new) data = getattr(self, "_params_for_{}".format(new._type))(new) self._client.record_create( - new.zone.name[:-1], rrset_name, new._type, data + new.zone.name[:-1], new.fqdn, new._type, data ) def _apply_update(self, change): self.log.info("updating: %s", change) new = change.new - rrset_name = self._build_rrset_name(new) data = getattr(self, "_params_for_{}".format(new._type))(new) self._client.record_update( - new.zone.name[:-1], rrset_name, new._type, data + new.zone.name[:-1], new.fqdn, new._type, data ) def _apply_delete(self, change): self.log.info("deleting: %s", change) existing = change.existing - rrset_name = self._build_rrset_name(existing) self._client.record_delete( - existing.zone.name[:-1], rrset_name, existing._type + existing.zone.name[:-1], existing.fqdn, existing._type ) def _apply(self, plan): @@ -299,9 +294,3 @@ class GCoreProvider(BaseProvider): for change in changes: class_name = change.__class__.__name__ getattr(self, "_apply_{}".format(class_name.lower()))(change) - - @staticmethod - def _build_rrset_name(record): - if len(record.name) > 0: - return "{}.{}".format(record.name, record.zone.name) - return record.zone.name diff --git a/tests/fixtures/gcore-no-changes.json b/tests/fixtures/gcore-no-changes.json index a70fb82..e5ff8c9 100644 --- a/tests/fixtures/gcore-no-changes.json +++ b/tests/fixtures/gcore-no-changes.json @@ -1,6 +1,6 @@ { "rrsets": [{ - "name": "unit.tests.", + "name": "unit.tests", "type": "A", "ttl": 300, "resource_records": [{ @@ -13,7 +13,7 @@ ] }] }, { - "name": "aaaa.unit.tests.", + "name": "aaaa.unit.tests", "type": "AAAA", "ttl": 600, "resource_records": [{ @@ -40,7 +40,7 @@ ] }] }, { - "name": "unit.tests.", + "name": "unit.tests", "type": "ns", "ttl": 300, "resource_records": [{ diff --git a/tests/fixtures/gcore-records.json b/tests/fixtures/gcore-records.json index 4d4685e..4086049 100644 --- a/tests/fixtures/gcore-records.json +++ b/tests/fixtures/gcore-records.json @@ -1,6 +1,6 @@ { "rrsets": [{ - "name": "unit.tests.", + "name": "unit.tests", "type": "A", "ttl": 300, "resource_records": [{ @@ -9,7 +9,7 @@ ] }] }, { - "name": "unit.tests.", + "name": "unit.tests", "type": "ns", "ttl": 300, "resource_records": [{ diff --git a/tests/fixtures/gcore-zone.json b/tests/fixtures/gcore-zone.json index 7f70275..925af72 100644 --- a/tests/fixtures/gcore-zone.json +++ b/tests/fixtures/gcore-zone.json @@ -11,7 +11,7 @@ "records": [ { "id": 12419, - "name": "unit.test.", + "name": "unit.test", "type": "ns", "ttl": 300, "short_answers": [ diff --git a/tests/test_octodns_provider_gcore.py b/tests/test_octodns_provider_gcore.py index aa536f1..06e199e 100644 --- a/tests/test_octodns_provider_gcore.py +++ b/tests/test_octodns_provider_gcore.py @@ -117,6 +117,9 @@ class TestGCoreProvider(TestCase): zone = Zone("unit.tests.", []) provider.populate(zone) self.assertEquals(4, len(zone.records)) + self.assertEquals( + {"aaaa", "www", "www.sub", ""}, {r.name for r in zone.records} + ) changes = self.expected.changes(zone, provider) self.assertEquals(0, len(changes)) From eb14873abbd6acbf08ce8ee08eaf70aa691343a7 Mon Sep 17 00:00:00 2001 From: Sham Date: Mon, 24 May 2021 19:01:17 -0700 Subject: [PATCH 231/358] Allow the option to not pass Host header in healthchecks --- octodns/provider/ns1.py | 4 +++- octodns/provider/route53.py | 2 +- tests/test_octodns_provider_ns1.py | 4 ++++ tests/test_octodns_provider_route53.py | 30 ++++++++++++++++++++++++++ 4 files changed, 38 insertions(+), 2 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index a910459..8408682 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -929,7 +929,9 @@ class Ns1Provider(BaseProvider): if record.healthcheck_protocol != 'TCP': # IF it's HTTP we need to send the request string path = record.healthcheck_path - host = record.healthcheck_host + # if host header is explicitly set to null in the yaml, + # just pass the domain (value) as the host header + host = record.healthcheck_host or value request = r'GET {path} HTTP/1.0\r\nHost: {host}\r\n' \ r'User-agent: NS1\r\n\r\n'.format(path=path, host=host) ret['config']['send'] = request diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index ad59eb0..0331847 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -1131,7 +1131,7 @@ class Route53Provider(BaseProvider): 'Type': healthcheck_protocol, } if healthcheck_protocol != 'TCP': - config['FullyQualifiedDomainName'] = healthcheck_host + config['FullyQualifiedDomainName'] = healthcheck_host or value config['ResourcePath'] = healthcheck_path if value: config['IPAddress'] = value diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index f54c7bd..4ae4757 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -757,6 +757,10 @@ class TestNs1ProviderDynamic(TestCase): self.assertFalse(monitor['config']['ssl']) self.assertEquals('host:unit.tests type:A', monitor['notes']) + record._octodns['healthcheck']['host'] = None + monitor = provider._monitor_gen(record, value) + self.assertTrue(r'\nHost: 3.4.5.6\r' in monitor['config']['send']) + record._octodns['healthcheck']['protocol'] = 'HTTPS' monitor = provider._monitor_gen(record, value) self.assertTrue(monitor['config']['ssl']) diff --git a/tests/test_octodns_provider_route53.py b/tests/test_octodns_provider_route53.py index 1e7210d..1bf3332 100644 --- a/tests/test_octodns_provider_route53.py +++ b/tests/test_octodns_provider_route53.py @@ -1166,6 +1166,31 @@ class TestRoute53Provider(TestCase): }) stubber.add_response('change_tags_for_resource', {}) + health_check_config = { + 'EnableSNI': False, + 'FailureThreshold': 6, + 'FullyQualifiedDomainName': '4.2.3.4', + 'IPAddress': '4.2.3.4', + 'MeasureLatency': True, + 'Port': 8080, + 'RequestInterval': 10, + 'ResourcePath': '/_status', + 'Type': 'HTTP' + } + stubber.add_response('create_health_check', { + 'HealthCheck': { + 'Id': '43', + 'CallerReference': self.caller_ref, + 'HealthCheckConfig': health_check_config, + 'HealthCheckVersion': 1, + }, + 'Location': 'http://url', + }, { + 'CallerReference': ANY, + 'HealthCheckConfig': health_check_config, + }) + stubber.add_response('change_tags_for_resource', {}) + record = Record.new(self.expected, '', { 'ttl': 61, 'type': 'A', @@ -1191,6 +1216,11 @@ class TestRoute53Provider(TestCase): # when allowed to create we do id = provider.get_health_check_id(record, value, True) self.assertEquals('42', id) + + # when allowed to create and when host is None + record._octodns['healthcheck']['host'] = None + id = provider.get_health_check_id(record, value, True) + self.assertEquals('43', id) stubber.assert_no_pending_responses() # A CNAME style healthcheck, without a value From f00766a77909c216249b72dcb4396ced16909ac2 Mon Sep 17 00:00:00 2001 From: Stan Brinkerhoff Date: Wed, 26 May 2021 22:45:07 -0400 Subject: [PATCH 232/358] Add support for ALIAS type --- octodns/provider/ultra.py | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/octodns/provider/ultra.py b/octodns/provider/ultra.py index 03b70de..f65c519 100644 --- a/octodns/provider/ultra.py +++ b/octodns/provider/ultra.py @@ -36,12 +36,12 @@ class UltraProvider(BaseProvider): ''' Neustar UltraDNS provider - Documentation for Ultra REST API requires a login: - https://portal.ultradns.com/static/docs/REST-API_User_Guide.pdf - Implemented to the May 20, 2020 version of the document (dated on page ii) - Also described as Version 2.83.0 (title page) + Documentation for Ultra REST API: + https://ultra-portalstatic.ultradns.com/static/docs/REST-API_User_Guide.pdf + Implemented to the May 26, 2021 version of the document (dated on page ii) + Also described as Version 3.18.0 (title page) - Tested against 3.0.0-20200627220036.81047f5 + Tested against 3.20.1-20210521075351.36b9297 As determined by querying https://api.ultradns.com/version ultra: @@ -57,6 +57,7 @@ class UltraProvider(BaseProvider): RECORDS_TO_TYPE = { 'A (1)': 'A', 'AAAA (28)': 'AAAA', + 'APEXALIAS (65282)': 'ALIAS', 'CAA (257)': 'CAA', 'CNAME (5)': 'CNAME', 'MX (15)': 'MX', @@ -72,6 +73,7 @@ class UltraProvider(BaseProvider): SUPPORTS_GEO = False SUPPORTS_DYNAMIC = False TIMEOUT = 5 + ZONE_REQUEST_LIMIT = 1000 def _request(self, method, path, params=None, data=None, json=None, json_response=True): @@ -151,7 +153,7 @@ class UltraProvider(BaseProvider): def zones(self): if self._zones is None: offset = 0 - limit = 100 + limit = self.ZONE_REQUEST_LIMIT zones = [] paging = True while paging: @@ -401,8 +403,15 @@ class UltraProvider(BaseProvider): def _gen_data(self, record): zone_name = self._remove_prefix(record.fqdn, record.name + '.') + + # UltraDNS treats the `APEXALIAS` type as the octodns `ALIAS`. + if record._type == "ALIAS": + record_type = "APEXALIAS" + else: + record_type = record._type + path = '/v2/zones/{}/rrsets/{}/{}'.format(zone_name, - record._type, + record_type, record.fqdn) contents_for = getattr(self, '_contents_for_{}'.format(record._type)) return path, contents_for(record) @@ -444,7 +453,12 @@ class UltraProvider(BaseProvider): existing._type == self.RECORDS_TO_TYPE[record['rrtype']]: zone_name = self._remove_prefix(existing.fqdn, existing.name + '.') - path = '/v2/zones/{}/rrsets/{}/{}'.format(zone_name, + + # UltraDNS treats the `APEXALIAS` type as the octodns `ALIAS`. + if existing._type == "ALIAS": + existing._type = "APEXALIAS" + + path = '/v2/zones/{}/rrsets/{}/{ }'.format(zone_name, existing._type, existing.fqdn) self._delete(path, json_response=False) From 4b6fd8b4a1e19fe889943aab705ecf23b6c19495 Mon Sep 17 00:00:00 2001 From: sbrinkerhoff Date: Thu, 27 May 2021 17:59:10 -0400 Subject: [PATCH 233/358] Remove type mutation Co-authored-by: Ross McFarland --- octodns/provider/ultra.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/octodns/provider/ultra.py b/octodns/provider/ultra.py index f65c519..a2eae3c 100644 --- a/octodns/provider/ultra.py +++ b/octodns/provider/ultra.py @@ -455,10 +455,11 @@ class UltraProvider(BaseProvider): existing.name + '.') # UltraDNS treats the `APEXALIAS` type as the octodns `ALIAS`. - if existing._type == "ALIAS": - existing._type = "APEXALIAS" + existing_type = existing._type + if existing_type == "ALIAS": + existing_type = "APEXALIAS" path = '/v2/zones/{}/rrsets/{}/{ }'.format(zone_name, - existing._type, + existing_type, existing.fqdn) self._delete(path, json_response=False) From e816e8e49b6c467cdf6a911b9d0a27bd2c8424e6 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Thu, 27 May 2021 15:19:33 -0700 Subject: [PATCH 234/358] Fixed extraneous whitespace in format in UltraDNS --- octodns/provider/ultra.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/ultra.py b/octodns/provider/ultra.py index a2eae3c..55c2c34 100644 --- a/octodns/provider/ultra.py +++ b/octodns/provider/ultra.py @@ -459,7 +459,7 @@ class UltraProvider(BaseProvider): if existing_type == "ALIAS": existing_type = "APEXALIAS" - path = '/v2/zones/{}/rrsets/{}/{ }'.format(zone_name, + path = '/v2/zones/{}/rrsets/{}/{}'.format(zone_name, existing_type, existing.fqdn) self._delete(path, json_response=False) From 60f916cd2400a27f23fb365f90978e6968cc088a Mon Sep 17 00:00:00 2001 From: Stan Brinkerhoff Date: Fri, 28 May 2021 00:52:48 -0400 Subject: [PATCH 235/358] add contents for alias method, test for alias --- octodns/provider/ultra.py | 4 +++- tests/test_octodns_provider_ultra.py | 15 ++++++++++++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/octodns/provider/ultra.py b/octodns/provider/ultra.py index 55c2c34..bc855d4 100644 --- a/octodns/provider/ultra.py +++ b/octodns/provider/ultra.py @@ -73,7 +73,7 @@ class UltraProvider(BaseProvider): SUPPORTS_GEO = False SUPPORTS_DYNAMIC = False TIMEOUT = 5 - ZONE_REQUEST_LIMIT = 1000 + ZONE_REQUEST_LIMIT = 100 def _request(self, method, path, params=None, data=None, json=None, json_response=True): @@ -213,6 +213,7 @@ class UltraProvider(BaseProvider): _data_for_PTR = _data_for_single _data_for_CNAME = _data_for_single + _data_for_ALIAS = _data_for_single def _data_for_CAA(self, _type, records): return { @@ -376,6 +377,7 @@ class UltraProvider(BaseProvider): } _contents_for_PTR = _contents_for_CNAME + _contents_for_ALIAS = _contents_for_CNAME def _contents_for_SRV(self, record): return { diff --git a/tests/test_octodns_provider_ultra.py b/tests/test_octodns_provider_ultra.py index b6d1017..52e0307 100644 --- a/tests/test_octodns_provider_ultra.py +++ b/tests/test_octodns_provider_ultra.py @@ -274,7 +274,7 @@ class TestUltraProvider(TestCase): self.assertTrue(provider.populate(zone)) self.assertEquals('octodns1.test.', zone.name) - self.assertEquals(11, len(zone.records)) + self.assertEquals(12, len(zone.records)) self.assertEquals(4, mock.call_count) def test_apply(self): @@ -352,8 +352,8 @@ class TestUltraProvider(TestCase): })) plan = provider.plan(wanted) - self.assertEquals(10, len(plan.changes)) - self.assertEquals(10, provider.apply(plan)) + self.assertEquals(11, len(plan.changes)) + self.assertEquals(11, provider.apply(plan)) self.assertTrue(plan.exists) provider._request.assert_has_calls([ @@ -492,6 +492,15 @@ class TestUltraProvider(TestCase): Record.new(zone, 'txt', {'ttl': 60, 'type': 'TXT', 'values': ['abc', 'def']})), + + # ALIAS + ('', 'ALIAS', + '/v2/zones/unit.tests./rrsets/APEXALIAS/unit.tests.', + {'ttl': 60, 'rdata': ['target.unit.tests.']}, + Record.new(zone, '', + {'ttl': 60, 'type': 'ALIAS', + 'value': 'target.unit.tests.'})), + ): # Validate path and payload based on record meet expectations path, payload = provider._gen_data(expected_record) From 6e71f2df76d454578bac1a84bf8920058350743f Mon Sep 17 00:00:00 2001 From: Stan Brinkerhoff Date: Fri, 28 May 2021 00:54:57 -0400 Subject: [PATCH 236/358] update test record --- tests/fixtures/ultra-records-page-2.json | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/fixtures/ultra-records-page-2.json b/tests/fixtures/ultra-records-page-2.json index abdc44f..274d95e 100644 --- a/tests/fixtures/ultra-records-page-2.json +++ b/tests/fixtures/ultra-records-page-2.json @@ -32,7 +32,16 @@ "rdata": [ "www.octodns1.test." ] + }, + { + "ownerName": "host1.octodns1.test.", + "rrtype": "RRSET (70)", + "ttl": 3600, + "rdata": [ + "E3B0C44298FC1C149AFBF4C8996FB92427AE41E4649B934CA495991B7852B855" + ] } + ], "resultInfo": { "totalCount": 13, From e97675fe3ddaa02ec4baa145fc7254f9b616fc28 Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Fri, 28 May 2021 12:48:09 -0700 Subject: [PATCH 237/358] check dns config when comparing profiles for equality --- octodns/provider/azuredns.py | 10 +++++++++- tests/test_octodns_provider_azuredns.py | 20 +++++++++++++------- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 8eeba02..65d9967 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -284,10 +284,18 @@ def _profile_is_match(have, desired): # compare basic attributes if have.name != desired.name or \ have.traffic_routing_method != desired.traffic_routing_method or \ - have.dns_config.ttl != desired.dns_config.ttl or \ len(have.endpoints) != len(desired.endpoints): return False + # compare dns config + dns_have = have.dns_config + dns_desired = desired.dns_config + if dns_have.ttl != dns_desired.ttl or \ + dns_have.relative_name is None or \ + dns_desired.relative_name is None or \ + dns_have.relative_name != dns_desired.relative_name: + return False + # compare monitoring configuration monitor_have = have.monitor_config monitor_desired = desired.monitor_config diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index e8e5281..0a1c366 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -453,6 +453,7 @@ class Test_ProfileIsMatch(TestCase): name = 'foo-unit-tests', ttl = 60, method = 'Geographic', + dns_name = None, monitor_proto = 'HTTPS', monitor_port = 4443, monitor_path = '/_ping', @@ -465,7 +466,7 @@ class Test_ProfileIsMatch(TestCase): weight = 1, priority = 1, ): - dns = DnsConfig(ttl=ttl) + dns = DnsConfig(relative_name=(dns_name or name), ttl=ttl) return Profile( name=name, traffic_routing_method=method, dns_config=dns, monitor_config=MonitorConfig( @@ -488,6 +489,7 @@ class Test_ProfileIsMatch(TestCase): self.assertFalse(is_match(profile(), profile(name='two'))) self.assertFalse(is_match(profile(), profile(endpoints=2))) + self.assertFalse(is_match(profile(), profile(dns_name='two'))) self.assertFalse(is_match(profile(), profile(monitor_proto='HTTP'))) self.assertFalse(is_match(profile(), profile(endpoint_name='a'))) self.assertFalse(is_match(profile(), profile(endpoint_type='b'))) @@ -596,7 +598,6 @@ class TestAzureDnsProvider(TestCase): id_format = base_id + '{}--' + suffix name_format = '{}--' + suffix - dns = DnsConfig(ttl=60) header = MonitorConfigCustomHeadersItem(name='Host', value='foo.unit.tests') monitor = MonitorConfig(protocol='HTTPS', port=4443, path='/_ping', @@ -604,12 +605,12 @@ class TestAzureDnsProvider(TestCase): external = 'Microsoft.Network/trafficManagerProfiles/externalEndpoints' nested = 'Microsoft.Network/trafficManagerProfiles/nestedEndpoints' - return [ + profiles = [ Profile( id=id_format.format('pool-two'), name=name_format.format('pool-two'), traffic_routing_method='Weighted', - dns_config=dns, + dns_config=DnsConfig(ttl=60), monitor_config=monitor, endpoints=[ Endpoint( @@ -630,7 +631,7 @@ class TestAzureDnsProvider(TestCase): id=id_format.format('rule-one'), name=name_format.format('rule-one'), traffic_routing_method='Priority', - dns_config=dns, + dns_config=DnsConfig(ttl=60), monitor_config=monitor, endpoints=[ Endpoint( @@ -663,7 +664,7 @@ class TestAzureDnsProvider(TestCase): id=id_format.format('rule-two'), name=name_format.format('rule-two'), traffic_routing_method='Priority', - dns_config=dns, + dns_config=DnsConfig(ttl=60), monitor_config=monitor, endpoints=[ Endpoint( @@ -690,7 +691,7 @@ class TestAzureDnsProvider(TestCase): id=base_id + suffix, name=suffix, traffic_routing_method='Geographic', - dns_config=dns, + dns_config=DnsConfig(ttl=60), monitor_config=monitor, endpoints=[ Endpoint( @@ -709,6 +710,11 @@ class TestAzureDnsProvider(TestCase): ), ] + for profile in profiles: + profile.dns_config.relative_name = profile.name + + return profiles + def _get_dynamic_package(self): '''Convenience function to setup a sample dynamic record. ''' From 18c0e5675978507063178cf9a83622ae25fe054b Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Fri, 28 May 2021 12:59:27 -0700 Subject: [PATCH 238/358] log.debug the reason for profile mismatch --- octodns/provider/azuredns.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 65d9967..689088c 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -281,11 +281,20 @@ def _profile_is_match(have, desired): if have is None or desired is None: return False + log = logging.getLogger('azuredns._profile_is_match').debug + + def false(have, desired, name=None): + prefix = 'profile={}'.format(name) if name else '' + attr = have.__class__.__name__ + log('%s have.%s = %s', prefix, attr, have) + log('%s desired.%s = %s', prefix, attr, desired) + return False + # compare basic attributes if have.name != desired.name or \ have.traffic_routing_method != desired.traffic_routing_method or \ len(have.endpoints) != len(desired.endpoints): - return False + return false(have, desired) # compare dns config dns_have = have.dns_config @@ -294,7 +303,7 @@ def _profile_is_match(have, desired): dns_have.relative_name is None or \ dns_desired.relative_name is None or \ dns_have.relative_name != dns_desired.relative_name: - return False + return false(dns_have, dns_desired, have.name) # compare monitoring configuration monitor_have = have.monitor_config @@ -303,7 +312,7 @@ def _profile_is_match(have, desired): monitor_have.port != monitor_desired.port or \ monitor_have.path != monitor_desired.path or \ monitor_have.custom_headers != monitor_desired.custom_headers: - return False + return false(monitor_have, monitor_desired, have.name) # compare endpoints method = have.traffic_routing_method @@ -321,26 +330,26 @@ def _profile_is_match(have, desired): for have_endpoint, desired_endpoint in endpoints: if have_endpoint.name != desired_endpoint.name or \ have_endpoint.type != desired_endpoint.type: - return False + return false(have_endpoint, desired_endpoint, have.name) target_type = have_endpoint.type.split('/')[-1] if target_type == 'externalEndpoints': # compare value, weight, priority if have_endpoint.target != desired_endpoint.target: - return False + return false(have_endpoint, desired_endpoint, have.name) if method == 'Weighted' and \ have_endpoint.weight != desired_endpoint.weight: - return False + return false(have_endpoint, desired_endpoint, have.name) elif target_type == 'nestedEndpoints': # compare targets if have_endpoint.target_resource_id != \ desired_endpoint.target_resource_id: - return False + return false(have_endpoint, desired_endpoint, have.name) # compare geos if method == 'Geographic': have_geos = sorted(have_endpoint.geo_mapping) desired_geos = sorted(desired_endpoint.geo_mapping) if have_geos != desired_geos: - return False + return false(have_endpoint, desired_endpoint, have.name) else: # unexpected, give up return False From aaffdb1388a0569710ca5f0063236d977fdeb9f8 Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Fri, 28 May 2021 13:23:35 -0700 Subject: [PATCH 239/358] minimize Azure Traffic Manager hops --- octodns/provider/azuredns.py | 259 +++++++---- tests/test_octodns_provider_azuredns.py | 585 ++++++++++++++++++------ 2 files changed, 625 insertions(+), 219 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 689088c..777e15b 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -646,14 +646,23 @@ class AzureProvider(BaseProvider): pools = defaultdict(lambda: {'fallback': None, 'values': []}) rules = [] - # top level geo profile - geo_profile = self._get_tm_profile_by_id(azrecord.target_resource.id) - for geo_ep in geo_profile.endpoints: + # top level profile + root_profile = self._get_tm_profile_by_id(azrecord.target_resource.id) + if root_profile.traffic_routing_method != 'Geographic': + # This record does not use geo fencing, so we skip the Geographic + # profile hop; let's pretend to be a geo-profile's only endpoint + geo_ep = Endpoint(target_resource_id=root_profile.id) + geo_ep.target_resource = root_profile + endpoints = [geo_ep] + else: + endpoints = root_profile.endpoints + + for geo_ep in endpoints: rule = {} # resolve list of regions - geo_map = list(geo_ep.geo_mapping) - if geo_map != ['WORLD']: + geo_map = list(geo_ep.geo_mapping or []) + if geo_map and geo_map != ['WORLD']: if 'GEO-ME' in geo_map: # Azure treats Middle East as a separate group, but # its part of Asia in octoDNS, so we need to remove GEO-ME @@ -662,7 +671,7 @@ class AzureProvider(BaseProvider): # profile was generated by octoDNS if 'GEO-AS' not in geo_map: msg = 'Profile={} for record {}: '.format( - geo_profile.name, azrecord.fqdn) + root_profile.name, azrecord.fqdn) msg += 'Middle East (GEO-ME) is not supported by ' + \ 'octoDNS. It needs to be either paired ' + \ 'with Asia (GEO-AS) or expanded into ' + \ @@ -690,18 +699,35 @@ class AzureProvider(BaseProvider): # country geos.append(GeoCodes.country_to_code(code)) - # second level priority profile + # build fallback chain from second level priority profile + if geo_ep.target_resource_id: + target = geo_ep.target_resource + if target.traffic_routing_method == 'Priority': + rule_endpoints = target.endpoints + rule_endpoints.sort(key=lambda e: e.priority) + else: + # Weighted + geo_ep.name = target.endpoints[0].name.split('--', 1)[0] + rule_endpoints = [geo_ep] + else: + # this geo directly points to the default, so we skip the + # Priority profile hop and directly use an external endpoint; + # let's pretend to be a Priority profile's only endpoint + rule_endpoints = [geo_ep] + pool = None - rule_endpoints = geo_ep.target_resource.endpoints - rule_endpoints = sorted(rule_endpoints, key=lambda e: e.priority) for rule_ep in rule_endpoints: pool_name = rule_ep.name # last/default pool - if pool_name == '--default--': + if pool_name.endswith('--default--'): default.add(rule_ep.target) - # this should be the last one, so let's break here - break + if pool_name == '--default--': + # this should be the last one, so let's break here + break + # last pool is a single value pool and its value is same + # as record's default value + pool_name = pool_name[:-len('--default--')] # set first priority endpoint as the rule's primary pool if 'pool' not in rule: @@ -711,32 +737,32 @@ class AzureProvider(BaseProvider): # set current pool as fallback of the previous pool pool['fallback'] = pool_name + if pool_name in pools: + # we've already populated the pool + continue + + # populate the pool from Weighted profile + # these should be leaf node entries with no further nesting pool = pools[pool_name] endpoints = [] - # these should be leaf node entries with no further nesting + if rule_ep.target_resource_id: # third (and last) level weighted RR profile endpoints = rule_ep.target_resource.endpoints else: - # single-value pool + # single-value pool, so we skip the Weighted profile hop + # and directly use an external endpoint; let's pretend to + # be a Weighted profile's only endpoint endpoints = [rule_ep] for pool_ep in endpoints: val = pool_ep.target - value_dict = { + pool['values'].append({ 'value': _check_endswith_dot(val), 'weight': pool_ep.weight or 1, - } - if value_dict not in pool['values']: - pool['values'].append(value_dict) - - if 'pool' not in rule or not default: - # this will happen if the priority profile does not have - # enough endpoints - msg = 'Expected at least 2 endpoints in {}, got {}'.format( - geo_ep.target_resource.name, len(rule_endpoints) - ) - raise AzureException(msg) + }) + if pool_ep.name.endswith('--default--'): + default.add(val) rules.append(rule) @@ -809,7 +835,7 @@ class AzureProvider(BaseProvider): elif ep.target: ep.type = endpoint_type_prefix + 'externalEndpoints' else: - msg = ('Invalid endpoint {} in profile {}, needs to have' + + msg = ('Invalid endpoint {} in profile {}, needs to have ' + 'either target or target_resource_id').format( ep.name, name) raise AzureException(msg) @@ -828,39 +854,83 @@ class AzureProvider(BaseProvider): location='global', ) + def _update_tm_name(self, profile, new_name): + profile.name = new_name + profile.id = self._profile_name_to_id(new_name) + profile.dns_config.relative_name = new_name + + return profile + def _generate_traffic_managers(self, record): traffic_managers = [] pools = record.dynamic.pools + default = record.value[:-1] tm_suffix = _traffic_manager_suffix(record) profile = self._generate_tm_profile geo_endpoints = [] + pool_profiles = {} for rule in record.dynamic.rules: + # Prepare the list of Traffic manager geos + rule_geos = rule.data.get('geos', []) + geos = [] + for geo in rule_geos: + if '-' in geo: + # country/state + geos.append(geo.split('-', 1)[-1]) + else: + # continent + if geo == 'AS': + # Middle East is part of Asia in octoDNS, but + # Azure treats it as a separate "group", so let's + # add it in the list of geo mappings. We will drop + # it when we later parse the list of regions. + geos.append('GEO-ME') + elif geo == 'OC': + # Azure uses Australia/Pacific (AP) instead of + # Oceania + geo = 'AP' + + geos.append('GEO-{}'.format(geo)) + if not geos: + geos.append('WORLD') + pool_name = rule.data['pool'] rule_endpoints = [] priority = 1 + default_seen = False while pool_name: # iterate until we reach end of fallback chain + default_seen = False pool = pools[pool_name].data - profile_name = 'pool-{}--{}'.format(pool_name, tm_suffix) if len(pool['values']) > 1: # create Weighted profile for multi-value pool - endpoints = [] - for val in pool['values']: - target = val['value'] - # strip trailing dot from CNAME value - target = target[:-1] - endpoints.append(Endpoint( - name=target, - target=target, - weight=val.get('weight', 1), - )) - pool_profile = profile(profile_name, 'Weighted', endpoints, - record) - traffic_managers.append(pool_profile) + pool_profile = pool_profiles.get(pool_name) + if pool_profile is None: + endpoints = [] + for val in pool['values']: + target = val['value'] + # strip trailing dot from CNAME value + target = target[:-1] + ep_name = '{}--{}'.format(pool_name, target) + if target == default: + # mark default + ep_name += '--default--' + default_seen = True + endpoints.append(Endpoint( + name=ep_name, + target=target, + weight=val.get('weight', 1), + )) + profile_name = 'pool-{}--{}'.format( + pool_name, tm_suffix) + pool_profile = profile(profile_name, 'Weighted', + endpoints, record) + traffic_managers.append(pool_profile) + pool_profiles[pool_name] = pool_profile # append pool to endpoint list of fallback rule profile rule_endpoints.append(Endpoint( @@ -869,8 +939,15 @@ class AzureProvider(BaseProvider): priority=priority, )) else: - # add single-value pool as an external endpoint + # Skip Weighted profile hop for single-value pool + # append its value as an external endpoint to fallback + # rule profile target = pool['values'][0]['value'][:-1] + ep_name = pool_name + if target == default: + # mark default + ep_name += '--default--' + default_seen = True rule_endpoints.append(Endpoint( name=pool_name, target=target, @@ -880,51 +957,61 @@ class AzureProvider(BaseProvider): priority += 1 pool_name = pool.get('fallback') - # append default profile to the end - rule_endpoints.append(Endpoint( - name='--default--', - target=record.value[:-1], - priority=priority, - )) - # create rule profile with fallback chain - rule_profile_name = 'rule-{}--{}'.format(rule.data['pool'], - tm_suffix) - rule_profile = profile(rule_profile_name, 'Priority', - rule_endpoints, record) - traffic_managers.append(rule_profile) - - # append rule profile to top-level geo profile - rule_geos = rule.data.get('geos', []) - geos = [] - if len(rule_geos) > 0: - for geo in rule_geos: - if '-' in geo: - # country or state - geos.append(geo.split('-', 1)[-1]) - else: - # continent - if geo == 'AS': - # Middle East is part of Asia in octoDNS, but - # Azure treats it as a separate "group", so let's - # add it in the list of geo mappings. We will drop - # it when we later parse the list of regions. - geos.append('GEO-ME') - elif geo == 'OC': - # Azure uses Australia/Pacific (AP) instead of - # Oceania - geo = 'AP' - - geos.append('GEO-{}'.format(geo)) + # append default endpoint unless it is already included in + # last pool of rule profile + if not default_seen: + rule_endpoints.append(Endpoint( + name='--default--', + target=default, + priority=priority, + )) + + if len(rule_endpoints) > 1: + # create rule profile with fallback chain + rule_profile_name = 'rule-{}--{}'.format( + rule.data['pool'], tm_suffix) + rule_profile = profile(rule_profile_name, 'Priority', + rule_endpoints, record) + traffic_managers.append(rule_profile) + + # append rule profile to top-level geo profile + geo_endpoints.append(Endpoint( + name='rule-{}'.format(rule.data['pool']), + target_resource_id=rule_profile.id, + geo_mapping=geos, + )) else: - geos.append('WORLD') - geo_endpoints.append(Endpoint( - name='rule-{}'.format(rule.data['pool']), - target_resource_id=rule_profile.id, - geo_mapping=geos, - )) - - geo_profile = profile(tm_suffix, 'Geographic', geo_endpoints, record) - traffic_managers.append(geo_profile) + # Priority profile has only one endpoint; skip the hop and + # append its only endpoint to the top-level profile + rule_ep = rule_endpoints[0] + if rule_ep.target_resource_id: + # point directly to the Weighted pool profile + geo_endpoints.append(Endpoint( + name='rule-{}'.format(rule.data['pool']), + target_resource_id=rule_ep.target_resource_id, + geo_mapping=geos, + )) + else: + # just add the value of single-value pool + geo_endpoints.append(Endpoint( + name=rule_ep.name + '--default--', + target=rule_ep.target, + geo_mapping=geos, + )) + + if len(geo_endpoints) == 1 and \ + geo_endpoints[0].geo_mapping == ['WORLD'] and \ + geo_endpoints[0].target_resource_id: + # Single WORLD rule does not require a Geographic profile, use + # the target profile as the root profile + target_profile_id = geo_endpoints[0].target_resource_id + profile_map = dict((tm.id, tm) for tm in traffic_managers) + target_profile = profile_map[target_profile_id] + self._update_tm_name(target_profile, tm_suffix) + else: + geo_profile = profile(tm_suffix, 'Geographic', geo_endpoints, + record) + traffic_managers.append(geo_profile) return traffic_managers diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index 0a1c366..2c998b6 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -614,13 +614,13 @@ class TestAzureDnsProvider(TestCase): monitor_config=monitor, endpoints=[ Endpoint( - name='two1.unit.tests', + name='two--two1.unit.tests', type=external, target='two1.unit.tests', weight=3, ), Endpoint( - name='two2.unit.tests', + name='two--two2.unit.tests', type=external, target='two2.unit.tests', weight=4, @@ -847,121 +847,6 @@ class TestAzureDnsProvider(TestCase): self.assertEquals(len(zone.records), 17) self.assertTrue(exists) - def test_populate_dynamic(self): - # Middle east without Asia raises exception - provider, zone, record = self._get_dynamic_package() - tm_suffix = _traffic_manager_suffix(record) - tm_id = provider._profile_name_to_id - tm_list = provider._tm_client.profiles.list_by_resource_group - rule_name = 'rule-one--{}'.format(tm_suffix) - nested = 'Microsoft.Network/trafficManagerProfiles/nestedEndpoints' - tm_list.return_value = [ - Profile( - id=tm_id(tm_suffix), - name=tm_suffix, - traffic_routing_method='Geographic', - endpoints=[ - Endpoint( - geo_mapping=['GEO-ME'], - ), - ], - ), - ] - azrecord = RecordSet( - ttl=60, - target_resource=SubResource(id=tm_id(tm_suffix)), - ) - azrecord.name = record.name or '@' - azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) - with self.assertRaises(AzureException) as ctx: - provider._populate_record(zone, azrecord) - self.assertTrue(text_type(ctx).startswith( - 'Middle East (GEO-ME) is not supported' - )) - - # empty priority profile raises exception - provider, zone, record = self._get_dynamic_package() - tm_list = provider._tm_client.profiles.list_by_resource_group - rule_name = 'rule-one--{}'.format(tm_suffix) - nested = 'Microsoft.Network/trafficManagerProfiles/nestedEndpoints' - tm_list.return_value = [ - Profile( - id=tm_id(rule_name), - name=rule_name, - traffic_routing_method='Priority', - endpoints=[], - ), - Profile( - id=tm_id(tm_suffix), - name=tm_suffix, - traffic_routing_method='Geographic', - endpoints=[ - Endpoint( - geo_mapping=['WORLD'], - name='rule-one', - type=nested, - target_resource_id=tm_id(rule_name), - ), - ], - ), - ] - with self.assertRaises(AzureException) as ctx: - provider._populate_record(zone, azrecord) - self.assertTrue(text_type(ctx).startswith( - 'Expected at least 2 endpoints' - )) - - # valid set of profiles produce expected dynamic record - provider, zone, record = self._get_dynamic_package() - root_profile_id = provider._profile_name_to_id( - _traffic_manager_suffix(record) - ) - azrecord = RecordSet( - ttl=60, - target_resource=SubResource(id=root_profile_id), - ) - azrecord.name = record.name or '@' - azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) - - record = provider._populate_record(zone, azrecord) - self.assertEqual(record.name, 'foo') - self.assertEqual(record.ttl, 60) - self.assertEqual(record.value, 'default.unit.tests.') - self.assertEqual(record.dynamic._data(), { - 'pools': { - 'one': { - 'values': [ - {'value': 'one.unit.tests.', 'weight': 1}, - ], - 'fallback': 'two', - }, - 'two': { - 'values': [ - {'value': 'two1.unit.tests.', 'weight': 3}, - {'value': 'two2.unit.tests.', 'weight': 4}, - ], - 'fallback': 'three', - }, - 'three': { - 'values': [ - {'value': 'three.unit.tests.', 'weight': 1}, - ], - 'fallback': None, - }, - }, - 'rules': [ - {'geos': ['AF', 'EU-DE', 'NA-US-CA', 'OC'], 'pool': 'one'}, - {'pool': 'two'}, - ], - }) - - # valid profiles with Middle East test case - geo_profile = provider._get_tm_for_dynamic_record(record) - geo_profile.endpoints[0].geo_mapping.extend(['GEO-ME', 'GEO-AS']) - record = provider._populate_record(zone, azrecord) - self.assertIn('AS', record.dynamic.rules[0].data['geos']) - self.assertNotIn('ME', record.dynamic.rules[0].data['geos']) - def test_populate_zone(self): provider = self._get_provider() @@ -1114,6 +999,7 @@ class TestAzureDnsProvider(TestCase): '/providers/Microsoft.Network/trafficManagerProfiles/' + name self.assertEqual(profile.id, expected_id) self.assertEqual(profile.name, name) + self.assertEqual(profile.name, profile.dns_config.relative_name) self.assertEqual(profile.traffic_routing_method, routing) self.assertEqual(profile.dns_config.ttl, record.ttl) self.assertEqual(len(profile.endpoints), len(endpoints)) @@ -1127,33 +1013,451 @@ class TestAzureDnsProvider(TestCase): 'Microsoft.Network/trafficManagerProfiles/nestedEndpoints' ) - def test_generate_traffic_managers(self): + def test_dynamic_record(self): provider, zone, record = self._get_dynamic_package() profiles = provider._generate_traffic_managers(record) - deduped = [] - seen = set() - for profile in profiles: - if profile.name not in seen: - deduped.append(profile) - seen.add(profile.name) # check that every profile is a match with what we expect expected_profiles = self._get_tm_profiles(provider) - self.assertEqual(len(expected_profiles), len(deduped)) - for have, expected in zip(deduped, expected_profiles): + self.assertEqual(len(expected_profiles), len(profiles)) + for have, expected in zip(profiles, expected_profiles): self.assertTrue(_profile_is_match(have, expected)) + # check that dynamic record is populated back from profiles + azrecord = RecordSet( + ttl=60, + target_resource=SubResource(id=profiles[-1].id), + ) + azrecord.name = record.name or '@' + azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) + record2 = provider._populate_record(zone, azrecord) + self.assertEqual(record2.dynamic._data(), record.dynamic._data()) + + def test_generate_traffic_managers_middle_east(self): # check Asia/Middle East test case + provider, zone, record = self._get_dynamic_package() record.dynamic._data()['rules'][0]['geos'].append('AS') profiles = provider._generate_traffic_managers(record) - geo_profile_name = _traffic_manager_suffix(record) - geo_profile = next( - profile - for profile in profiles - if profile.name == geo_profile_name + self.assertIn('GEO-ME', profiles[-1].endpoints[0].geo_mapping) + self.assertIn('GEO-AS', profiles[-1].endpoints[0].geo_mapping) + + def test_populate_dynamic_middle_east(self): + # Middle east without Asia raises exception + provider, zone, record = self._get_dynamic_package() + tm_suffix = _traffic_manager_suffix(record) + tm_id = provider._profile_name_to_id + tm_list = provider._tm_client.profiles.list_by_resource_group + tm_list.return_value = [ + Profile( + id=tm_id(tm_suffix), + name=tm_suffix, + traffic_routing_method='Geographic', + endpoints=[ + Endpoint( + geo_mapping=['GEO-ME'], + ), + ], + ), + ] + azrecord = RecordSet( + ttl=60, + target_resource=SubResource(id=tm_id(tm_suffix)), + ) + azrecord.name = record.name or '@' + azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) + with self.assertRaises(AzureException) as ctx: + provider._populate_record(zone, azrecord) + self.assertTrue(text_type(ctx).startswith( + 'Middle East (GEO-ME) is not supported' + )) + + # valid profiles with Middle East test case + provider, zone, record = self._get_dynamic_package() + geo_profile = provider._get_tm_for_dynamic_record(record) + geo_profile.endpoints[0].geo_mapping.extend(['GEO-ME', 'GEO-AS']) + record = provider._populate_record(zone, azrecord) + self.assertIn('AS', record.dynamic.rules[0].data['geos']) + self.assertNotIn('ME', record.dynamic.rules[0].data['geos']) + + def test_dynamic_no_geo(self): + # test that traffic managers are generated as expected + provider, zone, record = self._get_dynamic_package() + external = 'Microsoft.Network/trafficManagerProfiles/externalEndpoints' + + record = Record.new(zone, 'foo', data={ + 'type': 'CNAME', + 'ttl': 60, + 'value': 'default.unit.tests.', + 'dynamic': { + 'pools': { + 'one': { + 'values': [ + {'value': 'one.unit.tests.'}, + ], + }, + }, + 'rules': [ + {'pool': 'one'}, + ], + } + }) + profiles = provider._generate_traffic_managers(record) + + self.assertEqual(len(profiles), 1) + self.assertTrue(_profile_is_match(profiles[0], Profile( + name='foo-unit-tests', + traffic_routing_method='Priority', + dns_config=DnsConfig( + relative_name='foo-unit-tests', ttl=60), + monitor_config=_get_monitor(record), + endpoints=[ + Endpoint( + name='one', + type=external, + target='one.unit.tests', + priority=1, + ), + Endpoint( + name='--default--', + type=external, + target='default.unit.tests', + priority=2, + ), + ], + ))) + + # test that same record gets populated back from traffic managers + tm_list = provider._tm_client.profiles.list_by_resource_group + tm_list.return_value = profiles + azrecord = RecordSet( + ttl=60, + target_resource=SubResource(id=profiles[0].id), + ) + azrecord.name = record.name or '@' + azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) + record2 = provider._populate_record(zone, azrecord) + self.assertEqual(record2.dynamic._data(), record.dynamic._data()) + + def test_dynamic_fallback_is_default(self): + # test that traffic managers are generated as expected + provider, zone, record = self._get_dynamic_package() + external = 'Microsoft.Network/trafficManagerProfiles/externalEndpoints' + + record = Record.new(zone, 'foo', data={ + 'type': 'CNAME', + 'ttl': 60, + 'value': 'default.unit.tests.', + 'dynamic': { + 'pools': { + 'def': { + 'values': [ + {'value': 'default.unit.tests.'}, + ], + }, + }, + 'rules': [ + {'geos': ['AF'], 'pool': 'def'}, + ], + } + }) + profiles = provider._generate_traffic_managers(record) + + self.assertEqual(len(profiles), 1) + self.assertTrue(_profile_is_match(profiles[0], Profile( + name='foo-unit-tests', + traffic_routing_method='Geographic', + dns_config=DnsConfig( + relative_name='foo-unit-tests', ttl=60), + monitor_config=_get_monitor(record), + endpoints=[ + Endpoint( + name='def--default--', + type=external, + target='default.unit.tests', + geo_mapping=['GEO-AF'], + ), + ], + ))) + + # test that same record gets populated back from traffic managers + tm_list = provider._tm_client.profiles.list_by_resource_group + tm_list.return_value = profiles + azrecord = RecordSet( + ttl=60, + target_resource=SubResource(id=profiles[0].id), + ) + azrecord.name = record.name or '@' + azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) + record2 = provider._populate_record(zone, azrecord) + self.assertEqual(record2.dynamic._data(), record.dynamic._data()) + + def test_dynamic_pool_contains_default(self): + # test that traffic managers are generated as expected + provider, zone, record = self._get_dynamic_package() + tm_id = provider._profile_name_to_id + external = 'Microsoft.Network/trafficManagerProfiles/externalEndpoints' + nested = 'Microsoft.Network/trafficManagerProfiles/nestedEndpoints' + + record = Record.new(zone, 'foo', data={ + 'type': 'CNAME', + 'ttl': 60, + 'value': 'default.unit.tests.', + 'dynamic': { + 'pools': { + 'rr': { + 'values': [ + {'value': 'one.unit.tests.'}, + {'value': 'two.unit.tests.'}, + {'value': 'default.unit.tests.'}, + {'value': 'final.unit.tests.'}, + ], + }, + }, + 'rules': [ + {'geos': ['AF'], 'pool': 'rr'}, + ], + } + }) + profiles = provider._generate_traffic_managers(record) + + self.assertEqual(len(profiles), 2) + self.assertTrue(_profile_is_match(profiles[0], Profile( + name='pool-rr--foo-unit-tests', + traffic_routing_method='Weighted', + dns_config=DnsConfig( + relative_name='pool-rr--foo-unit-tests', ttl=60), + monitor_config=_get_monitor(record), + endpoints=[ + Endpoint( + name='rr--one.unit.tests', + type=external, + target='one.unit.tests', + weight=1, + ), + Endpoint( + name='rr--two.unit.tests', + type=external, + target='two.unit.tests', + weight=1, + ), + Endpoint( + name='rr--default.unit.tests--default--', + type=external, + target='default.unit.tests', + weight=1, + ), + Endpoint( + name='rr--final.unit.tests', + type=external, + target='final.unit.tests', + weight=1, + ), + ], + ))) + self.assertTrue(_profile_is_match(profiles[1], Profile( + name='foo-unit-tests', + traffic_routing_method='Geographic', + dns_config=DnsConfig( + relative_name='foo-unit-tests', ttl=60), + monitor_config=_get_monitor(record), + endpoints=[ + Endpoint( + name='rule-rr', + type=nested, + target_resource_id=tm_id('pool-rr--foo-unit-tests'), + geo_mapping=['GEO-AF'], + ), + ], + ))) + + # test that same record gets populated back from traffic managers + tm_list = provider._tm_client.profiles.list_by_resource_group + tm_list.return_value = profiles + azrecord = RecordSet( + ttl=60, + target_resource=SubResource(id=profiles[1].id), + ) + azrecord.name = record.name or '@' + azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) + record2 = provider._populate_record(zone, azrecord) + self.assertEqual(record2.dynamic._data(), record.dynamic._data()) + + def test_dynamic_pool_contains_default_no_geo(self): + # test that traffic managers are generated as expected + provider, zone, record = self._get_dynamic_package() + external = 'Microsoft.Network/trafficManagerProfiles/externalEndpoints' + + record = Record.new(zone, 'foo', data={ + 'type': 'CNAME', + 'ttl': 60, + 'value': 'default.unit.tests.', + 'dynamic': { + 'pools': { + 'rr': { + 'values': [ + {'value': 'one.unit.tests.'}, + {'value': 'two.unit.tests.'}, + {'value': 'default.unit.tests.'}, + {'value': 'final.unit.tests.'}, + ], + }, + }, + 'rules': [ + {'pool': 'rr'}, + ], + } + }) + profiles = provider._generate_traffic_managers(record) + + self.assertEqual(len(profiles), 1) + self.assertTrue(_profile_is_match(profiles[0], Profile( + name='foo-unit-tests', + traffic_routing_method='Weighted', + dns_config=DnsConfig( + relative_name='foo-unit-tests', ttl=60), + monitor_config=_get_monitor(record), + endpoints=[ + Endpoint( + name='rr--one.unit.tests', + type=external, + target='one.unit.tests', + weight=1, + ), + Endpoint( + name='rr--two.unit.tests', + type=external, + target='two.unit.tests', + weight=1, + ), + Endpoint( + name='rr--default.unit.tests--default--', + type=external, + target='default.unit.tests', + weight=1, + ), + Endpoint( + name='rr--final.unit.tests', + type=external, + target='final.unit.tests', + weight=1, + ), + ], + ))) + + # test that same record gets populated back from traffic managers + tm_list = provider._tm_client.profiles.list_by_resource_group + tm_list.return_value = profiles + azrecord = RecordSet( + ttl=60, + target_resource=SubResource(id=profiles[0].id), ) - self.assertIn('GEO-ME', geo_profile.endpoints[0].geo_mapping) - self.assertIn('GEO-AS', geo_profile.endpoints[0].geo_mapping) + azrecord.name = record.name or '@' + azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) + record2 = provider._populate_record(zone, azrecord) + self.assertEqual(record2.dynamic._data(), record.dynamic._data()) + + def test_dynamic_last_pool_contains_default_no_geo(self): + # test that traffic managers are generated as expected + provider, zone, record = self._get_dynamic_package() + tm_id = provider._profile_name_to_id + external = 'Microsoft.Network/trafficManagerProfiles/externalEndpoints' + nested = 'Microsoft.Network/trafficManagerProfiles/nestedEndpoints' + + record = Record.new(zone, 'foo', data={ + 'type': 'CNAME', + 'ttl': 60, + 'value': 'default.unit.tests.', + 'dynamic': { + 'pools': { + 'cloud': { + 'values': [ + {'value': 'cloud.unit.tests.'}, + ], + 'fallback': 'rr', + }, + 'rr': { + 'values': [ + {'value': 'one.unit.tests.'}, + {'value': 'two.unit.tests.'}, + {'value': 'default.unit.tests.'}, + {'value': 'final.unit.tests.'}, + ], + }, + }, + 'rules': [ + {'pool': 'cloud'}, + ], + } + }) + profiles = provider._generate_traffic_managers(record) + + self.assertEqual(len(profiles), 2) + self.assertTrue(_profile_is_match(profiles[0], Profile( + name='pool-rr--foo-unit-tests', + traffic_routing_method='Weighted', + dns_config=DnsConfig( + relative_name='pool-rr--foo-unit-tests', ttl=60), + monitor_config=_get_monitor(record), + endpoints=[ + Endpoint( + name='rr--one.unit.tests', + type=external, + target='one.unit.tests', + weight=1, + ), + Endpoint( + name='rr--two.unit.tests', + type=external, + target='two.unit.tests', + weight=1, + ), + Endpoint( + name='rr--default.unit.tests--default--', + type=external, + target='default.unit.tests', + weight=1, + ), + Endpoint( + name='rr--final.unit.tests', + type=external, + target='final.unit.tests', + weight=1, + ), + ], + ))) + self.assertTrue(_profile_is_match(profiles[1], Profile( + name='foo-unit-tests', + traffic_routing_method='Priority', + dns_config=DnsConfig( + relative_name='foo-unit-tests', ttl=60), + monitor_config=_get_monitor(record), + endpoints=[ + Endpoint( + name='cloud', + type=external, + target='cloud.unit.tests', + priority=1, + ), + Endpoint( + name='rr', + type=nested, + target_resource_id=tm_id('pool-rr--foo-unit-tests'), + priority=2, + ), + ], + ))) + + # test that same record gets populated back from traffic managers + tm_list = provider._tm_client.profiles.list_by_resource_group + tm_list.return_value = profiles + azrecord = RecordSet( + ttl=60, + target_resource=SubResource(id=profiles[1].id), + ) + azrecord.name = record.name or '@' + azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) + record2 = provider._populate_record(zone, azrecord) + self.assertEqual(record2.dynamic._data(), record.dynamic._data()) def test_sync_traffic_managers(self): provider, zone, record = self._get_dynamic_package() @@ -1194,6 +1498,21 @@ class TestAzureDnsProvider(TestCase): ) self.assertEqual(new_profile.endpoints[0].weight, 14) + @patch( + 'octodns.provider.azuredns.AzureProvider._generate_traffic_managers') + def test_sync_traffic_managers_duplicate(self, mock_gen_tms): + provider, zone, record = self._get_dynamic_package() + tm_sync = provider._tm_client.profiles.create_or_update + + # change and duplicate profiles + profile = self._get_tm_profiles(provider)[0] + profile.name = 'changing_this_to_trigger_sync' + mock_gen_tms.return_value = [profile, profile] + provider._sync_traffic_managers(record) + + # it should only be called once for duplicate profiles + tm_sync.assert_called_once() + def test_find_traffic_managers(self): provider, zone, record = self._get_dynamic_package() From f90d261133d11e10aff322a9526493315b503584 Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Wed, 9 Jun 2021 15:12:04 -0700 Subject: [PATCH 240/358] drop Azure DNS env vars in scripts --- script/coverage | 4 ++++ script/test | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/script/coverage b/script/coverage index bd6e4c9..db8e219 100755 --- a/script/coverage +++ b/script/coverage @@ -25,6 +25,10 @@ export DYN_CUSTOMER= export DYN_PASSWORD= export DYN_USERNAME= export GOOGLE_APPLICATION_CREDENTIALS= +export ARM_CLIENT_ID= +export ARM_CLIENT_SECRET= +export ARM_TENANT_ID= +export ARM_SUBSCRIPTION_ID= # Don't allow disabling coverage grep -r -I --line-number "# pragma: +no.*cover" octodns && { diff --git a/script/test b/script/test index 41edfd8..98bae20 100755 --- a/script/test +++ b/script/test @@ -25,5 +25,9 @@ export DYN_CUSTOMER= export DYN_PASSWORD= export DYN_USERNAME= export GOOGLE_APPLICATION_CREDENTIALS= +export ARM_CLIENT_ID= +export ARM_CLIENT_SECRET= +export ARM_TENANT_ID= +export ARM_SUBSCRIPTION_ID= nosetests "$@" From 5a943f0b13c08796137637dd55149fc2878ddc72 Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Wed, 9 Jun 2021 16:01:18 -0700 Subject: [PATCH 241/358] handle broken alias to ATM for dynamic records --- octodns/provider/azuredns.py | 9 +++++++-- tests/test_octodns_provider_azuredns.py | 21 +++++++++++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 777e15b..082fd42 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -613,8 +613,13 @@ class AzureProvider(BaseProvider): :type return: dict ''' - if azrecord.cname_record is None and azrecord.target_resource.id: - return self._data_for_dynamic(azrecord) + if azrecord.cname_record is None: + if azrecord.target_resource.id: + return self._data_for_dynamic(azrecord) + else: + # dynamic record alias is broken, return dummy value and apply + # will likely overwrite/fix it + return {'value': 'iam.invalid.'} return {'value': _check_endswith_dot(azrecord.cname_record.cname)} diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index 2c998b6..0332a72 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -1677,6 +1677,27 @@ class TestAzureDnsProvider(TestCase): dns_update.assert_not_called() tm_delete.assert_called_once() + # both are dynamic but alias is broken + provider, existing, record1 = self._get_dynamic_package() + azrecord = RecordSet( + ttl=record1.ttl, target_resource=SubResource(id=None)) + azrecord.name = record1.name or '@' + azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record1._type) + + record2 = provider._populate_record(zone, azrecord) + self.assertEqual(record2.value, 'iam.invalid.') + + change = Update(record2, record1) + provider._apply_Update(change) + tm_sync, dns_update, tm_delete = ( + provider._tm_client.profiles.create_or_update, + provider._dns_client.record_sets.create_or_update, + provider._tm_client.profiles.delete + ) + tm_sync.assert_not_called() + dns_update.assert_called_once() + tm_delete.assert_not_called() + def test_apply_delete_dynamic(self): provider, existing, record = self._get_dynamic_package() provider._populate_traffic_managers() From 7e0c6296fbcd62243f0b0d673b177e9c6a548ea1 Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Wed, 9 Jun 2021 17:25:06 -0700 Subject: [PATCH 242/358] ensure dynamic records map to unique ATM profiles --- octodns/provider/azuredns.py | 77 ++++++++----- tests/test_octodns_provider_azuredns.py | 145 ++++++++++++++++-------- 2 files changed, 147 insertions(+), 75 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 082fd42..bd555a7 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -259,8 +259,21 @@ def _parse_azure_type(string): return string.split('/')[-1] -def _traffic_manager_suffix(record): - return record.fqdn[:-1].replace('.', '-') +def _root_traffic_manager_name(record): + # ATM names can only have letters, numbers and hyphens + # replace dots with double hyphens to ensure unique mapping, + # hoping that real life FQDNs won't have double hyphens + return record.fqdn[:-1].replace('.', '--') + + +def _rule_traffic_manager_name(pool, record): + prefix = _root_traffic_manager_name(record) + return '{}-rule-{}'.format(prefix, pool) + + +def _pool_traffic_manager_name(pool, record): + prefix = _root_traffic_manager_name(record) + return '{}-pool-{}'.format(prefix, pool) def _get_monitor(record): @@ -533,7 +546,7 @@ class AzureProvider(BaseProvider): return self._get_tm_profile_by_id(profile_id) def _get_tm_for_dynamic_record(self, record): - name = _traffic_manager_suffix(record) + name = _root_traffic_manager_name(record) return self._get_tm_profile_by_name(name) def populate(self, zone, target=False, lenient=False): @@ -799,6 +812,7 @@ class AzureProvider(BaseProvider): raise AzureException(msg) log = self.log.info + seen_profiles = {} extra = [] for record in desired.records: if not getattr(record, 'dynamic', False): @@ -813,6 +827,16 @@ class AzureProvider(BaseProvider): profiles = self._generate_traffic_managers(record) for profile in profiles: name = profile.name + if name in seen_profiles: + # exit if a possible collision is detected, even though + # we've tried to ensure unique mapping + msg = 'Collision in Traffic Manager names detected' + msg = '{}: {} and {} both want to use {}'.format( + msg, seen_profiles[name], record.fqdn, name) + raise AzureException(msg) + else: + seen_profiles[name] = record.fqdn + active.add(name) existing_profile = self._get_tm_profile_by_name(name) if not _profile_is_match(existing_profile, profile): @@ -831,7 +855,14 @@ class AzureProvider(BaseProvider): return extra - def _generate_tm_profile(self, name, routing, endpoints, record): + def _generate_tm_profile(self, routing, endpoints, record, label=None): + # figure out profile name and Traffic Manager FQDN + name = _root_traffic_manager_name(record) + if routing == 'Weighted': + name = _pool_traffic_manager_name(label, record) + elif routing == 'Priority': + name = _rule_traffic_manager_name(label, record) + # set appropriate endpoint types endpoint_type_prefix = 'Microsoft.Network/trafficManagerProfiles/' for ep in endpoints: @@ -859,10 +890,10 @@ class AzureProvider(BaseProvider): location='global', ) - def _update_tm_name(self, profile, new_name): - profile.name = new_name - profile.id = self._profile_name_to_id(new_name) - profile.dns_config.relative_name = new_name + def _convert_tm_to_root(self, profile, record): + profile.name = _root_traffic_manager_name(record) + profile.id = self._profile_name_to_id(profile.name) + profile.dns_config.relative_name = profile.name return profile @@ -871,7 +902,6 @@ class AzureProvider(BaseProvider): pools = record.dynamic.pools default = record.value[:-1] - tm_suffix = _traffic_manager_suffix(record) profile = self._generate_tm_profile geo_endpoints = [] @@ -930,10 +960,8 @@ class AzureProvider(BaseProvider): target=target, weight=val.get('weight', 1), )) - profile_name = 'pool-{}--{}'.format( - pool_name, tm_suffix) - pool_profile = profile(profile_name, 'Weighted', - endpoints, record) + pool_profile = profile( + 'Weighted', endpoints, record, pool_name) traffic_managers.append(pool_profile) pool_profiles[pool_name] = pool_profile @@ -973,10 +1001,8 @@ class AzureProvider(BaseProvider): if len(rule_endpoints) > 1: # create rule profile with fallback chain - rule_profile_name = 'rule-{}--{}'.format( - rule.data['pool'], tm_suffix) - rule_profile = profile(rule_profile_name, 'Priority', - rule_endpoints, record) + rule_profile = profile( + 'Priority', rule_endpoints, record, rule.data['pool']) traffic_managers.append(rule_profile) # append rule profile to top-level geo profile @@ -1009,13 +1035,9 @@ class AzureProvider(BaseProvider): geo_endpoints[0].target_resource_id: # Single WORLD rule does not require a Geographic profile, use # the target profile as the root profile - target_profile_id = geo_endpoints[0].target_resource_id - profile_map = dict((tm.id, tm) for tm in traffic_managers) - target_profile = profile_map[target_profile_id] - self._update_tm_name(target_profile, tm_suffix) + self._convert_tm_to_root(traffic_managers[-1], record) else: - geo_profile = profile(tm_suffix, 'Geographic', geo_endpoints, - record) + geo_profile = profile('Geographic', geo_endpoints, record) traffic_managers.append(geo_profile) return traffic_managers @@ -1047,14 +1069,15 @@ class AzureProvider(BaseProvider): return seen def _find_traffic_managers(self, record): - tm_suffix = _traffic_manager_suffix(record) + tm_prefix = _root_traffic_manager_name(record) profiles = set() for profile_id in self._traffic_managers: - # match existing profiles with record's suffix + # match existing profiles with record's prefix name = profile_id.split('/')[-1] - if name == tm_suffix or \ - name.endswith('--{}'.format(tm_suffix)): + if name == tm_prefix or \ + name.startswith('{}-pool-'.format(tm_prefix)) or \ + name.startswith('{}-rule-'.format(tm_prefix)): profiles.add(name) return profiles diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index 0332a72..c7840d6 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -7,7 +7,7 @@ from __future__ import absolute_import, division, print_function, \ from octodns.record import Create, Update, Delete, Record from octodns.provider.azuredns import _AzureRecord, AzureProvider, \ - _check_endswith_dot, _parse_azure_type, _traffic_manager_suffix, \ + _check_endswith_dot, _parse_azure_type, _root_traffic_manager_name, \ _get_monitor, _profile_is_match, AzureException from octodns.zone import Zone from octodns.provider.base import Plan @@ -402,12 +402,12 @@ class Test_CheckEndswithDot(TestCase): self.assertEquals(expected, _check_endswith_dot(test)) -class Test_TrafficManagerSuffix(TestCase): - def test_traffic_manager_suffix(self): +class Test_RootTrafficManagerName(TestCase): + def test_root_traffic_manager_name(self): test = Record.new(zone, 'foo', data={ 'ttl': 60, 'type': 'CNAME', 'value': 'default.unit.tests.', }) - self.assertEqual(_traffic_manager_suffix(test), 'foo-unit-tests') + self.assertEqual(_root_traffic_manager_name(test), 'foo--unit--tests') class Test_GetMonitor(TestCase): @@ -594,9 +594,9 @@ class TestAzureDnsProvider(TestCase): base_id = '/subscriptions/' + sub + \ '/resourceGroups/' + rg + \ '/providers/Microsoft.Network/trafficManagerProfiles/' - suffix = 'foo-unit-tests' - id_format = base_id + '{}--' + suffix - name_format = '{}--' + suffix + prefix = 'foo--unit--tests' + name_format = prefix + '-{}' + id_format = base_id + name_format header = MonitorConfigCustomHeadersItem(name='Host', value='foo.unit.tests') @@ -688,8 +688,8 @@ class TestAzureDnsProvider(TestCase): ], ), Profile( - id=base_id + suffix, - name=suffix, + id=base_id + prefix, + name=prefix, traffic_routing_method='Geographic', dns_config=DnsConfig(ttl=60), monitor_config=monitor, @@ -899,10 +899,10 @@ class TestAzureDnsProvider(TestCase): # test unused TM produces the extra change for clean up sample_profile = self._get_tm_profiles(provider)[0] tm_id = provider._profile_name_to_id - root_profile_name = _traffic_manager_suffix(record) + root_profile_name = _root_traffic_manager_name(record) extra_profile = Profile( - id=tm_id('random--{}'.format(root_profile_name)), - name='random--{}'.format(root_profile_name), + id=tm_id('{}-pool-random'.format(root_profile_name)), + name='{}-pool-random'.format(root_profile_name), traffic_routing_method='Weighted', dns_config=sample_profile.dns_config, monitor_config=sample_profile.monitor_config, @@ -968,14 +968,40 @@ class TestAzureDnsProvider(TestCase): }) desired.add_record(a_dynamic) changes.append(Create(a_dynamic)) - with self.assertRaises(AzureException): + with self.assertRaises(AzureException) as ctx: + provider._extra_changes(existing, desired, changes) + self.assertTrue(text_type(ctx).endswith( + 'must be of type CNAME' + )) + desired._remove_record(a_dynamic) + + # test colliding ATM names throws exception + record1 = Record.new(desired, 'sub.www', data={ + 'type': record._type, + 'ttl': record.ttl, + 'value': record.value, + 'dynamic': record.dynamic._data(), + }) + record2 = Record.new(desired, 'sub--www', data={ + 'type': record._type, + 'ttl': record.ttl, + 'value': record.value, + 'dynamic': record.dynamic._data(), + }) + desired.add_record(record1) + desired.add_record(record2) + changes = [Create(record1), Create(record2)] + with self.assertRaises(AzureException) as ctx: provider._extra_changes(existing, desired, changes) + self.assertTrue(text_type(ctx).startswith( + 'Collision in Traffic Manager' + )) def test_generate_tm_profile(self): provider, zone, record = self._get_dynamic_package() profile_gen = provider._generate_tm_profile - name = 'foobar' + label = 'foobar' routing = 'Priority' endpoints = [ Endpoint(target='one.unit.tests'), @@ -985,20 +1011,22 @@ class TestAzureDnsProvider(TestCase): # invalid endpoint raises exception with self.assertRaises(AzureException): - profile_gen(name, routing, endpoints, record) + profile_gen(routing, endpoints, record, label) # regular test endpoints.pop() - profile = profile_gen(name, routing, endpoints, record) + profile = profile_gen(routing, endpoints, record, label) # implicitly tests _profile_name_to_id sub = provider._dns_client_subscription_id rg = provider._resource_group + expected_name = 'foo--unit--tests-rule-foobar' expected_id = '/subscriptions/' + sub + \ '/resourceGroups/' + rg + \ - '/providers/Microsoft.Network/trafficManagerProfiles/' + name + '/providers/Microsoft.Network/trafficManagerProfiles/' + \ + expected_name self.assertEqual(profile.id, expected_id) - self.assertEqual(profile.name, name) + self.assertEqual(profile.name, expected_name) self.assertEqual(profile.name, profile.dns_config.relative_name) self.assertEqual(profile.traffic_routing_method, routing) self.assertEqual(profile.dns_config.ttl, record.ttl) @@ -1044,7 +1072,7 @@ class TestAzureDnsProvider(TestCase): def test_populate_dynamic_middle_east(self): # Middle east without Asia raises exception provider, zone, record = self._get_dynamic_package() - tm_suffix = _traffic_manager_suffix(record) + tm_suffix = _root_traffic_manager_name(record) tm_id = provider._profile_name_to_id tm_list = provider._tm_client.profiles.list_by_resource_group tm_list.return_value = [ @@ -1105,10 +1133,10 @@ class TestAzureDnsProvider(TestCase): self.assertEqual(len(profiles), 1) self.assertTrue(_profile_is_match(profiles[0], Profile( - name='foo-unit-tests', + name='foo--unit--tests', traffic_routing_method='Priority', dns_config=DnsConfig( - relative_name='foo-unit-tests', ttl=60), + relative_name='foo--unit--tests', ttl=60), monitor_config=_get_monitor(record), endpoints=[ Endpoint( @@ -1164,10 +1192,10 @@ class TestAzureDnsProvider(TestCase): self.assertEqual(len(profiles), 1) self.assertTrue(_profile_is_match(profiles[0], Profile( - name='foo-unit-tests', + name='foo--unit--tests', traffic_routing_method='Geographic', dns_config=DnsConfig( - relative_name='foo-unit-tests', ttl=60), + relative_name='foo--unit--tests', ttl=60), monitor_config=_get_monitor(record), endpoints=[ Endpoint( @@ -1222,10 +1250,10 @@ class TestAzureDnsProvider(TestCase): self.assertEqual(len(profiles), 2) self.assertTrue(_profile_is_match(profiles[0], Profile( - name='pool-rr--foo-unit-tests', + name='foo--unit--tests-pool-rr', traffic_routing_method='Weighted', dns_config=DnsConfig( - relative_name='pool-rr--foo-unit-tests', ttl=60), + relative_name='foo--unit--tests-pool-rr', ttl=60), monitor_config=_get_monitor(record), endpoints=[ Endpoint( @@ -1255,16 +1283,16 @@ class TestAzureDnsProvider(TestCase): ], ))) self.assertTrue(_profile_is_match(profiles[1], Profile( - name='foo-unit-tests', + name='foo--unit--tests', traffic_routing_method='Geographic', dns_config=DnsConfig( - relative_name='foo-unit-tests', ttl=60), + relative_name='foo--unit--tests', ttl=60), monitor_config=_get_monitor(record), endpoints=[ Endpoint( name='rule-rr', type=nested, - target_resource_id=tm_id('pool-rr--foo-unit-tests'), + target_resource_id=tm_id(profiles[0].name), geo_mapping=['GEO-AF'], ), ], @@ -1311,10 +1339,10 @@ class TestAzureDnsProvider(TestCase): self.assertEqual(len(profiles), 1) self.assertTrue(_profile_is_match(profiles[0], Profile( - name='foo-unit-tests', + name='foo--unit--tests', traffic_routing_method='Weighted', dns_config=DnsConfig( - relative_name='foo-unit-tests', ttl=60), + relative_name='foo--unit--tests', ttl=60), monitor_config=_get_monitor(record), endpoints=[ Endpoint( @@ -1393,10 +1421,10 @@ class TestAzureDnsProvider(TestCase): self.assertEqual(len(profiles), 2) self.assertTrue(_profile_is_match(profiles[0], Profile( - name='pool-rr--foo-unit-tests', + name='foo--unit--tests-pool-rr', traffic_routing_method='Weighted', dns_config=DnsConfig( - relative_name='pool-rr--foo-unit-tests', ttl=60), + relative_name='foo--unit--tests-pool-rr', ttl=60), monitor_config=_get_monitor(record), endpoints=[ Endpoint( @@ -1426,10 +1454,10 @@ class TestAzureDnsProvider(TestCase): ], ))) self.assertTrue(_profile_is_match(profiles[1], Profile( - name='foo-unit-tests', + name='foo--unit--tests', traffic_routing_method='Priority', dns_config=DnsConfig( - relative_name='foo-unit-tests', ttl=60), + relative_name='foo--unit--tests', ttl=60), monitor_config=_get_monitor(record), endpoints=[ Endpoint( @@ -1441,7 +1469,7 @@ class TestAzureDnsProvider(TestCase): Endpoint( name='rr', type=nested, - target_resource_id=tm_id('pool-rr--foo-unit-tests'), + target_resource_id=tm_id(profiles[0].name), priority=2, ), ], @@ -1459,16 +1487,37 @@ class TestAzureDnsProvider(TestCase): record2 = provider._populate_record(zone, azrecord) self.assertEqual(record2.dynamic._data(), record.dynamic._data()) + def test_dynamic_unique_traffic_managers(self): + record = self._get_dynamic_record(zone) + data = { + 'type': record._type, + 'ttl': record.ttl, + 'value': record.value, + 'dynamic': record.dynamic._data() + } + record_names = [ + 'www.foo', 'www-foo' + ] + provider = self._get_provider() + + seen = set() + for name in record_names: + record = Record.new(zone, name, data=data) + tms = provider._generate_traffic_managers(record) + for tm in tms: + self.assertNotIn(tm.name, seen) + seen.add(tm.name) + def test_sync_traffic_managers(self): provider, zone, record = self._get_dynamic_package() provider._populate_traffic_managers() tm_sync = provider._tm_client.profiles.create_or_update - suffix = 'foo-unit-tests' + prefix = 'foo--unit--tests' expected_seen = { - suffix, 'pool-two--{}'.format(suffix), - 'rule-one--{}'.format(suffix), 'rule-two--{}'.format(suffix), + prefix, '{}-pool-two'.format(prefix), + '{}-rule-one'.format(prefix), '{}-rule-two'.format(prefix), } # test no change @@ -1494,7 +1543,7 @@ class TestAzureDnsProvider(TestCase): # test that new profile was successfully inserted in cache new_profile = provider._get_tm_profile_by_name( - 'pool-two--{}'.format(suffix) + '{}-pool-two'.format(prefix) ) self.assertEqual(new_profile.endpoints[0].weight, 14) @@ -1524,11 +1573,11 @@ class TestAzureDnsProvider(TestCase): 'ttl': record.ttl, 'value': record.value, }) - suffix2 = _traffic_manager_suffix(record2) + prefix2 = _root_traffic_manager_name(record2) tm_id = provider._profile_name_to_id extra_profile = Profile( - id=tm_id('random--{}'.format(suffix2)), - name='random--{}'.format(suffix2), + id=tm_id('{}-pool-random'.format(prefix2)), + name='{}-pool-random'.format(prefix2), traffic_routing_method='Weighted', dns_config=sample_profile.dns_config, monitor_config=sample_profile.monitor_config, @@ -1539,10 +1588,10 @@ class TestAzureDnsProvider(TestCase): provider._populate_traffic_managers() # implicitly asserts that non-matching profile is not included - suffix = _traffic_manager_suffix(record) + prefix = _root_traffic_manager_name(record) self.assertEqual(provider._find_traffic_managers(record), { - suffix, 'pool-two--{}'.format(suffix), - 'rule-one--{}'.format(suffix), 'rule-two--{}'.format(suffix), + prefix, '{}-pool-two'.format(prefix), + '{}-rule-one'.format(prefix), '{}-rule-two'.format(prefix), }) def test_traffic_manager_gc(self): @@ -1655,10 +1704,10 @@ class TestAzureDnsProvider(TestCase): provider, existing, dynamic_record = self._get_dynamic_package() sample_profile = self._get_tm_profiles(provider)[0] tm_id = provider._profile_name_to_id - root_profile_name = _traffic_manager_suffix(dynamic_record) + root_profile_name = _root_traffic_manager_name(dynamic_record) extra_profile = Profile( - id=tm_id('random--{}'.format(root_profile_name)), - name='random--{}'.format(root_profile_name), + id=tm_id('{}-pool-random'.format(root_profile_name)), + name='{}-pool-random'.format(root_profile_name), traffic_routing_method='Weighted', dns_config=sample_profile.dns_config, monitor_config=sample_profile.monitor_config, From 6734448462b79977ef0d0fa1391c716ad63bee6c Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Fri, 11 Jun 2021 11:43:35 -0700 Subject: [PATCH 243/358] log warning when dynamic CNAME has broken alias --- octodns/provider/azuredns.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index bd555a7..0fd3dae 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -632,6 +632,10 @@ class AzureProvider(BaseProvider): else: # dynamic record alias is broken, return dummy value and apply # will likely overwrite/fix it + self.log.warn('_data_for_CNAME: Missing Traffic Manager ' + 'alias for dynamic CNAME record %s, forcing ' + 're-link by setting an invalid value', + azrecord.fqdn) return {'value': 'iam.invalid.'} return {'value': _check_endswith_dot(azrecord.cname_record.cname)} From fec0aea14426b7f8cb9845063ec72bd4d89f6f8d Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Mon, 14 Jun 2021 13:56:19 -0700 Subject: [PATCH 244/358] minor clean up of azuredns tests --- tests/test_octodns_provider_azuredns.py | 26 ++++++++++++------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index c7840d6..81a3084 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -1109,7 +1109,7 @@ class TestAzureDnsProvider(TestCase): def test_dynamic_no_geo(self): # test that traffic managers are generated as expected - provider, zone, record = self._get_dynamic_package() + provider = self._get_provider() external = 'Microsoft.Network/trafficManagerProfiles/externalEndpoints' record = Record.new(zone, 'foo', data={ @@ -1159,7 +1159,7 @@ class TestAzureDnsProvider(TestCase): tm_list.return_value = profiles azrecord = RecordSet( ttl=60, - target_resource=SubResource(id=profiles[0].id), + target_resource=SubResource(id=profiles[-1].id), ) azrecord.name = record.name or '@' azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) @@ -1168,7 +1168,7 @@ class TestAzureDnsProvider(TestCase): def test_dynamic_fallback_is_default(self): # test that traffic managers are generated as expected - provider, zone, record = self._get_dynamic_package() + provider = self._get_provider() external = 'Microsoft.Network/trafficManagerProfiles/externalEndpoints' record = Record.new(zone, 'foo', data={ @@ -1212,7 +1212,7 @@ class TestAzureDnsProvider(TestCase): tm_list.return_value = profiles azrecord = RecordSet( ttl=60, - target_resource=SubResource(id=profiles[0].id), + target_resource=SubResource(id=profiles[-1].id), ) azrecord.name = record.name or '@' azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) @@ -1221,8 +1221,7 @@ class TestAzureDnsProvider(TestCase): def test_dynamic_pool_contains_default(self): # test that traffic managers are generated as expected - provider, zone, record = self._get_dynamic_package() - tm_id = provider._profile_name_to_id + provider = self._get_provider() external = 'Microsoft.Network/trafficManagerProfiles/externalEndpoints' nested = 'Microsoft.Network/trafficManagerProfiles/nestedEndpoints' @@ -1292,7 +1291,7 @@ class TestAzureDnsProvider(TestCase): Endpoint( name='rule-rr', type=nested, - target_resource_id=tm_id(profiles[0].name), + target_resource_id=profiles[0].id, geo_mapping=['GEO-AF'], ), ], @@ -1303,7 +1302,7 @@ class TestAzureDnsProvider(TestCase): tm_list.return_value = profiles azrecord = RecordSet( ttl=60, - target_resource=SubResource(id=profiles[1].id), + target_resource=SubResource(id=profiles[-1].id), ) azrecord.name = record.name or '@' azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) @@ -1312,7 +1311,7 @@ class TestAzureDnsProvider(TestCase): def test_dynamic_pool_contains_default_no_geo(self): # test that traffic managers are generated as expected - provider, zone, record = self._get_dynamic_package() + provider = self._get_provider() external = 'Microsoft.Network/trafficManagerProfiles/externalEndpoints' record = Record.new(zone, 'foo', data={ @@ -1377,7 +1376,7 @@ class TestAzureDnsProvider(TestCase): tm_list.return_value = profiles azrecord = RecordSet( ttl=60, - target_resource=SubResource(id=profiles[0].id), + target_resource=SubResource(id=profiles[-1].id), ) azrecord.name = record.name or '@' azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) @@ -1386,8 +1385,7 @@ class TestAzureDnsProvider(TestCase): def test_dynamic_last_pool_contains_default_no_geo(self): # test that traffic managers are generated as expected - provider, zone, record = self._get_dynamic_package() - tm_id = provider._profile_name_to_id + provider = self._get_provider() external = 'Microsoft.Network/trafficManagerProfiles/externalEndpoints' nested = 'Microsoft.Network/trafficManagerProfiles/nestedEndpoints' @@ -1469,7 +1467,7 @@ class TestAzureDnsProvider(TestCase): Endpoint( name='rr', type=nested, - target_resource_id=tm_id(profiles[0].name), + target_resource_id=profiles[0].id, priority=2, ), ], @@ -1480,7 +1478,7 @@ class TestAzureDnsProvider(TestCase): tm_list.return_value = profiles azrecord = RecordSet( ttl=60, - target_resource=SubResource(id=profiles[1].id), + target_resource=SubResource(id=profiles[-1].id), ) azrecord.name = record.name or '@' azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) From 725189af49791686542c5f580b2c30660ea653bb Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Mon, 14 Jun 2021 13:58:20 -0700 Subject: [PATCH 245/358] Handle re-used pools in Azure DNS dynamic records --- octodns/provider/azuredns.py | 28 ++++++++++- tests/test_octodns_provider_azuredns.py | 67 +++++++++++++++++++++++++ 2 files changed, 93 insertions(+), 2 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 0fd3dae..93478b8 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -717,6 +717,8 @@ class AzureProvider(BaseProvider): country, province = code.split('-', 1) country = GeoCodes.country_to_code(country) geos.append('{}-{}'.format(country, province)) + elif code == 'WORLD': + geos.append(code) else: # country geos.append(GeoCodes.country_to_code(code)) @@ -788,6 +790,13 @@ class AzureProvider(BaseProvider): rules.append(rule) + # add separate rule for re-used world pool + for rule in list(rules): + geos = rule.get('geos', []) + if len(geos) > 1 and 'WORLD' in geos: + geos.remove('WORLD') + rules.append({'pool': rule['pool']}) + # Order and convert to a list default = sorted(default) @@ -904,14 +913,29 @@ class AzureProvider(BaseProvider): def _generate_traffic_managers(self, record): traffic_managers = [] pools = record.dynamic.pools + rules = record.dynamic.rules default = record.value[:-1] profile = self._generate_tm_profile + # a pool can be re-used only with a world pool, record the pool + # to later consolidate it with a geo pool if one exists since we + # can't have multiple endpoints with the same target in ATM + world_pool = None + for rule in rules: + if not rule.data.get('geos', []): + world_pool = rule.data['pool'] + world_seen = False + geo_endpoints = [] pool_profiles = {} for rule in record.dynamic.rules: + pool_name = rule.data['pool'] + if pool_name == world_pool and world_seen: + # this world pool is already mentioned in another geo rule + continue + # Prepare the list of Traffic manager geos rule_geos = rule.data.get('geos', []) geos = [] @@ -933,10 +957,10 @@ class AzureProvider(BaseProvider): geo = 'AP' geos.append('GEO-{}'.format(geo)) - if not geos: + if not geos or pool_name == world_pool: geos.append('WORLD') + world_seen = True - pool_name = rule.data['pool'] rule_endpoints = [] priority = 1 default_seen = False diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index 81a3084..56a783d 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -1506,6 +1506,73 @@ class TestAzureDnsProvider(TestCase): self.assertNotIn(tm.name, seen) seen.add(tm.name) + def test_dynamic_reused_pool(self): + # test that traffic managers are generated as expected + provider = self._get_provider() + nested = 'Microsoft.Network/trafficManagerProfiles/nestedEndpoints' + + record = Record.new(zone, 'foo', data={ + 'type': 'CNAME', + 'ttl': 60, + 'value': 'default.unit.tests.', + 'dynamic': { + 'pools': { + 'iad': { + 'values': [ + {'value': 'iad.unit.tests.'}, + ], + 'fallback': 'lhr', + }, + 'lhr': { + 'values': [ + {'value': 'lhr.unit.tests.'}, + ], + }, + }, + 'rules': [ + {'geos': ['EU'], 'pool': 'iad'}, + {'geos': ['EU-GB'], 'pool': 'lhr'}, + {'pool': 'lhr'}, + ], + } + }) + profiles = provider._generate_traffic_managers(record) + + self.assertEqual(len(profiles), 3) + self.assertTrue(_profile_is_match(profiles[-1], Profile( + name='foo--unit--tests', + traffic_routing_method='Geographic', + dns_config=DnsConfig( + relative_name='foo--unit--tests', ttl=record.ttl), + monitor_config=_get_monitor(record), + endpoints=[ + Endpoint( + name='rule-iad', + type=nested, + target_resource_id=profiles[0].id, + geo_mapping=['GEO-EU'], + ), + Endpoint( + name='rule-lhr', + type=nested, + target_resource_id=profiles[1].id, + geo_mapping=['GB', 'WORLD'], + ), + ], + ))) + + # test that same record gets populated back from traffic managers + tm_list = provider._tm_client.profiles.list_by_resource_group + tm_list.return_value = profiles + azrecord = RecordSet( + ttl=60, + target_resource=SubResource(id=profiles[-1].id), + ) + azrecord.name = record.name or '@' + azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) + record2 = provider._populate_record(zone, azrecord) + self.assertEqual(record2.dynamic._data(), record.dynamic._data()) + def test_sync_traffic_managers(self): provider, zone, record = self._get_dynamic_package() provider._populate_traffic_managers() From a31fa189c131b8e45b5cddb309945e1a4bb7e3d7 Mon Sep 17 00:00:00 2001 From: Ella <72365100+eilla1@users.noreply.github.com> Date: Wed, 16 Jun 2021 00:32:43 -0700 Subject: [PATCH 246/358] fix: update syntax for commands for example: `$ octodns-sync --config-file=./config/production.yaml` was throwing an error. Then, when i did `$ octodns-dump`, it prompted me to put it in this format: `$ octodns-sync --config-file CONFIG_FILE` (without the = sign) --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 2ce9445..eb7789b 100644 --- a/README.md +++ b/README.md @@ -122,7 +122,7 @@ Further information can be found in [Records Documentation](/docs/records.md). We're ready to do a dry-run with our new setup to see what changes it would make. Since we're pretending here we'll act like there are no existing records for `example.com.` in our accounts on either provider. ```shell -$ octodns-sync --config-file=./config/production.yaml +$ octodns-sync --config-file ./config/production.yaml ... ******************************************************************************** * example.com. @@ -146,7 +146,7 @@ There will be other logging information presented on the screen, but successful Now it's time to tell OctoDNS to make things happen. We'll invoke it again with the same options and add a `--doit` on the end to tell it this time we actually want it to try and make the specified changes. ```shell -$ octodns-sync --config-file=./config/production.yaml --doit +$ octodns-sync --config-file ./config/production.yaml --doit ... ``` @@ -177,7 +177,7 @@ If that goes smoothly, you again see the expected changes, and verify them with Very few situations will involve starting with a blank slate which is why there's tooling built in to pull existing data out of providers into a matching config file. ```shell -$ octodns-dump --config-file=config/production.yaml --output-dir=tmp/ example.com. route53 +$ octodns-dump --config-file ./config/production.yaml --output-dir tmp/ example.com. route53 2017-03-15T13:33:34 INFO Manager __init__: config_file=tmp/production.yaml 2017-03-15T13:33:34 INFO Manager dump: zone=example.com., sources=('route53',) 2017-03-15T13:33:36 INFO Route53Provider[route53] populate: found 64 records From fb805ced62e649dd42529081055cf38bcb616bc8 Mon Sep 17 00:00:00 2001 From: Ella <72365100+eilla1@users.noreply.github.com> Date: Wed, 16 Jun 2021 00:39:01 -0700 Subject: [PATCH 247/358] chore: add alt text to images --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index eb7789b..3ad16d3 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ - +OctoDNS Logo ## DNS as code - Tools for managing DNS across multiple providers @@ -158,17 +158,17 @@ In the above case we manually ran OctoDNS from the command line. That works and The first step is to create a PR with your changes. -![](/docs/assets/pr.png) +![GitHub user interface of a pull request](/docs/assets/pr.png) Assuming the code tests and config validation statuses are green the next step is to do a noop deploy and verify that the changes OctoDNS plans to make are the ones you expect. -![](/docs/assets/noop.png) +![Output of a noop deployment command](/docs/assets/noop.png) After that comes a set of reviews. One from a teammate who should have full context on what you're trying to accomplish and visibility in to the changes you're making to do it. The other is from a member of the team here at GitHub that owns DNS, mostly as a sanity check and to make sure that best practices are being followed. As much of that as possible is baked into `octodns-validate`. After the reviews it's time to branch deploy the change. -![](/docs/assets/deploy.png) +![Output of a deployment command](/docs/assets/deploy.png) If that goes smoothly, you again see the expected changes, and verify them with `dig` and/or `octodns-report` you're good to hit the merge button. If there are problems you can quickly do a `.deploy dns/master` to go back to the previous state. From 501ae886738522b13e3c9e8c5f564c48d5c135d8 Mon Sep 17 00:00:00 2001 From: Ella <72365100+eilla1@users.noreply.github.com> Date: Wed, 16 Jun 2021 15:25:14 -0700 Subject: [PATCH 248/358] chore: use = --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 3ad16d3..91e04eb 100644 --- a/README.md +++ b/README.md @@ -122,7 +122,7 @@ Further information can be found in [Records Documentation](/docs/records.md). We're ready to do a dry-run with our new setup to see what changes it would make. Since we're pretending here we'll act like there are no existing records for `example.com.` in our accounts on either provider. ```shell -$ octodns-sync --config-file ./config/production.yaml +$ octodns-sync --config-file=./config/production.yaml ... ******************************************************************************** * example.com. @@ -146,7 +146,7 @@ There will be other logging information presented on the screen, but successful Now it's time to tell OctoDNS to make things happen. We'll invoke it again with the same options and add a `--doit` on the end to tell it this time we actually want it to try and make the specified changes. ```shell -$ octodns-sync --config-file ./config/production.yaml --doit +$ octodns-sync --config-file=./config/production.yaml --doit ... ``` @@ -177,7 +177,7 @@ If that goes smoothly, you again see the expected changes, and verify them with Very few situations will involve starting with a blank slate which is why there's tooling built in to pull existing data out of providers into a matching config file. ```shell -$ octodns-dump --config-file ./config/production.yaml --output-dir tmp/ example.com. route53 +$ octodns-dump --config-file=config/production.yaml --output-dir=tmp/ example.com. route53 2017-03-15T13:33:34 INFO Manager __init__: config_file=tmp/production.yaml 2017-03-15T13:33:34 INFO Manager dump: zone=example.com., sources=('route53',) 2017-03-15T13:33:36 INFO Route53Provider[route53] populate: found 64 records From 794fca0ee765b8ae8912ef557c66484f8253959f Mon Sep 17 00:00:00 2001 From: Iordanis Grigoriou Date: Thu, 17 Jun 2021 16:36:46 +0200 Subject: [PATCH 249/358] Fix typo --- octodns/provider/dyn.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/octodns/provider/dyn.py b/octodns/provider/dyn.py index da56a2e..2fbcadb 100644 --- a/octodns/provider/dyn.py +++ b/octodns/provider/dyn.py @@ -604,7 +604,7 @@ class DynProvider(BaseProvider): return record - def _is_traffic_director_dyanmic(self, td, rulesets): + def _is_traffic_director_dynamic(self, td, rulesets): for ruleset in rulesets: try: pieces = ruleset.label.split(':') @@ -632,7 +632,7 @@ class DynProvider(BaseProvider): continue # critical to call rulesets once, each call loads them :-( rulesets = td.rulesets - if self._is_traffic_director_dyanmic(td, rulesets): + if self._is_traffic_director_dynamic(td, rulesets): record = \ self._populate_dynamic_traffic_director(zone, fqdn, _type, td, From f147a3ab0fd2ddacc07964872c82d8c458e9e8ff Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Mon, 21 Jun 2021 15:36:57 -0700 Subject: [PATCH 250/358] Fix endpoint naming --- octodns/provider/azuredns.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 6629d6c..fcc9f54 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -1010,7 +1010,7 @@ class AzureProvider(BaseProvider): ep_name += '--default--' default_seen = True rule_endpoints.append(Endpoint( - name=pool_name, + name=ep_name, target=target, priority=priority, )) @@ -1053,7 +1053,7 @@ class AzureProvider(BaseProvider): else: # just add the value of single-value pool geo_endpoints.append(Endpoint( - name=rule_ep.name + '--default--', + name=rule_ep.name, target=rule_ep.target, geo_mapping=geos, )) From e7524ec1ad8341608c9ca1f96161adc05186ae53 Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Mon, 21 Jun 2021 15:46:27 -0700 Subject: [PATCH 251/358] Partial support for dynamic A+AAAA records on Azure --- octodns/provider/azuredns.py | 123 +++++-- tests/test_octodns_provider_azuredns.py | 412 +++++++++++++++++++++++- 2 files changed, 506 insertions(+), 29 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index fcc9f54..2790536 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -125,6 +125,9 @@ class _AzureRecord(object): self.params['ttl'] = record.ttl def _params_for_A(self, data, key_name, azure_class): + if self._record.dynamic and self.traffic_manager: + return {'target_resource': self.traffic_manager} + try: values = data['values'] except KeyError: @@ -132,6 +135,9 @@ class _AzureRecord(object): return {key_name: [azure_class(ipv4_address=v) for v in values]} def _params_for_AAAA(self, data, key_name, azure_class): + if self._record.dynamic and self.traffic_manager: + return {'target_resource': self.traffic_manager} + try: values = data['values'] except KeyError: @@ -263,7 +269,10 @@ def _root_traffic_manager_name(record): # ATM names can only have letters, numbers and hyphens # replace dots with double hyphens to ensure unique mapping, # hoping that real life FQDNs won't have double hyphens - return record.fqdn[:-1].replace('.', '--') + name = record.fqdn[:-1].replace('.', '--') + if record._type != 'CNAME': + name += '-{}'.format(record._type) + return name def _rule_traffic_manager_name(pool, record): @@ -608,9 +617,33 @@ class AzureProvider(BaseProvider): lenient=lenient) def _data_for_A(self, azrecord): + if azrecord.a_records is None: + if azrecord.target_resource.id: + return self._data_for_dynamic(azrecord) + else: + # dynamic record alias is broken, return dummy value and apply + # will likely overwrite/fix it + self.log.warn('_data_for_A: Missing Traffic Manager ' + 'alias for dynamic A record %s, forcing ' + 're-link by setting an invalid value', + azrecord.fqdn) + return {'values': ['255.255.255.255']} + return {'values': [ar.ipv4_address for ar in azrecord.a_records]} def _data_for_AAAA(self, azrecord): + if azrecord.aaaa_records is None: + if azrecord.target_resource.id: + return self._data_for_dynamic(azrecord) + else: + # dynamic record alias is broken, return dummy value and apply + # will likely overwrite/fix it + self.log.warn('_data_for_AAAA: Missing Traffic Manager ' + 'alias for dynamic AAAA record %s, forcing ' + 're-link by setting an invalid value', + azrecord.fqdn) + return {'values': ['::1']} + return {'values': [ar.ipv6_address for ar in azrecord.aaaa_records]} def _data_for_CAA(self, azrecord): @@ -667,6 +700,7 @@ class AzureProvider(BaseProvider): default = set() pools = defaultdict(lambda: {'fallback': None, 'values': []}) rules = [] + typ = _parse_azure_type(azrecord.type) # top level profile root_profile = self._get_tm_profile_by_id(azrecord.target_resource.id) @@ -781,8 +815,10 @@ class AzureProvider(BaseProvider): for pool_ep in endpoints: val = pool_ep.target + if typ == 'CNAME': + val = _check_endswith_dot(val) pool['values'].append({ - 'value': _check_endswith_dot(val), + 'value': val, 'weight': pool_ep.weight or 1, }) if pool_ep.name.endswith('--default--'): @@ -805,23 +841,58 @@ class AzureProvider(BaseProvider): 'pools': pools, 'rules': rules, }, - 'value': _check_endswith_dot(default[0]), } + if typ == 'CNAME': + data['value'] = _check_endswith_dot(default[0]) + else: + data['values'] = default + return data def _extra_changes(self, existing, desired, changes): changed = set() - # Abort if there are non-CNAME dynamic records + # Abort if there are unsupported dynamic records for change in changes: record = change.record changed.add(record) typ = record._type dynamic = getattr(record, 'dynamic', False) - if dynamic and typ != 'CNAME': - msg = '{}: Dynamic records in Azure must be of type CNAME' - msg = msg.format(record.fqdn) + if not dynamic: + continue + if typ in ['A', 'AAAA']: + data = dynamic._data() + values = set(record.values) + pools = data['pools'].values() + seen_values = set() + rr = False + fallback = False + for pool in pools: + vals = pool['values'] + if len(vals) > 1: + rr = True + pool_values = set(val['value'] for val in vals) + if pool.get('fallback'): + fallback = True + seen_values.update(pool_values) + + if values != seen_values: + msg = ('{} {}: All pool values of A/AAAA dynamic records ' + 'must be included in top-level \'values\'.' + .format(record.fqdn, record._type)) + raise AzureException(msg) + + geo = any(r.get('geos') for r in data['rules']) + + if [rr, fallback, geo].count(True) > 1: + msg = ('{} {}: A/AAAA dynamic records must use at most ' + 'one of round-robin, fallback and geo-fencing') + msg = msg.format(record.fqdn, record._type) + raise AzureException(msg) + elif typ != 'CNAME': + msg = ('{}: Dynamic records in Azure must be of type ' + 'A/AAAA/CNAME').format(record.fqdn) raise AzureException(msg) log = self.log.info @@ -833,11 +904,20 @@ class AzureProvider(BaseProvider): continue # let's walk through and show what will be changed even if - # the record is already be in list of changes + # the record is already in list of changes added = (record in changed) active = set() profiles = self._generate_traffic_managers(record) + + # this should not happen with above checks, still adding to block + # undesired changes + if record._type in ['A', 'AAAA'] and len(profiles) > 1: + msg = ('Unknown error: {} {} needs more than 1 Traffic ' + 'Managers which is not supported for A/AAAA dynamic ' + 'records').format(record.fqdn, record._type) + raise AzureException(msg) + for profile in profiles: name = profile.name if name in seen_profiles: @@ -871,9 +951,9 @@ class AzureProvider(BaseProvider): def _generate_tm_profile(self, routing, endpoints, record, label=None): # figure out profile name and Traffic Manager FQDN name = _root_traffic_manager_name(record) - if routing == 'Weighted': + if routing == 'Weighted' and label: name = _pool_traffic_manager_name(label, record) - elif routing == 'Priority': + elif routing == 'Priority' and label: name = _rule_traffic_manager_name(label, record) # set appropriate endpoint types @@ -895,7 +975,7 @@ class AzureProvider(BaseProvider): name=name, traffic_routing_method=routing, dns_config=DnsConfig( - relative_name=name, + relative_name=name.lower(), ttl=record.ttl, ), monitor_config=_get_monitor(record), @@ -906,7 +986,7 @@ class AzureProvider(BaseProvider): def _convert_tm_to_root(self, profile, record): profile.name = _root_traffic_manager_name(record) profile.id = self._profile_name_to_id(profile.name) - profile.dns_config.relative_name = profile.name + profile.dns_config.relative_name = profile.name.lower() return profile @@ -914,8 +994,12 @@ class AzureProvider(BaseProvider): traffic_managers = [] pools = record.dynamic.pools rules = record.dynamic.rules + typ = record._type - default = record.value[:-1] + if typ == 'CNAME': + defaults = [record.value[:-1]] + else: + defaults = record.values profile = self._generate_tm_profile # a pool can be re-used only with a world pool, record the pool @@ -977,9 +1061,10 @@ class AzureProvider(BaseProvider): for val in pool['values']: target = val['value'] # strip trailing dot from CNAME value - target = target[:-1] + if typ == 'CNAME': + target = target[:-1] ep_name = '{}--{}'.format(pool_name, target) - if target == default: + if target in defaults: # mark default ep_name += '--default--' default_seen = True @@ -1003,9 +1088,11 @@ class AzureProvider(BaseProvider): # Skip Weighted profile hop for single-value pool # append its value as an external endpoint to fallback # rule profile - target = pool['values'][0]['value'][:-1] + target = pool['values'][0]['value'] + if typ == 'CNAME': + target = target[:-1] ep_name = pool_name - if target == default: + if target in defaults: # mark default ep_name += '--default--' default_seen = True @@ -1023,7 +1110,7 @@ class AzureProvider(BaseProvider): if not default_seen: rule_endpoints.append(Endpoint( name='--default--', - target=default, + target=defaults[0], priority=priority, )) diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index 3248b97..41454e9 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -4,6 +4,7 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals +from logging import debug from octodns.record import Create, Update, Delete, Record from octodns.provider.azuredns import _AzureRecord, AzureProvider, \ @@ -883,12 +884,13 @@ class TestAzureDnsProvider(TestCase): # test simple records produce no extra changes desired = Zone(name=existing.name, sub_zones=[]) - desired.add_record(Record.new(desired, 'simple', data={ + simple = Record.new(desired, 'simple', data={ 'type': record._type, 'ttl': record.ttl, 'value': record.value, - })) - extra = provider._extra_changes(desired, desired, []) + }) + desired.add_record(simple) + extra = provider._extra_changes(desired, desired, [Create(simple)]) self.assertEqual(len(extra), 0) # test an unchanged dynamic record produces no extra changes @@ -952,28 +954,28 @@ class TestAzureDnsProvider(TestCase): self.assertIsInstance(extra, Update) self.assertEqual(extra.new, update_dynamic) - # test non-CNAME dynamic record throws exception - a_dynamic = Record.new(desired, record.name + '3', data={ - 'type': 'A', + # test dynamic record of unsupported type throws exception + unsupported_dynamic = Record.new(desired, record.name + '3', data={ + 'type': 'DNAME', 'ttl': record.ttl, - 'values': ['1.1.1.1'], + 'value': 'default.unit.tests.', 'dynamic': { 'pools': { - 'one': {'values': [{'value': '2.2.2.2'}]}, + 'one': {'values': [{'value': 'one.unit.tests.'}]}, }, 'rules': [ {'pool': 'one'}, ], }, }) - desired.add_record(a_dynamic) - changes.append(Create(a_dynamic)) + desired.add_record(unsupported_dynamic) + changes = [Create(unsupported_dynamic)] with self.assertRaises(AzureException) as ctx: provider._extra_changes(existing, desired, changes) self.assertTrue(text_type(ctx).endswith( 'must be of type CNAME' )) - desired._remove_record(a_dynamic) + desired._remove_record(unsupported_dynamic) # test colliding ATM names throws exception record1 = Record.new(desired, 'sub.www', data={ @@ -997,6 +999,100 @@ class TestAzureDnsProvider(TestCase): 'Collision in Traffic Manager' )) + def test_extra_changes_invalid_dynamic_a(self): + provider = self._get_provider() + + desired = Zone(name=zone.name, sub_zones=[]) + + # too many test case combinations, here's a method to generate them + def record_data(all_values=True, rr=True, fallback=True, geo=True): + data = { + 'type': 'A', + 'ttl': 60, + 'dynamic': { + 'pools': { + 'one': { + 'values': [ + {'value': '11.11.11.11'}, + {'value': '12.12.12.12'}, + ], + 'fallback': 'two', + }, + 'two': { + 'values': [ + {'value': '2.2.2.2'}, + ], + }, + }, + 'rules': [ + {'geos': ['EU'], 'pool': 'two'}, + {'pool': 'one'}, + ], + } + } + dynamic = data['dynamic'] + if not rr: + dynamic['pools']['one']['values'].pop() + if not fallback: + dynamic['pools']['one'].pop('fallback') + if not geo: + rule = dynamic['rules'].pop(0) + if not fallback: + dynamic['pools'].pop(rule['pool']) + # put all pool values in default + data['values'] = [ + v['value'] + for p in dynamic['pools'].values() + for v in p['values'] + ] + if not all_values: + rm = list(dynamic['pools'].values())[0]['values'][0]['value'] + data['values'].remove(rm) + return data + + # test all combinations + values = [True, False] + combos = [ + [arg1, arg2, arg3, arg4] + for arg1 in values + for arg2 in values + for arg3 in values + for arg4 in values + ] + for all_values, rr, fallback, geo in combos: + args = [all_values, rr, fallback, geo] + + if not any(args): + # all False, invalid use-case + continue + + debug('[all_values, rr, fallback, geo] = %s', args) + data = record_data(*args) + record = Record.new(desired, 'foo', data) + + features = args[1:] + if all_values and features.count(True) <= 1: + # assert does not raise exception + provider._extra_changes(zone, desired, [Create(record)]) + continue + + with self.assertRaises(AzureException) as ctx: + msg = text_type(ctx) + provider._extra_changes(zone, desired, [Create(record)]) + if not all_values: + self.assertTrue('included in top-level \'values\'' in msg) + else: + self.assertTrue('at most one of' in msg) + + # multiple profiles should also raise + record = Record.new(desired, 'foo', + record_data(True, True, True, True)) + desired.add_record(record) + with self.assertRaises(AzureException) as ctx: + # bypass above check by setting changes to empty + provider._extra_changes(zone, desired, []) + self.assertTrue('more than 1 Traffic Managers' in text_type(ctx)) + def test_generate_tm_profile(self): provider, zone, record = self._get_dynamic_package() profile_gen = provider._generate_tm_profile @@ -1573,6 +1669,300 @@ class TestAzureDnsProvider(TestCase): record2 = provider._populate_record(zone, azrecord) self.assertEqual(record2.dynamic._data(), record.dynamic._data()) + def test_dynamic_a_geo(self): + provider = self._get_provider() + external = 'Microsoft.Network/trafficManagerProfiles/externalEndpoints' + + record = Record.new(zone, 'foo', data={ + 'type': 'A', + 'ttl': 60, + 'values': ['1.1.1.1', '2.2.2.2', '3.3.3.3'], + 'dynamic': { + 'pools': { + 'one': { + 'values': [ + {'value': '1.1.1.1'}, + ], + }, + 'two': { + 'values': [ + {'value': '2.2.2.2'}, + ], + }, + 'three': { + 'values': [ + {'value': '3.3.3.3'}, + ], + }, + }, + 'rules': [ + {'geos': ['AS'], 'pool': 'one'}, + {'geos': ['AF'], 'pool': 'two'}, + {'pool': 'three'}, + ], + } + }) + + # test that extra_changes doesn't complain + changes = [Create(record)] + provider._extra_changes(zone, zone, changes) + + profiles = provider._generate_traffic_managers(record) + + self.assertEqual(len(profiles), 1) + self.assertTrue(_profile_is_match(profiles[0], Profile( + name='foo--unit--tests-A', + traffic_routing_method='Geographic', + dns_config=DnsConfig( + relative_name='foo--unit--tests-a', ttl=record.ttl), + monitor_config=_get_monitor(record), + endpoints=[ + Endpoint( + name='one--default--', + type=external, + target='1.1.1.1', + geo_mapping=['GEO-AS'], + ), + Endpoint( + name='two--default--', + type=external, + target='2.2.2.2', + geo_mapping=['GEO-AF'], + ), + Endpoint( + name='three--default--', + type=external, + target='3.3.3.3', + geo_mapping=['WORLD'], + ), + ], + ))) + + # test that the record and ATM profile gets created + tm_sync = provider._tm_client.profiles.create_or_update + create = provider._dns_client.record_sets.create_or_update + provider._apply_Create(changes[0]) + # A dynamic record can only have 1 profile + tm_sync.assert_called_once() + create.assert_called_once() + + # test broken alias + azrecord = RecordSet( + ttl=60, target_resource=SubResource(id=None)) + azrecord.name = record.name or '@' + azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) + record2 = provider._populate_record(zone, azrecord) + self.assertEqual(record2.values, ['255.255.255.255']) + + # test that same record gets populated back from traffic managers + tm_list = provider._tm_client.profiles.list_by_resource_group + tm_list.return_value = profiles + azrecord = RecordSet( + ttl=60, + target_resource=SubResource(id=profiles[-1].id), + ) + azrecord.name = record.name or '@' + azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) + record2 = provider._populate_record(zone, azrecord) + self.assertEqual(record2.dynamic._data(), record.dynamic._data()) + + def test_dynamic_a_fallback(self): + provider = self._get_provider() + external = 'Microsoft.Network/trafficManagerProfiles/externalEndpoints' + + record = Record.new(zone, 'foo', data={ + 'type': 'A', + 'ttl': 60, + 'values': ['8.8.8.8'], + 'dynamic': { + 'pools': { + 'one': { + 'values': [ + {'value': '1.1.1.1'}, + ], + 'fallback': 'two', + }, + 'two': { + 'values': [ + {'value': '2.2.2.2'}, + ], + }, + }, + 'rules': [ + {'pool': 'one'}, + ], + } + }) + profiles = provider._generate_traffic_managers(record) + + self.assertEqual(len(profiles), 1) + self.assertTrue(_profile_is_match(profiles[0], Profile( + name='foo--unit--tests-A', + traffic_routing_method='Priority', + dns_config=DnsConfig( + relative_name='foo--unit--tests-a', ttl=record.ttl), + monitor_config=_get_monitor(record), + endpoints=[ + Endpoint( + name='one', + type=external, + target='1.1.1.1', + priority=1, + ), + Endpoint( + name='two', + type=external, + target='2.2.2.2', + priority=2, + ), + Endpoint( + name='--default--', + type=external, + target='8.8.8.8', + priority=3, + ), + ], + ))) + + # test that same record gets populated back from traffic managers + tm_list = provider._tm_client.profiles.list_by_resource_group + tm_list.return_value = profiles + azrecord = RecordSet( + ttl=60, + target_resource=SubResource(id=profiles[-1].id), + ) + azrecord.name = record.name or '@' + azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) + record2 = provider._populate_record(zone, azrecord) + self.assertEqual(record2.dynamic._data(), record.dynamic._data()) + + def test_dynamic_a_weighted_rr(self): + provider = self._get_provider() + external = 'Microsoft.Network/trafficManagerProfiles/externalEndpoints' + + record = Record.new(zone, 'foo', data={ + 'type': 'A', + 'ttl': 60, + 'values': ['1.1.1.1', '8.8.8.8'], + 'dynamic': { + 'pools': { + 'one': { + 'values': [ + {'value': '1.1.1.1', 'weight': 11}, + {'value': '8.8.8.8', 'weight': 8}, + ], + }, + }, + 'rules': [ + {'pool': 'one'}, + ], + } + }) + profiles = provider._generate_traffic_managers(record) + + self.assertEqual(len(profiles), 1) + self.assertTrue(_profile_is_match(profiles[0], Profile( + name='foo--unit--tests-A', + traffic_routing_method='Weighted', + dns_config=DnsConfig( + relative_name='foo--unit--tests-a', ttl=record.ttl), + monitor_config=_get_monitor(record), + endpoints=[ + Endpoint( + name='one--1.1.1.1--default--', + type=external, + target='1.1.1.1', + weight=11, + ), + Endpoint( + name='one--8.8.8.8--default--', + type=external, + target='8.8.8.8', + weight=8, + ), + ], + ))) + + # test that same record gets populated back from traffic managers + tm_list = provider._tm_client.profiles.list_by_resource_group + tm_list.return_value = profiles + azrecord = RecordSet( + ttl=60, + target_resource=SubResource(id=profiles[-1].id), + ) + azrecord.name = record.name or '@' + azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) + record2 = provider._populate_record(zone, azrecord) + self.assertEqual(record2.dynamic._data(), record.dynamic._data()) + + def test_dynamic_aaaa(self): + provider = self._get_provider() + external = 'Microsoft.Network/trafficManagerProfiles/externalEndpoints' + + record = Record.new(zone, 'foo', data={ + 'type': 'AAAA', + 'ttl': 60, + 'values': ['1::1'], + 'dynamic': { + 'pools': { + 'one': { + 'values': [ + {'value': '1::1'}, + ], + }, + }, + 'rules': [ + {'pool': 'one'}, + ], + } + }) + profiles = provider._generate_traffic_managers(record) + + self.assertEqual(len(profiles), 1) + self.assertTrue(_profile_is_match(profiles[0], Profile( + name='foo--unit--tests-AAAA', + traffic_routing_method='Geographic', + dns_config=DnsConfig( + relative_name='foo--unit--tests-aaaa', ttl=record.ttl), + monitor_config=_get_monitor(record), + endpoints=[ + Endpoint( + name='one--default--', + type=external, + target='1::1', + geo_mapping=['WORLD'], + ), + ], + ))) + + # test that the record and ATM profile gets created + tm_sync = provider._tm_client.profiles.create_or_update + create = provider._dns_client.record_sets.create_or_update + provider._apply_Create(Create(record)) + # A dynamic record can only have 1 profile + tm_sync.assert_called_once() + create.assert_called_once() + + # test broken alias + azrecord = RecordSet( + ttl=60, target_resource=SubResource(id=None)) + azrecord.name = record.name or '@' + azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) + record2 = provider._populate_record(zone, azrecord) + self.assertEqual(record2.values, ['::1']) + + # test that same record gets populated back from traffic managers + tm_list = provider._tm_client.profiles.list_by_resource_group + tm_list.return_value = profiles + azrecord = RecordSet( + ttl=60, + target_resource=SubResource(id=profiles[-1].id), + ) + azrecord.name = record.name or '@' + azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) + record2 = provider._populate_record(zone, azrecord) + self.assertEqual(record2.dynamic._data(), record.dynamic._data()) + def test_sync_traffic_managers(self): provider, zone, record = self._get_dynamic_package() provider._populate_traffic_managers() From c0161fb228fdb3191225b9bcc39ba1036b173c44 Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Tue, 22 Jun 2021 15:48:38 -0700 Subject: [PATCH 252/358] Split out dynamic record validty check for readability --- octodns/provider/azuredns.py | 92 +++++++++++++------------ tests/test_octodns_provider_azuredns.py | 41 +++++++++-- 2 files changed, 83 insertions(+), 50 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 2790536..4551c6c 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -299,6 +299,47 @@ def _get_monitor(record): return monitor +def _check_valid_dynamic(record): + typ = record._type + dynamic = record.dynamic + if typ in ['A', 'AAAA']: + # A/AAAA records cannot be aliased to Traffic Managers that contain + # other nested Traffic Managers. Due to this limitation, A/AAAA + # dynamic records can do only one of geo-fencing, fallback and + # weighted RR. So let's validate that the record adheres to this + # limitation. + data = dynamic._data() + values = set(record.values) + pools = data['pools'].values() + seen_values = set() + rr = False + fallback = False + for pool in pools: + vals = pool['values'] + if len(vals) > 1: + rr = True + pool_values = set(val['value'] for val in vals) + if pool.get('fallback'): + fallback = True + seen_values.update(pool_values) + + if values != seen_values: + msg = ('{} {}: All pool values of A/AAAA dynamic records must be ' + 'included in top-level \'values\'.') + raise AzureException(msg.format(record.fqdn, record._type)) + + geo = any(r.get('geos') for r in data['rules']) + + if [rr, fallback, geo].count(True) > 1: + msg = ('{} {}: A/AAAA dynamic records must use at most one of ' + 'round-robin, fallback and geo-fencing') + raise AzureException(msg.format(record.fqdn, record._type)) + elif typ != 'CNAME': + # dynamic records of unsupported type + msg = '{}: Dynamic records in Azure must be of type A/AAAA/CNAME' + raise AzureException(msg.format(record.fqdn)) + + def _profile_is_match(have, desired): if have is None or desired is None: return False @@ -851,49 +892,7 @@ class AzureProvider(BaseProvider): return data def _extra_changes(self, existing, desired, changes): - changed = set() - - # Abort if there are unsupported dynamic records - for change in changes: - record = change.record - changed.add(record) - typ = record._type - dynamic = getattr(record, 'dynamic', False) - if not dynamic: - continue - if typ in ['A', 'AAAA']: - data = dynamic._data() - values = set(record.values) - pools = data['pools'].values() - seen_values = set() - rr = False - fallback = False - for pool in pools: - vals = pool['values'] - if len(vals) > 1: - rr = True - pool_values = set(val['value'] for val in vals) - if pool.get('fallback'): - fallback = True - seen_values.update(pool_values) - - if values != seen_values: - msg = ('{} {}: All pool values of A/AAAA dynamic records ' - 'must be included in top-level \'values\'.' - .format(record.fqdn, record._type)) - raise AzureException(msg) - - geo = any(r.get('geos') for r in data['rules']) - - if [rr, fallback, geo].count(True) > 1: - msg = ('{} {}: A/AAAA dynamic records must use at most ' - 'one of round-robin, fallback and geo-fencing') - msg = msg.format(record.fqdn, record._type) - raise AzureException(msg) - elif typ != 'CNAME': - msg = ('{}: Dynamic records in Azure must be of type ' - 'A/AAAA/CNAME').format(record.fqdn) - raise AzureException(msg) + changed = set(c.record for c in changes) log = self.log.info seen_profiles = {} @@ -903,6 +902,9 @@ class AzureProvider(BaseProvider): # Already changed, or not dynamic, no need to check it continue + # Abort if there are unsupported dynamic record configurations + _check_valid_dynamic(record) + # let's walk through and show what will be changed even if # the record is already in list of changes added = (record in changed) @@ -910,8 +912,8 @@ class AzureProvider(BaseProvider): active = set() profiles = self._generate_traffic_managers(record) - # this should not happen with above checks, still adding to block - # undesired changes + # this should not happen with above check, check again here to + # prevent undesired changes if record._type in ['A', 'AAAA'] and len(profiles) > 1: msg = ('Unknown error: {} {} needs more than 1 Traffic ' 'Managers which is not supported for A/AAAA dynamic ' diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index 41454e9..16fa5b1 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -1002,8 +1002,6 @@ class TestAzureDnsProvider(TestCase): def test_extra_changes_invalid_dynamic_a(self): provider = self._get_provider() - desired = Zone(name=zone.name, sub_zones=[]) - # too many test case combinations, here's a method to generate them def record_data(all_values=True, rr=True, fallback=True, geo=True): data = { @@ -1068,7 +1066,9 @@ class TestAzureDnsProvider(TestCase): debug('[all_values, rr, fallback, geo] = %s', args) data = record_data(*args) + desired = Zone(name=zone.name, sub_zones=[]) record = Record.new(desired, 'foo', data) + desired.add_record(record) features = args[1:] if all_values and features.count(True) <= 1: @@ -1084,9 +1084,40 @@ class TestAzureDnsProvider(TestCase): else: self.assertTrue('at most one of' in msg) - # multiple profiles should also raise - record = Record.new(desired, 'foo', - record_data(True, True, True, True)) + @patch( + 'octodns.provider.azuredns._check_valid_dynamic') + def test_extra_changes_dynamic_a_multiple_profiles(self, mock_cvd): + provider = self._get_provider() + + # bypass validity check to trigger mutliple-profiles check + mock_cvd.return_value = True + + desired = Zone(name=zone.name, sub_zones=[]) + record = Record.new(desired, 'foo', { + 'type': 'A', + 'ttl': 60, + 'values': ['11.11.11.11', '12.12.12.12', '2.2.2.2'], + 'dynamic': { + 'pools': { + 'one': { + 'values': [ + {'value': '11.11.11.11'}, + {'value': '12.12.12.12'}, + ], + 'fallback': 'two', + }, + 'two': { + 'values': [ + {'value': '2.2.2.2'}, + ], + }, + }, + 'rules': [ + {'geos': ['EU'], 'pool': 'two'}, + {'pool': 'one'}, + ], + } + }) desired.add_record(record) with self.assertRaises(AzureException) as ctx: # bypass above check by setting changes to empty From 568e6860d32fbcbaa5e8487646c4c6ff57fd868e Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Tue, 22 Jun 2021 15:56:08 -0700 Subject: [PATCH 253/358] drop comment --- tests/test_octodns_provider_azuredns.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index 16fa5b1..b577aa4 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -1120,8 +1120,7 @@ class TestAzureDnsProvider(TestCase): }) desired.add_record(record) with self.assertRaises(AzureException) as ctx: - # bypass above check by setting changes to empty - provider._extra_changes(zone, desired, []) + provider._extra_changes(zone, desired, [Create(record)]) self.assertTrue('more than 1 Traffic Managers' in text_type(ctx)) def test_generate_tm_profile(self): From 500097542be7bef9d2ca61a3ec16ae16e5277eec Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Tue, 22 Jun 2021 16:11:28 -0700 Subject: [PATCH 254/358] drop unneeded line break --- tests/test_octodns_provider_azuredns.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index b577aa4..697b30c 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -1084,8 +1084,7 @@ class TestAzureDnsProvider(TestCase): else: self.assertTrue('at most one of' in msg) - @patch( - 'octodns.provider.azuredns._check_valid_dynamic') + @patch('octodns.provider.azuredns._check_valid_dynamic') def test_extra_changes_dynamic_a_multiple_profiles(self, mock_cvd): provider = self._get_provider() From a8a9c7c6d12fa2aa4462db1e1042eae151d28476 Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Wed, 23 Jun 2021 10:52:14 -0700 Subject: [PATCH 255/358] better test method names --- tests/test_octodns_provider_azuredns.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index 697b30c..eb74007 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -999,7 +999,7 @@ class TestAzureDnsProvider(TestCase): 'Collision in Traffic Manager' )) - def test_extra_changes_invalid_dynamic_a(self): + def test_extra_changes_invalid_dynamic_A(self): provider = self._get_provider() # too many test case combinations, here's a method to generate them @@ -1085,7 +1085,7 @@ class TestAzureDnsProvider(TestCase): self.assertTrue('at most one of' in msg) @patch('octodns.provider.azuredns._check_valid_dynamic') - def test_extra_changes_dynamic_a_multiple_profiles(self, mock_cvd): + def test_extra_changes_dynamic_A_multiple_profiles(self, mock_cvd): provider = self._get_provider() # bypass validity check to trigger mutliple-profiles check @@ -1698,7 +1698,7 @@ class TestAzureDnsProvider(TestCase): record2 = provider._populate_record(zone, azrecord) self.assertEqual(record2.dynamic._data(), record.dynamic._data()) - def test_dynamic_a_geo(self): + def test_dynamic_A_geo(self): provider = self._get_provider() external = 'Microsoft.Network/trafficManagerProfiles/externalEndpoints' @@ -1795,7 +1795,7 @@ class TestAzureDnsProvider(TestCase): record2 = provider._populate_record(zone, azrecord) self.assertEqual(record2.dynamic._data(), record.dynamic._data()) - def test_dynamic_a_fallback(self): + def test_dynamic_A_fallback(self): provider = self._get_provider() external = 'Microsoft.Network/trafficManagerProfiles/externalEndpoints' @@ -1865,7 +1865,7 @@ class TestAzureDnsProvider(TestCase): record2 = provider._populate_record(zone, azrecord) self.assertEqual(record2.dynamic._data(), record.dynamic._data()) - def test_dynamic_a_weighted_rr(self): + def test_dynamic_A_weighted_rr(self): provider = self._get_provider() external = 'Microsoft.Network/trafficManagerProfiles/externalEndpoints' @@ -1924,7 +1924,7 @@ class TestAzureDnsProvider(TestCase): record2 = provider._populate_record(zone, azrecord) self.assertEqual(record2.dynamic._data(), record.dynamic._data()) - def test_dynamic_aaaa(self): + def test_dynamic_AAAA(self): provider = self._get_provider() external = 'Microsoft.Network/trafficManagerProfiles/externalEndpoints' From fe7f57e2adeb42cd080680b4ec6bcb089fd7a6df Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Wed, 23 Jun 2021 11:26:57 -0700 Subject: [PATCH 256/358] update README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 91e04eb..3f36e73 100644 --- a/README.md +++ b/README.md @@ -192,7 +192,7 @@ The above command pulled the existing data out of Route53 and placed the results | Provider | Requirements | Record Support | Dynamic | Notes | |--|--|--|--|--| -| [AzureProvider](/octodns/provider/azuredns.py) | azure-identity, azure-mgmt-dns, azure-mgmt-trafficmanager | A, AAAA, CAA, CNAME, MX, NS, PTR, SRV, TXT | Alpha (CNAMEs only) | | +| [AzureProvider](/octodns/provider/azuredns.py) | azure-identity, azure-mgmt-dns, azure-mgmt-trafficmanager | A, AAAA, CAA, CNAME, MX, NS, PTR, SRV, TXT | Alpha (CNAMEs and partial A/AAAA) | | | [Akamai](/octodns/provider/edgedns.py) | edgegrid-python | A, AAAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, SSHFP, TXT | No | | | [CloudflareProvider](/octodns/provider/cloudflare.py) | | A, AAAA, ALIAS, CAA, CNAME, LOC, MX, NS, PTR, SPF, SRV, TXT | No | CAA tags restricted | | [ConstellixProvider](/octodns/provider/constellix.py) | | A, AAAA, ALIAS (ANAME), CAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | CAA tags restricted | From 7e0733a4e4ed953a72eb75d62449b674a66aa831 Mon Sep 17 00:00:00 2001 From: Sham Date: Wed, 23 Jun 2021 12:37:40 -0700 Subject: [PATCH 257/358] fix for NA continent geo target limitation on NS1 --- octodns/provider/ns1.py | 50 +++++++++++++++++------------- tests/test_octodns_provider_ns1.py | 28 ++++++++++++++--- 2 files changed, 51 insertions(+), 27 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 9313d0d..cfa7be7 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -350,8 +350,6 @@ class Ns1Provider(BaseProvider): 'AS': ('ASIAPAC',), 'EU': ('EUROPE',), 'SA': ('SOUTH-AMERICA',), - # TODO: what about CA, MX, and all the other NA countries? - 'NA': ('US-CENTRAL', 'US-EAST', 'US-WEST'), } # Necessary for handling unsupported continents in _CONTINENT_TO_REGIONS @@ -359,6 +357,11 @@ class Ns1Provider(BaseProvider): 'OC': {'FJ', 'NC', 'PG', 'SB', 'VU', 'AU', 'NF', 'NZ', 'FM', 'GU', 'KI', 'MH', 'MP', 'NR', 'PW', 'AS', 'CK', 'NU', 'PF', 'PN', 'TK', 'TO', 'TV', 'WF', 'WS'}, + 'NA': {'DO', 'DM', 'BB', 'BL', 'BM', 'HT', 'KN', 'JM', 'VC', 'HN', + 'BS', 'BZ', 'PR', 'NI', 'LC', 'TT', 'VG', 'PA', 'TC', 'PM', + 'GT', 'AG', 'GP', 'AI', 'VI', 'CA', 'GD', 'AW', 'CR', 'GL', + 'CU', 'MF', 'SV', 'US', 'UM', 'MQ', 'MS', 'KY', 'MX', 'CW', + 'BQ', 'SX'} } def __init__(self, id, api_key, retry_count=4, monitor_regions=None, @@ -549,42 +552,45 @@ class Ns1Provider(BaseProvider): geos = set() - # continents are mapped (imperfectly) to regions, but what about - # Canada/North America for georegion in meta.get('georegion', []): geos.add(self._REGION_TO_CONTINENT[georegion]) # Countries are easy enough to map, we just have to find their # continent # - # NOTE: Special handling for Oceania - # NS1 doesn't support Oceania as a region. So the Oceania countries - # will be present in meta['country']. If all the countries in the - # Oceania countries list are found, set the region to OC and remove - # individual oceania country entries - - oc_countries = set() + # NOTE: Some continents need special handling since NS1 + # does not supprt them as regions. These are defined under + # _CONTINENT_TO_LIST_OF_COUNTRIES. So the countries for these + # regions will be present in meta['country']. If all the countries + # in _CONTINENT_TO_LIST_OF_COUNTRIES[] list are found, + # set the continent as the region and remove individual countries + + special_continents = dict() for country in meta.get('country', []): - # country_alpha2_to_continent_code fails for Pitcairn ('PN') + # country_alpha2_to_continent_code fails for Pitcairn ('PN'), + # United States Minor Outlying Islands ('UM') and + # Sint Maarten ('SX') if country == 'PN': con = 'OC' + elif country in ['SX', 'UM']: + con = 'NA' else: con = country_alpha2_to_continent_code(country) - if con == 'OC': - oc_countries.add(country) + if con in self._CONTINENT_TO_LIST_OF_COUNTRIES: + special_continents.setdefault(con, set()).add(country) else: - # Adding only non-OC countries here to geos geos.add('{}-{}'.format(con, country)) - if oc_countries: - if oc_countries == self._CONTINENT_TO_LIST_OF_COUNTRIES['OC']: - # All OC countries found, so add 'OC' to geos - geos.add('OC') + for continent, countries in special_continents.items(): + if countries == self._CONTINENT_TO_LIST_OF_COUNTRIES[ + continent]: + # All countries found, so add it to geos + geos.add(continent) else: - # Partial OC countries found, just add them as-is to geos - for c in oc_countries: - geos.add('{}-{}'.format('OC', c)) + # Partial countries found, so just add them as-is to geos + for c in countries: + geos.add('{}-{}'.format(continent, c)) # States are easy too, just assume NA-US (CA providences aren't # supported by octoDNS currently) diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index 4ae4757..8535999 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -1034,7 +1034,7 @@ class TestNs1ProviderDynamic(TestCase): rule0 = record.data['dynamic']['rules'][0] rule1 = record.data['dynamic']['rules'][1] rule0['geos'] = ['AF', 'EU'] - rule1['geos'] = ['NA'] + rule1['geos'] = ['AS'] ret, monitor_ids = provider._params_for_A(record) self.assertEquals(10, len(ret['answers'])) self.assertEquals(ret['filters'], @@ -1048,7 +1048,7 @@ class TestNs1ProviderDynamic(TestCase): }, 'iad__georegion': { 'meta': { - 'georegion': ['US-CENTRAL', 'US-EAST', 'US-WEST'], + 'georegion': ['ASIAPAC'], 'note': 'rule-order:1' } }, @@ -1150,7 +1150,7 @@ class TestNs1ProviderDynamic(TestCase): rule0 = record.data['dynamic']['rules'][0] rule1 = record.data['dynamic']['rules'][1] rule0['geos'] = ['AF', 'EU', 'NA-US-CA'] - rule1['geos'] = ['NA', 'NA-US'] + rule1['geos'] = ['AS', 'AS-IN'] ret, _ = provider._params_for_A(record) self.assertEquals(17, len(ret['answers'])) @@ -1210,13 +1210,13 @@ class TestNs1ProviderDynamic(TestCase): }, 'iad__country': { 'meta': { - 'country': ['US'], + 'country': ['IN'], 'note': 'rule-order:1' } }, 'iad__georegion': { 'meta': { - 'georegion': ['US-CENTRAL', 'US-EAST', 'US-WEST'], + 'georegion': ['ASIAPAC'], 'note': 'rule-order:1' } }, @@ -1562,6 +1562,24 @@ class TestNs1ProviderDynamic(TestCase): self.assertTrue( 'OC-{}'.format(c) in data4['dynamic']['rules'][0]['geos']) + # NA test cases + # 1. Full list of countries should return 'NA' in geos + na_countries = Ns1Provider._CONTINENT_TO_LIST_OF_COUNTRIES['NA'] + del ns1_record['regions']['lhr__country']['meta']['us_state'] + ns1_record['regions']['lhr__country']['meta']['country'] = \ + list(na_countries) + data5 = provider._data_for_A('A', ns1_record) + self.assertTrue('NA' in data5['dynamic']['rules'][0]['geos']) + + # 2. Partial list of countries should return just those + partial_na_cntry_list = list(na_countries)[:5] + ns1_record['regions']['lhr__country']['meta']['country'] = \ + partial_na_cntry_list + data6 = provider._data_for_A('A', ns1_record) + for c in partial_na_cntry_list: + self.assertTrue( + 'NA-{}'.format(c) in data6['dynamic']['rules'][0]['geos']) + # Test out fallback only pools and new-style notes ns1_record = { 'answers': [{ From c3f0bf677a9775bcb55ce3a25b7f622deba6798d Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Wed, 23 Jun 2021 18:49:19 -0700 Subject: [PATCH 258/358] Validate processor config sections --- octodns/manager.py | 11 +++++++++-- tests/config/unknown-processor.yaml | 17 +++++++++++++++++ tests/test_octodns_manager.py | 5 +++++ 3 files changed, 31 insertions(+), 2 deletions(-) create mode 100644 tests/config/unknown-processor.yaml diff --git a/octodns/manager.py b/octodns/manager.py index c7173c6..dcbf083 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -573,8 +573,15 @@ class Manager(object): if isinstance(source, YamlProvider): source.populate(zone) - # TODO: validate - # processors = config.get('processors', []) + # check that processors are in order if any are specified + processors = config.get('processors', []) + try: + # same as above, but for processors this time + for processor in processors: + collected.append(self.processors[processor]) + except KeyError: + raise ManagerException('Zone {}, unknown processor: {}' + .format(zone_name, processor)) def get_zone(self, zone_name): if not zone_name[-1] == '.': diff --git a/tests/config/unknown-processor.yaml b/tests/config/unknown-processor.yaml new file mode 100644 index 0000000..4aff713 --- /dev/null +++ b/tests/config/unknown-processor.yaml @@ -0,0 +1,17 @@ +manager: + max_workers: 2 +providers: + in: + class: octodns.provider.yaml.YamlProvider + directory: tests/config + dump: + class: octodns.provider.yaml.YamlProvider + directory: env/YAML_TMP_DIR +zones: + unit.tests.: + sources: + - in + processors: + - missing + targets: + - dump diff --git a/tests/test_octodns_manager.py b/tests/test_octodns_manager.py index 069bc0b..8bada06 100644 --- a/tests/test_octodns_manager.py +++ b/tests/test_octodns_manager.py @@ -339,6 +339,11 @@ class TestManager(TestCase): Manager(get_config_filename('simple-alias-zone.yaml')) \ .validate_configs() + with self.assertRaises(ManagerException) as ctx: + Manager(get_config_filename('unknown-processor.yaml')) \ + .validate_configs() + self.assertTrue('unknown processor' in text_type(ctx.exception)) + def test_get_zone(self): Manager(get_config_filename('simple.yaml')).get_zone('unit.tests.') From 8f3ee159c25389bf961a2c5e6401f28a3931412a Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Wed, 23 Jun 2021 18:56:20 -0700 Subject: [PATCH 259/358] Changelog entry for processors --- CHANGELOG.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d16544..0b9e498 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,13 @@ +## v0.9.13 - 2021-..-.. - + +#### Noteworthy changes + +* Alpha support for Processors has been added. Processors allow for hooking + into the source, target, and planing process to make nearly arbitrary changes + to data. See the [octodns/processors/](/octodns/processors) directory for + examples. The change has been designed to have no impact on the process + unless the `processors` key is present in zone configs. + ## v0.9.12 - 2021-04-30 - Enough time has passed #### Noteworthy changes From a7659c308669fa163ff647afb85f544a9b64033f Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Wed, 23 Jun 2021 19:02:26 -0700 Subject: [PATCH 260/358] processor not processors --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b9e498..f70d67c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ * Alpha support for Processors has been added. Processors allow for hooking into the source, target, and planing process to make nearly arbitrary changes - to data. See the [octodns/processors/](/octodns/processors) directory for + to data. See the [octodns/processor/](/octodns/processor) directory for examples. The change has been designed to have no impact on the process unless the `processors` key is present in zone configs. From 2c72f840c16c650e7282cee371c8ee3090317454 Mon Sep 17 00:00:00 2001 From: Sham Date: Wed, 23 Jun 2021 21:46:39 -0700 Subject: [PATCH 261/358] CHANGELOG entry --- CHANGELOG.md | 3 +++ README.md | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f70d67c..10e87b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ to data. See the [octodns/processor/](/octodns/processor) directory for examples. The change has been designed to have no impact on the process unless the `processors` key is present in zone configs. +* Fixes NS1 provider's geotarget limitation of using `NA` continent. Now, when + `NA` is used in geos it considers **all** the countries of `North America` + insted of just `us-east`, `us-west` and `us-central` regions ## v0.9.12 - 2021-04-30 - Enough time has passed diff --git a/README.md b/README.md index 3f36e73..8486799 100644 --- a/README.md +++ b/README.md @@ -207,7 +207,7 @@ The above command pulled the existing data out of Route53 and placed the results | [GoogleCloudProvider](/octodns/provider/googlecloud.py) | google-cloud-dns | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | No | | | [HetznerProvider](/octodns/provider/hetzner.py) | | A, AAAA, CAA, CNAME, MX, NS, SRV, TXT | No | | | [MythicBeastsProvider](/octodns/provider/mythicbeasts.py) | Mythic Beasts | A, AAAA, ALIAS, CNAME, MX, NS, SRV, SSHFP, CAA, TXT | No | | -| [Ns1Provider](/octodns/provider/ns1.py) | ns1-python | All | Yes | Missing `NA` geo target | +| [Ns1Provider](/octodns/provider/ns1.py) | ns1-python | All | Yes | | | [OVH](/octodns/provider/ovh.py) | ovh | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, SSHFP, TXT, DKIM | No | | | [PowerDnsProvider](/octodns/provider/powerdns.py) | | All | No | | | [Rackspace](/octodns/provider/rackspace.py) | | A, AAAA, ALIAS, CNAME, MX, NS, PTR, SPF, TXT | No | | From 3cc0fac817023e8d22f1d69146ccfe8bea00f5d4 Mon Sep 17 00:00:00 2001 From: blanariu Date: Thu, 24 Jun 2021 12:39:56 +0300 Subject: [PATCH 262/358] Fix bug in Manager when using Python 2.7 In Python 2.7 the if statement would catch both cases from the test test_populate_lenient_fallback, so the test was failing. These are the error strings differences between Python 2 and 3: Python 2: NoLenient: populate() got an unexpected keyword argument 'lenient' NoZone: populate() got multiple values for keyword argument 'lenient' Python 3: NoLenient: populate() got an unexpected keyword argument 'lenient' NoZone: populate() got multiple values for argument 'lenient' --- octodns/manager.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/octodns/manager.py b/octodns/manager.py index 9b10196..dcae0a7 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -243,7 +243,8 @@ class Manager(object): try: source.populate(zone, lenient=lenient) except TypeError as e: - if "keyword argument 'lenient'" not in text_type(e): + if ("unexpected keyword argument 'lenient'" + not in text_type(e)): raise self.log.warn(': provider %s does not accept lenient ' 'param', source.__class__.__name__) From 749f0bd90fae302f23c9d0951c5447b279782fd6 Mon Sep 17 00:00:00 2001 From: blanariu Date: Thu, 24 Jun 2021 12:45:22 +0300 Subject: [PATCH 263/358] Fix bug in envvar source tests when run in Python 2.7 Due to an import error the envvar source tests would not run in Python 2.7 --- tests/test_octodns_source_envvar.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/test_octodns_source_envvar.py b/tests/test_octodns_source_envvar.py index 0714883..38b63dd 100644 --- a/tests/test_octodns_source_envvar.py +++ b/tests/test_octodns_source_envvar.py @@ -1,6 +1,12 @@ from six import text_type from unittest import TestCase -from unittest.mock import patch + +try: + # Python 3 + from unittest.mock import patch +except ImportError: + # Python 2 + from mock import patch from octodns.source.envvar import EnvVarSource from octodns.source.envvar import EnvironmentVariableNotFoundException From e934ea0423b25c656420948a73f1922db2d61b41 Mon Sep 17 00:00:00 2001 From: blanariu Date: Thu, 24 Jun 2021 12:51:45 +0300 Subject: [PATCH 264/358] Fix bug in ultra provider tests when run in Python 2.7 The test_login test from TestUltraProvider would fail in Python 2.7 due to the dictionary insertion order not being preserved in 2.7 and early 3.x versions. Comparing the dictionaries containing the query parameters solves this. Snippet from test failure: - username=user&password=rightpass&grant_type=password + grant_type=password&username=user&password=rightpass --- tests/test_octodns_provider_ultra.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/test_octodns_provider_ultra.py b/tests/test_octodns_provider_ultra.py index 52e0307..0597d85 100644 --- a/tests/test_octodns_provider_ultra.py +++ b/tests/test_octodns_provider_ultra.py @@ -1,3 +1,12 @@ +from __future__ import unicode_literals + +try: + # Python 3 + from urllib.parse import parse_qs +except ImportError: + # Python 2 + from urlparse import parse_qs + from mock import Mock, call from os.path import dirname, join from requests import HTTPError @@ -55,7 +64,8 @@ class TestUltraProvider(TestCase): self.assertEquals(1, mock.call_count) expected_payload = "grant_type=password&username=user&"\ "password=rightpass" - self.assertEquals(mock.last_request.text, expected_payload) + self.assertEquals(parse_qs(mock.last_request.text), + parse_qs(expected_payload)) def test_get_zones(self): provider = _get_provider() From efc4d99d8dc5409ae870ecd41e20729498019ba3 Mon Sep 17 00:00:00 2001 From: blanariu Date: Thu, 24 Jun 2021 19:17:09 +0300 Subject: [PATCH 265/358] Replaced conditional imports with six --- tests/test_octodns_provider_ultra.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/tests/test_octodns_provider_ultra.py b/tests/test_octodns_provider_ultra.py index 0597d85..a22a489 100644 --- a/tests/test_octodns_provider_ultra.py +++ b/tests/test_octodns_provider_ultra.py @@ -1,17 +1,11 @@ from __future__ import unicode_literals -try: - # Python 3 - from urllib.parse import parse_qs -except ImportError: - # Python 2 - from urlparse import parse_qs - from mock import Mock, call from os.path import dirname, join from requests import HTTPError from requests_mock import ANY, mock as requests_mock from six import text_type +from six.moves.urllib import parse from unittest import TestCase from json import load as json_load @@ -64,8 +58,8 @@ class TestUltraProvider(TestCase): self.assertEquals(1, mock.call_count) expected_payload = "grant_type=password&username=user&"\ "password=rightpass" - self.assertEquals(parse_qs(mock.last_request.text), - parse_qs(expected_payload)) + self.assertEquals(parse.parse_qs(mock.last_request.text), + parse.parse_qs(expected_payload)) def test_get_zones(self): provider = _get_provider() From 29532302e28fd16886d782b5eb9544f62b02ea46 Mon Sep 17 00:00:00 2001 From: blanariu Date: Thu, 24 Jun 2021 19:28:53 +0300 Subject: [PATCH 266/358] Leave just importing from mock --- tests/test_octodns_source_envvar.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/tests/test_octodns_source_envvar.py b/tests/test_octodns_source_envvar.py index 38b63dd..ac66a22 100644 --- a/tests/test_octodns_source_envvar.py +++ b/tests/test_octodns_source_envvar.py @@ -1,13 +1,7 @@ +from mock import patch from six import text_type from unittest import TestCase -try: - # Python 3 - from unittest.mock import patch -except ImportError: - # Python 2 - from mock import patch - from octodns.source.envvar import EnvVarSource from octodns.source.envvar import EnvironmentVariableNotFoundException from octodns.zone import Zone From e55da245d9414f4250463b66c3421ad65df40025 Mon Sep 17 00:00:00 2001 From: Sham Date: Thu, 24 Jun 2021 11:29:34 -0700 Subject: [PATCH 267/358] comment for why US-* need to continue to exist under _REGION_TO_CONTINENT --- octodns/provider/ns1.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index cfa7be7..c296ab2 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -341,6 +341,9 @@ class Ns1Provider(BaseProvider): 'ASIAPAC': 'AS', 'EUROPE': 'EU', 'SOUTH-AMERICA': 'SA', + # continent NA has been handled as part of Geofence Country filter + # starting from v0.9.13. These below US-* just need to continue to + # exist here so it doesn't break the ugrade path 'US-CENTRAL': 'NA', 'US-EAST': 'NA', 'US-WEST': 'NA', From d6deabcc522f8a18cfdef39acf0c02abc377f818 Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Fri, 25 Jun 2021 10:52:35 -0700 Subject: [PATCH 268/358] Fix duplicate endpoints in Azure DNS dynamic records --- octodns/provider/azuredns.py | 11 +++++- tests/test_octodns_provider_azuredns.py | 45 +++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 4551c6c..c69c723 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -922,6 +922,16 @@ class AzureProvider(BaseProvider): for profile in profiles: name = profile.name + + endpoints = set() + for ep in profile.endpoints: + if not ep.target: + continue + if ep.target in endpoints: + msg = '{} contains duplicate endpoint {}' + raise AzureException(msg.format(name, ep.target)) + endpoints.add(ep.target) + if name in seen_profiles: # exit if a possible collision is detected, even though # we've tried to ensure unique mapping @@ -1053,7 +1063,6 @@ class AzureProvider(BaseProvider): while pool_name: # iterate until we reach end of fallback chain - default_seen = False pool = pools[pool_name].data if len(pool['values']) > 1: # create Weighted profile for multi-value pool diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index eb74007..13c1bf4 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -999,6 +999,51 @@ class TestAzureDnsProvider(TestCase): 'Collision in Traffic Manager' )) + @patch( + 'octodns.provider.azuredns.AzureProvider._generate_traffic_managers') + def test_extra_changes_non_last_fallback_contains_default(self, mock_gtm): + provider = self._get_provider() + + desired = Zone(zone.name, sub_zones=[]) + record = Record.new(desired, 'foo', { + 'type': 'CNAME', + 'ttl': 60, + 'value': 'default.unit.tests.', + 'dynamic': { + 'pools': { + 'one': { + 'values': [{'value': 'one.unit.tests.'}], + 'fallback': 'def', + }, + 'def': { + 'values': [{'value': 'default.unit.tests.'}], + 'fallback': 'two', + }, + 'two': { + 'values': [{'value': 'two.unit.tests.'}], + }, + }, + 'rules': [ + {'pool': 'one'}, + ] + } + }) + desired.add_record(record) + changes = [Create(record)] + + # assert that no exception is raised + provider._extra_changes(zone, desired, changes) + + # simulate duplicate endpoint and assert exception + endpoint = Endpoint(target='dup.unit.tests.') + mock_gtm.return_value = [Profile( + name='test-profile', + endpoints=[endpoint, endpoint], + )] + with self.assertRaises(AzureException) as ctx: + provider._extra_changes(zone, desired, changes) + self.assertTrue('duplicate endpoint' in text_type(ctx)) + def test_extra_changes_invalid_dynamic_A(self): provider = self._get_provider() From 4848246712d56268b7f074e151466846533c5639 Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Fri, 25 Jun 2021 13:58:00 -0700 Subject: [PATCH 269/358] Fix partially re-used fallback chain --- octodns/provider/azuredns.py | 4 ++-- tests/test_octodns_provider_azuredns.py | 15 ++++++++++++++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index c69c723..87bbef0 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -837,8 +837,8 @@ class AzureProvider(BaseProvider): pool['fallback'] = pool_name if pool_name in pools: - # we've already populated the pool - continue + # we've already populated this and subsequent pools + break # populate the pool from Weighted profile # these should be leaf node entries with no further nesting diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index 13c1bf4..e09de28 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -1687,6 +1687,12 @@ class TestAzureDnsProvider(TestCase): 'value': 'default.unit.tests.', 'dynamic': { 'pools': { + 'sto': { + 'values': [ + {'value': 'sto.unit.tests.'}, + ], + 'fallback': 'iad', + }, 'iad': { 'values': [ {'value': 'iad.unit.tests.'}, @@ -1702,13 +1708,14 @@ class TestAzureDnsProvider(TestCase): 'rules': [ {'geos': ['EU'], 'pool': 'iad'}, {'geos': ['EU-GB'], 'pool': 'lhr'}, + {'geos': ['EU-SE'], 'pool': 'sto'}, {'pool': 'lhr'}, ], } }) profiles = provider._generate_traffic_managers(record) - self.assertEqual(len(profiles), 3) + self.assertEqual(len(profiles), 4) self.assertTrue(_profile_is_match(profiles[-1], Profile( name='foo--unit--tests', traffic_routing_method='Geographic', @@ -1728,6 +1735,12 @@ class TestAzureDnsProvider(TestCase): target_resource_id=profiles[1].id, geo_mapping=['GB', 'WORLD'], ), + Endpoint( + name='rule-sto', + type=nested, + target_resource_id=profiles[2].id, + geo_mapping=['SE'], + ), ], ))) From 832c22a513df4d8bceefbbb1c57c18330cecff22 Mon Sep 17 00:00:00 2001 From: Sham Date: Sun, 27 Jun 2021 01:24:13 -0700 Subject: [PATCH 270/358] added SX and UM to partial list of countries test --- octodns/provider/ns1.py | 3 +-- tests/test_octodns_provider_ns1.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index c296ab2..bf08358 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -363,8 +363,7 @@ class Ns1Provider(BaseProvider): 'NA': {'DO', 'DM', 'BB', 'BL', 'BM', 'HT', 'KN', 'JM', 'VC', 'HN', 'BS', 'BZ', 'PR', 'NI', 'LC', 'TT', 'VG', 'PA', 'TC', 'PM', 'GT', 'AG', 'GP', 'AI', 'VI', 'CA', 'GD', 'AW', 'CR', 'GL', - 'CU', 'MF', 'SV', 'US', 'UM', 'MQ', 'MS', 'KY', 'MX', 'CW', - 'BQ', 'SX'} + 'CU', 'MF', 'SV', 'US', 'MQ', 'MS', 'KY', 'MX', 'CW', 'BQ'} } def __init__(self, id, api_key, retry_count=4, monitor_regions=None, diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index 8535999..df517a2 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -1572,7 +1572,7 @@ class TestNs1ProviderDynamic(TestCase): self.assertTrue('NA' in data5['dynamic']['rules'][0]['geos']) # 2. Partial list of countries should return just those - partial_na_cntry_list = list(na_countries)[:5] + partial_na_cntry_list = list(na_countries)[:5] + ['SX', 'UM'] ns1_record['regions']['lhr__country']['meta']['country'] = \ partial_na_cntry_list data6 = provider._data_for_A('A', ns1_record) From 5c4a46f63ff90bbfa5036593d6827776213d057b Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sun, 27 Jun 2021 13:17:52 -0700 Subject: [PATCH 271/358] Add note about SX & UM country code support --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 10e87b9..ce876a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ * Fixes NS1 provider's geotarget limitation of using `NA` continent. Now, when `NA` is used in geos it considers **all** the countries of `North America` insted of just `us-east`, `us-west` and `us-central` regions +* `SX' & 'UM` country support added, not yet in the North America list for + backwards compatibility reasons. They will be added in the next releaser. ## v0.9.12 - 2021-04-30 - Enough time has passed From 0dcbb11bcbe80317448d6d5437d46f950bb5be9c Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sun, 27 Jun 2021 14:51:32 -0700 Subject: [PATCH 272/358] Clarify new NA countries are NS1Provider specific --- CHANGELOG.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ce876a6..e6dcb48 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,8 +10,9 @@ * Fixes NS1 provider's geotarget limitation of using `NA` continent. Now, when `NA` is used in geos it considers **all** the countries of `North America` insted of just `us-east`, `us-west` and `us-central` regions -* `SX' & 'UM` country support added, not yet in the North America list for - backwards compatibility reasons. They will be added in the next releaser. +* `SX' & 'UM` country support added to NS1Provider, not yet in the North + America list for backwards compatibility reasons. They will be added in the + next releaser. ## v0.9.12 - 2021-04-30 - Enough time has passed From 1d55124a5c612c2ead3b66779d87b983fa5e402b Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Mon, 28 Jun 2021 10:13:59 -0700 Subject: [PATCH 273/358] Fix repetitive updates due to endpoint name overwrite Also add _extra_changes to dynamic record tests to make them more exhausitve. Simplify root profile's endpoint names. --- octodns/provider/azuredns.py | 27 ++++--- tests/test_octodns_provider_azuredns.py | 96 +++++++++++++++++++------ 2 files changed, 89 insertions(+), 34 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 87bbef0..107866e 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -748,7 +748,10 @@ class AzureProvider(BaseProvider): if root_profile.traffic_routing_method != 'Geographic': # This record does not use geo fencing, so we skip the Geographic # profile hop; let's pretend to be a geo-profile's only endpoint - geo_ep = Endpoint(target_resource_id=root_profile.id) + geo_ep = Endpoint( + name=root_profile.endpoints[0].name.split('--', 1)[0], + target_resource_id=root_profile.id + ) geo_ep.target_resource = root_profile endpoints = [geo_ep] else: @@ -799,18 +802,14 @@ class AzureProvider(BaseProvider): geos.append(GeoCodes.country_to_code(code)) # build fallback chain from second level priority profile - if geo_ep.target_resource_id: - target = geo_ep.target_resource - if target.traffic_routing_method == 'Priority': - rule_endpoints = target.endpoints - rule_endpoints.sort(key=lambda e: e.priority) - else: - # Weighted - geo_ep.name = target.endpoints[0].name.split('--', 1)[0] - rule_endpoints = [geo_ep] + if geo_ep.target_resource_id and \ + geo_ep.target_resource.traffic_routing_method == 'Priority': + rule_endpoints = geo_ep.target_resource.endpoints + rule_endpoints.sort(key=lambda e: e.priority) else: - # this geo directly points to the default, so we skip the - # Priority profile hop and directly use an external endpoint; + # this geo directly points to a pool containing the default + # so we skip the Priority profile hop and directly use an + # external endpoint or Weighted profile # let's pretend to be a Priority profile's only endpoint rule_endpoints = [geo_ep] @@ -1133,7 +1132,7 @@ class AzureProvider(BaseProvider): # append rule profile to top-level geo profile geo_endpoints.append(Endpoint( - name='rule-{}'.format(rule.data['pool']), + name=rule.data['pool'], target_resource_id=rule_profile.id, geo_mapping=geos, )) @@ -1144,7 +1143,7 @@ class AzureProvider(BaseProvider): if rule_ep.target_resource_id: # point directly to the Weighted pool profile geo_endpoints.append(Endpoint( - name='rule-{}'.format(rule.data['pool']), + name=rule_ep.name, target_resource_id=rule_ep.target_resource_id, geo_mapping=geos, )) diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index e09de28..b92372f 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -697,13 +697,13 @@ class TestAzureDnsProvider(TestCase): endpoints=[ Endpoint( geo_mapping=['GEO-AF', 'DE', 'US-CA', 'GEO-AP'], - name='rule-one', + name='one', type=nested, target_resource_id=id_format.format('rule-one'), ), Endpoint( geo_mapping=['WORLD'], - name='rule-two', + name='two', type=nested, target_resource_id=id_format.format('rule-two'), ), @@ -1231,6 +1231,12 @@ class TestAzureDnsProvider(TestCase): record2 = provider._populate_record(zone, azrecord) self.assertEqual(record2.dynamic._data(), record.dynamic._data()) + # test that extra changes doesn't show any changes + desired = Zone(zone.name, sub_zones=[]) + desired.add_record(record) + changes = provider._extra_changes(zone, desired, []) + self.assertEqual(len(changes), 0) + def test_generate_traffic_managers_middle_east(self): # check Asia/Middle East test case provider, zone, record = self._get_dynamic_package() @@ -1336,6 +1342,12 @@ class TestAzureDnsProvider(TestCase): record2 = provider._populate_record(zone, azrecord) self.assertEqual(record2.dynamic._data(), record.dynamic._data()) + # test that extra changes doesn't show any changes + desired = Zone(zone.name, sub_zones=[]) + desired.add_record(record) + changes = provider._extra_changes(zone, desired, []) + self.assertEqual(len(changes), 0) + def test_dynamic_fallback_is_default(self): # test that traffic managers are generated as expected provider = self._get_provider() @@ -1389,6 +1401,12 @@ class TestAzureDnsProvider(TestCase): record2 = provider._populate_record(zone, azrecord) self.assertEqual(record2.dynamic._data(), record.dynamic._data()) + # test that extra changes doesn't show any changes + desired = Zone(zone.name, sub_zones=[]) + desired.add_record(record) + changes = provider._extra_changes(zone, desired, []) + self.assertEqual(len(changes), 0) + def test_dynamic_pool_contains_default(self): # test that traffic managers are generated as expected provider = self._get_provider() @@ -1459,7 +1477,7 @@ class TestAzureDnsProvider(TestCase): monitor_config=_get_monitor(record), endpoints=[ Endpoint( - name='rule-rr', + name='rr', type=nested, target_resource_id=profiles[0].id, geo_mapping=['GEO-AF'], @@ -1479,6 +1497,12 @@ class TestAzureDnsProvider(TestCase): record2 = provider._populate_record(zone, azrecord) self.assertEqual(record2.dynamic._data(), record.dynamic._data()) + # test that extra changes doesn't show any changes + desired = Zone(zone.name, sub_zones=[]) + desired.add_record(record) + changes = provider._extra_changes(zone, desired, []) + self.assertEqual(len(changes), 0) + def test_dynamic_pool_contains_default_no_geo(self): # test that traffic managers are generated as expected provider = self._get_provider() @@ -1553,6 +1577,12 @@ class TestAzureDnsProvider(TestCase): record2 = provider._populate_record(zone, azrecord) self.assertEqual(record2.dynamic._data(), record.dynamic._data()) + # test that extra changes doesn't show any changes + desired = Zone(zone.name, sub_zones=[]) + desired.add_record(record) + changes = provider._extra_changes(zone, desired, []) + self.assertEqual(len(changes), 0) + def test_dynamic_last_pool_contains_default_no_geo(self): # test that traffic managers are generated as expected provider = self._get_provider() @@ -1655,6 +1685,12 @@ class TestAzureDnsProvider(TestCase): record2 = provider._populate_record(zone, azrecord) self.assertEqual(record2.dynamic._data(), record.dynamic._data()) + # test that extra changes doesn't show any changes + desired = Zone(zone.name, sub_zones=[]) + desired.add_record(record) + changes = provider._extra_changes(zone, desired, []) + self.assertEqual(len(changes), 0) + def test_dynamic_unique_traffic_managers(self): record = self._get_dynamic_record(zone) data = { @@ -1724,19 +1760,19 @@ class TestAzureDnsProvider(TestCase): monitor_config=_get_monitor(record), endpoints=[ Endpoint( - name='rule-iad', + name='iad', type=nested, target_resource_id=profiles[0].id, geo_mapping=['GEO-EU'], ), Endpoint( - name='rule-lhr', + name='lhr', type=nested, target_resource_id=profiles[1].id, geo_mapping=['GB', 'WORLD'], ), Endpoint( - name='rule-sto', + name='sto', type=nested, target_resource_id=profiles[2].id, geo_mapping=['SE'], @@ -1756,6 +1792,12 @@ class TestAzureDnsProvider(TestCase): record2 = provider._populate_record(zone, azrecord) self.assertEqual(record2.dynamic._data(), record.dynamic._data()) + # test that extra changes doesn't show any changes + desired = Zone(zone.name, sub_zones=[]) + desired.add_record(record) + changes = provider._extra_changes(zone, desired, []) + self.assertEqual(len(changes), 0) + def test_dynamic_A_geo(self): provider = self._get_provider() external = 'Microsoft.Network/trafficManagerProfiles/externalEndpoints' @@ -1790,10 +1832,6 @@ class TestAzureDnsProvider(TestCase): } }) - # test that extra_changes doesn't complain - changes = [Create(record)] - provider._extra_changes(zone, zone, changes) - profiles = provider._generate_traffic_managers(record) self.assertEqual(len(profiles), 1) @@ -1828,7 +1866,7 @@ class TestAzureDnsProvider(TestCase): # test that the record and ATM profile gets created tm_sync = provider._tm_client.profiles.create_or_update create = provider._dns_client.record_sets.create_or_update - provider._apply_Create(changes[0]) + provider._apply_Create(Create(record)) # A dynamic record can only have 1 profile tm_sync.assert_called_once() create.assert_called_once() @@ -1853,6 +1891,12 @@ class TestAzureDnsProvider(TestCase): record2 = provider._populate_record(zone, azrecord) self.assertEqual(record2.dynamic._data(), record.dynamic._data()) + # test that extra changes doesn't show any changes + desired = Zone(zone.name, sub_zones=[]) + desired.add_record(record) + changes = provider._extra_changes(zone, desired, []) + self.assertEqual(len(changes), 0) + def test_dynamic_A_fallback(self): provider = self._get_provider() external = 'Microsoft.Network/trafficManagerProfiles/externalEndpoints' @@ -1860,7 +1904,7 @@ class TestAzureDnsProvider(TestCase): record = Record.new(zone, 'foo', data={ 'type': 'A', 'ttl': 60, - 'values': ['8.8.8.8'], + 'values': ['1.1.1.1', '2.2.2.2'], 'dynamic': { 'pools': { 'one': { @@ -1891,23 +1935,17 @@ class TestAzureDnsProvider(TestCase): monitor_config=_get_monitor(record), endpoints=[ Endpoint( - name='one', + name='one--default--', type=external, target='1.1.1.1', priority=1, ), Endpoint( - name='two', + name='two--default--', type=external, target='2.2.2.2', priority=2, ), - Endpoint( - name='--default--', - type=external, - target='8.8.8.8', - priority=3, - ), ], ))) @@ -1923,6 +1961,12 @@ class TestAzureDnsProvider(TestCase): record2 = provider._populate_record(zone, azrecord) self.assertEqual(record2.dynamic._data(), record.dynamic._data()) + # test that extra changes doesn't show any changes + desired = Zone(zone.name, sub_zones=[]) + desired.add_record(record) + changes = provider._extra_changes(zone, desired, []) + self.assertEqual(len(changes), 0) + def test_dynamic_A_weighted_rr(self): provider = self._get_provider() external = 'Microsoft.Network/trafficManagerProfiles/externalEndpoints' @@ -1982,6 +2026,12 @@ class TestAzureDnsProvider(TestCase): record2 = provider._populate_record(zone, azrecord) self.assertEqual(record2.dynamic._data(), record.dynamic._data()) + # test that extra changes doesn't show any changes + desired = Zone(zone.name, sub_zones=[]) + desired.add_record(record) + changes = provider._extra_changes(zone, desired, []) + self.assertEqual(len(changes), 0) + def test_dynamic_AAAA(self): provider = self._get_provider() external = 'Microsoft.Network/trafficManagerProfiles/externalEndpoints' @@ -2050,6 +2100,12 @@ class TestAzureDnsProvider(TestCase): record2 = provider._populate_record(zone, azrecord) self.assertEqual(record2.dynamic._data(), record.dynamic._data()) + # test that extra changes doesn't show any changes + desired = Zone(zone.name, sub_zones=[]) + desired.add_record(record) + changes = provider._extra_changes(zone, desired, []) + self.assertEqual(len(changes), 0) + def test_sync_traffic_managers(self): provider, zone, record = self._get_dynamic_package() provider._populate_traffic_managers() From 1b95724e1706cf1656854edad8880da1e50dc5a5 Mon Sep 17 00:00:00 2001 From: Sham Date: Mon, 28 Jun 2021 14:18:47 -0700 Subject: [PATCH 274/358] Adding SX and UM to NA countries --- octodns/provider/ns1.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index bf08358..9e720cd 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -363,7 +363,8 @@ class Ns1Provider(BaseProvider): 'NA': {'DO', 'DM', 'BB', 'BL', 'BM', 'HT', 'KN', 'JM', 'VC', 'HN', 'BS', 'BZ', 'PR', 'NI', 'LC', 'TT', 'VG', 'PA', 'TC', 'PM', 'GT', 'AG', 'GP', 'AI', 'VI', 'CA', 'GD', 'AW', 'CR', 'GL', - 'CU', 'MF', 'SV', 'US', 'MQ', 'MS', 'KY', 'MX', 'CW', 'BQ'} + 'CU', 'MF', 'SV', 'US', 'MQ', 'MS', 'KY', 'MX', 'CW', 'BQ', + 'SX', 'UM'} } def __init__(self, id, api_key, retry_count=4, monitor_regions=None, From 0f0d0d12e2dd9464d01d8ff9c8298b4dfb68160f Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Tue, 29 Jun 2021 15:36:48 -0700 Subject: [PATCH 275/358] Drop colons of IPv6 values from endpoint names --- octodns/provider/azuredns.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 107866e..fd5555d 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -1074,6 +1074,9 @@ class AzureProvider(BaseProvider): if typ == 'CNAME': target = target[:-1] ep_name = '{}--{}'.format(pool_name, target) + # Endpoint names cannot have colons, drop them + # from IPv6 addresses + ep_name = ep_name.replace(':', '-') if target in defaults: # mark default ep_name += '--default--' From 303e439a541731ed0bbe16ff7f16e0f78bbbe62e Mon Sep 17 00:00:00 2001 From: Sham Date: Wed, 30 Jun 2021 12:11:54 -0700 Subject: [PATCH 276/358] skip monitors that are not managed by OctoDNS --- octodns/provider/ns1.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index bf08358..510b099 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -849,6 +849,8 @@ class Ns1Provider(BaseProvider): for monitor in self._client.monitors.values(): data = self._parse_notes(monitor['notes']) + if not data: + continue if expected_host == data['host'] and \ expected_type == data['type']: # This monitor does not belong to this record From 7f3aafe6a416212ca24900223a7b60ccd8801fb6 Mon Sep 17 00:00:00 2001 From: Sham Date: Thu, 1 Jul 2021 00:31:17 -0700 Subject: [PATCH 277/358] added tests for both non-confirming notes as well as empty notes in existing monitors --- tests/test_octodns_provider_ns1.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index df517a2..e21fe0d 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -659,6 +659,18 @@ class TestNs1ProviderDynamic(TestCase): }, 'four': monitor_four, 'five': monitor_five, + 'six': { + 'config': { + 'host': '10.10.10.10', + }, + 'notes': 'non-conforming notes', + }, + 'seven': { + 'config': { + 'host': '11.11.11.11', + }, + 'notes': None, + }, } # Would match, but won't get there b/c it's not dynamic From 2d21125645b3170b023e9ceeb8f3b1867862099c Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Sun, 4 Jul 2021 04:55:18 -0700 Subject: [PATCH 278/358] Full support for A/AAAA dynamic records in Azure --- octodns/provider/azuredns.py | 97 +++-- tests/test_octodns_provider_azuredns.py | 472 +++++++++++------------- 2 files changed, 255 insertions(+), 314 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index fd5555d..845d4d5 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -301,38 +301,10 @@ def _get_monitor(record): def _check_valid_dynamic(record): typ = record._type - dynamic = record.dynamic if typ in ['A', 'AAAA']: - # A/AAAA records cannot be aliased to Traffic Managers that contain - # other nested Traffic Managers. Due to this limitation, A/AAAA - # dynamic records can do only one of geo-fencing, fallback and - # weighted RR. So let's validate that the record adheres to this - # limitation. - data = dynamic._data() - values = set(record.values) - pools = data['pools'].values() - seen_values = set() - rr = False - fallback = False - for pool in pools: - vals = pool['values'] - if len(vals) > 1: - rr = True - pool_values = set(val['value'] for val in vals) - if pool.get('fallback'): - fallback = True - seen_values.update(pool_values) - - if values != seen_values: - msg = ('{} {}: All pool values of A/AAAA dynamic records must be ' - 'included in top-level \'values\'.') - raise AzureException(msg.format(record.fqdn, record._type)) - - geo = any(r.get('geos') for r in data['rules']) - - if [rr, fallback, geo].count(True) > 1: - msg = ('{} {}: A/AAAA dynamic records must use at most one of ' - 'round-robin, fallback and geo-fencing') + if len(record.values) > 1: + # we don't yet support multi-value defaults + msg = '{} {}: A/AAAA dynamic records can only have a single value' raise AzureException(msg.format(record.fqdn, record._type)) elif typ != 'CNAME': # dynamic records of unsupported type @@ -911,14 +883,6 @@ class AzureProvider(BaseProvider): active = set() profiles = self._generate_traffic_managers(record) - # this should not happen with above check, check again here to - # prevent undesired changes - if record._type in ['A', 'AAAA'] and len(profiles) > 1: - msg = ('Unknown error: {} {} needs more than 1 Traffic ' - 'Managers which is not supported for A/AAAA dynamic ' - 'records').format(record.fqdn, record._type) - raise AzureException(msg) - for profile in profiles: name = profile.name @@ -1170,8 +1134,7 @@ class AzureProvider(BaseProvider): return traffic_managers - def _sync_traffic_managers(self, record): - desired_profiles = self._generate_traffic_managers(record) + def _sync_traffic_managers(self, desired_profiles): seen = set() tm_sync = self._tm_client.profiles.create_or_update @@ -1230,12 +1193,22 @@ class AzureProvider(BaseProvider): record = change.new dynamic = getattr(record, 'dynamic', False) + root_profile = None + endpoints = [] if dynamic: - self._sync_traffic_managers(record) + profiles = self._generate_traffic_managers(record) + root_profile = profiles[-1] + if record._type in ['A', 'AAAA'] and len(profiles) > 1: + # A/AAAA records cannot be aliased to Traffic Managers that + # contain other nested Traffic Managers. To work around this + # limitation, we remove nesting before adding the record, and + # then add the nested endpoints later. + endpoints = root_profile.endpoints + root_profile.endpoints = [] + self._sync_traffic_managers(profiles) - profile = self._get_tm_for_dynamic_record(record) ar = _AzureRecord(self._resource_group, record, - traffic_manager=profile) + traffic_manager=root_profile) create = self._dns_client.record_sets.create_or_update create(resource_group_name=ar.resource_group, @@ -1244,6 +1217,12 @@ class AzureProvider(BaseProvider): record_type=ar.record_type, parameters=ar.params) + if endpoints: + # add nested endpoints for A/AAAA dynamic record limitation after + # record creation + root_profile.endpoints = endpoints + self._sync_traffic_managers([root_profile]) + self.log.debug('* Success Create: {}'.format(record)) def _apply_Update(self, change): @@ -1256,19 +1235,35 @@ class AzureProvider(BaseProvider): ''' existing = change.existing new = change.new + typ = new._type existing_is_dynamic = getattr(existing, 'dynamic', False) new_is_dynamic = getattr(new, 'dynamic', False) update_record = True if new_is_dynamic: - active = self._sync_traffic_managers(new) - # only TTL is configured in record, everything else goes inside - # traffic managers, so no need to update if TTL is unchanged - # and existing record is already aliased to its traffic manager - if existing.ttl == new.ttl and existing_is_dynamic: + endpoints = [] + profiles = self._generate_traffic_managers(new) + root_profile = profiles[-1] + + if typ in ['A', 'AAAA']: + if existing_is_dynamic: + # update to the record is not needed + update_record = False + elif len(profiles) > 1: + # record needs to aliased; remove nested endpoints, we + # will add them at the end + endpoints = root_profile.endpoints + root_profile.endpoints = [] + elif existing.ttl == new.ttl and existing_is_dynamic: + # CNAME dynamic records only have TTL in them, everything else + # goes inside the aliased traffic managers; skip update if TTL + # is unchanged and existing record is already aliased to its + # traffic manager update_record = False + active = self._sync_traffic_managers(profiles) + if update_record: profile = self._get_tm_for_dynamic_record(new) ar = _AzureRecord(self._resource_group, new, @@ -1282,6 +1277,10 @@ class AzureProvider(BaseProvider): parameters=ar.params) if new_is_dynamic: + # add any pending nested endpoints + if endpoints: + root_profile.endpoints = endpoints + self._sync_traffic_managers([root_profile]) # let's cleanup unused traffic managers self._traffic_managers_gc(new, active) elif existing_is_dynamic: diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index b92372f..85fa572 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -4,7 +4,6 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -from logging import debug from octodns.record import Create, Update, Delete, Record from octodns.provider.azuredns import _AzureRecord, AzureProvider, \ @@ -543,7 +542,14 @@ class TestAzureDnsProvider(TestCase): tm_sync = provider._tm_client.profiles.create_or_update def side_effect(rg, name, profile): - return profile + return Profile( + id=profile.id, + name=profile.name, + traffic_routing_method=profile.traffic_routing_method, + dns_config=profile.dns_config, + monitor_config=profile.monitor_config, + endpoints=profile.endpoints, + ) tm_sync.side_effect = side_effect @@ -1044,128 +1050,31 @@ class TestAzureDnsProvider(TestCase): provider._extra_changes(zone, desired, changes) self.assertTrue('duplicate endpoint' in text_type(ctx)) - def test_extra_changes_invalid_dynamic_A(self): + def test_extra_changes_A_multi_defaults(self): provider = self._get_provider() - # too many test case combinations, here's a method to generate them - def record_data(all_values=True, rr=True, fallback=True, geo=True): - data = { - 'type': 'A', - 'ttl': 60, - 'dynamic': { - 'pools': { - 'one': { - 'values': [ - {'value': '11.11.11.11'}, - {'value': '12.12.12.12'}, - ], - 'fallback': 'two', - }, - 'two': { - 'values': [ - {'value': '2.2.2.2'}, - ], - }, - }, - 'rules': [ - {'geos': ['EU'], 'pool': 'two'}, - {'pool': 'one'}, - ], - } - } - dynamic = data['dynamic'] - if not rr: - dynamic['pools']['one']['values'].pop() - if not fallback: - dynamic['pools']['one'].pop('fallback') - if not geo: - rule = dynamic['rules'].pop(0) - if not fallback: - dynamic['pools'].pop(rule['pool']) - # put all pool values in default - data['values'] = [ - v['value'] - for p in dynamic['pools'].values() - for v in p['values'] - ] - if not all_values: - rm = list(dynamic['pools'].values())[0]['values'][0]['value'] - data['values'].remove(rm) - return data - - # test all combinations - values = [True, False] - combos = [ - [arg1, arg2, arg3, arg4] - for arg1 in values - for arg2 in values - for arg3 in values - for arg4 in values - ] - for all_values, rr, fallback, geo in combos: - args = [all_values, rr, fallback, geo] - - if not any(args): - # all False, invalid use-case - continue - - debug('[all_values, rr, fallback, geo] = %s', args) - data = record_data(*args) - desired = Zone(name=zone.name, sub_zones=[]) - record = Record.new(desired, 'foo', data) - desired.add_record(record) - - features = args[1:] - if all_values and features.count(True) <= 1: - # assert does not raise exception - provider._extra_changes(zone, desired, [Create(record)]) - continue - - with self.assertRaises(AzureException) as ctx: - msg = text_type(ctx) - provider._extra_changes(zone, desired, [Create(record)]) - if not all_values: - self.assertTrue('included in top-level \'values\'' in msg) - else: - self.assertTrue('at most one of' in msg) - - @patch('octodns.provider.azuredns._check_valid_dynamic') - def test_extra_changes_dynamic_A_multiple_profiles(self, mock_cvd): - provider = self._get_provider() - - # bypass validity check to trigger mutliple-profiles check - mock_cvd.return_value = True - - desired = Zone(name=zone.name, sub_zones=[]) - record = Record.new(desired, 'foo', { + record = Record.new(zone, 'foo', data={ 'type': 'A', 'ttl': 60, - 'values': ['11.11.11.11', '12.12.12.12', '2.2.2.2'], + 'values': ['1.1.1.1', '8.8.8.8'], 'dynamic': { 'pools': { 'one': { - 'values': [ - {'value': '11.11.11.11'}, - {'value': '12.12.12.12'}, - ], - 'fallback': 'two', - }, - 'two': { - 'values': [ - {'value': '2.2.2.2'}, - ], + 'values': [{'value': '1.1.1.1'}], }, }, 'rules': [ - {'geos': ['EU'], 'pool': 'two'}, {'pool': 'one'}, ], } }) + + # test that extra changes doesn't show any changes + desired = Zone(zone.name, sub_zones=[]) desired.add_record(record) with self.assertRaises(AzureException) as ctx: - provider._extra_changes(zone, desired, [Create(record)]) - self.assertTrue('more than 1 Traffic Managers' in text_type(ctx)) + provider._extra_changes(zone, desired, []) + self.assertEqual('single value' in text_type(ctx)) def test_generate_tm_profile(self): provider, zone, record = self._get_dynamic_package() @@ -1798,225 +1707,150 @@ class TestAzureDnsProvider(TestCase): changes = provider._extra_changes(zone, desired, []) self.assertEqual(len(changes), 0) - def test_dynamic_A_geo(self): + def test_dynamic_A(self): provider = self._get_provider() external = 'Microsoft.Network/trafficManagerProfiles/externalEndpoints' + nested = 'Microsoft.Network/trafficManagerProfiles/nestedEndpoints' record = Record.new(zone, 'foo', data={ 'type': 'A', 'ttl': 60, - 'values': ['1.1.1.1', '2.2.2.2', '3.3.3.3'], + 'values': ['9.9.9.9'], 'dynamic': { 'pools': { 'one': { 'values': [ - {'value': '1.1.1.1'}, + {'value': '11.11.11.11'}, + {'value': '12.12.12.12'}, ], + 'fallback': 'two' }, 'two': { 'values': [ {'value': '2.2.2.2'}, ], }, - 'three': { - 'values': [ - {'value': '3.3.3.3'}, - ], - }, }, 'rules': [ - {'geos': ['AS'], 'pool': 'one'}, - {'geos': ['AF'], 'pool': 'two'}, - {'pool': 'three'}, + {'geos': ['AF'], 'pool': 'one'}, + {'pool': 'two'}, ], } }) profiles = provider._generate_traffic_managers(record) - self.assertEqual(len(profiles), 1) + self.assertEqual(len(profiles), 4) self.assertTrue(_profile_is_match(profiles[0], Profile( - name='foo--unit--tests-A', - traffic_routing_method='Geographic', + name='foo--unit--tests-A-pool-one', + traffic_routing_method='Weighted', dns_config=DnsConfig( - relative_name='foo--unit--tests-a', ttl=record.ttl), + relative_name='foo--unit--tests-a-pool-one', ttl=record.ttl), monitor_config=_get_monitor(record), endpoints=[ Endpoint( - name='one--default--', + name='one--11.11.11.11', + type=external, + target='11.11.11.11', + weight=1, + ), + Endpoint( + name='one--12.12.12.12', type=external, - target='1.1.1.1', - geo_mapping=['GEO-AS'], + target='12.12.12.12', + weight=1, + ), + ], + ))) + self.assertTrue(_profile_is_match(profiles[1], Profile( + name='foo--unit--tests-A-rule-one', + traffic_routing_method='Priority', + dns_config=DnsConfig( + relative_name='foo--unit--tests-a-rule-one', ttl=record.ttl), + monitor_config=_get_monitor(record), + endpoints=[ + Endpoint( + name='one', + type=nested, + target_resource_id=profiles[0].id, + priority=1, ), Endpoint( - name='two--default--', + name='two', type=external, target='2.2.2.2', - geo_mapping=['GEO-AF'], + priority=2, ), Endpoint( - name='three--default--', + name='--default--', type=external, - target='3.3.3.3', - geo_mapping=['WORLD'], + target='9.9.9.9', + priority=3, ), ], ))) - - # test that the record and ATM profile gets created - tm_sync = provider._tm_client.profiles.create_or_update - create = provider._dns_client.record_sets.create_or_update - provider._apply_Create(Create(record)) - # A dynamic record can only have 1 profile - tm_sync.assert_called_once() - create.assert_called_once() - - # test broken alias - azrecord = RecordSet( - ttl=60, target_resource=SubResource(id=None)) - azrecord.name = record.name or '@' - azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) - record2 = provider._populate_record(zone, azrecord) - self.assertEqual(record2.values, ['255.255.255.255']) - - # test that same record gets populated back from traffic managers - tm_list = provider._tm_client.profiles.list_by_resource_group - tm_list.return_value = profiles - azrecord = RecordSet( - ttl=60, - target_resource=SubResource(id=profiles[-1].id), - ) - azrecord.name = record.name or '@' - azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) - record2 = provider._populate_record(zone, azrecord) - self.assertEqual(record2.dynamic._data(), record.dynamic._data()) - - # test that extra changes doesn't show any changes - desired = Zone(zone.name, sub_zones=[]) - desired.add_record(record) - changes = provider._extra_changes(zone, desired, []) - self.assertEqual(len(changes), 0) - - def test_dynamic_A_fallback(self): - provider = self._get_provider() - external = 'Microsoft.Network/trafficManagerProfiles/externalEndpoints' - - record = Record.new(zone, 'foo', data={ - 'type': 'A', - 'ttl': 60, - 'values': ['1.1.1.1', '2.2.2.2'], - 'dynamic': { - 'pools': { - 'one': { - 'values': [ - {'value': '1.1.1.1'}, - ], - 'fallback': 'two', - }, - 'two': { - 'values': [ - {'value': '2.2.2.2'}, - ], - }, - }, - 'rules': [ - {'pool': 'one'}, - ], - } - }) - profiles = provider._generate_traffic_managers(record) - - self.assertEqual(len(profiles), 1) - self.assertTrue(_profile_is_match(profiles[0], Profile( - name='foo--unit--tests-A', + self.assertTrue(_profile_is_match(profiles[2], Profile( + name='foo--unit--tests-A-rule-two', traffic_routing_method='Priority', dns_config=DnsConfig( - relative_name='foo--unit--tests-a', ttl=record.ttl), + relative_name='foo--unit--tests-a-rule-two', ttl=record.ttl), monitor_config=_get_monitor(record), endpoints=[ Endpoint( - name='one--default--', + name='two', type=external, - target='1.1.1.1', + target='2.2.2.2', priority=1, ), Endpoint( - name='two--default--', + name='--default--', type=external, - target='2.2.2.2', + target='9.9.9.9', priority=2, ), ], ))) - - # test that same record gets populated back from traffic managers - tm_list = provider._tm_client.profiles.list_by_resource_group - tm_list.return_value = profiles - azrecord = RecordSet( - ttl=60, - target_resource=SubResource(id=profiles[-1].id), - ) - azrecord.name = record.name or '@' - azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) - record2 = provider._populate_record(zone, azrecord) - self.assertEqual(record2.dynamic._data(), record.dynamic._data()) - - # test that extra changes doesn't show any changes - desired = Zone(zone.name, sub_zones=[]) - desired.add_record(record) - changes = provider._extra_changes(zone, desired, []) - self.assertEqual(len(changes), 0) - - def test_dynamic_A_weighted_rr(self): - provider = self._get_provider() - external = 'Microsoft.Network/trafficManagerProfiles/externalEndpoints' - - record = Record.new(zone, 'foo', data={ - 'type': 'A', - 'ttl': 60, - 'values': ['1.1.1.1', '8.8.8.8'], - 'dynamic': { - 'pools': { - 'one': { - 'values': [ - {'value': '1.1.1.1', 'weight': 11}, - {'value': '8.8.8.8', 'weight': 8}, - ], - }, - }, - 'rules': [ - {'pool': 'one'}, - ], - } - }) - profiles = provider._generate_traffic_managers(record) - - self.assertEqual(len(profiles), 1) - self.assertTrue(_profile_is_match(profiles[0], Profile( + self.assertTrue(_profile_is_match(profiles[3], Profile( name='foo--unit--tests-A', - traffic_routing_method='Weighted', + traffic_routing_method='Geographic', dns_config=DnsConfig( relative_name='foo--unit--tests-a', ttl=record.ttl), monitor_config=_get_monitor(record), endpoints=[ Endpoint( - name='one--1.1.1.1--default--', - type=external, - target='1.1.1.1', - weight=11, + name='one', + type=nested, + target_resource_id=profiles[1].id, + geo_mapping=['GEO-AF'], ), Endpoint( - name='one--8.8.8.8--default--', - type=external, - target='8.8.8.8', - weight=8, + name='two', + type=nested, + target_resource_id=profiles[2].id, + geo_mapping=['WORLD'], ), ], ))) + # test that the record and ATM profile gets created + tm_sync = provider._tm_client.profiles.create_or_update + create = provider._dns_client.record_sets.create_or_update + provider._apply_Create(Create(record)) + self.assertEqual(tm_sync.call_count, len(profiles) + 1) + create.assert_called_once() + + # test broken alias + azrecord = RecordSet( + ttl=60, target_resource=SubResource(id=None)) + azrecord.name = record.name or '@' + azrecord.type = 'Microsoft.Network/dnszones/{}'.format(record._type) + record2 = provider._populate_record(zone, azrecord) + self.assertEqual(record2.values, ['255.255.255.255']) + # test that same record gets populated back from traffic managers tm_list = provider._tm_client.profiles.list_by_resource_group tm_list.return_value = profiles + provider._populate_traffic_managers() azrecord = RecordSet( ttl=60, target_resource=SubResource(id=profiles[-1].id), @@ -2119,7 +1953,8 @@ class TestAzureDnsProvider(TestCase): } # test no change - seen = provider._sync_traffic_managers(record) + profiles = provider._generate_traffic_managers(record) + seen = provider._sync_traffic_managers(profiles) self.assertEqual(seen, expected_seen) tm_sync.assert_not_called() @@ -2135,7 +1970,8 @@ class TestAzureDnsProvider(TestCase): } new_record = Record.new(zone, record.name, data) tm_sync.reset_mock() - seen2 = provider._sync_traffic_managers(new_record) + profiles = provider._generate_traffic_managers(new_record) + seen2 = provider._sync_traffic_managers(profiles) self.assertEqual(seen2, expected_seen) tm_sync.assert_called_once() @@ -2145,17 +1981,14 @@ class TestAzureDnsProvider(TestCase): ) self.assertEqual(new_profile.endpoints[0].weight, 14) - @patch( - 'octodns.provider.azuredns.AzureProvider._generate_traffic_managers') - def test_sync_traffic_managers_duplicate(self, mock_gen_tms): + def test_sync_traffic_managers_duplicate(self): provider, zone, record = self._get_dynamic_package() tm_sync = provider._tm_client.profiles.create_or_update # change and duplicate profiles profile = self._get_tm_profiles(provider)[0] profile.name = 'changing_this_to_trigger_sync' - mock_gen_tms.return_value = [profile, profile] - provider._sync_traffic_managers(record) + provider._sync_traffic_managers([profile, profile]) # it should only be called once for duplicate profiles tm_sync.assert_called_once() @@ -2232,7 +2065,6 @@ class TestAzureDnsProvider(TestCase): tm_sync = provider._tm_client.profiles.create_or_update - zone = Zone(name='unit.tests.', sub_zones=[]) record = self._get_dynamic_record(zone) profiles = self._get_tm_profiles(provider) @@ -2345,6 +2177,116 @@ class TestAzureDnsProvider(TestCase): dns_update.assert_called_once() tm_delete.assert_not_called() + def test_apply_update_dynamic_A(self): + # existing is simple, new is dynamic + provider = self._get_provider() + simple_record = Record.new(zone, 'foo', data={ + 'type': 'A', + 'ttl': 3600, + 'values': ['1.1.1.1', '2.2.2.2'], + }) + dynamic_record = Record.new(zone, simple_record.name, data={ + 'type': 'A', + 'ttl': 60, + 'values': ['1.1.1.1'], + 'dynamic': { + 'pools': { + 'one': { + 'values': [ + {'value': '8.8.8.8'}, + {'value': '4.4.4.4'}, + ], + 'fallback': 'two', + }, + 'two': { + 'values': [{'value': '9.9.9.9'}], + }, + }, + 'rules': [ + {'geos': ['AF'], 'pool': 'two'}, + {'pool': 'one'}, + ], + } + }) + num_tms = len(provider._generate_traffic_managers(dynamic_record)) + change = Update(simple_record, dynamic_record) + provider._apply_Update(change) + tm_sync, dns_update, tm_delete = ( + provider._tm_client.profiles.create_or_update, + provider._dns_client.record_sets.create_or_update, + provider._tm_client.profiles.delete + ) + # sync is called once for each profile, plus 1 at the end for nested + # endpoints to workaround A/AAAA nesting limitation in Azure + self.assertEqual(tm_sync.call_count, num_tms + 1) + dns_update.assert_called_once() + tm_delete.assert_not_called() + + # both are dynamic, healthcheck port is changed to trigger sync on + # all profiles + provider = self._get_provider() + dynamic_record2 = Record.new(zone, dynamic_record.name, data={ + 'type': dynamic_record._type, + 'ttl': 300, + 'values': dynamic_record.values, + 'dynamic': dynamic_record.dynamic._data(), + 'octodns': { + 'healthcheck': {'port': 4433}, + } + }) + change = Update(dynamic_record, dynamic_record2) + provider._apply_Update(change) + tm_sync, dns_update, tm_delete = ( + provider._tm_client.profiles.create_or_update, + provider._dns_client.record_sets.create_or_update, + provider._tm_client.profiles.delete + ) + # sync is called once for each profile, extra call at the end is not + # needed when existing dynamic record is already aliased to its root + # profile + self.assertEqual(tm_sync.call_count, num_tms) + dns_update.assert_not_called() + tm_delete.assert_not_called() + + def test_apply_update_dynamic_A_singluar(self): + # existing is simple, new is dynamic that needs only one profile + provider = self._get_provider() + simple_record = Record.new(zone, 'foo', data={ + 'type': 'A', + 'ttl': 3600, + 'values': ['1.1.1.1', '2.2.2.2'], + }) + dynamic_record = Record.new(zone, simple_record.name, data={ + 'type': 'A', + 'ttl': 60, + 'values': ['1.1.1.1'], + 'dynamic': { + 'pools': { + 'one': { + 'values': [ + {'value': '8.8.8.8'}, + {'value': '1.1.1.1'}, + ], + }, + }, + 'rules': [ + {'pool': 'one'}, + ], + } + }) + num_tms = len(provider._generate_traffic_managers(dynamic_record)) + self.assertEqual(num_tms, 1) + change = Update(simple_record, dynamic_record) + provider._apply_Update(change) + tm_sync, dns_update, tm_delete = ( + provider._tm_client.profiles.create_or_update, + provider._dns_client.record_sets.create_or_update, + provider._tm_client.profiles.delete + ) + self.assertEqual(tm_sync.call_count, num_tms) + dns_update.assert_called_once() + tm_delete.assert_not_called() + def test_apply_delete_dynamic(self): provider, existing, record = self._get_dynamic_package() provider._populate_traffic_managers() From 6f5df26e883b79ce2daf058c05d5086cd53c7d20 Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Sun, 4 Jul 2021 23:52:16 -0700 Subject: [PATCH 279/358] allow multiple values if all pool values are included --- octodns/provider/azuredns.py | 19 +++++++++++++++---- tests/test_octodns_provider_azuredns.py | 15 +++++++++++---- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 845d4d5..59bd8c3 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -302,10 +302,21 @@ def _get_monitor(record): def _check_valid_dynamic(record): typ = record._type if typ in ['A', 'AAAA']: - if len(record.values) > 1: - # we don't yet support multi-value defaults - msg = '{} {}: A/AAAA dynamic records can only have a single value' - raise AzureException(msg.format(record.fqdn, record._type)) + defaults = set(record.values) + if len(defaults) > 1: + pools = record.dynamic.pools + vals = set( + v['value'] + for _, pool in pools.items() + for v in pool._data()['values'] + ) + if defaults != vals: + # we don't yet support multi-value defaults, specifying all + # pool values allows for Traffic Manager profile optimization + msg = ('{} {}: Values of A/AAAA dynamic records must either ' + 'have a single value or contain all values from all ' + 'pools') + raise AzureException(msg.format(record.fqdn, record._type)) elif typ != 'CNAME': # dynamic records of unsupported type msg = '{}: Dynamic records in Azure must be of type A/AAAA/CNAME' diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index 85fa572..0af8665 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -1873,12 +1873,13 @@ class TestAzureDnsProvider(TestCase): record = Record.new(zone, 'foo', data={ 'type': 'AAAA', 'ttl': 60, - 'values': ['1::1'], + 'values': ['1::1', '2::2'], 'dynamic': { 'pools': { 'one': { 'values': [ {'value': '1::1'}, + {'value': '2::2'}, ], }, }, @@ -1892,16 +1893,22 @@ class TestAzureDnsProvider(TestCase): self.assertEqual(len(profiles), 1) self.assertTrue(_profile_is_match(profiles[0], Profile( name='foo--unit--tests-AAAA', - traffic_routing_method='Geographic', + traffic_routing_method='Weighted', dns_config=DnsConfig( relative_name='foo--unit--tests-aaaa', ttl=record.ttl), monitor_config=_get_monitor(record), endpoints=[ Endpoint( - name='one--default--', + name='one--1--1--default--', type=external, target='1::1', - geo_mapping=['WORLD'], + weight=1, + ), + Endpoint( + name='one--2--2--default--', + type=external, + target='2::2', + weight=1, ), ], ))) From b684b54b3e2a07613623c85a195ac44dacb01a8a Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Sun, 4 Jul 2021 23:57:45 -0700 Subject: [PATCH 280/358] update README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8486799..733b087 100644 --- a/README.md +++ b/README.md @@ -192,7 +192,7 @@ The above command pulled the existing data out of Route53 and placed the results | Provider | Requirements | Record Support | Dynamic | Notes | |--|--|--|--|--| -| [AzureProvider](/octodns/provider/azuredns.py) | azure-identity, azure-mgmt-dns, azure-mgmt-trafficmanager | A, AAAA, CAA, CNAME, MX, NS, PTR, SRV, TXT | Alpha (CNAMEs and partial A/AAAA) | | +| [AzureProvider](/octodns/provider/azuredns.py) | azure-identity, azure-mgmt-dns, azure-mgmt-trafficmanager | A, AAAA, CAA, CNAME, MX, NS, PTR, SRV, TXT | Alpha (A, AAAA, CNAME) | | | [Akamai](/octodns/provider/edgedns.py) | edgegrid-python | A, AAAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, SSHFP, TXT | No | | | [CloudflareProvider](/octodns/provider/cloudflare.py) | | A, AAAA, ALIAS, CAA, CNAME, LOC, MX, NS, PTR, SPF, SRV, TXT | No | CAA tags restricted | | [ConstellixProvider](/octodns/provider/constellix.py) | | A, AAAA, ALIAS (ANAME), CAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | CAA tags restricted | From 3cde73ce1c2dd9ca47616ba535f857277e2dcec8 Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Mon, 5 Jul 2021 10:20:39 -0700 Subject: [PATCH 281/358] small cleanup --- octodns/provider/azuredns.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 59bd8c3..6edc1ba 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -1246,7 +1246,6 @@ class AzureProvider(BaseProvider): ''' existing = change.existing new = change.new - typ = new._type existing_is_dynamic = getattr(existing, 'dynamic', False) new_is_dynamic = getattr(new, 'dynamic', False) @@ -1257,7 +1256,7 @@ class AzureProvider(BaseProvider): profiles = self._generate_traffic_managers(new) root_profile = profiles[-1] - if typ in ['A', 'AAAA']: + if new._type in ['A', 'AAAA']: if existing_is_dynamic: # update to the record is not needed update_record = False From fec33bbb10730d7efaf7f1ca2a7ef0aef5f0b0b4 Mon Sep 17 00:00:00 2001 From: jozo Date: Mon, 12 Jul 2021 11:29:59 +0200 Subject: [PATCH 282/358] Fix missing dot in example --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 733b087..999d8b4 100644 --- a/README.md +++ b/README.md @@ -89,7 +89,7 @@ zones: - dyn - route53 - example.net: + example.net.: alias: example.com. ``` From 1a1391bf67502a455f7c284c83bfc3c07f55ab3d Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Mon, 12 Jul 2021 12:49:31 -0700 Subject: [PATCH 283/358] Fix git URL in readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8486799..0056ba3 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ $ mkdir config If you'd like to install a version that has not yet been released in a repetable/safe manner you can do the following. In general octoDNS is fairly stable inbetween releases thanks to the plan and apply process, but care should be taken regardless. ```shell -$ pip install -e git+https://git@github.com/github/octodns.git@#egg=octodns +$ pip install -e git+https://git@github.com/octodns/octodns.git@#egg=octodns ``` ### Config From bb5dbd7d420a27107383d22f27cf789d344921a5 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sun, 18 Jul 2021 14:04:12 -0700 Subject: [PATCH 284/358] v0.9.13 version bump and CHANGELOG update --- CHANGELOG.md | 12 +++++++++++- octodns/__init__.py | 2 +- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e6dcb48..dca36a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## v0.9.13 - 2021-..-.. - +## v0.9.13 - 2021-07-18 - Processors Alpha #### Noteworthy changes @@ -14,6 +14,16 @@ America list for backwards compatibility reasons. They will be added in the next releaser. +#### Stuff + +* Lots of progress on the partial/beta support for dynamic records in Azure, + still not production ready. +* NS1 fix for when a pool only exists as a fallback +* Zone level lenient flag +* Validate weight makes sense for pools with a single record +* UltraDNS support for aliases and general fixes/improvements +* Misc doc fixes and improvements + ## v0.9.12 - 2021-04-30 - Enough time has passed #### Noteworthy changes diff --git a/octodns/__init__.py b/octodns/__init__.py index 1885d42..16ec066 100644 --- a/octodns/__init__.py +++ b/octodns/__init__.py @@ -3,4 +3,4 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -__VERSION__ = '0.9.12' +__VERSION__ = '0.9.13' From 1de2521c7a37923e0bc6c0482de5a8de7a49ce3a Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sun, 18 Jul 2021 14:25:44 -0700 Subject: [PATCH 285/358] Changelog entry about SX and UM inclusion in NA for NS1 --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index dca36a4..55a88c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## v0.9.14 - 2021-??-?? - ... + +#### Noteworthy changes + +* NS1 NA target now includes `SX` and `UM`. If `NA` continent is in use in + dynamic records care must be taken to upgrade/downgrade to v0.9.13. + ## v0.9.13 - 2021-07-18 - Processors Alpha #### Noteworthy changes From 5361cadd1c736e9b7dcd4f2c6287d49e824d899d Mon Sep 17 00:00:00 2001 From: Brian E Clow Date: Thu, 27 May 2021 15:36:07 -0700 Subject: [PATCH 286/358] Adding URLFWD to record framework --- octodns/record/__init__.py | 86 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/octodns/record/__init__.py b/octodns/record/__init__.py index 3ab8263..15cb143 100644 --- a/octodns/record/__init__.py +++ b/octodns/record/__init__.py @@ -106,6 +106,7 @@ class Record(EqualityTupleMixin): 'SRV': SrvRecord, 'SSHFP': SshfpRecord, 'TXT': TxtRecord, + 'URLFWD': UrlfwdRecord, }[_type] except KeyError: raise Exception('Unknown record type: "{}"'.format(_type)) @@ -1467,3 +1468,88 @@ class _TxtValue(_ChunkedValue): class TxtRecord(_ChunkedValuesMixin, Record): _type = 'TXT' _value_type = _TxtValue + + +class UrlfwdValue(EqualityTupleMixin): + # TODO: should have defaults for path, code, masking, and query + + VALID_CODES = (301, 302) + VALID_MASKS = (0, 1, 2) + VALID_QUERY = (0, 1) + + @classmethod + def validate(cls, data, _type): + if not isinstance(data, (list, tuple)): + data = (data,) + reasons = [] + for value in data: + try: + code = int(value['code']) + if code not in cls.VALID_CODES: + reasons.append('unrecognized return code "{}"' + .format(code)) + except KeyError: + reasons.append('missing code') + except ValueError: + reasons.append('invalid return code "{}"' + .format(value['code'])) + try: + masking = int(value['masking']) + if masking not in cls.VALID_MASKS: + reasons.append('unrecognized masking setting "{}"' + .format(masking)) + except KeyError: + reasons.append('missing masking') + except ValueError: + reasons.append('invalid masking setting "{}"' + .format(value['masking'])) + try: + query = int(value['query']) + if query not in cls.VALID_QUERY: + reasons.append('unrecognized query setting "{}"' + .format(query)) + except KeyError: + reasons.append('missing query') + except ValueError: + reasons.append('invalid query setting "{}"' + .format(value['query'])) + for k in ('path', 'target'): + if k not in value: + reasons.append('missing {}'.format(k)) + return reasons + + @classmethod + def process(cls, values): + return [UrlfwdValue(v) for v in values] + + def __init__(self, value): + self.path = value['path'] + self.target = value['target'] + self.code = int(value['code']) + self.masking = int(value['masking']) + self.query = int(value['query']) + + @property + def data(self): + return { + 'path': self.path, + 'target': self.target, + 'code': self.code, + 'masking': self.masking, + 'query': self.query, + } + + def __hash__(self): + return hash(self.__repr__()) + + def _equality_tuple(self): + return (self.path, self.target, self.code, self.masking, self.query) + + def __repr__(self): + return '"{}" "{}" {} {} {}'.format(self.path, self.target, self.code, + self.masking, self.query) + + +class UrlfwdRecord(_ValuesMixin, Record): + _type = 'URLFWD' + _value_type = UrlfwdValue From 2a6480bc05e15ef30a8fa578a58da5c229a840cf Mon Sep 17 00:00:00 2001 From: Brian E Clow Date: Thu, 27 May 2021 15:37:55 -0700 Subject: [PATCH 287/358] Adding URLFWD record testing --- tests/test_octodns_record.py | 307 ++++++++++++++++++++++++++++++++++- 1 file changed, 305 insertions(+), 2 deletions(-) diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index 315670e..3bd48e5 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -12,8 +12,8 @@ from octodns.record import ARecord, AaaaRecord, AliasRecord, CaaRecord, \ CaaValue, CnameRecord, DnameRecord, Create, Delete, GeoValue, LocRecord, \ LocValue, MxRecord, MxValue, NaptrRecord, NaptrValue, NsRecord, \ PtrRecord, Record, SshfpRecord, SshfpValue, SpfRecord, SrvRecord, \ - SrvValue, TxtRecord, Update, ValidationError, _Dynamic, _DynamicPool, \ - _DynamicRule + SrvValue, TxtRecord, Update, UrlfwdRecord, UrlfwdValue, ValidationError, \ + _Dynamic, _DynamicPool, _DynamicRule from octodns.zone import Zone from helpers import DynamicProvider, GeoProvider, SimpleProvider @@ -884,6 +884,112 @@ class TestRecord(TestCase): b_value = 'b other' self.assertMultipleValues(TxtRecord, a_values, b_value) + def test_urlfwd(self): + a_values = [{ + 'path': '/', + 'target': 'http://foo', + 'code': 301, + 'masking': 2, + 'query': 0, + }, { + 'path': '/target', + 'target': 'http://target', + 'code': 302, + 'masking': 2, + 'query': 0, + }] + a_data = {'ttl': 30, 'values': a_values} + a = UrlfwdRecord(self.zone, 'a', a_data) + self.assertEquals('a', a.name) + self.assertEquals('a.unit.tests.', a.fqdn) + self.assertEquals(30, a.ttl) + self.assertEquals(a_values[0]['path'], a.values[0].path) + self.assertEquals(a_values[0]['target'], a.values[0].target) + self.assertEquals(a_values[0]['code'], a.values[0].code) + self.assertEquals(a_values[0]['masking'], a.values[0].masking) + self.assertEquals(a_values[0]['query'], a.values[0].query) + self.assertEquals(a_values[1]['path'], a.values[1].path) + self.assertEquals(a_values[1]['target'], a.values[1].target) + self.assertEquals(a_values[1]['code'], a.values[1].code) + self.assertEquals(a_values[1]['masking'], a.values[1].masking) + self.assertEquals(a_values[1]['query'], a.values[1].query) + self.assertEquals(a_data, a.data) + + b_value = { + 'path': '/', + 'target': 'http://location', + 'code': 301, + 'masking': 2, + 'query': 0, + } + b_data = {'ttl': 30, 'value': b_value} + b = UrlfwdRecord(self.zone, 'b', b_data) + self.assertEquals(b_value['path'], b.values[0].path) + self.assertEquals(b_value['target'], b.values[0].target) + self.assertEquals(b_value['code'], b.values[0].code) + self.assertEquals(b_value['masking'], b.values[0].masking) + self.assertEquals(b_value['query'], b.values[0].query) + self.assertEquals(b_data, b.data) + + target = SimpleProvider() + # No changes with self + self.assertFalse(a.changes(a, target)) + # Diff in path causes change + other = UrlfwdRecord(self.zone, 'a', {'ttl': 30, 'values': a_values}) + other.values[0].path = '/change' + change = a.changes(other, target) + self.assertEqual(change.existing, a) + self.assertEqual(change.new, other) + # Diff in target causes change + other = UrlfwdRecord(self.zone, 'a', {'ttl': 30, 'values': a_values}) + other.values[0].target = 'http://target' + change = a.changes(other, target) + self.assertEqual(change.existing, a) + self.assertEqual(change.new, other) + # Diff in code causes change + other = UrlfwdRecord(self.zone, 'a', {'ttl': 30, 'values': a_values}) + other.values[0].code = 302 + change = a.changes(other, target) + self.assertEqual(change.existing, a) + self.assertEqual(change.new, other) + # Diff in masking causes change + other = UrlfwdRecord(self.zone, 'a', {'ttl': 30, 'values': a_values}) + other.values[0].masking = 0 + change = a.changes(other, target) + self.assertEqual(change.existing, a) + self.assertEqual(change.new, other) + # Diff in query causes change + other = UrlfwdRecord(self.zone, 'a', {'ttl': 30, 'values': a_values}) + other.values[0].query = 1 + change = a.changes(other, target) + self.assertEqual(change.existing, a) + self.assertEqual(change.new, other) + + # hash + v = UrlfwdValue({ + 'path': '/', + 'target': 'http://place', + 'code': 301, + 'masking': 2, + 'query': 0, + }) + o = UrlfwdValue({ + 'path': '/location', + 'target': 'http://redirect', + 'code': 302, + 'masking': 2, + 'query': 0, + }) + values = set() + values.add(v) + self.assertTrue(v in values) + self.assertFalse(o in values) + values.add(o) + self.assertTrue(o in values) + + # __repr__ doesn't blow up + a.__repr__() + def test_record_new(self): txt = Record.new(self.zone, 'txt', { 'ttl': 44, @@ -3019,6 +3125,203 @@ class TestRecordValidation(TestCase): # should be chunked values, with quoting self.assertEquals(single.chunked_values, chunked.chunked_values) + def test_URLFWD(self): + # doesn't blow up + Record.new(self.zone, '', { + 'type': 'URLFWD', + 'ttl': 600, + 'value': { + 'path': '/', + 'target': 'http://foo', + 'code': 301, + 'masking': 2, + 'query': 0, + } + }) + Record.new(self.zone, '', { + 'type': 'URLFWD', + 'ttl': 600, + 'values': [{ + 'path': '/', + 'target': 'http://foo', + 'code': 301, + 'masking': 2, + 'query': 0, + }, { + 'path': '/target', + 'target': 'http://target', + 'code': 302, + 'masking': 2, + 'query': 0, + }] + }) + + # missing path + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'URLFWD', + 'ttl': 600, + 'value': { + 'target': 'http://foo', + 'code': 301, + 'masking': 2, + 'query': 0, + } + }) + self.assertEquals(['missing path'], ctx.exception.reasons) + + # missing target + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'URLFWD', + 'ttl': 600, + 'value': { + 'path': '/', + 'code': 301, + 'masking': 2, + 'query': 0, + } + }) + self.assertEquals(['missing target'], ctx.exception.reasons) + + # missing code + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'URLFWD', + 'ttl': 600, + 'value': { + 'path': '/', + 'target': 'http://foo', + 'masking': 2, + 'query': 0, + } + }) + self.assertEquals(['missing code'], ctx.exception.reasons) + + # invalid code + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'URLFWD', + 'ttl': 600, + 'value': { + 'path': '/', + 'target': 'http://foo', + 'code': 'nope', + 'masking': 2, + 'query': 0, + } + }) + self.assertEquals(['invalid return code "nope"'], + ctx.exception.reasons) + + # unrecognized code + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'URLFWD', + 'ttl': 600, + 'value': { + 'path': '/', + 'target': 'http://foo', + 'code': 3, + 'masking': 2, + 'query': 0, + } + }) + self.assertEquals(['unrecognized return code "3"'], + ctx.exception.reasons) + + # missing masking + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'URLFWD', + 'ttl': 600, + 'value': { + 'path': '/', + 'target': 'http://foo', + 'code': 301, + 'query': 0, + } + }) + self.assertEquals(['missing masking'], ctx.exception.reasons) + + # invalid masking + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'URLFWD', + 'ttl': 600, + 'value': { + 'path': '/', + 'target': 'http://foo', + 'code': 301, + 'masking': 'nope', + 'query': 0, + } + }) + self.assertEquals(['invalid masking setting "nope"'], + ctx.exception.reasons) + + # unrecognized masking + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'URLFWD', + 'ttl': 600, + 'value': { + 'path': '/', + 'target': 'http://foo', + 'code': 301, + 'masking': 3, + 'query': 0, + } + }) + self.assertEquals(['unrecognized masking setting "3"'], + ctx.exception.reasons) + + # missing query + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'URLFWD', + 'ttl': 600, + 'value': { + 'path': '/', + 'target': 'http://foo', + 'code': 301, + 'masking': 2, + } + }) + self.assertEquals(['missing query'], ctx.exception.reasons) + + # invalid query + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'URLFWD', + 'ttl': 600, + 'value': { + 'path': '/', + 'target': 'http://foo', + 'code': 301, + 'masking': 2, + 'query': 'nope', + } + }) + self.assertEquals(['invalid query setting "nope"'], + ctx.exception.reasons) + + # unrecognized query + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'URLFWD', + 'ttl': 600, + 'value': { + 'path': '/', + 'target': 'http://foo', + 'code': 301, + 'masking': 2, + 'query': 3, + } + }) + self.assertEquals(['unrecognized query setting "3"'], + ctx.exception.reasons) + class TestDynamicRecords(TestCase): zone = Zone('unit.tests.', []) From c5efba89fe68551b1029899b65af66416ef652ef Mon Sep 17 00:00:00 2001 From: Brian E Clow Date: Thu, 27 May 2021 15:40:44 -0700 Subject: [PATCH 288/358] Adding yaml support and testing for URLFWD --- octodns/provider/yaml.py | 3 ++- tests/config/unit.tests.yaml | 14 ++++++++++++++ tests/test_octodns_provider_yaml.py | 20 +++++++++++--------- 3 files changed, 27 insertions(+), 10 deletions(-) diff --git a/octodns/provider/yaml.py b/octodns/provider/yaml.py index 8314f38..b3dd2d9 100644 --- a/octodns/provider/yaml.py +++ b/octodns/provider/yaml.py @@ -105,7 +105,8 @@ class YamlProvider(BaseProvider): SUPPORTS_GEO = True SUPPORTS_DYNAMIC = True SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CAA', 'CNAME', 'DNAME', 'LOC', 'MX', - 'NAPTR', 'NS', 'PTR', 'SSHFP', 'SPF', 'SRV', 'TXT')) + 'NAPTR', 'NS', 'PTR', 'SSHFP', 'SPF', 'SRV', 'TXT', + 'URLFWD')) def __init__(self, id, directory, default_ttl=3600, enforce_order=True, populate_should_replace=False, *args, **kwargs): diff --git a/tests/config/unit.tests.yaml b/tests/config/unit.tests.yaml index 39e5326..c70b20c 100644 --- a/tests/config/unit.tests.yaml +++ b/tests/config/unit.tests.yaml @@ -169,6 +169,20 @@ txt: - Bah bah black sheep - have you any wool. - 'v=DKIM1\;k=rsa\;s=email\;h=sha256\;p=A/kinda+of/long/string+with+numb3rs' +urlfwd: + ttl: 300 + type: URLFWD + values: + - code: 302 + masking: 2 + path: '/' + query: 0 + target: 'http://www.unit.tests' + - code: 301 + masking: 2 + path: '/target' + query: 0 + target: 'http://target.unit.tests' www: ttl: 300 type: A diff --git a/tests/test_octodns_provider_yaml.py b/tests/test_octodns_provider_yaml.py index 872fcca..7e4f6f7 100644 --- a/tests/test_octodns_provider_yaml.py +++ b/tests/test_octodns_provider_yaml.py @@ -35,7 +35,7 @@ class TestYamlProvider(TestCase): # without it we see everything source.populate(zone) - self.assertEquals(22, len(zone.records)) + self.assertEquals(23, len(zone.records)) source.populate(dynamic_zone) self.assertEquals(6, len(dynamic_zone.records)) @@ -58,12 +58,12 @@ class TestYamlProvider(TestCase): # We add everything plan = target.plan(zone) - self.assertEquals(19, len([c for c in plan.changes + self.assertEquals(20, len([c for c in plan.changes if isinstance(c, Create)])) self.assertFalse(isfile(yaml_file)) # Now actually do it - self.assertEquals(19, target.apply(plan)) + self.assertEquals(20, target.apply(plan)) self.assertTrue(isfile(yaml_file)) # Dynamic plan @@ -87,7 +87,7 @@ class TestYamlProvider(TestCase): # A 2nd sync should still create everything plan = target.plan(zone) - self.assertEquals(19, len([c for c in plan.changes + self.assertEquals(20, len([c for c in plan.changes if isinstance(c, Create)])) with open(yaml_file) as fh: @@ -107,6 +107,7 @@ class TestYamlProvider(TestCase): self.assertTrue('values' in data.pop('sub')) self.assertTrue('values' in data.pop('txt')) self.assertTrue('values' in data.pop('loc')) + self.assertTrue('values' in data.pop('urlfwd')) # these are stored as singular 'value' self.assertTrue('value' in data.pop('_imap._tcp')) self.assertTrue('value' in data.pop('_pop3._tcp')) @@ -248,7 +249,7 @@ class TestSplitYamlProvider(TestCase): # without it we see everything source.populate(zone) - self.assertEquals(19, len(zone.records)) + self.assertEquals(20, len(zone.records)) source.populate(dynamic_zone) self.assertEquals(5, len(dynamic_zone.records)) @@ -263,12 +264,12 @@ class TestSplitYamlProvider(TestCase): # We add everything plan = target.plan(zone) - self.assertEquals(16, len([c for c in plan.changes + self.assertEquals(17, len([c for c in plan.changes if isinstance(c, Create)])) self.assertFalse(isdir(zone_dir)) # Now actually do it - self.assertEquals(16, target.apply(plan)) + self.assertEquals(17, target.apply(plan)) # Dynamic plan plan = target.plan(dynamic_zone) @@ -291,7 +292,7 @@ class TestSplitYamlProvider(TestCase): # A 2nd sync should still create everything plan = target.plan(zone) - self.assertEquals(16, len([c for c in plan.changes + self.assertEquals(17, len([c for c in plan.changes if isinstance(c, Create)])) yaml_file = join(zone_dir, '$unit.tests.yaml') @@ -306,7 +307,8 @@ class TestSplitYamlProvider(TestCase): # These records are stored as plural "values." Check each file to # ensure correctness. - for record_name in ('_srv._tcp', 'mx', 'naptr', 'sub', 'txt'): + for record_name in ('_srv._tcp', 'mx', 'naptr', 'sub', 'txt', + 'urlfwd'): yaml_file = join(zone_dir, '{}.yaml'.format(record_name)) self.assertTrue(isfile(yaml_file)) with open(yaml_file) as fh: From 9be1195d47c72a4377a9af3bc45a334e12c6ba9e Mon Sep 17 00:00:00 2001 From: Brian E Clow Date: Thu, 27 May 2021 15:42:27 -0700 Subject: [PATCH 289/358] SplitYAML testing --- tests/config/split/unit.tests.tst/urlfwd.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 tests/config/split/unit.tests.tst/urlfwd.yaml diff --git a/tests/config/split/unit.tests.tst/urlfwd.yaml b/tests/config/split/unit.tests.tst/urlfwd.yaml new file mode 100644 index 0000000..778b9b5 --- /dev/null +++ b/tests/config/split/unit.tests.tst/urlfwd.yaml @@ -0,0 +1,15 @@ +--- +urlfwd: + ttl: 300 + type: URLFWD + values: + - code: 302 + masking: 2 + path: '/' + query: 0 + target: 'http://www.unit.tests' + - code: 301 + masking: 2 + path: '/target' + query: 0 + target: 'http://target.unit.tests' From 21fcff920e7cf15ac39a4dbf0b62477d5b00faec Mon Sep 17 00:00:00 2001 From: Brian E Clow Date: Thu, 27 May 2021 15:43:36 -0700 Subject: [PATCH 290/358] Adding NS1 URLFWD support and testing --- octodns/provider/ns1.py | 24 +++++++++++++++++++++++- tests/test_octodns_provider_ns1.py | 18 +++++++++++++++++- 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 4886c00..6d4f84d 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -234,7 +234,7 @@ class Ns1Provider(BaseProvider): SUPPORTS_GEO = True SUPPORTS_DYNAMIC = True SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CAA', 'CNAME', 'MX', 'NAPTR', - 'NS', 'PTR', 'SPF', 'SRV', 'TXT')) + 'NS', 'PTR', 'SPF', 'SRV', 'TXT', 'URLFWD')) ZONE_NOT_FOUND_MESSAGE = 'server error: zone not found' @@ -749,6 +749,23 @@ class Ns1Provider(BaseProvider): 'values': values, } + def _data_for_URLFWD(self, _type, record): + values = [] + for answer in record['short_answers']: + path, target, code, masking, query = answer.split(' ', 4) + values.append({ + 'path': path, + 'target': target, + 'code': code, + 'masking': masking, + 'query': query, + }) + return { + 'ttl': record['ttl'], + 'type': _type, + 'values': values, + } + def populate(self, zone, target=False, lenient=False): self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, @@ -1244,6 +1261,11 @@ class Ns1Provider(BaseProvider): for v in record.values] return {'answers': values, 'ttl': record.ttl}, None + def _params_for_URLFWD(self, record): + values = [(v.path, v.target, v.code, v.masking, v.query) + for v in record.values] + return {'answers': values, 'ttl': record.ttl}, None + def _get_ns1_filters(self, ns1_zone_name): ns1_filters = {} ns1_zone = {} diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index e21fe0d..875ebbf 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -109,6 +109,17 @@ class TestNs1Provider(TestCase): 'value': 'ca.unit.tests', }, })) + expected.add(Record.new(zone, 'urlfwd', { + 'ttl': 41, + 'type': 'URLFWD', + 'value': { + 'path': '/', + 'target': 'http://foo.unit.tests', + 'code': 301, + 'masking': 2, + 'query': 0, + }, + })) ns1_records = [{ 'type': 'A', @@ -164,6 +175,11 @@ class TestNs1Provider(TestCase): 'ttl': 40, 'short_answers': ['0 issue ca.unit.tests'], 'domain': 'unit.tests.', + }, { + 'type': 'URLFWD', + 'ttl': 41, + 'short_answers': ['/ http://foo.unit.tests 301 2 0'], + 'domain': 'urlfwd.unit.tests.', }] @patch('ns1.rest.records.Records.retrieve') @@ -345,7 +361,7 @@ class TestNs1Provider(TestCase): # Test out the create rate-limit handling, then 9 successes record_create_mock.side_effect = [ RateLimitException('boo', period=0), - ] + ([None] * 9) + ] + ([None] * 10) got_n = provider.apply(plan) self.assertEquals(expected_n, got_n) From f4caa35caa80dd1b8d898d8d1989130bad528f6e Mon Sep 17 00:00:00 2001 From: Brian E Clow Date: Thu, 27 May 2021 15:45:41 -0700 Subject: [PATCH 291/358] Ignoring URLFWD and adjusting test counts for other providers --- tests/test_octodns_manager.py | 14 +++++++------- tests/test_octodns_provider_constellix.py | 2 +- tests/test_octodns_provider_digitalocean.py | 2 +- tests/test_octodns_provider_dnsimple.py | 2 +- tests/test_octodns_provider_dnsmadeeasy.py | 2 +- tests/test_octodns_provider_easydns.py | 2 +- tests/test_octodns_provider_gandi.py | 2 +- tests/test_octodns_provider_hetzner.py | 2 +- tests/test_octodns_provider_powerdns.py | 4 ++-- 9 files changed, 16 insertions(+), 16 deletions(-) diff --git a/tests/test_octodns_manager.py b/tests/test_octodns_manager.py index 8bada06..96f67fd 100644 --- a/tests/test_octodns_manager.py +++ b/tests/test_octodns_manager.py @@ -121,12 +121,12 @@ class TestManager(TestCase): environ['YAML_TMP_DIR'] = tmpdir.dirname tc = Manager(get_config_filename('simple.yaml')) \ .sync(dry_run=False) - self.assertEquals(25, tc) + self.assertEquals(26, tc) # try with just one of the zones tc = Manager(get_config_filename('simple.yaml')) \ .sync(dry_run=False, eligible_zones=['unit.tests.']) - self.assertEquals(19, tc) + self.assertEquals(20, tc) # the subzone, with 2 targets tc = Manager(get_config_filename('simple.yaml')) \ @@ -141,18 +141,18 @@ class TestManager(TestCase): # Again with force tc = Manager(get_config_filename('simple.yaml')) \ .sync(dry_run=False, force=True) - self.assertEquals(25, tc) + self.assertEquals(26, tc) # Again with max_workers = 1 tc = Manager(get_config_filename('simple.yaml'), max_workers=1) \ .sync(dry_run=False, force=True) - self.assertEquals(25, tc) + self.assertEquals(26, tc) # Include meta tc = Manager(get_config_filename('simple.yaml'), max_workers=1, include_meta=True) \ .sync(dry_run=False, force=True) - self.assertEquals(29, tc) + self.assertEquals(30, tc) def test_eligible_sources(self): with TemporaryDirectory() as tmpdir: @@ -218,13 +218,13 @@ class TestManager(TestCase): fh.write('---\n{}') changes = manager.compare(['in'], ['dump'], 'unit.tests.') - self.assertEquals(19, len(changes)) + self.assertEquals(20, len(changes)) # Compound sources with varying support changes = manager.compare(['in', 'nosshfp'], ['dump'], 'unit.tests.') - self.assertEquals(18, len(changes)) + self.assertEquals(19, len(changes)) with self.assertRaises(ManagerException) as ctx: manager.compare(['nope'], ['dump'], 'unit.tests.') diff --git a/tests/test_octodns_provider_constellix.py b/tests/test_octodns_provider_constellix.py index e9ece0e..38b7ab9 100644 --- a/tests/test_octodns_provider_constellix.py +++ b/tests/test_octodns_provider_constellix.py @@ -132,7 +132,7 @@ class TestConstellixProvider(TestCase): plan = provider.plan(self.expected) # No root NS, no ignored, no excluded, no unsupported - n = len(self.expected.records) - 7 + n = len(self.expected.records) - 8 self.assertEquals(n, len(plan.changes)) self.assertEquals(n, provider.apply(plan)) diff --git a/tests/test_octodns_provider_digitalocean.py b/tests/test_octodns_provider_digitalocean.py index affd140..9ed54bf 100644 --- a/tests/test_octodns_provider_digitalocean.py +++ b/tests/test_octodns_provider_digitalocean.py @@ -163,7 +163,7 @@ class TestDigitalOceanProvider(TestCase): plan = provider.plan(self.expected) # No root NS, no ignored, no excluded, no unsupported - n = len(self.expected.records) - 9 + n = len(self.expected.records) - 10 self.assertEquals(n, len(plan.changes)) self.assertEquals(n, provider.apply(plan)) self.assertFalse(plan.exists) diff --git a/tests/test_octodns_provider_dnsimple.py b/tests/test_octodns_provider_dnsimple.py index be881e4..0b8d209 100644 --- a/tests/test_octodns_provider_dnsimple.py +++ b/tests/test_octodns_provider_dnsimple.py @@ -137,7 +137,7 @@ class TestDnsimpleProvider(TestCase): plan = provider.plan(self.expected) # No root NS, no ignored, no excluded - n = len(self.expected.records) - 7 + n = len(self.expected.records) - 8 self.assertEquals(n, len(plan.changes)) self.assertEquals(n, provider.apply(plan)) self.assertFalse(plan.exists) diff --git a/tests/test_octodns_provider_dnsmadeeasy.py b/tests/test_octodns_provider_dnsmadeeasy.py index dc104b7..9efc81d 100644 --- a/tests/test_octodns_provider_dnsmadeeasy.py +++ b/tests/test_octodns_provider_dnsmadeeasy.py @@ -134,7 +134,7 @@ class TestDnsMadeEasyProvider(TestCase): plan = provider.plan(self.expected) # No root NS, no ignored, no excluded, no unsupported - n = len(self.expected.records) - 9 + n = len(self.expected.records) - 10 self.assertEquals(n, len(plan.changes)) self.assertEquals(n, provider.apply(plan)) diff --git a/tests/test_octodns_provider_easydns.py b/tests/test_octodns_provider_easydns.py index a6de8a9..85492eb 100644 --- a/tests/test_octodns_provider_easydns.py +++ b/tests/test_octodns_provider_easydns.py @@ -374,7 +374,7 @@ class TestEasyDNSProvider(TestCase): plan = provider.plan(self.expected) # No root NS, no ignored, no excluded, no unsupported - n = len(self.expected.records) - 8 + n = len(self.expected.records) - 9 self.assertEquals(n, len(plan.changes)) self.assertEquals(n, provider.apply(plan)) self.assertFalse(plan.exists) diff --git a/tests/test_octodns_provider_gandi.py b/tests/test_octodns_provider_gandi.py index 26ffeee..f2e3028 100644 --- a/tests/test_octodns_provider_gandi.py +++ b/tests/test_octodns_provider_gandi.py @@ -193,7 +193,7 @@ class TestGandiProvider(TestCase): plan = provider.plan(self.expected) # No root NS, no ignored, no excluded, no LOC - n = len(self.expected.records) - 5 + n = len(self.expected.records) - 6 self.assertEquals(n, len(plan.changes)) self.assertEquals(n, provider.apply(plan)) self.assertFalse(plan.exists) diff --git a/tests/test_octodns_provider_hetzner.py b/tests/test_octodns_provider_hetzner.py index 4167944..218a6b2 100644 --- a/tests/test_octodns_provider_hetzner.py +++ b/tests/test_octodns_provider_hetzner.py @@ -108,7 +108,7 @@ class TestHetznerProvider(TestCase): plan = provider.plan(self.expected) # No root NS, no ignored, no excluded, no unsupported - n = len(self.expected.records) - 9 + n = len(self.expected.records) - 10 self.assertEquals(n, len(plan.changes)) self.assertEquals(n, provider.apply(plan)) self.assertFalse(plan.exists) diff --git a/tests/test_octodns_provider_powerdns.py b/tests/test_octodns_provider_powerdns.py index 5775f41..92211d1 100644 --- a/tests/test_octodns_provider_powerdns.py +++ b/tests/test_octodns_provider_powerdns.py @@ -185,7 +185,7 @@ class TestPowerDnsProvider(TestCase): expected = Zone('unit.tests.', []) source = YamlProvider('test', join(dirname(__file__), 'config')) source.populate(expected) - expected_n = len(expected.records) - 3 + expected_n = len(expected.records) - 4 self.assertEquals(19, expected_n) # No diffs == no changes @@ -291,7 +291,7 @@ class TestPowerDnsProvider(TestCase): expected = Zone('unit.tests.', []) source = YamlProvider('test', join(dirname(__file__), 'config')) source.populate(expected) - self.assertEquals(22, len(expected.records)) + self.assertEquals(23, len(expected.records)) # A small change to a single record with requests_mock() as mock: From 75423fd786f70e4fc42ae79f4c1737d2bd5d1777 Mon Sep 17 00:00:00 2001 From: Brian E Clow Date: Thu, 27 May 2021 15:46:42 -0700 Subject: [PATCH 292/358] Documentation update? --- docs/records.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/records.md b/docs/records.md index c6b2a77..1457546 100644 --- a/docs/records.md +++ b/docs/records.md @@ -19,6 +19,7 @@ OctoDNS supports the following record types: * `SRV` * `SSHFP` * `TXT` +* `URLFWD` Underlying provider support for each of these varies and some providers have extra requirements or limitations. In cases where a record type is not supported by a provider OctoDNS will ignore it there and continue to manage the record elsewhere. For example `SSHFP` is supported by Dyn, but not Route53. If your source data includes an SSHFP record OctoDNS will keep it in sync on Dyn, but not consider it when evaluating the state of Route53. The best way to find out what types are supported by a provider is to look for its `supports` method. If that method exists the logic will drive which records are supported and which are ignored. If the provider does not implement the method it will fall back to `BaseProvider.supports` which indicates full support. From 07aad177b5ddd736867b6d4ff1362f9bbcebd3c5 Mon Sep 17 00:00:00 2001 From: Brian E Clow Date: Fri, 28 May 2021 13:03:35 -0700 Subject: [PATCH 293/358] Update docs/records.md Co-authored-by: Ross McFarland --- docs/records.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/records.md b/docs/records.md index 1457546..f210846 100644 --- a/docs/records.md +++ b/docs/records.md @@ -19,7 +19,7 @@ OctoDNS supports the following record types: * `SRV` * `SSHFP` * `TXT` -* `URLFWD` +* `URLFWD` Underlying provider support for each of these varies and some providers have extra requirements or limitations. In cases where a record type is not supported by a provider OctoDNS will ignore it there and continue to manage the record elsewhere. For example `SSHFP` is supported by Dyn, but not Route53. If your source data includes an SSHFP record OctoDNS will keep it in sync on Dyn, but not consider it when evaluating the state of Route53. The best way to find out what types are supported by a provider is to look for its `supports` method. If that method exists the logic will drive which records are supported and which are ignored. If the provider does not implement the method it will fall back to `BaseProvider.supports` which indicates full support. From 096785855415636c0502a037d82c1cdf230f7048 Mon Sep 17 00:00:00 2001 From: Brian E Clow Date: Wed, 23 Jun 2021 13:38:58 -0700 Subject: [PATCH 294/358] Accounting for CloudFlare TTL alias --- octodns/provider/cloudflare.py | 21 ++++++++++++--------- tests/test_octodns_provider_cloudflare.py | 8 ++++++++ 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/octodns/provider/cloudflare.py b/octodns/provider/cloudflare.py index 4f9ba64..9c206c0 100644 --- a/octodns/provider/cloudflare.py +++ b/octodns/provider/cloudflare.py @@ -170,6 +170,9 @@ class CloudflareProvider(BaseProvider): return self._zones + def _ttl_data(self, ttl): + return 300 if ttl == 1 else ttl + def _data_for_cdn(self, name, _type, records): self.log.info('CDN rewrite for %s', records[0]['name']) _type = "CNAME" @@ -177,14 +180,14 @@ class CloudflareProvider(BaseProvider): _type = "ALIAS" return { - 'ttl': records[0]['ttl'], + 'ttl': self._ttl_data(records[0]['ttl']), 'type': _type, 'value': '{}.cdn.cloudflare.net.'.format(records[0]['name']), } def _data_for_multiple(self, _type, records): return { - 'ttl': records[0]['ttl'], + 'ttl': self._ttl_data(records[0]['ttl']), 'type': _type, 'values': [r['content'] for r in records], } @@ -195,7 +198,7 @@ class CloudflareProvider(BaseProvider): def _data_for_TXT(self, _type, records): return { - 'ttl': records[0]['ttl'], + 'ttl': self._ttl_data(records[0]['ttl']), 'type': _type, 'values': [r['content'].replace(';', '\\;') for r in records], } @@ -206,7 +209,7 @@ class CloudflareProvider(BaseProvider): data = r['data'] values.append(data) return { - 'ttl': records[0]['ttl'], + 'ttl': self._ttl_data(records[0]['ttl']), 'type': _type, 'values': values, } @@ -214,7 +217,7 @@ class CloudflareProvider(BaseProvider): def _data_for_CNAME(self, _type, records): only = records[0] return { - 'ttl': only['ttl'], + 'ttl': self._ttl_data(only['ttl']), 'type': _type, 'value': '{}.'.format(only['content']) } @@ -241,7 +244,7 @@ class CloudflareProvider(BaseProvider): 'precision_vert': float(r['precision_vert']), }) return { - 'ttl': records[0]['ttl'], + 'ttl': self._ttl_data(records[0]['ttl']), 'type': _type, 'values': values } @@ -254,14 +257,14 @@ class CloudflareProvider(BaseProvider): 'exchange': '{}.'.format(r['content']), }) return { - 'ttl': records[0]['ttl'], + 'ttl': self._ttl_data(records[0]['ttl']), 'type': _type, 'values': values, } def _data_for_NS(self, _type, records): return { - 'ttl': records[0]['ttl'], + 'ttl': self._ttl_data(records[0]['ttl']), 'type': _type, 'values': ['{}.'.format(r['content']) for r in records], } @@ -279,7 +282,7 @@ class CloudflareProvider(BaseProvider): }) return { 'type': _type, - 'ttl': records[0]['ttl'], + 'ttl': self._ttl_data(records[0]['ttl']), 'values': values } diff --git a/tests/test_octodns_provider_cloudflare.py b/tests/test_octodns_provider_cloudflare.py index 8843843..52c261e 100644 --- a/tests/test_octodns_provider_cloudflare.py +++ b/tests/test_octodns_provider_cloudflare.py @@ -1410,3 +1410,11 @@ class TestCloudflareProvider(TestCase): with self.assertRaises(CloudflareRateLimitError) as ctx: provider.zone_records(zone) self.assertEquals('last', text_type(ctx.exception)) + + def test_ttl_mapping(self): + provider = CloudflareProvider('test', 'email', 'token') + + self.assertEquals(120, provider._ttl_data(120)) + self.assertEquals(120, provider._ttl_data(120)) + self.assertEquals(3600, provider._ttl_data(3600)) + self.assertEquals(300, provider._ttl_data(1)) From a3b94cfed32d2ae02407958f02d32c809d9f87a8 Mon Sep 17 00:00:00 2001 From: Brian E Clow Date: Thu, 22 Jul 2021 14:33:06 -0700 Subject: [PATCH 295/358] Adding URLFWD type to CloudeFlare provider + testing updates --- octodns/provider/cloudflare.py | 187 +++++++++++++--- tests/test_octodns_provider_cloudflare.py | 251 ++++++++++++++++++++-- 2 files changed, 393 insertions(+), 45 deletions(-) diff --git a/octodns/provider/cloudflare.py b/octodns/provider/cloudflare.py index 9c206c0..9462704 100644 --- a/octodns/provider/cloudflare.py +++ b/octodns/provider/cloudflare.py @@ -10,6 +10,7 @@ from copy import deepcopy from logging import getLogger from requests import Session from time import sleep +from urllib.parse import urlsplit from ..record import Record, Update from .base import BaseProvider @@ -76,7 +77,7 @@ class CloudflareProvider(BaseProvider): SUPPORTS_GEO = False SUPPORTS_DYNAMIC = False SUPPORTS = set(('ALIAS', 'A', 'AAAA', 'CAA', 'CNAME', 'LOC', 'MX', 'NS', - 'PTR', 'SRV', 'SPF', 'TXT')) + 'PTR', 'SRV', 'SPF', 'TXT', 'URLFWD')) MIN_TTL = 120 TIMEOUT = 15 @@ -286,6 +287,22 @@ class CloudflareProvider(BaseProvider): 'values': values } + def _data_for_URLFWD(self, _type, records): + values = [] + for r in records: + values.append({ + 'path': r['path'], + 'target': r['url'], + 'code': r['status_code'], + 'masking': 2, + 'query': 0, + }) + return { + 'type': _type, + 'ttl': 3600, # ttl does not exist for this type, forcing a setting + 'values': values + } + def zone_records(self, zone): if zone.name not in self._zone_records: zone_id = self.zones.get(zone.name, False) @@ -305,6 +322,10 @@ class CloudflareProvider(BaseProvider): else: page = None + path = '/zones/{}/pagerules'.format(zone_id) + resp = self._try_request('GET', path, params={'status': 'active'}) + records += resp['result'] + self._zone_records[zone.name] = records return self._zone_records[zone.name] @@ -341,10 +362,30 @@ class CloudflareProvider(BaseProvider): exists = True values = defaultdict(lambda: defaultdict(list)) for record in records: - name = zone.hostname_from_fqdn(record['name']) - _type = record['type'] - if _type in self.SUPPORTS: - values[name][record['type']].append(record) + if 'targets' in record: + # assumption, targets will always contain 1 target + # API documentation only indicates 'url' as the only target + # if record['targets'][0]['target'] == 'url': + uri = record['targets'][0]['constraint']['value'] + uri = '//' + uri if not uri.startswith('http') else uri + parsed_uri = urlsplit(uri) + name = zone.hostname_from_fqdn(parsed_uri.netloc) + _path = parsed_uri.path + _type = 'URLFWD' + # assumption, actions will always contain 1 action + if record['actions'][0]['id'] == 'forwarding_url': + _values = record['actions'][0]['value'] + _values['path'] = _path + # no ttl set by pagerule, creating one + _values['ttl'] = 3600 + values[name][_type].append(_values) + # the dns_records branch + # elif 'name' in record: + else: + name = zone.hostname_from_fqdn(record['name']) + _type = record['type'] + if _type in self.SUPPORTS: + values[name][record['type']].append(record) for name, types in values.items(): for _type, records in types.items(): @@ -473,6 +514,31 @@ class CloudflareProvider(BaseProvider): } } + def _contents_for_URLFWD(self, record): + name = record.fqdn[:-1] + for value in record.values: + yield { + 'targets': [ + { + 'target': 'url', + 'constraint': { + 'operator': 'matches', + 'value': name + value.path + } + } + ], + 'actions': [ + { + 'id': 'forwarding_url', + 'value': { + 'url': value.target, + 'status_code': value.code, + } + } + ], + 'status': 'active', + } + def _record_is_proxied(self, record): return ( not self.cdn and @@ -488,20 +554,25 @@ class CloudflareProvider(BaseProvider): if _type == 'ALIAS': _type = 'CNAME' - contents_for = getattr(self, '_contents_for_{}'.format(_type)) - for content in contents_for(record): - content.update({ - 'name': name, - 'type': _type, - 'ttl': ttl, - }) - - if _type in _PROXIABLE_RECORD_TYPES: + if _type == 'URLFWD': + contents_for = getattr(self, '_contents_for_{}'.format(_type)) + for content in contents_for(record): + yield content + else: + contents_for = getattr(self, '_contents_for_{}'.format(_type)) + for content in contents_for(record): content.update({ - 'proxied': self._record_is_proxied(record) + 'name': name, + 'type': _type, + 'ttl': ttl, }) - yield content + if _type in _PROXIABLE_RECORD_TYPES: + content.update({ + 'proxied': self._record_is_proxied(record) + }) + + yield content def _gen_key(self, data): # Note that most CF record data has a `content` field the value of @@ -515,7 +586,11 @@ class CloudflareProvider(BaseProvider): # BUT... there are exceptions. MX, CAA, LOC and SRV don't have a simple # content as things are currently implemented so we need to handle # those explicitly and create unique/hashable strings for them. - _type = data['type'] + # AND... for URLFWD/Redirects additional adventures are created. + if 'targets' in data: + _type = 'URLFWD' + else: + _type = data['type'] if _type == 'MX': return '{priority} {content}'.format(**data) elif _type == 'CAA': @@ -540,12 +615,28 @@ class CloudflareProvider(BaseProvider): '{precision_horz}', '{precision_vert}') return ' '.join(loc).format(**data) + elif _type == 'URLFWD': + uri = data['targets'][0]['constraint']['value'] + uri = '//' + uri if not uri.startswith('http') else uri + parsed_uri = urlsplit(uri) + ret = {} + ret.update(data['actions'][0]['value']) + ret.update({'name': parsed_uri.netloc, 'path': parsed_uri.path}) + urlfwd = ( + '{name}', + '{path}', + '{url}', + '{status_code}') + return ' '.join(urlfwd).format(**ret) return data['content'] def _apply_Create(self, change): new = change.new zone_id = self.zones[new.zone.name] - path = '/zones/{}/dns_records'.format(zone_id) + if new._type == 'URLFWD': + path = '/zones/{}/pagerules'.format(zone_id) + else: + path = '/zones/{}/dns_records'.format(zone_id) for content in self._gen_data(new): self._try_request('POST', path, data=content) @@ -558,14 +649,28 @@ class CloudflareProvider(BaseProvider): existing = {} # Find all of the existing CF records for this name & type for record in self.zone_records(zone): - name = zone.hostname_from_fqdn(record['name']) + if 'targets' in record: + uri = record['targets'][0]['constraint']['value'] + uri = '//' + uri if not uri.startswith('http') else uri + parsed_uri = urlsplit(uri) + name = zone.hostname_from_fqdn(parsed_uri.netloc) + _path = parsed_uri.path + # assumption, populate will catch this contition + # if record['actions'][0]['id'] == 'forwarding_url': + _values = record['actions'][0]['value'] + _values['path'] = _path + _values['ttl'] = 3600 + _values['type'] = 'URLFWD' + record.update(_values) + else: + name = zone.hostname_from_fqdn(record['name']) # Use the _record_for so that we include all of standard # conversion logic r = self._record_for(zone, name, record['type'], [record], True) if hostname == r.name and _type == r._type: - # Round trip the single value through a record to contents flow - # to get a consistent _gen_data result that matches what - # went in to new_contents + # Round trip the single value through a record to contents + # flow to get a consistent _gen_data result that matches + # what went in to new_contents data = next(self._gen_data(r)) # Record the record_id and data for this existing record @@ -633,7 +738,10 @@ class CloudflareProvider(BaseProvider): # otherwise required, just makes things deterministic # Creates - path = '/zones/{}/dns_records'.format(zone_id) + if _type == 'URLFWD': + path = '/zones/{}/pagerules'.format(zone_id) + else: + path = '/zones/{}/dns_records'.format(zone_id) for _, data in sorted(creates.items()): self.log.debug('_apply_Update: creating %s', data) self._try_request('POST', path, data=data) @@ -643,7 +751,10 @@ class CloudflareProvider(BaseProvider): record_id = info['record_id'] data = info['data'] old_data = info['old_data'] - path = '/zones/{}/dns_records/{}'.format(zone_id, record_id) + if _type == 'URLFWD': + path = '/zones/{}/pagerules/{}'.format(zone_id, record_id) + else: + path = '/zones/{}/dns_records/{}'.format(zone_id, record_id) self.log.debug('_apply_Update: updating %s, %s -> %s', record_id, data, old_data) self._try_request('PUT', path, data=data) @@ -652,7 +763,10 @@ class CloudflareProvider(BaseProvider): for _, info in sorted(deletes.items()): record_id = info['record_id'] old_data = info['data'] - path = '/zones/{}/dns_records/{}'.format(zone_id, record_id) + if _type == 'URLFWD': + path = '/zones/{}/pagerules/{}'.format(zone_id, record_id) + else: + path = '/zones/{}/dns_records/{}'.format(zone_id, record_id) self.log.debug('_apply_Update: removing %s, %s', record_id, old_data) self._try_request('DELETE', path) @@ -664,11 +778,24 @@ class CloudflareProvider(BaseProvider): existing_type = 'CNAME' if existing._type == 'ALIAS' \ else existing._type for record in self.zone_records(existing.zone): - if existing_name == record['name'] and \ - existing_type == record['type']: - path = '/zones/{}/dns_records/{}'.format(record['zone_id'], - record['id']) - self._try_request('DELETE', path) + if 'targets' in record: + uri = record['targets'][0]['constraint']['value'] + uri = '//' + uri if not uri.startswith('http') else uri + parsed_uri = urlsplit(uri) + record_name = parsed_uri.netloc + record_type = 'URLFWD' + zone_id = self.zones.get(existing.zone.name, False) + if existing_name == record_name and \ + existing_type == record_type: + path = '/zones/{}/pagerules/{}' \ + .format(zone_id, record['id']) + self._try_request('DELETE', path) + else: + if existing_name == record['name'] and \ + existing_type == record['type']: + path = '/zones/{}/dns_records/{}' \ + .format(record['zone_id'], record['id']) + self._try_request('DELETE', path) def _apply(self, plan): desired = plan.desired diff --git a/tests/test_octodns_provider_cloudflare.py b/tests/test_octodns_provider_cloudflare.py index 52c261e..c2addf0 100644 --- a/tests/test_octodns_provider_cloudflare.py +++ b/tests/test_octodns_provider_cloudflare.py @@ -166,9 +166,15 @@ class TestCloudflareProvider(TestCase): json={'result': [], 'result_info': {'count': 0, 'per_page': 0}}) + base = '{}/234234243423aaabb334342aaa343435'.format(base) + + # pagerules/URLFWD + with open('tests/fixtures/cloudflare-pagerules.json') as fh: + mock.get('{}/pagerules?status=active'.format(base), + status_code=200, text=fh.read()) + # records - base = '{}/234234243423aaabb334342aaa343435/dns_records' \ - .format(base) + base = '{}/dns_records'.format(base) with open('tests/fixtures/cloudflare-dns_records-' 'page-1.json') as fh: mock.get('{}?page=1'.format(base), status_code=200, @@ -184,16 +190,16 @@ class TestCloudflareProvider(TestCase): zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals(16, len(zone.records)) + self.assertEquals(19, len(zone.records)) changes = self.expected.changes(zone, provider) - self.assertEquals(0, len(changes)) + self.assertEquals(4, len(changes)) # re-populating the same zone/records comes out of cache, no calls again = Zone('unit.tests.', []) provider.populate(again) - self.assertEquals(16, len(again.records)) + self.assertEquals(19, len(again.records)) def test_apply(self): provider = CloudflareProvider('test', 'email', 'token', retry_period=0) @@ -207,12 +213,12 @@ class TestCloudflareProvider(TestCase): 'id': 42, } }, # zone create - ] + [None] * 25 # individual record creates + ] + [None] * 27 # individual record creates # non-existent zone, create everything plan = provider.plan(self.expected) - self.assertEquals(16, len(plan.changes)) - self.assertEquals(16, provider.apply(plan)) + self.assertEquals(17, len(plan.changes)) + self.assertEquals(17, provider.apply(plan)) self.assertFalse(plan.exists) provider._request.assert_has_calls([ @@ -236,9 +242,31 @@ class TestCloudflareProvider(TestCase): 'name': 'txt.unit.tests', 'ttl': 600 }), + # create at least one pagerules + call('POST', '/zones/42/pagerules', data={ + 'targets': [ + { + 'target': 'url', + 'constraint': { + 'operator': 'matches', + 'value': 'urlfwd.unit.tests/' + } + } + ], + 'actions': [ + { + 'id': 'forwarding_url', + 'value': { + 'url': 'http://www.unit.tests', + 'status_code': 302 + } + } + ], + 'status': 'active' + }), ], True) # expected number of total calls - self.assertEquals(27, provider._request.call_count) + self.assertEquals(29, provider._request.call_count) provider._request.reset_mock() @@ -311,6 +339,56 @@ class TestCloudflareProvider(TestCase): "auto_added": False } }, + { + "id": "2a9140b17ffb0e6aed826049eec970b7", + "targets": [ + { + "target": "url", + "constraint": { + "operator": "matches", + "value": "urlfwd.unit.tests/" + } + } + ], + "actions": [ + { + "id": "forwarding_url", + "value": { + "url": "https://www.unit.tests", + "status_code": 302 + } + } + ], + "priority": 1, + "status": "active", + "created_on": "2021-06-25T20:10:50.000000Z", + "modified_on": "2021-06-28T22:38:10.000000Z" + }, + { + "id": "2a9141b18ffb0e6aed826050eec970b8", + "targets": [ + { + "target": "url", + "constraint": { + "operator": "matches", + "value": "urlfwdother.unit.tests/target" + } + } + ], + "actions": [ + { + "id": "forwarding_url", + "value": { + "url": "https://target.unit.tests", + "status_code": 301 + } + } + ], + "priority": 2, + "status": "active", + "created_on": "2021-06-25T20:10:50.000000Z", + "modified_on": "2021-06-28T22:38:10.000000Z" + }, ]) # we don't care about the POST/create return values @@ -319,7 +397,7 @@ class TestCloudflareProvider(TestCase): # Test out the create rate-limit handling, then 9 successes provider._request.side_effect = [ CloudflareRateLimitError('{}'), - ] + ([None] * 3) + ] + ([None] * 5) wanted = Zone('unit.tests.', []) wanted.add_record(Record.new(wanted, 'nc', { @@ -332,14 +410,27 @@ class TestCloudflareProvider(TestCase): 'type': 'A', 'value': '3.2.3.4' })) + wanted.add_record(Record.new(wanted, 'urlfwd', { + 'ttl': 3600, + 'type': 'URLFWD', + 'value': { + 'path': '/*', # path change + 'target': 'https://www.unit.tests/', # target change + 'code': 301, # status_code change + 'masking': '2', + 'query': 0, + } + })) plan = provider.plan(wanted) # only see the delete & ttl update, below min-ttl is filtered out - self.assertEquals(2, len(plan.changes)) - self.assertEquals(2, provider.apply(plan)) + self.assertEquals(4, len(plan.changes)) + self.assertEquals(4, provider.apply(plan)) self.assertTrue(plan.exists) # creates a the new value and then deletes all the old provider._request.assert_has_calls([ + call('DELETE', '/zones/42/' + 'pagerules/2a9141b18ffb0e6aed826050eec970b8'), call('DELETE', '/zones/ff12ab34cd5611334422ab3322997650/' 'dns_records/fc12ab34cd5611334422ab3322997653'), call('DELETE', '/zones/ff12ab34cd5611334422ab3322997650/' @@ -351,7 +442,29 @@ class TestCloudflareProvider(TestCase): 'name': 'ttl.unit.tests', 'proxied': False, 'ttl': 300 - }) + }), + call('PUT', '/zones/42/pagerules/' + '2a9140b17ffb0e6aed826049eec970b7', data={ + 'targets': [ + { + 'target': 'url', + 'constraint': { + 'operator': 'matches', + 'value': 'urlfwd.unit.tests/*' + } + } + ], + 'actions': [ + { + 'id': 'forwarding_url', + 'value': { + 'url': 'https://www.unit.tests/', + 'status_code': 301 + } + } + ], + 'status': 'active', + }), ]) def test_update_add_swap(self): @@ -500,6 +613,56 @@ class TestCloudflareProvider(TestCase): "auto_added": False } }, + { + "id": "2a9140b17ffb0e6aed826049eec974b7", + "targets": [ + { + "target": "url", + "constraint": { + "operator": "matches", + "value": "urlfwd1.unit.tests/" + } + } + ], + "actions": [ + { + "id": "forwarding_url", + "value": { + "url": "https://www.unit.tests", + "status_code": 302 + } + } + ], + "priority": 1, + "status": "active", + "created_on": "2021-06-25T20:10:50.000000Z", + "modified_on": "2021-06-28T22:38:10.000000Z" + }, + { + "id": "2a9141b18ffb0e6aed826054eec970b8", + "targets": [ + { + "target": "url", + "constraint": { + "operator": "matches", + "value": "urlfwd1.unit.tests/target" + } + } + ], + "actions": [ + { + "id": "forwarding_url", + "value": { + "url": "https://target.unit.tests", + "status_code": 301 + } + } + ], + "priority": 2, + "status": "active", + "created_on": "2021-06-25T20:10:50.000000Z", + "modified_on": "2021-06-28T22:38:10.000000Z" + }, ]) provider._request = Mock() @@ -513,6 +676,8 @@ class TestCloudflareProvider(TestCase): }, # zone create None, None, + None, + None, ] # Add something and delete something @@ -523,14 +688,46 @@ class TestCloudflareProvider(TestCase): # This matches the zone data above, one to delete, one to leave 'values': ['ns1.foo.bar.', 'ns2.foo.bar.'], }) + exstingurlfwd = Record.new(zone, 'urlfwd1', { + 'ttl': 3600, + 'type': 'URLFWD', + 'values': [ + { + 'path': '/', + 'target': 'https://www.unit.tests', + 'code': 302, + 'masking': '2', + 'query': 0, + }, + { + 'path': '/target', + 'target': 'https://target.unit.tests', + 'code': 301, + 'masking': '2', + 'query': 0, + } + ] + }) new = Record.new(zone, '', { 'ttl': 300, 'type': 'NS', # This leaves one and deletes one 'value': 'ns2.foo.bar.', }) + newurlfwd = Record.new(zone, 'urlfwd1', { + 'ttl': 3600, + 'type': 'URLFWD', + 'value': { + 'path': '/', + 'target': 'https://www.unit.tests', + 'code': 302, + 'masking': '2', + 'query': 0, + } + }) change = Update(existing, new) - plan = Plan(zone, zone, [change], True) + changeurlfwd = Update(exstingurlfwd, newurlfwd) + plan = Plan(zone, zone, [change, changeurlfwd], True) provider._apply(plan) # Get zones, create zone, create a record, delete a record @@ -548,7 +745,31 @@ class TestCloudflareProvider(TestCase): 'ttl': 300 }), call('DELETE', '/zones/42/dns_records/' - 'fc12ab34cd5611334422ab3322997653') + 'fc12ab34cd5611334422ab3322997653'), + call('PUT', '/zones/42/pagerules/' + '2a9140b17ffb0e6aed826049eec974b7', data={ + 'targets': [ + { + 'target': 'url', + 'constraint': { + 'operator': 'matches', + 'value': 'urlfwd1.unit.tests/' + } + } + ], + 'actions': [ + { + 'id': 'forwarding_url', + 'value': { + 'url': 'https://www.unit.tests', + 'status_code': 302 + } + } + ], + 'status': 'active' + }), + call('DELETE', '/zones/42/pagerules/' + '2a9141b18ffb0e6aed826054eec970b8'), ]) def test_ptr(self): From 5eae6164b659e8e0cb4c3ae30b9735f3c6ed7744 Mon Sep 17 00:00:00 2001 From: Brian E Clow Date: Thu, 22 Jul 2021 14:34:10 -0700 Subject: [PATCH 296/358] Adding CloudFlare URLFWD/pagerules fixtures for testing updates --- tests/fixtures/cloudflare-pagerules.json | 103 +++++++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 tests/fixtures/cloudflare-pagerules.json diff --git a/tests/fixtures/cloudflare-pagerules.json b/tests/fixtures/cloudflare-pagerules.json new file mode 100644 index 0000000..7efa018 --- /dev/null +++ b/tests/fixtures/cloudflare-pagerules.json @@ -0,0 +1,103 @@ +{ + "result": [ + { + "id": "2b1ec1793185213139f22059a165376e", + "targets": [ + { + "target": "url", + "constraint": { + "operator": "matches", + "value": "urlfwd0.unit.tests/" + } + } + ], + "actions": [ + { + "id": "always_use_https" + } + ], + "priority": 4, + "status": "active", + "created_on": "2021-06-29T17:14:28.000000Z", + "modified_on": "2021-06-29T17:15:33.000000Z" + }, + { + "id": "2b1ec1793185213139f22059a165376f", + "targets": [ + { + "target": "url", + "constraint": { + "operator": "matches", + "value": "urlfwd0.unit.tests/*" + } + } + ], + "actions": [ + { + "id": "forwarding_url", + "value": { + "url": "https://www.unit.tests/", + "status_code": 301 + } + } + ], + "priority": 3, + "status": "active", + "created_on": "2021-06-29T17:07:12.000000Z", + "modified_on": "2021-06-29T17:15:12.000000Z" + }, + { + "id": "2b1ec1793185213139f22059a165377e", + "targets": [ + { + "target": "url", + "constraint": { + "operator": "matches", + "value": "urlfwd1.unit.tests/*" + } + } + ], + "actions": [ + { + "id": "forwarding_url", + "value": { + "url": "https://www.unit.tests/", + "status_code": 302 + } + } + ], + "priority": 2, + "status": "active", + "created_on": "2021-06-28T22:42:27.000000Z", + "modified_on": "2021-06-28T22:43:13.000000Z" + }, + { + "id": "2a9140b17ffb0e6aed826049eec970b8", + "targets": [ + { + "target": "url", + "constraint": { + "operator": "matches", + "value": "urlfwd2.unit.tests/*" + } + } + ], + "actions": [ + { + "id": "forwarding_url", + "value": { + "url": "https://www.unit.tests/", + "status_code": 301 + } + } + ], + "priority": 1, + "status": "active", + "created_on": "2021-06-25T20:10:50.000000Z", + "modified_on": "2021-06-28T22:38:10.000000Z" + } + ], + "success": true, + "errors": [], + "messages": [] +} From afc46f67eba9c840c2968bc95d1357dca8d06c2b Mon Sep 17 00:00:00 2001 From: Brian E Clow Date: Thu, 22 Jul 2021 15:23:09 -0700 Subject: [PATCH 297/358] When you filter is important --- octodns/provider/cloudflare.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/octodns/provider/cloudflare.py b/octodns/provider/cloudflare.py index 9462704..c94bffb 100644 --- a/octodns/provider/cloudflare.py +++ b/octodns/provider/cloudflare.py @@ -324,7 +324,10 @@ class CloudflareProvider(BaseProvider): path = '/zones/{}/pagerules'.format(zone_id) resp = self._try_request('GET', path, params={'status': 'active'}) - records += resp['result'] + for r in resp['result']: + # assumption, base on API guide, will only contain 1 action + if r['actions'][0]['id'] == 'forwarding_url': + records += [r] self._zone_records[zone.name] = records @@ -373,12 +376,11 @@ class CloudflareProvider(BaseProvider): _path = parsed_uri.path _type = 'URLFWD' # assumption, actions will always contain 1 action - if record['actions'][0]['id'] == 'forwarding_url': - _values = record['actions'][0]['value'] - _values['path'] = _path - # no ttl set by pagerule, creating one - _values['ttl'] = 3600 - values[name][_type].append(_values) + _values = record['actions'][0]['value'] + _values['path'] = _path + # no ttl set by pagerule, creating one + _values['ttl'] = 3600 + values[name][_type].append(_values) # the dns_records branch # elif 'name' in record: else: @@ -655,8 +657,7 @@ class CloudflareProvider(BaseProvider): parsed_uri = urlsplit(uri) name = zone.hostname_from_fqdn(parsed_uri.netloc) _path = parsed_uri.path - # assumption, populate will catch this contition - # if record['actions'][0]['id'] == 'forwarding_url': + # assumption, actions will always contain 1 action _values = record['actions'][0]['value'] _values['path'] = _path _values['ttl'] = 3600 From 5e2f7d43845d48fbf2e92b7bbaf2bb5b0e529b33 Mon Sep 17 00:00:00 2001 From: Brian E Clow Date: Thu, 22 Jul 2021 15:26:21 -0700 Subject: [PATCH 298/358] Removing the TODO note from record --- octodns/record/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/octodns/record/__init__.py b/octodns/record/__init__.py index 15cb143..5acffab 100644 --- a/octodns/record/__init__.py +++ b/octodns/record/__init__.py @@ -618,7 +618,6 @@ class _DynamicMixin(object): else: seen_default = False - # TODO: don't allow 'default' as a pool name, reserved for i, rule in enumerate(rules): rule_num = i + 1 try: From f4ccaaa7919c695aeebe2e26118fd144c6396e35 Mon Sep 17 00:00:00 2001 From: Brian E Clow Date: Fri, 23 Jul 2021 14:45:26 -0700 Subject: [PATCH 299/358] Apply suggestions from code review Thank you, Ross Co-authored-by: Ross McFarland --- octodns/provider/cloudflare.py | 37 +++++++++++++--------------------- 1 file changed, 14 insertions(+), 23 deletions(-) diff --git a/octodns/provider/cloudflare.py b/octodns/provider/cloudflare.py index c94bffb..c79b5ce 100644 --- a/octodns/provider/cloudflare.py +++ b/octodns/provider/cloudflare.py @@ -373,13 +373,13 @@ class CloudflareProvider(BaseProvider): uri = '//' + uri if not uri.startswith('http') else uri parsed_uri = urlsplit(uri) name = zone.hostname_from_fqdn(parsed_uri.netloc) - _path = parsed_uri.path + path = parsed_uri.path _type = 'URLFWD' # assumption, actions will always contain 1 action - _values = record['actions'][0]['value'] - _values['path'] = _path + values = record['actions'][0]['value'] + values['path'] = path # no ttl set by pagerule, creating one - _values['ttl'] = 3600 + values['ttl'] = 3600 values[name][_type].append(_values) # the dns_records branch # elif 'name' in record: @@ -589,10 +589,7 @@ class CloudflareProvider(BaseProvider): # content as things are currently implemented so we need to handle # those explicitly and create unique/hashable strings for them. # AND... for URLFWD/Redirects additional adventures are created. - if 'targets' in data: - _type = 'URLFWD' - else: - _type = data['type'] + _type = data.get('type', 'URLFWD') if _type == 'MX': return '{priority} {content}'.format(**data) elif _type == 'CAA': @@ -621,15 +618,9 @@ class CloudflareProvider(BaseProvider): uri = data['targets'][0]['constraint']['value'] uri = '//' + uri if not uri.startswith('http') else uri parsed_uri = urlsplit(uri) - ret = {} - ret.update(data['actions'][0]['value']) - ret.update({'name': parsed_uri.netloc, 'path': parsed_uri.path}) - urlfwd = ( - '{name}', - '{path}', - '{url}', - '{status_code}') - return ' '.join(urlfwd).format(**ret) + return '{name} {path} {url} {status_code}'.format(name=parsed_uri.netloc, + path=parsed_uri.path, + **data['actions'][0]['value']) return data['content'] def _apply_Create(self, change): @@ -656,13 +647,13 @@ class CloudflareProvider(BaseProvider): uri = '//' + uri if not uri.startswith('http') else uri parsed_uri = urlsplit(uri) name = zone.hostname_from_fqdn(parsed_uri.netloc) - _path = parsed_uri.path + path = parsed_uri.path # assumption, actions will always contain 1 action - _values = record['actions'][0]['value'] - _values['path'] = _path - _values['ttl'] = 3600 - _values['type'] = 'URLFWD' - record.update(_values) + values = record['actions'][0]['value'] + values['path'] = path + values['ttl'] = 3600 + values['type'] = 'URLFWD' + record.update(values) else: name = zone.hostname_from_fqdn(record['name']) # Use the _record_for so that we include all of standard From 8ca7070186c5609b4abde2c4d7ecb0fa6204c9ef Mon Sep 17 00:00:00 2001 From: Brian E Clow Date: Fri, 23 Jul 2021 15:03:51 -0700 Subject: [PATCH 300/358] Formatting, lingering pr comments, fixing resulting errors --- octodns/provider/cloudflare.py | 25 ++++++++++++----------- octodns/record/__init__.py | 2 -- tests/test_octodns_provider_cloudflare.py | 6 +++--- 3 files changed, 16 insertions(+), 17 deletions(-) diff --git a/octodns/provider/cloudflare.py b/octodns/provider/cloudflare.py index c79b5ce..01710b6 100644 --- a/octodns/provider/cloudflare.py +++ b/octodns/provider/cloudflare.py @@ -299,7 +299,7 @@ class CloudflareProvider(BaseProvider): }) return { 'type': _type, - 'ttl': 3600, # ttl does not exist for this type, forcing a setting + 'ttl': 300, # ttl does not exist for this type, forcing a setting 'values': values } @@ -376,10 +376,10 @@ class CloudflareProvider(BaseProvider): path = parsed_uri.path _type = 'URLFWD' # assumption, actions will always contain 1 action - values = record['actions'][0]['value'] - values['path'] = path + _values = record['actions'][0]['value'] + _values['path'] = path # no ttl set by pagerule, creating one - values['ttl'] = 3600 + _values['ttl'] = 300 values[name][_type].append(_values) # the dns_records branch # elif 'name' in record: @@ -618,9 +618,10 @@ class CloudflareProvider(BaseProvider): uri = data['targets'][0]['constraint']['value'] uri = '//' + uri if not uri.startswith('http') else uri parsed_uri = urlsplit(uri) - return '{name} {path} {url} {status_code}'.format(name=parsed_uri.netloc, - path=parsed_uri.path, - **data['actions'][0]['value']) + return '{name} {path} {url} {status_code}' \ + .format(name=parsed_uri.netloc, + path=parsed_uri.path, + **data['actions'][0]['value']) return data['content'] def _apply_Create(self, change): @@ -649,11 +650,11 @@ class CloudflareProvider(BaseProvider): name = zone.hostname_from_fqdn(parsed_uri.netloc) path = parsed_uri.path # assumption, actions will always contain 1 action - values = record['actions'][0]['value'] - values['path'] = path - values['ttl'] = 3600 - values['type'] = 'URLFWD' - record.update(values) + _values = record['actions'][0]['value'] + _values['path'] = path + _values['ttl'] = 300 + _values['type'] = 'URLFWD' + record.update(_values) else: name = zone.hostname_from_fqdn(record['name']) # Use the _record_for so that we include all of standard diff --git a/octodns/record/__init__.py b/octodns/record/__init__.py index 5acffab..7714b27 100644 --- a/octodns/record/__init__.py +++ b/octodns/record/__init__.py @@ -1470,8 +1470,6 @@ class TxtRecord(_ChunkedValuesMixin, Record): class UrlfwdValue(EqualityTupleMixin): - # TODO: should have defaults for path, code, masking, and query - VALID_CODES = (301, 302) VALID_MASKS = (0, 1, 2) VALID_QUERY = (0, 1) diff --git a/tests/test_octodns_provider_cloudflare.py b/tests/test_octodns_provider_cloudflare.py index c2addf0..2cc11cb 100644 --- a/tests/test_octodns_provider_cloudflare.py +++ b/tests/test_octodns_provider_cloudflare.py @@ -411,7 +411,7 @@ class TestCloudflareProvider(TestCase): 'value': '3.2.3.4' })) wanted.add_record(Record.new(wanted, 'urlfwd', { - 'ttl': 3600, + 'ttl': 300, 'type': 'URLFWD', 'value': { 'path': '/*', # path change @@ -689,7 +689,7 @@ class TestCloudflareProvider(TestCase): 'values': ['ns1.foo.bar.', 'ns2.foo.bar.'], }) exstingurlfwd = Record.new(zone, 'urlfwd1', { - 'ttl': 3600, + 'ttl': 300, 'type': 'URLFWD', 'values': [ { @@ -715,7 +715,7 @@ class TestCloudflareProvider(TestCase): 'value': 'ns2.foo.bar.', }) newurlfwd = Record.new(zone, 'urlfwd1', { - 'ttl': 3600, + 'ttl': 300, 'type': 'URLFWD', 'value': { 'path': '/', From a9fe3b53983a99847a68f71df281085f99643e13 Mon Sep 17 00:00:00 2001 From: Brian E Clow Date: Fri, 23 Jul 2021 15:24:09 -0700 Subject: [PATCH 301/358] Adding URLFWD ttl exception handling for cloudflare --- octodns/provider/cloudflare.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/octodns/provider/cloudflare.py b/octodns/provider/cloudflare.py index 01710b6..ad057eb 100644 --- a/octodns/provider/cloudflare.py +++ b/octodns/provider/cloudflare.py @@ -419,6 +419,11 @@ class CloudflareProvider(BaseProvider): existing.update({ 'ttl': new['ttl'] }) + elif change.new._type == 'URLFWD': + existing = deepcopy(change.existing.data) + existing.update({ + 'ttl': new['ttl'] + }) else: existing = change.existing.data From c6486ce3d1ac29ff706ebfb95a9a4c3fbe78b8be Mon Sep 17 00:00:00 2001 From: "Yaroshevich, Denis" Date: Mon, 26 Jul 2021 17:57:16 +0300 Subject: [PATCH 302/358] add support for NS, MX, TXT, SRV, CNAME, PTR --- README.md | 2 +- octodns/provider/gcore.py | 117 ++++++++++- tests/fixtures/gcore-no-changes.json | 299 ++++++++++++++++++++++----- tests/fixtures/gcore-records.json | 227 +++++++++++++++++--- tests/test_octodns_provider_gcore.py | 145 +++++++++++-- 5 files changed, 687 insertions(+), 103 deletions(-) diff --git a/README.md b/README.md index 52d3a25..2323633 100644 --- a/README.md +++ b/README.md @@ -196,7 +196,7 @@ The above command pulled the existing data out of Route53 and placed the results | [EtcHostsProvider](/octodns/provider/etc_hosts.py) | | A, AAAA, ALIAS, CNAME | No | | | [EnvVarSource](/octodns/source/envvar.py) | | TXT | No | read-only environment variable injection | | [GandiProvider](/octodns/provider/gandi.py) | | A, AAAA, ALIAS, CAA, CNAME, DNAME, MX, NS, PTR, SPF, SRV, SSHFP, TXT | No | | -| [GCoreProvider](/octodns/provider/gcore.py) | | A, AAAA | No | | +| [GCoreProvider](/octodns/provider/gcore.py) | | A, AAAA, NS, MX, TXT, SRV, CNAME, PTR | No | | | [GoogleCloudProvider](/octodns/provider/googlecloud.py) | google-cloud-dns | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | No | | | [MythicBeastsProvider](/octodns/provider/mythicbeasts.py) | Mythic Beasts | A, AAAA, ALIAS, CNAME, MX, NS, SRV, SSHFP, CAA, TXT | No | | | [Ns1Provider](/octodns/provider/ns1.py) | ns1-python | All | Yes | Missing `NA` geo target | diff --git a/octodns/provider/gcore.py b/octodns/provider/gcore.py index d5c7fa0..c551f03 100644 --- a/octodns/provider/gcore.py +++ b/octodns/provider/gcore.py @@ -161,7 +161,7 @@ class GCoreProvider(BaseProvider): SUPPORTS_GEO = False SUPPORTS_DYNAMIC = False - SUPPORTS = set(("A", "AAAA")) + SUPPORTS = set(("A", "AAAA", "NS", "MX", "TXT", "SRV", "CNAME", "PTR")) def __init__(self, id, *args, **kwargs): token = kwargs.pop("token", None) @@ -183,7 +183,22 @@ class GCoreProvider(BaseProvider): password=password, ) + def _add_dot_if_need(self, value): + return "{}.".format(value) if not value.endswith(".") else value + def _data_for_single(self, _type, record): + return { + "ttl": record["ttl"], + "type": _type, + "value": self._add_dot_if_need( + record["resource_records"][0]["content"][0] + ), + } + + _data_for_CNAME = _data_for_single + _data_for_PTR = _data_for_single + + def _data_for_multiple(self, _type, record): return { "ttl": record["ttl"], "type": _type, @@ -194,8 +209,62 @@ class GCoreProvider(BaseProvider): ], } - _data_for_A = _data_for_single - _data_for_AAAA = _data_for_single + _data_for_A = _data_for_multiple + _data_for_AAAA = _data_for_multiple + + def _data_for_TXT(self, _type, record): + return { + "ttl": record["ttl"], + "type": _type, + "values": [ + rr_value.replace(";", "\\;") + for resource_record in record["resource_records"] + for rr_value in resource_record["content"] + ], + } + + def _data_for_MX(self, _type, record): + return { + "ttl": record["ttl"], + "type": _type, + "values": [ + dict( + preference=preference, + exchange=self._add_dot_if_need(exchange), + ) + for preference, exchange in map( + lambda x: x["content"], record["resource_records"] + ) + ], + } + + def _data_for_NS(self, _type, record): + return { + "ttl": record["ttl"], + "type": _type, + "values": [ + self._add_dot_if_need(rr_value) + for resource_record in record["resource_records"] + for rr_value in resource_record["content"] + ], + } + + def _data_for_SRV(self, _type, record): + return { + "ttl": record["ttl"], + "type": _type, + "values": [ + dict( + priority=priority, + weight=weight, + port=port, + target=self._add_dot_if_need(target), + ) + for priority, weight, port, target in map( + lambda x: x["content"], record["resource_records"] + ) + ], + } def zone_records(self, zone): try: @@ -241,6 +310,15 @@ class GCoreProvider(BaseProvider): return exists def _params_for_single(self, record): + return { + "ttl": record.ttl, + "resource_records": [{"content": [record.value]}], + } + + _params_for_CNAME = _params_for_single + _params_for_PTR = _params_for_single + + def _params_for_multiple(self, record): return { "ttl": record.ttl, "resource_records": [ @@ -248,8 +326,37 @@ class GCoreProvider(BaseProvider): ], } - _params_for_A = _params_for_single - _params_for_AAAA = _params_for_single + _params_for_A = _params_for_multiple + _params_for_AAAA = _params_for_multiple + _params_for_NS = _params_for_multiple + + def _params_for_TXT(self, record): + # print(record.values) + return { + "ttl": record.ttl, + "resource_records": [ + {"content": [value.replace("\\;", ";")]} + for value in record.values + ], + } + + def _params_for_MX(self, record): + return { + "ttl": record.ttl, + "resource_records": [ + {"content": [rec.preference, rec.exchange]} + for rec in record.values + ], + } + + def _params_for_SRV(self, record): + return { + "ttl": record.ttl, + "resource_records": [ + {"content": [rec.priority, rec.weight, rec.port, rec.target]} + for rec in record.values + ], + } def _apply_create(self, change): self.log.info("creating: %s", change) diff --git a/tests/fixtures/gcore-no-changes.json b/tests/fixtures/gcore-no-changes.json index e5ff8c9..b1a3b25 100644 --- a/tests/fixtures/gcore-no-changes.json +++ b/tests/fixtures/gcore-no-changes.json @@ -1,56 +1,245 @@ { - "rrsets": [{ - "name": "unit.tests", - "type": "A", - "ttl": 300, - "resource_records": [{ - "content": [ - "1.2.3.4" - ] - }, { - "content": [ - "1.2.3.5" - ] - }] - }, { - "name": "aaaa.unit.tests", - "type": "AAAA", - "ttl": 600, - "resource_records": [{ - "content": [ - "2601:644:500:e210:62f8:1dff:feb8:947a" - ] - }] - }, { - "name": "www.unit.tests.", - "type": "A", - "ttl": 300, - "resource_records": [{ - "content": [ - "2.2.3.6" - ] - }] - }, { - "name": "www.sub.unit.tests.", - "type": "A", - "ttl": 300, - "resource_records": [{ - "content": [ - "2.2.3.6" - ] - }] - }, { - "name": "unit.tests", - "type": "ns", - "ttl": 300, - "resource_records": [{ - "content": [ - "ns2.gcdn.services" - ] - }, { - "content": [ - "ns1.gcorelabs.net" - ] - }] - }] -} + "rrsets": [ + { + "name": "unit.tests", + "type": "A", + "ttl": 300, + "resource_records": [ + { + "content": [ + "1.2.3.4" + ] + }, + { + "content": [ + "1.2.3.5" + ] + } + ] + }, + { + "name": "unit.tests", + "type": "NS", + "ttl": 300, + "resource_records": [ + { + "content": [ + "ns2.gcdn.services" + ] + }, + { + "content": [ + "ns1.gcorelabs.net" + ] + } + ] + }, + { + "name": "_imap._tcp", + "type": "SRV", + "ttl": 600, + "resource_records": [ + { + "content": [ + 0, + 0, + 0, + "." + ] + } + ] + }, + { + "name": "_pop3._tcp", + "type": "SRV", + "ttl": 600, + "resource_records": [ + { + "content": [ + 0, + 0, + 0, + "." + ] + } + ] + }, + { + "name": "_srv._tcp", + "type": "SRV", + "ttl": 600, + "resource_records": [ + { + "content": [ + 12, + 20, + 30, + "foo-2.unit.tests" + ] + }, + { + "content": [ + 10, + 20, + 30, + "foo-1.unit.tests" + ] + } + ] + }, + { + "name": "aaaa.unit.tests", + "type": "AAAA", + "ttl": 600, + "resource_records": [ + { + "content": [ + "2601:644:500:e210:62f8:1dff:feb8:947a" + ] + } + ] + }, + { + "name": "cname.unit.tests", + "type": "CNAME", + "ttl": 300, + "resource_records": [ + { + "content": [ + "unit.tests." + ] + } + ] + }, + { + "name": "excluded.unit.tests", + "type": "CNAME", + "ttl": 3600, + "resource_records": [ + { + "content": [ + "unit.tests." + ] + } + ] + }, + { + "name": "mx.unit.tests", + "type": "MX", + "ttl": 300, + "resource_records": [ + { + "content": [ + 40, + "smtp-1.unit.tests." + ] + }, + { + "content": [ + 20, + "smtp-2.unit.tests." + ] + }, + { + "content": [ + 30, + "smtp-3.unit.tests." + ] + }, + { + "content": [ + 10, + "smtp-4.unit.tests." + ] + } + ] + }, + { + "name": "ptr.unit.tests.", + "type": "PTR", + "ttl": 300, + "resource_records": [ + { + "content": [ + "foo.bar.com" + ] + } + ] + }, + { + "name": "sub.unit.tests", + "type": "NS", + "ttl": 3600, + "resource_records": [ + { + "content": [ + "6.2.3.4" + ] + }, + { + "content": [ + "7.2.3.4" + ] + } + ] + }, + { + "name": "txt.unit.tests", + "type": "TXT", + "ttl": 600, + "resource_records": [ + { + "content": [ + "Bah bah black sheep" + ] + }, + { + "content": [ + "have you any wool." + ] + }, + { + "content": [ + "v=DKIM1;k=rsa;s=email;h=sha256;p=A/kinda+of/long/string+with+numb3rs" + ] + } + ] + }, + { + "name": "www.unit.tests.", + "type": "A", + "ttl": 300, + "resource_records": [ + { + "content": [ + "2.2.3.6" + ] + } + ] + }, + { + "name": "www.sub.unit.tests.", + "type": "A", + "ttl": 300, + "resource_records": [ + { + "content": [ + "2.2.3.6" + ] + } + ] + }, + { + "name": "spf.sub.unit.tests.", + "type": "SPF", + "ttl": 600, + "resource_records": [ + { + "content": [ + "v=spf1 ip4:192.168.0.1/16-all" + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/fixtures/gcore-records.json b/tests/fixtures/gcore-records.json index 4086049..ba29246 100644 --- a/tests/fixtures/gcore-records.json +++ b/tests/fixtures/gcore-records.json @@ -1,25 +1,204 @@ { - "rrsets": [{ - "name": "unit.tests", - "type": "A", - "ttl": 300, - "resource_records": [{ - "content": [ - "1.2.3.4" - ] - }] - }, { - "name": "unit.tests", - "type": "ns", - "ttl": 300, - "resource_records": [{ - "content": [ - "ns2.gcdn.services" - ] - }, { - "content": [ - "ns1.gcorelabs.net" - ] - }] - }] -} + "rrsets": [ + { + "name": "unit.tests", + "type": "A", + "ttl": 300, + "resource_records": [ + { + "content": [ + "1.2.3.4" + ] + } + ] + }, + { + "name": "unit.tests", + "type": "NS", + "ttl": 300, + "resource_records": [ + { + "content": [ + "ns2.gcdn.services" + ] + }, + { + "content": [ + "ns1.gcorelabs.net" + ] + } + ] + }, + { + "name": "_imap._tcp", + "type": "SRV", + "ttl": 1200, + "resource_records": [ + { + "content": [ + 0, + 0, + 0, + "." + ] + } + ] + }, + { + "name": "_pop3._tcp", + "type": "SRV", + "ttl": 1200, + "resource_records": [ + { + "content": [ + 0, + 0, + 0, + "." + ] + } + ] + }, + { + "name": "_srv._tcp", + "type": "SRV", + "ttl": 1200, + "resource_records": [ + { + "content": [ + 12, + 20, + 30, + "foo-2.unit.tests." + ] + }, + { + "content": [ + 10, + 20, + 30, + "foo-1.unit.tests." + ] + } + ] + }, + { + "name": "aaaa.unit.tests", + "type": "AAAA", + "ttl": 600, + "resource_records": [ + { + "content": [ + "2601:644:500:e210:62f8:1dff:feb8:947a" + ] + } + ] + }, + { + "name": "cname.unit.tests", + "type": "CNAME", + "ttl": 300, + "resource_records": [ + { + "content": [ + "unit.tests." + ] + } + ] + }, + { + "name": "mx.unit.tests", + "type": "MX", + "ttl": 600, + "resource_records": [ + { + "content": [ + 40, + "smtp-1.unit.tests." + ] + }, + { + "content": [ + 20, + "smtp-2.unit.tests." + ] + } + ] + }, + { + "name": "ptr.unit.tests.", + "type": "PTR", + "ttl": 300, + "resource_records": [ + { + "content": [ + "foo.bar.com" + ] + } + ] + }, + { + "name": "sub.unit.tests", + "type": "NS", + "ttl": 300, + "resource_records": [ + { + "content": [ + "6.2.3.4" + ] + }, + { + "content": [ + "7.2.3.4" + ] + } + ] + }, + { + "name": "txt.unit.tests", + "type": "TXT", + "ttl": 300, + "resource_records": [ + { + "content": [ + "\"Bah bah black sheep\"" + ] + }, + { + "content": [ + "\"have you any wool.\"" + ] + }, + { + "content": [ + "\"v=DKIM1;k=rsa;s=email;h=sha256;p=A/kinda+of/long/string+with+numb3rs\"" + ] + } + ] + }, + { + "name": "www.unit.tests.", + "type": "A", + "ttl": 300, + "resource_records": [ + { + "content": [ + "2.2.3.6" + ] + } + ] + }, + { + "name": "www.sub.unit.tests.", + "type": "A", + "ttl": 300, + "resource_records": [ + { + "content": [ + "2.2.3.6" + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/test_octodns_provider_gcore.py b/tests/test_octodns_provider_gcore.py index 06e199e..2ddbb59 100644 --- a/tests/test_octodns_provider_gcore.py +++ b/tests/test_octodns_provider_gcore.py @@ -64,13 +64,13 @@ class TestGCoreProvider(TestCase): with self.assertRaises(GCoreClientException) as ctx: zone = Zone("unit.tests.", []) provider.populate(zone) - self.assertEquals("Things caught fire", text_type(ctx.exception)) + self.assertEqual("Things caught fire", text_type(ctx.exception)) # No credentials or token error with requests_mock() as mock: with self.assertRaises(ValueError) as ctx: GCoreProvider("test_id") - self.assertEquals( + self.assertEqual( "either token or login & password must be set", text_type(ctx.exception), ) @@ -116,14 +116,29 @@ class TestGCoreProvider(TestCase): zone = Zone("unit.tests.", []) provider.populate(zone) - self.assertEquals(4, len(zone.records)) - self.assertEquals( - {"aaaa", "www", "www.sub", ""}, {r.name for r in zone.records} + self.assertEqual(14, len(zone.records)) + self.assertEqual( + { + "", + "_imap._tcp", + "_pop3._tcp", + "_srv._tcp", + "aaaa", + "cname", + "excluded", + "mx", + "ptr", + "sub", + "txt", + "www", + "www.sub", + }, + {r.name for r in zone.records}, ) changes = self.expected.changes(zone, provider) - self.assertEquals(0, len(changes)) + self.assertEqual(0, len(changes)) - # 3 removed + 1 modified + # 1 removed + 7 modified with requests_mock() as mock: base = "https://dnsapi.gcorelabs.com/v2/zones/unit.tests/rrsets" with open("tests/fixtures/gcore-records.json") as fh: @@ -131,14 +146,14 @@ class TestGCoreProvider(TestCase): zone = Zone("unit.tests.", []) provider.populate(zone) - self.assertEquals(1, len(zone.records)) + self.assertEqual(13, len(zone.records)) changes = self.expected.changes(zone, provider) - self.assertEquals(4, len(changes)) - self.assertEquals( - 3, len([c for c in changes if isinstance(c, Delete)]) + self.assertEqual(8, len(changes)) + self.assertEqual( + 1, len([c for c in changes if isinstance(c, Delete)]) ) - self.assertEquals( - 1, len([c for c in changes if isinstance(c, Update)]) + self.assertEqual( + 7, len([c for c in changes if isinstance(c, Update)]) ) def test_apply(self): @@ -192,8 +207,8 @@ class TestGCoreProvider(TestCase): plan = provider.plan(self.expected) # create all - self.assertEquals(4, len(plan.changes)) - self.assertEquals(4, provider.apply(plan)) + self.assertEqual(13, len(plan.changes)) + self.assertEqual(13, provider.apply(plan)) self.assertFalse(plan.exists) provider._client._request.assert_has_calls( @@ -221,6 +236,73 @@ class TestGCoreProvider(TestCase): "resource_records": [{"content": ["2.2.3.6"]}], }, ), + call( + "POST", + "http://api/zones/unit.tests/txt.unit.tests./TXT", + data={ + "ttl": 600, + "resource_records": [ + {"content": ["Bah bah black sheep"]}, + {"content": ["have you any wool."]}, + { + "content": [ + "v=DKIM1;k=rsa;s=email;h=sha256;p=A/kinda+" + "of/long/string+with+numb3rs" + ] + }, + ], + }, + ), + call( + "POST", + "http://api/zones/unit.tests/sub.unit.tests./NS", + data={ + "ttl": 3600, + "resource_records": [ + {"content": ["6.2.3.4."]}, + {"content": ["7.2.3.4."]}, + ], + }, + ), + call( + "POST", + "http://api/zones/unit.tests/ptr.unit.tests./PTR", + data={ + "ttl": 300, + "resource_records": [ + {"content": ["foo.bar.com."]}, + ], + }, + ), + call( + "POST", + "http://api/zones/unit.tests/mx.unit.tests./MX", + data={ + "ttl": 300, + "resource_records": [ + {"content": [10, "smtp-4.unit.tests."]}, + {"content": [20, "smtp-2.unit.tests."]}, + {"content": [30, "smtp-3.unit.tests."]}, + {"content": [40, "smtp-1.unit.tests."]}, + ], + }, + ), + call( + "POST", + "http://api/zones/unit.tests/excluded.unit.tests./CNAME", + data={ + "ttl": 3600, + "resource_records": [{"content": ["unit.tests."]}], + }, + ), + call( + "POST", + "http://api/zones/unit.tests/cname.unit.tests./CNAME", + data={ + "ttl": 300, + "resource_records": [{"content": ["unit.tests."]}], + }, + ), call( "POST", "http://api/zones/unit.tests/aaaa.unit.tests./AAAA", @@ -235,6 +317,33 @@ class TestGCoreProvider(TestCase): ], }, ), + call( + "POST", + "http://api/zones/unit.tests/_srv._tcp.unit.tests./SRV", + data={ + "ttl": 600, + "resource_records": [ + {"content": [10, 20, 30, "foo-1.unit.tests."]}, + {"content": [12, 20, 30, "foo-2.unit.tests."]}, + ], + }, + ), + call( + "POST", + "http://api/zones/unit.tests/_pop3._tcp.unit.tests./SRV", + data={ + "ttl": 600, + "resource_records": [{"content": [0, 0, 0, "."]}], + }, + ), + call( + "POST", + "http://api/zones/unit.tests/_imap._tcp.unit.tests./SRV", + data={ + "ttl": 600, + "resource_records": [{"content": [0, 0, 0, "."]}], + }, + ), call( "POST", "http://api/zones/unit.tests/unit.tests./A", @@ -249,7 +358,7 @@ class TestGCoreProvider(TestCase): ] ) # expected number of total calls - self.assertEquals(7, provider._client._request.call_count) + self.assertEqual(16, provider._client._request.call_count) provider._client._request.reset_mock() @@ -283,8 +392,8 @@ class TestGCoreProvider(TestCase): plan = provider.plan(wanted) self.assertTrue(plan.exists) - self.assertEquals(2, len(plan.changes)) - self.assertEquals(2, provider.apply(plan)) + self.assertEqual(2, len(plan.changes)) + self.assertEqual(2, provider.apply(plan)) provider._client._request.assert_has_calls( [ From f9ccd4342cce4c6e2fc2dfaea7447cce949ac3fa Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Wed, 28 Jul 2021 18:49:52 -0700 Subject: [PATCH 303/358] create FUNDING.yml w/ross --- .github/FUNDING.yml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .github/FUNDING.yml diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000..047c9ed --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,3 @@ +# These are supported funding model platforms + +github: ross From 65056fb4cd89f542317f8642a16ef8d2ac7b7cd9 Mon Sep 17 00:00:00 2001 From: Sham Date: Fri, 30 Jul 2021 12:37:21 -0700 Subject: [PATCH 304/358] check for identical monitor and skip creating one if found --- octodns/provider/ns1.py | 14 ++++++++++---- tests/test_octodns_provider_ns1.py | 4 ++++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 6d4f84d..87013fa 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -1162,15 +1162,21 @@ class Ns1Provider(BaseProvider): # Build a list of primary values for each pool, including their # feed_id (monitor) + value_feed = dict() pool_answers = defaultdict(list) for pool_name, pool in sorted(pools.items()): for value in pool.data['values']: weight = value['weight'] value = value['value'] - existing = existing_monitors.get(value) - monitor_id, feed_id = self._monitor_sync(record, value, - existing) - active_monitors.add(monitor_id) + feed_id = value_feed.get(value) + # check for identical monitor and skip creating one if found + if not feed_id: + existing = existing_monitors.get(value) + monitor_id, feed_id = self._monitor_sync(record, value, + existing) + value_feed[value] = feed_id + active_monitors.add(monitor_id) + pool_answers[pool_name].append({ 'answer': [value], 'weight': weight, diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index 875ebbf..da7f3de 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -1332,6 +1332,10 @@ class TestNs1ProviderDynamic(TestCase): # This indirectly calls into _params_for_dynamic and tests the # handling to get there record = self.record() + # copy an existing answer from a different pool to 'lhr' so + # in order to test answer repetition across pools (monitor reuse) + record.dynamic._data()['pools']['lhr']['values'].append( + record.dynamic._data()['pools']['iad']['values'][0]) ret, _ = provider._params_for_A(record) # Given that record has both country and region in the rules, From 7f59c476e00837409f55b3242b8821c3e58facc1 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sun, 1 Aug 2021 12:17:29 -0700 Subject: [PATCH 305/358] NS1 _apply_Update needs to gc monitors w/existing --- octodns/provider/ns1.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 6d4f84d..c114199 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -1358,7 +1358,9 @@ class Ns1Provider(BaseProvider): params, active_monitor_ids = \ getattr(self, '_params_for_{}'.format(_type))(new) self._client.records_update(zone, domain, _type, **params) - self._monitors_gc(new, active_monitor_ids) + # If we're cleaning up we need to send in the old record since it'd + # have anything that needs cleaning up + self._monitors_gc(change.existing, active_monitor_ids) def _apply_Delete(self, ns1_zone, change): existing = change.existing From e2139f92c070c9ea418092a9638e38f0dbf30298 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sun, 1 Aug 2021 14:40:51 -0700 Subject: [PATCH 306/358] Suport for shared_notifylist in Ns1Provider --- octodns/provider/ns1.py | 75 ++++++++++--- tests/test_octodns_provider_ns1.py | 165 ++++++++++++++++++++++++++++- 2 files changed, 220 insertions(+), 20 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index c114199..1e8e972 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -77,8 +77,10 @@ class Ns1Client(object): self._datafeed = client.datafeed() self._datasource_id = None + self._feeds_for_monitors = None self._monitors_cache = None + self._notifylists_cache = None @property def datasource_id(self): @@ -121,6 +123,14 @@ class Ns1Client(object): {m['id']: m for m in self.monitors_list()} return self._monitors_cache + @property + def notifylists(self): + if self._notifylists_cache is None: + self.log.debug('notifylists: fetching & building') + self._notifylists_cache = \ + {l['name']: l for l in self.notifylists_list()} + return self._notifylists_cache + def datafeed_create(self, sourceid, name, config): ret = self._try(self._datafeed.create, sourceid, name, config) self.feeds_for_monitors[config['jobid']] = ret['id'] @@ -163,10 +173,17 @@ class Ns1Client(object): return ret def notifylists_delete(self, nlid): + for name, nl in self.notifylists.items(): + if nl['id'] == nlid: + del self._notifylists_cache[name] + break return self._try(self._notifylists.delete, nlid) def notifylists_create(self, **body): - return self._try(self._notifylists.create, body) + nl = self._try(self._notifylists.create, body) + # cache it + self.notifylists[nl['name']] = nl + return nl def notifylists_list(self): return self._try(self._notifylists.list) @@ -216,6 +233,11 @@ class Ns1Provider(BaseProvider): # Only required if using dynamic records monitor_regions: - lga + # Optional. Default: false. true is Recommended, but not the default + # for backwards compatibility reasons. If true, all NS1 monitors will + # use a shared notify list rather than one per record & value + # combination. + shared_notifylist: false # Optional. Default: None. If set, back off in advance to avoid 429s # from rate-limiting. Generally this should be set to the number # of processes or workers hitting the API, e.g. the value of @@ -237,6 +259,7 @@ class Ns1Provider(BaseProvider): 'NS', 'PTR', 'SPF', 'SRV', 'TXT', 'URLFWD')) ZONE_NOT_FOUND_MESSAGE = 'server error: zone not found' + SHARED_NOTIFYLIST_NAME = 'octoDNS NS1 Notify List' def _update_filter(self, filter, with_disabled): if with_disabled: @@ -368,7 +391,8 @@ class Ns1Provider(BaseProvider): } def __init__(self, id, api_key, retry_count=4, monitor_regions=None, - parallelism=None, client_config=None, *args, **kwargs): + parallelism=None, client_config=None, shared_notifylist=False, + *args, **kwargs): self.log = getLogger('Ns1Provider[{}]'.format(id)) self.log.debug('__init__: id=%s, api_key=***, retry_count=%d, ' 'monitor_regions=%s, parallelism=%s, client_config=%s', @@ -376,6 +400,7 @@ class Ns1Provider(BaseProvider): client_config) super(Ns1Provider, self).__init__(id, *args, **kwargs) self.monitor_regions = monitor_regions + self.shared_notifylist = shared_notifylist self._client = Ns1Client(api_key, parallelism, retry_count, client_config) @@ -902,22 +927,36 @@ class Ns1Provider(BaseProvider): return feed_id + def _notifylists_find_or_create(self, name): + self.log.debug('_notifylists_find_or_create: name="%s"', name) + try: + nl = self._client.notifylists[name] + self.log.debug('_notifylists_find_or_create: existing=%s', + nl['id']) + except KeyError: + notify_list = [{ + 'config': { + 'sourceid': self._client.datasource_id, + }, + 'type': 'datafeed', + }] + nl = self._client.notifylists_create(name=name, + notify_list=notify_list) + self.log.debug('_notifylists_find_or_create: created=%s', + nl['id']) + + return nl + def _monitor_create(self, monitor): self.log.debug('_monitor_create: monitor="%s"', monitor['name']) - # Create the notify list - notify_list = [{ - 'config': { - 'sourceid': self._client.datasource_id, - }, - 'type': 'datafeed', - }] - nl = self._client.notifylists_create(name=monitor['name'], - notify_list=notify_list) - nl_id = nl['id'] - self.log.debug('_monitor_create: notify_list=%s', nl_id) + + # Find the right notifylist + nl_name = self.SHARED_NOTIFYLIST_NAME \ + if self.shared_notifylist else monitor['name'] + nl = self._notifylists_find_or_create(nl_name) # Create the monitor - monitor['notify_list'] = nl_id + monitor['notify_list'] = nl['id'] monitor = self._client.monitors_create(**monitor) monitor_id = monitor['id'] self.log.debug('_monitor_create: monitor=%s', monitor_id) @@ -1028,7 +1067,13 @@ class Ns1Provider(BaseProvider): self._client.monitors_delete(monitor_id) notify_list_id = monitor['notify_list'] - self._client.notifylists_delete(notify_list_id) + for nl_name, nl in self._client.notifylists.items(): + if nl['id'] == notify_list_id: + # We've found the that might need deleting + if nl['name'] != self.SHARED_NOTIFYLIST_NAME: + # It's not shared so is safe to delete + self._client.notifylists_delete(notify_list_id) + break def _add_answers_for_pool(self, answers, default_answers, pool_name, pool_label, pool_answers, pools, priority): diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index 875ebbf..1f69465 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -768,12 +768,70 @@ class TestNs1ProviderDynamic(TestCase): monitor = { 'name': 'test monitor', } + provider._client._notifylists_cache = {} monitor_id, feed_id = provider._monitor_create(monitor) self.assertEquals('mon-id', monitor_id) self.assertEquals('feed-id', feed_id) monitors_create_mock.assert_has_calls([call(name='test monitor', notify_list='nl-id')]) + @patch('octodns.provider.ns1.Ns1Provider._feed_create') + @patch('octodns.provider.ns1.Ns1Client.monitors_create') + @patch('octodns.provider.ns1.Ns1Client._try') + def test_monitor_create_shared_notifylist(self, try_mock, + monitors_create_mock, + feed_create_mock): + provider = Ns1Provider('test', 'api-key', shared_notifylist=True) + + # pre-fill caches to avoid extranious calls (things we're testing + # elsewhere) + provider._client._datasource_id = 'foo' + provider._client._feeds_for_monitors = {} + + # First time we'll need to create the share list + provider._client._notifylists_cache = {} + try_mock.reset_mock() + monitors_create_mock.reset_mock() + feed_create_mock.reset_mock() + try_mock.side_effect = [{ + 'id': 'nl-id', + 'name': provider.SHARED_NOTIFYLIST_NAME, + }] + monitors_create_mock.side_effect = [{ + 'id': 'mon-id', + }] + feed_create_mock.side_effect = ['feed-id'] + monitor = { + 'name': 'test monitor', + } + monitor_id, feed_id = provider._monitor_create(monitor) + self.assertEquals('mon-id', monitor_id) + self.assertEquals('feed-id', feed_id) + monitors_create_mock.assert_has_calls([call(name='test monitor', + notify_list='nl-id')]) + try_mock.assert_called_once() + # The shared notifylist should be cached now + self.assertEquals([provider.SHARED_NOTIFYLIST_NAME], + list(provider._client._notifylists_cache.keys())) + + # Second time we'll use the cached version + try_mock.reset_mock() + monitors_create_mock.reset_mock() + feed_create_mock.reset_mock() + monitors_create_mock.side_effect = [{ + 'id': 'mon-id', + }] + feed_create_mock.side_effect = ['feed-id'] + monitor = { + 'name': 'test monitor', + } + monitor_id, feed_id = provider._monitor_create(monitor) + self.assertEquals('mon-id', monitor_id) + self.assertEquals('feed-id', feed_id) + monitors_create_mock.assert_has_calls([call(name='test monitor', + notify_list='nl-id')]) + try_mock.assert_not_called() + def test_monitor_gen(self): provider = Ns1Provider('test', 'api-key') @@ -986,6 +1044,12 @@ class TestNs1ProviderDynamic(TestCase): 'notify_list': 'nl-id', } }] + provider._client._notifylists_cache = { + 'not shared': { + 'id': 'nl-id', + 'name': 'not shared', + } + } provider._monitors_gc(record) monitors_for_mock.assert_has_calls([call(record)]) datafeed_delete_mock.assert_has_calls([call('foo', 'feed-id')]) @@ -1025,12 +1089,75 @@ class TestNs1ProviderDynamic(TestCase): 'notify_list': 'nl-id2', }, }] + provider._client._notifylists_cache = { + 'not shared': { + 'id': 'nl-id', + 'name': 'not shared', + }, + 'not shared 2': { + 'id': 'nl-id2', + 'name': 'not shared 2', + } + } provider._monitors_gc(record, {'mon-id'}) monitors_for_mock.assert_has_calls([call(record)]) datafeed_delete_mock.assert_not_called() monitors_delete_mock.assert_has_calls([call('mon-id2')]) notifylists_delete_mock.assert_has_calls([call('nl-id2')]) + # Non-active monitor w/o a notifylist, generally shouldn't happen, but + # code should handle it just in case someone gets clicky in the UI + monitors_for_mock.reset_mock() + datafeed_delete_mock.reset_mock() + monitors_delete_mock.reset_mock() + notifylists_delete_mock.reset_mock() + monitors_for_mock.side_effect = [{ + 'y': { + 'id': 'mon-id2', + 'notify_list': 'nl-id2', + }, + }] + provider._client._notifylists_cache = { + 'not shared a': { + 'id': 'nl-ida', + 'name': 'not shared a', + }, + 'not shared b': { + 'id': 'nl-idb', + 'name': 'not shared b', + } + } + provider._monitors_gc(record, {'mon-id'}) + monitors_for_mock.assert_has_calls([call(record)]) + datafeed_delete_mock.assert_not_called() + monitors_delete_mock.assert_has_calls([call('mon-id2')]) + notifylists_delete_mock.assert_not_called() + + # Non-active monitor with a shared notifylist, monitor deleted, but + # notifylist is left alone + provider.shared_notifylist = True + monitors_for_mock.reset_mock() + datafeed_delete_mock.reset_mock() + monitors_delete_mock.reset_mock() + notifylists_delete_mock.reset_mock() + monitors_for_mock.side_effect = [{ + 'y': { + 'id': 'mon-id2', + 'notify_list': 'shared', + }, + }] + provider._client._notifylists_cache = { + 'shared': { + 'id': 'shared', + 'name': provider.SHARED_NOTIFYLIST_NAME, + }, + } + provider._monitors_gc(record, {'mon-id'}) + monitors_for_mock.assert_has_calls([call(record)]) + datafeed_delete_mock.assert_not_called() + monitors_delete_mock.assert_has_calls([call('mon-id2')]) + notifylists_delete_mock.assert_not_called() + @patch('octodns.provider.ns1.Ns1Provider._monitor_sync') @patch('octodns.provider.ns1.Ns1Provider._monitors_for') def test_params_for_dynamic_region_only(self, monitors_for_mock, @@ -2325,17 +2452,22 @@ class TestNs1Client(TestCase): notifylists_list_mock.reset_mock() notifylists_create_mock.reset_mock() notifylists_delete_mock.reset_mock() - notifylists_create_mock.side_effect = ['bar'] + notifylists_list_mock.side_effect = [{}] + expected = { + 'id': 'nl-id', + 'name': 'bar', + } + notifylists_create_mock.side_effect = [expected] notify_list = [{ 'config': { 'sourceid': 'foo', }, 'type': 'datafeed', }] - nl = client.notifylists_create(name='some name', - notify_list=notify_list) - self.assertEquals('bar', nl) - notifylists_list_mock.assert_not_called() + got = client.notifylists_create(name='some name', + notify_list=notify_list) + self.assertEquals(expected, got) + notifylists_list_mock.assert_called_once() notifylists_create_mock.assert_has_calls([ call({'name': 'some name', 'notify_list': notify_list}) ]) @@ -2349,6 +2481,29 @@ class TestNs1Client(TestCase): notifylists_create_mock.assert_not_called() notifylists_delete_mock.assert_has_calls([call('nlid')]) + # Delete again, this time with a cache item that needs cleaned out and + # another that needs to be ignored + notifylists_list_mock.reset_mock() + notifylists_create_mock.reset_mock() + notifylists_delete_mock.reset_mock() + client._notifylists_cache = { + 'another': { + 'id': 'notid', + 'name': 'another', + }, + # This one comes 2nd on purpose + 'the-one': { + 'id': 'nlid', + 'name': 'the-one', + }, + } + client.notifylists_delete('nlid') + notifylists_list_mock.assert_not_called() + notifylists_create_mock.assert_not_called() + notifylists_delete_mock.assert_has_calls([call('nlid')]) + # Only another left + self.assertEquals(['another'], list(client._notifylists_cache.keys())) + notifylists_list_mock.reset_mock() notifylists_create_mock.reset_mock() notifylists_delete_mock.reset_mock() From 006a61e4d870f1393ed21ff56964050d89646e2e Mon Sep 17 00:00:00 2001 From: "Yaroshevich, Denis" Date: Mon, 2 Aug 2021 18:47:11 +0300 Subject: [PATCH 307/358] add support for dynamic A, AAAA, CNAME records --- README.md | 2 +- octodns/provider/gcore.py | 246 ++++++++++++++++++-- tests/fixtures/gcore-records.json | 224 +++++++++++++++++++ tests/test_octodns_provider_gcore.py | 320 +++++++++++++++++++++++++-- 4 files changed, 763 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index 9764caa..9b4031e 100644 --- a/README.md +++ b/README.md @@ -204,7 +204,7 @@ The above command pulled the existing data out of Route53 and placed the results | [EtcHostsProvider](/octodns/provider/etc_hosts.py) | | A, AAAA, ALIAS, CNAME | No | | | [EnvVarSource](/octodns/source/envvar.py) | | TXT | No | read-only environment variable injection | | [GandiProvider](/octodns/provider/gandi.py) | | A, AAAA, ALIAS, CAA, CNAME, DNAME, MX, NS, PTR, SPF, SRV, SSHFP, TXT | No | | -| [GCoreProvider](/octodns/provider/gcore.py) | | A, AAAA, NS, MX, TXT, SRV, CNAME, PTR | No | | +| [GCoreProvider](/octodns/provider/gcore.py) | | A, AAAA, NS, MX, TXT, SRV, CNAME, PTR | Dynamic | | | [GoogleCloudProvider](/octodns/provider/googlecloud.py) | google-cloud-dns | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | No | | | [HetznerProvider](/octodns/provider/hetzner.py) | | A, AAAA, CAA, CNAME, MX, NS, SRV, TXT | No | | | [MythicBeastsProvider](/octodns/provider/mythicbeasts.py) | Mythic Beasts | A, AAAA, ALIAS, CNAME, MX, NS, SRV, SSHFP, CAA, TXT | No | | diff --git a/octodns/provider/gcore.py b/octodns/provider/gcore.py index c551f03..821d109 100644 --- a/octodns/provider/gcore.py +++ b/octodns/provider/gcore.py @@ -15,6 +15,7 @@ import http import logging import urllib.parse +from ..record import GeoCodes from ..record import Record from .base import BaseProvider @@ -157,10 +158,11 @@ class GCoreProvider(BaseProvider): password: XXXXXXXXXXXX # auth_url: https://api.gcdn.co # url: https://dnsapi.gcorelabs.com/v2 + # records_per_response: 1 """ SUPPORTS_GEO = False - SUPPORTS_DYNAMIC = False + SUPPORTS_DYNAMIC = True SUPPORTS = set(("A", "AAAA", "NS", "MX", "TXT", "SRV", "CNAME", "PTR")) def __init__(self, id, *args, **kwargs): @@ -170,6 +172,7 @@ class GCoreProvider(BaseProvider): password = kwargs.pop("password", None) api_url = kwargs.pop("url", "https://dnsapi.gcorelabs.com/v2") auth_url = kwargs.pop("auth_url", "https://api.gcdn.co") + self.records_per_response = kwargs.pop("records_per_response", 1) self.log = logging.getLogger("GCoreProvider[{}]".format(id)) self.log.debug("__init__: id=%s", id) super(GCoreProvider, self).__init__(id, *args, **kwargs) @@ -186,6 +189,86 @@ class GCoreProvider(BaseProvider): def _add_dot_if_need(self, value): return "{}.".format(value) if not value.endswith(".") else value + def _build_pools(self, record, default_pool_name, value_transform_fn): + defaults = [] + geo_sets, pool_idx = dict(), 0 + pools = defaultdict(lambda: {"values": []}) + for rr in record["resource_records"]: + meta = rr.get("meta", {}) + value = {"value": value_transform_fn(rr["content"][0])} + countries = meta.get("countries", []) + continents = meta.get("continents", []) + + if meta.get("default", False): + pools[default_pool_name]["values"].append(value) + defaults.append(value["value"]) + continue + # defaults is false or missing and no conties or continents + elif len(continents) == 0 and len(countries) == 0: + defaults.append(value["value"]) + continue + + # RR with the same set of countries and continents are + # combined in single pool + geo_set = frozenset( + [GeoCodes.country_to_code(cc.upper()) for cc in countries] + ) | frozenset(cc.upper() for cc in continents) + if geo_set not in geo_sets: + geo_sets[geo_set] = "pool-{}".format(pool_idx) + pool_idx += 1 + + pools[geo_sets[geo_set]]["values"].append(value) + + return pools, geo_sets, defaults + + def _build_rules(self, pools, geo_sets): + rules = [] + for name, _ in pools.items(): + rule = {"pool": name} + geo_set = next( + ( + geo_set + for geo_set, pool_name in geo_sets.items() + if pool_name == name + ), + {}, + ) + if len(geo_set) > 0: + rule["geos"] = list(geo_set) + rules.append(rule) + + return sorted(rules, key=lambda x: x["pool"]) + + def _data_for_dynamic(self, record, value_transform_fn=lambda x: x): + default_pool = "other" + pools, geo_sets, defaults = self._build_pools( + record, default_pool, value_transform_fn + ) + if len(pools) == 0: + raise RuntimeError( + "filter is enabled, but no pools where built for {}".format( + record + ) + ) + + # defaults can't be empty, so use first pool values + if len(defaults) == 0: + defaults = [ + value_transform_fn(v["value"]) + for v in next(iter(pools.values()))["values"] + ] + + # if at least one default RR was found then setup fallback for + # other pools to default + if default_pool in pools: + for pool_name, pool in pools.items(): + if pool_name == default_pool: + continue + pool["fallback"] = default_pool + + rules = self._build_rules(pools, geo_sets) + return pools, rules, defaults + def _data_for_single(self, _type, record): return { "ttl": record["ttl"], @@ -195,18 +278,42 @@ class GCoreProvider(BaseProvider): ), } - _data_for_CNAME = _data_for_single _data_for_PTR = _data_for_single + def _data_for_CNAME(self, _type, record): + if record.get("filters") is None: + return self._data_for_single(_type, record) + + pools, rules, defaults = self._data_for_dynamic( + record, self._add_dot_if_need + ) + return { + "ttl": record["ttl"], + "type": _type, + "dynamic": {"pools": pools, "rules": rules}, + "value": self._add_dot_if_need(defaults[0]), + } + def _data_for_multiple(self, _type, record): + extra = dict() + if record.get("filters") is not None: + pools, rules, defaults = self._data_for_dynamic(record) + extra = { + "dynamic": {"pools": pools, "rules": rules}, + "values": defaults, + } + else: + extra = { + "values": [ + rr_value + for resource_record in record["resource_records"] + for rr_value in resource_record["content"] + ] + } return { "ttl": record["ttl"], "type": _type, - "values": [ - rr_value - for resource_record in record["resource_records"] - for rr_value in resource_record["content"] - ], + **extra, } _data_for_A = _data_for_multiple @@ -286,6 +393,8 @@ class GCoreProvider(BaseProvider): _type = record["type"].upper() if _type not in self.SUPPORTS: continue + if self._should_ignore(record): + continue rr_name = zone.hostname_from_fqdn(record["name"]) values[rr_name][_type] = record @@ -309,29 +418,140 @@ class GCoreProvider(BaseProvider): ) return exists + def _should_ignore(self, record): + name = record.get("name", "name-not-defined") + if record.get("filters") is None: + return False + want_filters = 3 + filters = record.get("filters", []) + if len(filters) != want_filters: + self.log.info( + "ignore %s has filters and their count is not %d", + name, + want_filters, + ) + return True + types = [v.get("type") for v in filters] + for i, want_type in enumerate(["geodns", "first_n", "default"]): + if types[i] != want_type: + self.log.info( + "ignore %s, filters.%d.type is %s, want %s", + name, + i, + types[i], + want_type, + ) + return True + limits = [filters[i].get("limit", 1) for i in [1, 2]] + if limits[0] != limits[1]: + self.log.info( + "ignore %s, filters.1.limit (%d) != filters.2.limit (%d)", + name, + limits[0], + limits[1], + ) + return True + return False + + def _params_for_dymanic(self, record): + records = [] + default_pool_found = False + default_values = set( + record.values if hasattr(record, "values") else [record.value] + ) + for rule in record.dynamic.rules: + meta = dict() + # build meta tags if geos information present + if len(rule.data.get("geos", [])) > 0: + for geo_code in rule.data["geos"]: + geo = GeoCodes.parse(geo_code) + + country = geo["country_code"] + continent = geo["continent_code"] + if country is not None: + meta.setdefault("countries", []).append(country) + else: + meta.setdefault("continents", []).append(continent) + else: + meta["default"] = True + + pool_values = set() + pool_name = rule.data["pool"] + for value in record.dynamic.pools[pool_name].data["values"]: + v = value["value"] + records.append({"content": [v], "meta": meta}) + pool_values.add(v) + + default_pool_found |= default_values == pool_values + + # if default values doesn't match any pool values, then just add this + # values with no any meta + if not default_pool_found: + for value in default_values: + records.append({"content": [value]}) + + return records + def _params_for_single(self, record): return { "ttl": record.ttl, "resource_records": [{"content": [record.value]}], } - _params_for_CNAME = _params_for_single _params_for_PTR = _params_for_single - def _params_for_multiple(self, record): + def _params_for_CNAME(self, record): + if not record.dynamic: + return self._params_for_single(record) + return { "ttl": record.ttl, - "resource_records": [ - {"content": [value]} for value in record.values + "resource_records": self._params_for_dymanic(record), + "filters": [ + {"type": "geodns"}, + {"type": "first_n", "limit": self.records_per_response}, + { + "type": "default", + "limit": self.records_per_response, + "strict": False, + }, ], } + def _params_for_multiple(self, record): + extra = dict() + if record.dynamic: + extra["resource_records"] = self._params_for_dymanic(record) + extra["filters"] = [ + {"type": "geodns"}, + {"type": "first_n", "limit": self.records_per_response}, + { + "type": "default", + "limit": self.records_per_response, + "strict": False, + }, + ] + else: + extra["resource_records"] = [ + {"content": [value]} for value in record.values + ] + return { + "ttl": record.ttl, + **extra, + } + _params_for_A = _params_for_multiple _params_for_AAAA = _params_for_multiple - _params_for_NS = _params_for_multiple + + def _params_for_NS(self, record): + return { + "ttl": record.ttl, + "resource_records": [ + {"content": [value]} for value in record.values + ], + } def _params_for_TXT(self, record): - # print(record.values) return { "ttl": record.ttl, "resource_records": [ diff --git a/tests/fixtures/gcore-records.json b/tests/fixtures/gcore-records.json index ba29246..570b358 100644 --- a/tests/fixtures/gcore-records.json +++ b/tests/fixtures/gcore-records.json @@ -199,6 +199,230 @@ ] } ] + }, + { + "name": "geo-A-single.unit.tests.", + "type": "A", + "ttl": 300, + "filters": [ + { + "type": "geodns" + }, + { + "limit": 1, + "type": "first_n" + }, + { + "limit": 1, + "strict": false, + "type": "default" + } + ], + "resource_records": [ + { + "content": [ + "7.7.7.7" + ], + "meta": { + "countries": [ + "RU" + ] + } + }, + { + "content": [ + "8.8.8.8" + ], + "meta": { + "countries": [ + "RU" + ] + } + }, + { + "content": [ + "9.9.9.9" + ], + "meta": { + "continents": [ + "EU" + ] + } + }, + { + "content": [ + "10.10.10.10" + ], + "meta": { + "default": true + } + } + ] + }, + { + "name": "geo-no-def.unit.tests.", + "type": "A", + "ttl": 300, + "filters": [ + { + "type": "geodns" + }, + { + "limit": 1, + "type": "first_n" + }, + { + "limit": 1, + "strict": false, + "type": "default" + } + ], + "resource_records": [ + { + "content": [ + "7.7.7.7" + ], + "meta": { + "countries": [ + "RU" + ] + } + } + ] + }, + { + "name": "geo-CNAME.unit.tests.", + "type": "CNAME", + "ttl": 300, + "filters": [ + { + "type": "geodns" + }, + { + "limit": 1, + "type": "first_n" + }, + { + "limit": 1, + "strict": false, + "type": "default" + } + ], + "resource_records": [ + { + "content": [ + "ru-1.unit.tests" + ], + "meta": { + "countries": [ + "RU" + ] + } + }, + { + "content": [ + "ru-2.unit.tests" + ], + "meta": { + "countries": [ + "RU" + ] + } + }, + { + "content": [ + "eu.unit.tests" + ], + "meta": { + "continents": [ + "EU" + ] + } + }, + { + "content": [ + "any.unit.tests." + ], + "meta": { + "default": true + } + } + ] + }, + { + "name": "geo-ignore-len-filters.unit.tests.", + "type": "A", + "ttl": 300, + "filters": [ + { + "limit": 1, + "type": "first_n" + }, + { + "limit": 1, + "strict": false, + "type": "default" + } + ], + "resource_records": [ + { + "content": [ + "7.7.7.7" + ] + } + ] + }, + { + "name": "geo-ignore-types.unit.tests.", + "type": "A", + "ttl": 300, + "filters": [ + { + "type": "geodistance" + }, + { + "limit": 1, + "type": "first_n" + }, + { + "limit": 1, + "strict": false, + "type": "default" + } + ], + "resource_records": [ + { + "content": [ + "7.7.7.7" + ] + } + ] + }, + { + "name": "geo-ignore-limits.unit.tests.", + "type": "A", + "ttl": 300, + "filters": [ + { + "type": "geodns" + }, + { + "limit": 1, + "type": "first_n" + }, + { + "limit": 2, + "strict": false, + "type": "default" + } + ], + "resource_records": [ + { + "content": [ + "7.7.7.7" + ] + } + ] } ] } \ No newline at end of file diff --git a/tests/test_octodns_provider_gcore.py b/tests/test_octodns_provider_gcore.py index 2ddbb59..14c0137 100644 --- a/tests/test_octodns_provider_gcore.py +++ b/tests/test_octodns_provider_gcore.py @@ -15,7 +15,7 @@ from requests_mock import ANY, mock as requests_mock from six import text_type from unittest import TestCase -from octodns.record import Record, Update, Delete +from octodns.record import Record, Update, Delete, Create from octodns.provider.gcore import ( GCoreProvider, GCoreClientBadRequest, @@ -35,7 +35,7 @@ class TestGCoreProvider(TestCase): provider = GCoreProvider("test_id", token="token") - # 400 - Bad Request. + # TC: 400 - Bad Request. with requests_mock() as mock: mock.get(ANY, status_code=400, text='{"error":"bad body"}') @@ -44,7 +44,7 @@ class TestGCoreProvider(TestCase): provider.populate(zone) self.assertIn('"error":"bad body"', text_type(ctx.exception)) - # 404 - Not Found. + # TC: 404 - Not Found. with requests_mock() as mock: mock.get( ANY, status_code=404, text='{"error":"zone is not found"}' @@ -57,7 +57,7 @@ class TestGCoreProvider(TestCase): '"error":"zone is not found"', text_type(ctx.exception) ) - # General error + # TC: General error with requests_mock() as mock: mock.get(ANY, status_code=500, text="Things caught fire") @@ -66,7 +66,7 @@ class TestGCoreProvider(TestCase): provider.populate(zone) self.assertEqual("Things caught fire", text_type(ctx.exception)) - # No credentials or token error + # TC: No credentials or token error with requests_mock() as mock: with self.assertRaises(ValueError) as ctx: GCoreProvider("test_id") @@ -75,7 +75,7 @@ class TestGCoreProvider(TestCase): text_type(ctx.exception), ) - # Auth with login password + # TC: Auth with login password with requests_mock() as mock: def match_body(request): @@ -108,7 +108,7 @@ class TestGCoreProvider(TestCase): zone = Zone("unit.tests.", []) assert not providerPassword.populate(zone) - # No diffs == no changes + # TC: No diffs == no changes with requests_mock() as mock: base = "https://dnsapi.gcorelabs.com/v2/zones/unit.tests/rrsets" with open("tests/fixtures/gcore-no-changes.json") as fh: @@ -138,7 +138,7 @@ class TestGCoreProvider(TestCase): changes = self.expected.changes(zone, provider) self.assertEqual(0, len(changes)) - # 1 removed + 7 modified + # TC: 4 create (dynamic) + 1 removed + 7 modified with requests_mock() as mock: base = "https://dnsapi.gcorelabs.com/v2/zones/unit.tests/rrsets" with open("tests/fixtures/gcore-records.json") as fh: @@ -146,9 +146,12 @@ class TestGCoreProvider(TestCase): zone = Zone("unit.tests.", []) provider.populate(zone) - self.assertEqual(13, len(zone.records)) + self.assertEqual(16, len(zone.records)) changes = self.expected.changes(zone, provider) - self.assertEqual(8, len(changes)) + self.assertEqual(11, len(changes)) + self.assertEqual( + 3, len([c for c in changes if isinstance(c, Create)]) + ) self.assertEqual( 1, len([c for c in changes if isinstance(c, Delete)]) ) @@ -156,10 +159,47 @@ class TestGCoreProvider(TestCase): 7, len([c for c in changes if isinstance(c, Update)]) ) + # TC: no pools can be built + with requests_mock() as mock: + base = "https://dnsapi.gcorelabs.com/v2/zones/unit.tests/rrsets" + mock.get( + base, + json={ + "rrsets": [ + { + "name": "unit.tests.", + "type": "A", + "ttl": 300, + "filters": [ + {"type": "geodns"}, + {"limit": 1, "type": "first_n"}, + { + "limit": 1, + "strict": False, + "type": "default", + }, + ], + "resource_records": [{"content": ["7.7.7.7"]}], + } + ] + }, + ) + + zone = Zone("unit.tests.", []) + with self.assertRaises(RuntimeError) as ctx: + provider.populate(zone) + + self.assertTrue( + str(ctx.exception).startswith( + "filter is enabled, but no pools where built for" + ), + "{} - is not start from desired text".format(ctx.exception), + ) + def test_apply(self): provider = GCoreProvider("test_id", url="http://api", token="token") - # Zone does not exists but can be created. + # TC: Zone does not exists but can be created. with requests_mock() as mock: mock.get( ANY, status_code=404, text='{"error":"zone is not found"}' @@ -169,7 +209,7 @@ class TestGCoreProvider(TestCase): plan = provider.plan(self.expected) provider.apply(plan) - # Zone does not exists and can't be created. + # TC: Zone does not exists and can't be created. with requests_mock() as mock: mock.get( ANY, status_code=404, text='{"error":"zone is not found"}' @@ -206,7 +246,7 @@ class TestGCoreProvider(TestCase): ] plan = provider.plan(self.expected) - # create all + # TC: create all self.assertEqual(13, len(plan.changes)) self.assertEqual(13, provider.apply(plan)) self.assertFalse(plan.exists) @@ -360,9 +400,8 @@ class TestGCoreProvider(TestCase): # expected number of total calls self.assertEqual(16, provider._client._request.call_count) + # TC: delete 1 and update 1 provider._client._request.reset_mock() - - # delete 1 and update 1 provider._client.zone_records = Mock( return_value=[ { @@ -410,3 +449,254 @@ class TestGCoreProvider(TestCase): ), ] ) + + # TC: create dynamics + provider._client._request.reset_mock() + provider._client.zone_records = Mock(return_value=[]) + + # Domain exists, we don't care about return + resp.json.side_effect = ["{}"] + + wanted = Zone("unit.tests.", []) + wanted.add_record( + Record.new( + wanted, + "geo-simple", + { + "ttl": 300, + "type": "A", + "value": "3.3.3.3", + "dynamic": { + "pools": { + "pool-1": { + "fallback": "other", + "values": [ + {"value": "1.1.1.1"}, + {"value": "1.1.1.2"}, + ], + }, + "pool-2": { + "fallback": "other", + "values": [ + {"value": "2.2.2.1"}, + ], + }, + "other": {"values": [{"value": "3.3.3.3"}]}, + }, + "rules": [ + {"pool": "pool-1", "geos": ["EU-RU"]}, + {"pool": "pool-2", "geos": ["EU"]}, + {"pool": "other"}, + ], + }, + }, + ), + ) + wanted.add_record( + Record.new( + wanted, + "geo-defaults", + { + "ttl": 300, + "type": "A", + "value": "3.2.3.4", + "dynamic": { + "pools": { + "pool-1": { + "values": [ + {"value": "2.2.2.1"}, + ], + }, + }, + "rules": [ + {"pool": "pool-1", "geos": ["EU"]}, + ], + }, + }, + ), + ) + wanted.add_record( + Record.new( + wanted, + "geo-cname-simple", + { + "ttl": 300, + "type": "CNAME", + "value": "en.unit.tests.", + "dynamic": { + "pools": { + "pool-1": { + "fallback": "other", + "values": [ + {"value": "ru-1.unit.tests."}, + {"value": "ru-2.unit.tests."}, + ], + }, + "pool-2": { + "fallback": "other", + "values": [ + {"value": "eu.unit.tests."}, + ], + }, + "other": {"values": [{"value": "en.unit.tests."}]}, + }, + "rules": [ + {"pool": "pool-1", "geos": ["EU-RU"]}, + {"pool": "pool-2", "geos": ["EU"]}, + {"pool": "other"}, + ], + }, + }, + ), + ) + wanted.add_record( + Record.new( + wanted, + "geo-cname-defaults", + { + "ttl": 300, + "type": "CNAME", + "value": "en.unit.tests.", + "dynamic": { + "pools": { + "pool-1": { + "values": [ + {"value": "eu.unit.tests."}, + ], + }, + }, + "rules": [ + {"pool": "pool-1", "geos": ["EU"]}, + ], + }, + }, + ), + ) + + plan = provider.plan(wanted) + self.assertTrue(plan.exists) + self.assertEqual(4, len(plan.changes)) + self.assertEqual(4, provider.apply(plan)) + + provider._client._request.assert_has_calls( + [ + call( + "POST", + "http://api/zones/unit.tests/geo-simple.unit.tests./A", + data={ + "ttl": 300, + "filters": [ + {"type": "geodns"}, + {"type": "first_n", "limit": 1}, + { + "type": "default", + "limit": 1, + "strict": False, + }, + ], + "resource_records": [ + { + "content": ["1.1.1.1"], + "meta": {"countries": ["RU"]}, + }, + { + "content": ["1.1.1.2"], + "meta": {"countries": ["RU"]}, + }, + { + "content": ["2.2.2.1"], + "meta": {"continents": ["EU"]}, + }, + { + "content": ["3.3.3.3"], + "meta": {"default": True}, + }, + ], + }, + ), + call( + "POST", + "http://api/zones/unit.tests/geo-defaults.unit.tests./A", + data={ + "ttl": 300, + "filters": [ + {"type": "geodns"}, + {"type": "first_n", "limit": 1}, + { + "type": "default", + "limit": 1, + "strict": False, + }, + ], + "resource_records": [ + { + "content": ["2.2.2.1"], + "meta": {"continents": ["EU"]}, + }, + { + "content": ["3.2.3.4"], + }, + ], + }, + ), + call( + "POST", + "http://api/zones/unit.tests/geo-cname-simple.unit.tests./CNAME", + data={ + "ttl": 300, + "filters": [ + {"type": "geodns"}, + {"type": "first_n", "limit": 1}, + { + "type": "default", + "limit": 1, + "strict": False, + }, + ], + "resource_records": [ + { + "content": ["ru-1.unit.tests."], + "meta": {"countries": ["RU"]}, + }, + { + "content": ["ru-2.unit.tests."], + "meta": {"countries": ["RU"]}, + }, + { + "content": ["eu.unit.tests."], + "meta": {"continents": ["EU"]}, + }, + { + "content": ["en.unit.tests."], + "meta": {"default": True}, + }, + ], + }, + ), + call( + "POST", + "http://api/zones/unit.tests/geo-cname-defaults.unit.tests./CNAME", + data={ + "ttl": 300, + "filters": [ + {"type": "geodns"}, + {"type": "first_n", "limit": 1}, + { + "type": "default", + "limit": 1, + "strict": False, + }, + ], + "resource_records": [ + { + "content": ["eu.unit.tests."], + "meta": {"continents": ["EU"]}, + }, + { + "content": ["en.unit.tests."], + }, + ], + }, + ), + ] + ) From e8ebe936bee81d1a894a247c28ecfeeda75efdb7 Mon Sep 17 00:00:00 2001 From: "Roger D. Winans" Date: Mon, 2 Aug 2021 12:32:29 -0400 Subject: [PATCH 308/358] Clarify Mythic Beasts uses DNS API v1 and fix the provider YAML in `class MythicBeastsProvider(BaseProvider)` --- octodns/provider/mythicbeasts.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/octodns/provider/mythicbeasts.py b/octodns/provider/mythicbeasts.py index b255a74..63e4d4d 100644 --- a/octodns/provider/mythicbeasts.py +++ b/octodns/provider/mythicbeasts.py @@ -70,8 +70,8 @@ class MythicBeastsProvider(BaseProvider): ... mythicbeasts: class: octodns.provider.mythicbeasts.MythicBeastsProvider - passwords: - my.domain.: 'password' + passwords: + my.domain.: 'DNS API v1 password' zones: my.domain.: From b873997ef5599478f22f6fe2c27bfe3971b84ddd Mon Sep 17 00:00:00 2001 From: "Roger D. Winans" Date: Mon, 2 Aug 2021 12:53:43 -0400 Subject: [PATCH 309/358] Match zone target to provider ID --- octodns/provider/mythicbeasts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/mythicbeasts.py b/octodns/provider/mythicbeasts.py index 63e4d4d..e1a2b04 100644 --- a/octodns/provider/mythicbeasts.py +++ b/octodns/provider/mythicbeasts.py @@ -76,7 +76,7 @@ class MythicBeastsProvider(BaseProvider): zones: my.domain.: targets: - - mythic + - mythicbeasts ''' RE_MX = re.compile(r'^(?P[0-9]+)\s+(?P\S+)$', From cfde1e908c7e2ab526d340321b3ebcaeba24e47d Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 2 Aug 2021 17:12:13 -0700 Subject: [PATCH 310/358] notifylists have a 64 char limit, not monitors --- octodns/provider/ns1.py | 1 - 1 file changed, 1 deletion(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 1e8e972..89e3798 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -913,7 +913,6 @@ class Ns1Provider(BaseProvider): def _feed_create(self, monitor): monitor_id = monitor['id'] self.log.debug('_feed_create: monitor=%s', monitor_id) - # TODO: looks like length limit is 64 char name = '{} - {}'.format(monitor['name'], self._uuid()[:6]) # Create the data feed From 33d56b8357ebde9b513925e929440c581c92c03a Mon Sep 17 00:00:00 2001 From: "Yaroshevich, Denis" Date: Tue, 3 Aug 2021 10:46:04 +0300 Subject: [PATCH 311/358] filters must be ordered as 'geodns', 'defaults', 'first_n' --- octodns/provider/gcore.py | 12 ++--- tests/fixtures/gcore-records.json | 26 +++++------ tests/test_octodns_provider_gcore.py | 68 ++++++++-------------------- 3 files changed, 38 insertions(+), 68 deletions(-) diff --git a/octodns/provider/gcore.py b/octodns/provider/gcore.py index 821d109..bbbf81f 100644 --- a/octodns/provider/gcore.py +++ b/octodns/provider/gcore.py @@ -194,10 +194,10 @@ class GCoreProvider(BaseProvider): geo_sets, pool_idx = dict(), 0 pools = defaultdict(lambda: {"values": []}) for rr in record["resource_records"]: - meta = rr.get("meta", {}) + meta = rr.get("meta", {}) or {} value = {"value": value_transform_fn(rr["content"][0])} - countries = meta.get("countries", []) - continents = meta.get("continents", []) + countries = meta.get("countries", []) or [] + continents = meta.get("continents", []) or [] if meta.get("default", False): pools[default_pool_name]["values"].append(value) @@ -432,7 +432,7 @@ class GCoreProvider(BaseProvider): ) return True types = [v.get("type") for v in filters] - for i, want_type in enumerate(["geodns", "first_n", "default"]): + for i, want_type in enumerate(["geodns", "default", "first_n"]): if types[i] != want_type: self.log.info( "ignore %s, filters.%d.type is %s, want %s", @@ -509,12 +509,12 @@ class GCoreProvider(BaseProvider): "resource_records": self._params_for_dymanic(record), "filters": [ {"type": "geodns"}, - {"type": "first_n", "limit": self.records_per_response}, { "type": "default", "limit": self.records_per_response, "strict": False, }, + {"type": "first_n", "limit": self.records_per_response}, ], } @@ -524,12 +524,12 @@ class GCoreProvider(BaseProvider): extra["resource_records"] = self._params_for_dymanic(record) extra["filters"] = [ {"type": "geodns"}, - {"type": "first_n", "limit": self.records_per_response}, { "type": "default", "limit": self.records_per_response, "strict": False, }, + {"type": "first_n", "limit": self.records_per_response}, ] else: extra["resource_records"] = [ diff --git a/tests/fixtures/gcore-records.json b/tests/fixtures/gcore-records.json index 570b358..9bf58d7 100644 --- a/tests/fixtures/gcore-records.json +++ b/tests/fixtures/gcore-records.json @@ -210,12 +210,12 @@ }, { "limit": 1, - "type": "first_n" + "strict": false, + "type": "default" }, { "limit": 1, - "strict": false, - "type": "default" + "type": "first_n" } ], "resource_records": [ @@ -269,12 +269,12 @@ }, { "limit": 1, - "type": "first_n" + "strict": false, + "type": "default" }, { "limit": 1, - "strict": false, - "type": "default" + "type": "first_n" } ], "resource_records": [ @@ -300,12 +300,12 @@ }, { "limit": 1, - "type": "first_n" + "strict": false, + "type": "default" }, { "limit": 1, - "strict": false, - "type": "default" + "type": "first_n" } ], "resource_records": [ @@ -406,14 +406,14 @@ { "type": "geodns" }, - { - "limit": 1, - "type": "first_n" - }, { "limit": 2, "strict": false, "type": "default" + }, + { + "limit": 1, + "type": "first_n" } ], "resource_records": [ diff --git a/tests/test_octodns_provider_gcore.py b/tests/test_octodns_provider_gcore.py index 14c0137..2151440 100644 --- a/tests/test_octodns_provider_gcore.py +++ b/tests/test_octodns_provider_gcore.py @@ -31,6 +31,16 @@ class TestGCoreProvider(TestCase): source = YamlProvider("test", join(dirname(__file__), "config")) source.populate(expected) + default_filters = [ + {"type": "geodns"}, + { + "type": "default", + "limit": 1, + "strict": False, + }, + {"type": "first_n", "limit": 1}, + ] + def test_populate(self): provider = GCoreProvider("test_id", token="token") @@ -170,15 +180,7 @@ class TestGCoreProvider(TestCase): "name": "unit.tests.", "type": "A", "ttl": 300, - "filters": [ - {"type": "geodns"}, - {"limit": 1, "type": "first_n"}, - { - "limit": 1, - "strict": False, - "type": "default", - }, - ], + "filters": self.default_filters, "resource_records": [{"content": ["7.7.7.7"]}], } ] @@ -518,7 +520,7 @@ class TestGCoreProvider(TestCase): wanted.add_record( Record.new( wanted, - "geo-cname-simple", + "cname-smpl", { "ttl": 300, "type": "CNAME", @@ -552,7 +554,7 @@ class TestGCoreProvider(TestCase): wanted.add_record( Record.new( wanted, - "geo-cname-defaults", + "cname-dflt", { "ttl": 300, "type": "CNAME", @@ -585,15 +587,7 @@ class TestGCoreProvider(TestCase): "http://api/zones/unit.tests/geo-simple.unit.tests./A", data={ "ttl": 300, - "filters": [ - {"type": "geodns"}, - {"type": "first_n", "limit": 1}, - { - "type": "default", - "limit": 1, - "strict": False, - }, - ], + "filters": self.default_filters, "resource_records": [ { "content": ["1.1.1.1"], @@ -619,15 +613,7 @@ class TestGCoreProvider(TestCase): "http://api/zones/unit.tests/geo-defaults.unit.tests./A", data={ "ttl": 300, - "filters": [ - {"type": "geodns"}, - {"type": "first_n", "limit": 1}, - { - "type": "default", - "limit": 1, - "strict": False, - }, - ], + "filters": self.default_filters, "resource_records": [ { "content": ["2.2.2.1"], @@ -641,18 +627,10 @@ class TestGCoreProvider(TestCase): ), call( "POST", - "http://api/zones/unit.tests/geo-cname-simple.unit.tests./CNAME", + "http://api/zones/unit.tests/cname-smpl.unit.tests./CNAME", data={ "ttl": 300, - "filters": [ - {"type": "geodns"}, - {"type": "first_n", "limit": 1}, - { - "type": "default", - "limit": 1, - "strict": False, - }, - ], + "filters": self.default_filters, "resource_records": [ { "content": ["ru-1.unit.tests."], @@ -675,18 +653,10 @@ class TestGCoreProvider(TestCase): ), call( "POST", - "http://api/zones/unit.tests/geo-cname-defaults.unit.tests./CNAME", + "http://api/zones/unit.tests/cname-dflt.unit.tests./CNAME", data={ "ttl": 300, - "filters": [ - {"type": "geodns"}, - {"type": "first_n", "limit": 1}, - { - "type": "default", - "limit": 1, - "strict": False, - }, - ], + "filters": self.default_filters, "resource_records": [ { "content": ["eu.unit.tests."], From b21821dc88a30a064c49730af9611aaeb0ceb1ed Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Tue, 3 Aug 2021 08:18:55 -0700 Subject: [PATCH 312/358] Further info on Ns1Provider shared_notifylist usage --- CHANGELOG.md | 10 +++++++++- octodns/provider/ns1.py | 3 ++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 55a88c2..ca42a68 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ * NS1 NA target now includes `SX` and `UM`. If `NA` continent is in use in dynamic records care must be taken to upgrade/downgrade to v0.9.13. +* Ns1Provider now supports a new parameter, shared_notifylist, which results in + all dynamic record monitors using a shared notify list named 'octoDNS NS1 + Notify List'. Only newly created record values will use the shared notify + list. It should be safe to enable this functionality, but existing records + will not be converted. Note: Once this option is enabled downgrades to + previous versions of octoDNS are discouraged and may result in undefined + behavior and broken records. See https://github.com/octodns/octodns/pull/749 + for related discussion. ## v0.9.13 - 2021-07-18 - Processors Alpha @@ -17,7 +25,7 @@ * Fixes NS1 provider's geotarget limitation of using `NA` continent. Now, when `NA` is used in geos it considers **all** the countries of `North America` insted of just `us-east`, `us-west` and `us-central` regions -* `SX' & 'UM` country support added to NS1Provider, not yet in the North +* `SX' & 'UM` country support added to NS1Provider, not yet in the North America list for backwards compatibility reasons. They will be added in the next releaser. diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 89e3798..23ee4e8 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -236,7 +236,8 @@ class Ns1Provider(BaseProvider): # Optional. Default: false. true is Recommended, but not the default # for backwards compatibility reasons. If true, all NS1 monitors will # use a shared notify list rather than one per record & value - # combination. + # combination. See CHANGELOG for more information before enabling this + # behavior. shared_notifylist: false # Optional. Default: None. If set, back off in advance to avoid 429s # from rate-limiting. Generally this should be set to the number From c873824aff42e304c7b54cbac5e0513befb4a460 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Wed, 4 Aug 2021 12:28:59 -0700 Subject: [PATCH 313/358] Link the changelog from ns1 shared_notifylist docstring --- octodns/provider/ns1.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 23ee4e8..5488d60 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -236,8 +236,9 @@ class Ns1Provider(BaseProvider): # Optional. Default: false. true is Recommended, but not the default # for backwards compatibility reasons. If true, all NS1 monitors will # use a shared notify list rather than one per record & value - # combination. See CHANGELOG for more information before enabling this - # behavior. + # combination. See CHANGELOG, + # https://github.com/octodns/octodns/blob/master/CHANGELOG.md, for more + # information before enabling this behavior. shared_notifylist: false # Optional. Default: None. If set, back off in advance to avoid 429s # from rate-limiting. Generally this should be set to the number From 44b7c6880dcf4e793fc18d05cf0fb2afe163dc29 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Thu, 5 Aug 2021 08:57:29 -0700 Subject: [PATCH 314/358] Implement AcmeIgnoringProcessor --- octodns/processor/acme.py | 33 ++++++++++++++++++ tests/test_octodns_processor_acme.py | 51 ++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+) create mode 100644 octodns/processor/acme.py create mode 100644 tests/test_octodns_processor_acme.py diff --git a/octodns/processor/acme.py b/octodns/processor/acme.py new file mode 100644 index 0000000..e786859 --- /dev/null +++ b/octodns/processor/acme.py @@ -0,0 +1,33 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from logging import getLogger + +from .base import BaseProcessor + + +class AcmeIgnoringProcessor(BaseProcessor): + log = getLogger('AcmeIgnoringProcessor') + + def __init__(self, name): + super(AcmeIgnoringProcessor, self).__init__(name) + + def _process(self, zone, *args, **kwargs): + ret = self._clone_zone(zone) + for record in zone.records: + # Uses a startswith rather than == to ignore subdomain challenges, + # e.g. _acme-challenge.foo.domain.com when managing domain.com + if record._type == 'TXT' and \ + record.name.startswith('_acme-challenge'): + self.log.info('_process: ignoring %s', record.fqdn) + continue + ret.add_record(record) + + return ret + + process_source_zone = _process + process_target_zone = _process diff --git a/tests/test_octodns_processor_acme.py b/tests/test_octodns_processor_acme.py new file mode 100644 index 0000000..aae55ba --- /dev/null +++ b/tests/test_octodns_processor_acme.py @@ -0,0 +1,51 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from unittest import TestCase + +from octodns.processor.acme import AcmeIgnoringProcessor +from octodns.record import Record +from octodns.zone import Zone + +zone = Zone('unit.tests.', []) +for record in [ + # Will be ignored + Record.new(zone, '_acme-challenge', { + 'ttl': 30, + 'type': 'TXT', + 'value': 'magic bit', + }), + # Not TXT so will live + Record.new(zone, '_acme-challenge.aaaa', { + 'ttl': 30, + 'type': 'AAAA', + 'value': '::1', + }), + # Will be ignored + Record.new(zone, '_acme-challenge.foo', { + 'ttl': 30, + 'type': 'TXT', + 'value': 'magic bit', + }), + # Not acme-challenge so will live + Record.new(zone, 'txt', { + 'ttl': 30, + 'type': 'TXT', + 'value': 'Hello World!', + }), +]: + zone.add_record(record) + + +class TestAcmeIgnoringProcessor(TestCase): + + def test_basics(self): + acme = AcmeIgnoringProcessor('acme') + + got = acme.process_source_zone(zone) + self.assertEquals(['_acme-challenge.aaaa', 'txt'], + sorted([r.name for r in got.records])) From 181dcec71bc079bf905ba00915836f700424b222 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Wed, 11 Aug 2021 07:21:37 -0700 Subject: [PATCH 315/358] Allow octoDNS managed _acme-challenge records to exist by marking them --- octodns/processor/acme.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/octodns/processor/acme.py b/octodns/processor/acme.py index e786859..f50d924 100644 --- a/octodns/processor/acme.py +++ b/octodns/processor/acme.py @@ -16,18 +16,29 @@ class AcmeIgnoringProcessor(BaseProcessor): def __init__(self, name): super(AcmeIgnoringProcessor, self).__init__(name) - def _process(self, zone, *args, **kwargs): + def process_source_zone(self, zone, *args, **kwargs): + ret = self._clone_zone(zone) + for record in zone.records: + if record._type == 'TXT' and \ + record.name.startswith('_acme-challenge'): + # We have a managed acme challenge record (owned by octoDNS) so + # we should mark it as such + record = record.copy() + record.values.append('*octoDNS*') + record.values.sort() + ret.add_record(record) + return ret + + def process_target_zone(self, zone, *args, **kwargs): ret = self._clone_zone(zone) for record in zone.records: # Uses a startswith rather than == to ignore subdomain challenges, # e.g. _acme-challenge.foo.domain.com when managing domain.com if record._type == 'TXT' and \ - record.name.startswith('_acme-challenge'): + record.name.startswith('_acme-challenge') and \ + '*octoDNS*' not in record.values: self.log.info('_process: ignoring %s', record.fqdn) continue ret.add_record(record) return ret - - process_source_zone = _process - process_target_zone = _process From ea5093935b9988c3edb6abb6cf9f1d34cafb985f Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Wed, 11 Aug 2021 09:31:54 -0700 Subject: [PATCH 316/358] Support taking ownership of acme records --- octodns/processor/acme.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/octodns/processor/acme.py b/octodns/processor/acme.py index f50d924..8516ce2 100644 --- a/octodns/processor/acme.py +++ b/octodns/processor/acme.py @@ -16,6 +16,8 @@ class AcmeIgnoringProcessor(BaseProcessor): def __init__(self, name): super(AcmeIgnoringProcessor, self).__init__(name) + self._owned = set() + def process_source_zone(self, zone, *args, **kwargs): ret = self._clone_zone(zone) for record in zone.records: @@ -26,6 +28,7 @@ class AcmeIgnoringProcessor(BaseProcessor): record = record.copy() record.values.append('*octoDNS*') record.values.sort() + self._owned.add(record) ret.add_record(record) return ret @@ -36,7 +39,8 @@ class AcmeIgnoringProcessor(BaseProcessor): # e.g. _acme-challenge.foo.domain.com when managing domain.com if record._type == 'TXT' and \ record.name.startswith('_acme-challenge') and \ - '*octoDNS*' not in record.values: + '*octoDNS*' not in record.values and \ + record not in self._owned: self.log.info('_process: ignoring %s', record.fqdn) continue ret.add_record(record) From 5c1ea34bc6ed1a58cc4f97e63bd97dbc81317177 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Wed, 11 Aug 2021 10:41:52 -0700 Subject: [PATCH 317/358] Tests for acme processor with ownership --- tests/test_octodns_processor_acme.py | 84 ++++++++++++++++++++++------ 1 file changed, 68 insertions(+), 16 deletions(-) diff --git a/tests/test_octodns_processor_acme.py b/tests/test_octodns_processor_acme.py index aae55ba..8d30368 100644 --- a/tests/test_octodns_processor_acme.py +++ b/tests/test_octodns_processor_acme.py @@ -12,40 +12,92 @@ from octodns.record import Record from octodns.zone import Zone zone = Zone('unit.tests.', []) -for record in [ - # Will be ignored - Record.new(zone, '_acme-challenge', { +records = { + 'root-unowned': Record.new(zone, '_acme-challenge', { 'ttl': 30, 'type': 'TXT', 'value': 'magic bit', }), - # Not TXT so will live - Record.new(zone, '_acme-challenge.aaaa', { + 'sub-unowned': Record.new(zone, '_acme-challenge.sub-unowned', { + 'ttl': 30, + 'type': 'TXT', + 'value': 'magic bit', + }), + 'not-txt': Record.new(zone, '_acme-challenge.not-txt', { 'ttl': 30, 'type': 'AAAA', 'value': '::1', }), - # Will be ignored - Record.new(zone, '_acme-challenge.foo', { + 'not-acme': Record.new(zone, 'not-acme', { + 'ttl': 30, + 'type': 'TXT', + 'value': 'Hello World!', + }), + 'managed': Record.new(zone, '_acme-challenge.managed', { 'ttl': 30, 'type': 'TXT', 'value': 'magic bit', }), - # Not acme-challenge so will live - Record.new(zone, 'txt', { + 'owned': Record.new(zone, '_acme-challenge.owned', { 'ttl': 30, 'type': 'TXT', - 'value': 'Hello World!', + 'values': ['*octoDNS*', 'magic bit'], + }), + 'going-away': Record.new(zone, '_acme-challenge.going-away', { + 'ttl': 30, + 'type': 'TXT', + 'values': ['*octoDNS*', 'magic bit'], }), -]: - zone.add_record(record) +} class TestAcmeIgnoringProcessor(TestCase): - def test_basics(self): + def test_process_zones(self): acme = AcmeIgnoringProcessor('acme') - got = acme.process_source_zone(zone) - self.assertEquals(['_acme-challenge.aaaa', 'txt'], - sorted([r.name for r in got.records])) + source = Zone(zone.name, []) + # Unrelated stuff that should be untouched + source.add_record(records['not-txt']) + source.add_record(records['not-acme']) + # A managed acme that will have ownership value added + source.add_record(records['managed']) + + got = acme.process_source_zone(source) + self.assertEquals([ + '_acme-challenge.managed', + '_acme-challenge.not-txt', + 'not-acme', + ], sorted([r.name for r in got.records])) + managed = None + for record in got.records: + print(record.name) + if record.name.endswith('managed'): + managed = record + break + self.assertTrue(managed) + # Ownership was marked with an extra value + self.assertEquals(['*octoDNS*', 'magic bit'], record.values) + + existing = Zone(zone.name, []) + # Unrelated stuff that should be untouched + existing.add_record(records['not-txt']) + existing.add_record(records['not-acme']) + # Stuff that will be ignored + existing.add_record(records['root-unowned']) + existing.add_record(records['sub-unowned']) + # A managed acme that needs ownership value added + existing.add_record(records['managed']) + # A managed acme that has ownershp managed + existing.add_record(records['owned']) + # A managed acme that needs to go away + existing.add_record(records['going-away']) + + got = acme.process_target_zone(existing) + self.assertEquals([ + '_acme-challenge.going-away', + '_acme-challenge.managed', + '_acme-challenge.not-txt', + '_acme-challenge.owned', + 'not-acme' + ], sorted([r.name for r in got.records])) From c0c3a93d4aa6d9670a8193f4216306e74e038a2f Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Wed, 11 Aug 2021 20:52:23 -0700 Subject: [PATCH 318/358] Rename to AcmeManagingProcess, doc usage --- octodns/processor/acme.py | 22 +++++++++++++++++++--- tests/test_octodns_processor_acme.py | 6 +++--- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/octodns/processor/acme.py b/octodns/processor/acme.py index 8516ce2..685130d 100644 --- a/octodns/processor/acme.py +++ b/octodns/processor/acme.py @@ -10,11 +10,25 @@ from logging import getLogger from .base import BaseProcessor -class AcmeIgnoringProcessor(BaseProcessor): - log = getLogger('AcmeIgnoringProcessor') +class AcmeMangingProcessor(BaseProcessor): + log = getLogger('AcmeMangingProcessor') def __init__(self, name): - super(AcmeIgnoringProcessor, self).__init__(name) + ''' + processors: + acme: + class: octodns.processor.acme.AcmeMangingProcessor + + ... + + zones: + something.com.: + ... + processors: + - acme + ... + ''' + super(AcmeMangingProcessor, self).__init__(name) self._owned = set() @@ -28,6 +42,8 @@ class AcmeIgnoringProcessor(BaseProcessor): record = record.copy() record.values.append('*octoDNS*') record.values.sort() + # This assumes we'll see things as sources before targets, + # which is the case... self._owned.add(record) ret.add_record(record) return ret diff --git a/tests/test_octodns_processor_acme.py b/tests/test_octodns_processor_acme.py index 8d30368..c927608 100644 --- a/tests/test_octodns_processor_acme.py +++ b/tests/test_octodns_processor_acme.py @@ -7,7 +7,7 @@ from __future__ import absolute_import, division, print_function, \ from unittest import TestCase -from octodns.processor.acme import AcmeIgnoringProcessor +from octodns.processor.acme import AcmeMangingProcessor from octodns.record import Record from octodns.zone import Zone @@ -51,10 +51,10 @@ records = { } -class TestAcmeIgnoringProcessor(TestCase): +class TestAcmeMangingProcessor(TestCase): def test_process_zones(self): - acme = AcmeIgnoringProcessor('acme') + acme = AcmeMangingProcessor('acme') source = Zone(zone.name, []) # Unrelated stuff that should be untouched From 49bff426b7ee4d5a4554b2c6e1603e6384d32949 Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Thu, 12 Aug 2021 21:43:01 -0700 Subject: [PATCH 319/358] Multi-value PTR records --- octodns/provider/azuredns.py | 4 +-- octodns/provider/ns1.py | 24 ++++++++++++----- octodns/record/__init__.py | 31 ++++++++++++++++++++-- tests/config/split/unit.tests.tst/ptr.yaml | 2 +- tests/config/unit.tests.yaml | 2 +- tests/test_octodns_provider_ns1.py | 17 ++++++++++-- tests/test_octodns_record.py | 12 ++++++++- 7 files changed, 76 insertions(+), 16 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 6edc1ba..c75ef4b 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -707,8 +707,8 @@ class AzureProvider(BaseProvider): return {'values': [_check_endswith_dot(val) for val in vals]} def _data_for_PTR(self, azrecord): - ptrdname = azrecord.ptr_records[0].ptrdname - return {'value': _check_endswith_dot(ptrdname)} + vals = [ar.ptrdname for ar in azrecord.ptr_records] + return {'values': [_check_endswith_dot(val) for val in vals]} def _data_for_SRV(self, azrecord): return {'values': [{'priority': ar.priority, 'weight': ar.weight, diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 0ebb36b..72e0bc9 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -20,6 +20,10 @@ from ..record import Record, Update from .base import BaseProvider +def _check_endswith_dot(string): + return string if string.endswith('.') else '{}.'.format(string) + + class Ns1Exception(Exception): pass @@ -717,7 +721,6 @@ class Ns1Provider(BaseProvider): } _data_for_ALIAS = _data_for_CNAME - _data_for_PTR = _data_for_CNAME def _data_for_MX(self, _type, record): values = [] @@ -756,10 +759,11 @@ class Ns1Provider(BaseProvider): return { 'ttl': record['ttl'], 'type': _type, - 'values': [a if a.endswith('.') else '{}.'.format(a) - for a in record['short_answers']], + 'values': record['short_answers'], } + _data_for_PTR = _data_for_NS + def _data_for_SRV(self, _type, record): values = [] for answer in record['short_answers']: @@ -809,9 +813,10 @@ class Ns1Provider(BaseProvider): for record in ns1_zone['records']: if record['type'] in ['ALIAS', 'CNAME', 'MX', 'NS', 'PTR', 'SRV']: - for i, a in enumerate(record['short_answers']): - if not a.endswith('.'): - record['short_answers'][i] = '{}.'.format(a) + record['short_answers'] = [ + _check_endswith_dot(a) + for a in record['short_answers'] + ] if record.get('tier', 1) > 1: # Need to get the full record data for geo records @@ -1297,7 +1302,6 @@ class Ns1Provider(BaseProvider): return {'answers': [record.value], 'ttl': record.ttl}, None _params_for_ALIAS = _params_for_CNAME - _params_for_PTR = _params_for_CNAME def _params_for_MX(self, record): values = [(v.preference, v.exchange) for v in record.values] @@ -1308,6 +1312,12 @@ class Ns1Provider(BaseProvider): v.replacement) for v in record.values] return {'answers': values, 'ttl': record.ttl}, None + def _params_for_PTR(self, record): + return { + 'answers': record.values, + 'ttl': record.ttl, + }, None + def _params_for_SRV(self, record): values = [(v.priority, v.weight, v.port, v.target) for v in record.values] diff --git a/octodns/record/__init__.py b/octodns/record/__init__.py index 7714b27..93f6660 100644 --- a/octodns/record/__init__.py +++ b/octodns/record/__init__.py @@ -1256,13 +1256,40 @@ class NsRecord(_ValuesMixin, Record): class PtrValue(_TargetValue): - pass + @classmethod + def validate(cls, values, _type): + if not isinstance(values, list): + values = [values] + + reasons = [] + + if not values: + reasons.append('missing values') + + for value in values: + reasons.extend(super(PtrValue, cls).validate(value, _type)) + + return reasons -class PtrRecord(_ValueMixin, Record): + @classmethod + def process(cls, values): + return [super(PtrValue, cls).process(v) for v in values] + + +class PtrRecord(_ValuesMixin, Record): _type = 'PTR' _value_type = PtrValue + # This is for backward compatibility with providers that don't support + # multi-value PTR records. + @property + def value(self): + if len(self.values) == 1: + return self.values[0] + + raise AttributeError("Multi-value PTR record has no attribute 'value'") + class SshfpValue(EqualityTupleMixin): VALID_ALGORITHMS = (1, 2, 3, 4) diff --git a/tests/config/split/unit.tests.tst/ptr.yaml b/tests/config/split/unit.tests.tst/ptr.yaml index 0098b57..cffb50b 100644 --- a/tests/config/split/unit.tests.tst/ptr.yaml +++ b/tests/config/split/unit.tests.tst/ptr.yaml @@ -2,4 +2,4 @@ ptr: ttl: 300 type: PTR - value: foo.bar.com. + values: [foo.bar.com.] diff --git a/tests/config/unit.tests.yaml b/tests/config/unit.tests.yaml index c70b20c..aa28ee5 100644 --- a/tests/config/unit.tests.yaml +++ b/tests/config/unit.tests.yaml @@ -152,7 +152,7 @@ naptr: ptr: ttl: 300 type: PTR - value: foo.bar.com. + values: [foo.bar.com.] spf: ttl: 600 type: SPF diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index 932e4c4..3e239b3 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -120,6 +120,11 @@ class TestNs1Provider(TestCase): 'query': 0, }, })) + expected.add(Record.new(zone, '1.2.3.4', { + 'ttl': 42, + 'type': 'PTR', + 'values': ['one.one.one.one.', 'two.two.two.two.'], + })) ns1_records = [{ 'type': 'A', @@ -180,6 +185,11 @@ class TestNs1Provider(TestCase): 'ttl': 41, 'short_answers': ['/ http://foo.unit.tests 301 2 0'], 'domain': 'urlfwd.unit.tests.', + }, { + 'type': 'PTR', + 'ttl': 42, + 'short_answers': ['one.one.one.one.', 'two.two.two.two.'], + 'domain': '1.2.3.4.unit.tests.', }] @patch('ns1.rest.records.Records.retrieve') @@ -358,10 +368,10 @@ class TestNs1Provider(TestCase): ResourceException('server error: zone not found') zone_create_mock.side_effect = ['foo'] - # Test out the create rate-limit handling, then 9 successes + # Test out the create rate-limit handling, then successes for the rest record_create_mock.side_effect = [ RateLimitException('boo', period=0), - ] + ([None] * 10) + ] + ([None] * len(self.expected)) got_n = provider.apply(plan) self.assertEquals(expected_n, got_n) @@ -379,6 +389,9 @@ class TestNs1Provider(TestCase): call('unit.tests', 'unit.tests', 'MX', answers=[ (10, 'mx1.unit.tests.'), (20, 'mx2.unit.tests.') ], ttl=35), + call('unit.tests', '1.2.3.4.unit.tests', 'PTR', answers=[ + 'one.one.one.one.', 'two.two.two.two.', + ], ttl=42), ]) # Update & delete diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index 3bd48e5..2d8e000 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -2733,7 +2733,7 @@ class TestRecordValidation(TestCase): 'type': 'PTR', 'ttl': 600, }) - self.assertEquals(['missing value'], ctx.exception.reasons) + self.assertEquals(['missing values'], ctx.exception.reasons) # not a valid FQDN with self.assertRaises(ValidationError) as ctx: @@ -2755,6 +2755,16 @@ class TestRecordValidation(TestCase): self.assertEquals(['PTR value "foo.bar" missing trailing .'], ctx.exception.reasons) + # multi-value requesting single-value + with self.assertRaises(AttributeError) as ctx: + Record.new(self.zone, '', { + 'type': 'PTR', + 'ttl': 600, + 'values': ['foo.com.', 'bar.net.'], + }).value + self.assertEquals("Multi-value PTR record has no attribute 'value'", + text_type(ctx.exception)) + def test_SSHFP(self): # doesn't blow up Record.new(self.zone, '', { From ec41e0377e9f689ed5865977b32c22cf643a688b Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Thu, 12 Aug 2021 22:15:46 -0700 Subject: [PATCH 320/358] multi-value PTR tests for Azure --- octodns/record/__init__.py | 2 +- tests/test_octodns_provider_azuredns.py | 30 +++++++++++++++++++------ 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/octodns/record/__init__.py b/octodns/record/__init__.py index 93f6660..2e17948 100644 --- a/octodns/record/__init__.py +++ b/octodns/record/__init__.py @@ -1286,7 +1286,7 @@ class PtrRecord(_ValuesMixin, Record): @property def value(self): if len(self.values) == 1: - return self.values[0] + return self.data['value'] raise AttributeError("Multi-value PTR record has no attribute 'value'") diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index 0af8665..b3b52e5 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -150,6 +150,11 @@ octo_records.append(Record.new(zone, 'txt3', { 'type': 'TXT', 'values': ['txt multiple test', long_txt]})) +octo_records.append(Record.new(zone, 'ptr2', { + 'ttl': 11, + 'type': 'PTR', + 'values': ['ptr21.unit.tests.', 'ptr22.unit.tests.']})) + azure_records = [] _base0 = _AzureRecord('TestAzure', octo_records[0]) _base0.zone_name = 'unit.tests' @@ -338,6 +343,15 @@ _base18.params['txt_records'] = [TxtRecord(value=['txt multiple test']), TxtRecord(value=[long_txt_az1, long_txt_az2])] azure_records.append(_base18) +_base19 = _AzureRecord('TestAzure', octo_records[19]) +_base19.zone_name = 'unit.tests' +_base19.relative_record_set_name = 'ptr2' +_base19.record_type = 'PTR' +_base19.params['ttl'] = 11 +_base19.params['ptr_records'] = [PtrRecord(ptrdname='ptr21.unit.tests.'), + PtrRecord(ptrdname='ptr22.unit.tests.')] +azure_records.append(_base19) + class Test_AzureRecord(TestCase): def test_azure_record(self): @@ -2054,15 +2068,16 @@ class TestAzureDnsProvider(TestCase): def test_apply(self): provider = self._get_provider() - half = int(len(octo_records) / 2) + expected_n = len(octo_records) + half = int(expected_n / 2) changes = [Create(r) for r in octo_records[:half]] + \ [Update(r, r) for r in octo_records[half:]] deletes = [Delete(r) for r in octo_records] - self.assertEquals(19, provider.apply(Plan(None, zone, - changes, True))) - self.assertEquals(19, provider.apply(Plan(zone, zone, - deletes, True))) + self.assertEquals(expected_n, provider.apply(Plan(None, zone, + changes, True))) + self.assertEquals(expected_n, provider.apply(Plan(zone, zone, + deletes, True))) def test_apply_create_dynamic(self): provider = self._get_provider() @@ -2320,8 +2335,9 @@ class TestAzureDnsProvider(TestCase): _get = provider._dns_client.zones.get _get.side_effect = CloudError(Mock(status=404), err_msg) - self.assertEquals(19, provider.apply(Plan(None, desired, changes, - True))) + expected_n = len(octo_records) + self.assertEquals(expected_n, provider.apply(Plan(None, desired, + changes, True))) def test_check_zone_no_create(self): provider = self._get_provider() From 73ad7000cab36088245dfafcae5db1f5b4df85f6 Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Thu, 12 Aug 2021 22:33:10 -0700 Subject: [PATCH 321/358] better helper method name --- octodns/provider/ns1.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 72e0bc9..9209ede 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -20,7 +20,7 @@ from ..record import Record, Update from .base import BaseProvider -def _check_endswith_dot(string): +def _ensure_endswith_dot(string): return string if string.endswith('.') else '{}.'.format(string) @@ -814,7 +814,7 @@ class Ns1Provider(BaseProvider): if record['type'] in ['ALIAS', 'CNAME', 'MX', 'NS', 'PTR', 'SRV']: record['short_answers'] = [ - _check_endswith_dot(a) + _ensure_endswith_dot(a) for a in record['short_answers'] ] From 6d302af719d1036dac0c1a2be2e2f055a8458077 Mon Sep 17 00:00:00 2001 From: Sham Date: Fri, 13 Aug 2021 02:34:35 -0700 Subject: [PATCH 322/358] adds support for CA provinces --- octodns/provider/ns1.py | 19 +++++++++++++------ tests/test_octodns_provider_ns1.py | 13 ++++++++----- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 0ebb36b..812e859 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -622,11 +622,14 @@ class Ns1Provider(BaseProvider): for c in countries: geos.add('{}-{}'.format(continent, c)) - # States are easy too, just assume NA-US (CA providences aren't - # supported by octoDNS currently) + # States and provinces are easy too, + # just assume NA-US or NA-CA for state in meta.get('us_state', []): geos.add('NA-US-{}'.format(state)) + for province in meta.get('ca_province', []): + geos.add('NA-CA-{}'.format(province)) + if geos: # There are geos, combine them with any existing geos for this # pool and recorded the sorted unique set of them @@ -1144,12 +1147,15 @@ class Ns1Provider(BaseProvider): country = set() georegion = set() us_state = set() + ca_province = set() for geo in rule.data.get('geos', []): n = len(geo) if n == 8: # US state, e.g. NA-US-KY - us_state.add(geo[-2:]) + # CA province, e.g. NA-CA-NL + us_state.add(geo[-2:]) if "NA-US" in geo \ + else ca_province.add(geo[-2:]) # For filtering. State filtering is done by the country # filter has_country = True @@ -1182,7 +1188,7 @@ class Ns1Provider(BaseProvider): 'meta': georegion_meta, } - if country or us_state: + if country or us_state or ca_province: # If there's country and/or states its a country pool, # countries and states can coexist as they're handled by the # same step in the filterchain (countries and georegions @@ -1193,11 +1199,12 @@ class Ns1Provider(BaseProvider): country_state_meta['country'] = sorted(country) if us_state: country_state_meta['us_state'] = sorted(us_state) + if ca_province: + country_state_meta['ca_province'] = sorted(ca_province) regions['{}__country'.format(pool_name)] = { 'meta': country_state_meta, } - - if not georegion and not country and not us_state: + elif not georegion: # If there's no targeting it's a catchall regions['{}__catchall'.format(pool_name)] = { 'meta': meta, diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index 932e4c4..d716be6 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -1247,7 +1247,7 @@ class TestNs1ProviderDynamic(TestCase): rule0 = record.data['dynamic']['rules'][0] rule1 = record.data['dynamic']['rules'][1] rule0['geos'] = ['AF', 'EU'] - rule1['geos'] = ['NA-US-CA'] + rule1['geos'] = ['NA-US-CA', 'NA-CA-NL'] ret, _ = provider._params_for_A(record) self.assertEquals(10, len(ret['answers'])) exp = Ns1Provider._FILTER_CHAIN_WITH_REGION_AND_COUNTRY(provider, @@ -1262,7 +1262,8 @@ class TestNs1ProviderDynamic(TestCase): 'iad__country': { 'meta': { 'note': 'rule-order:1', - 'us_state': ['CA'] + 'us_state': ['CA'], + 'ca_province': ['NL'] } }, 'lhr__georegion': { @@ -1624,8 +1625,9 @@ class TestNs1ProviderDynamic(TestCase): 'lhr__country': { 'meta': { 'note': 'rule-order:1 fallback:iad', - 'country': ['CA'], + 'country': ['MX'], 'us_state': ['OR'], + 'ca_province': ['NL'] }, }, # iad will use the old style "plain" region naming. We won't @@ -1669,8 +1671,9 @@ class TestNs1ProviderDynamic(TestCase): '_order': '1', 'geos': [ 'AF', - 'NA-CA', - 'NA-US-OR', + 'NA-CA-NL', + 'NA-MX', + 'NA-US-OR' ], 'pool': 'lhr', }, { From 6e9ce3ac3c71ddfa7d56fae4bf295e974c58f492 Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Mon, 16 Aug 2021 16:39:23 -0700 Subject: [PATCH 323/358] pick first PTR value instead of erroring out --- octodns/manager.py | 26 ++++++++++++++++++++++++-- octodns/provider/azuredns.py | 1 + octodns/provider/ns1.py | 1 + octodns/record/__init__.py | 5 +---- octodns/source/base.py | 2 ++ tests/test_octodns_record.py | 10 ---------- 6 files changed, 29 insertions(+), 16 deletions(-) diff --git a/octodns/manager.py b/octodns/manager.py index 104e445..a991b4b 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -294,8 +294,30 @@ class Manager(object): for processor in processors: plan = processor.process_plan(plan, sources=sources, target=target) - if plan: - plans.append((target, plan)) + if not plan: + continue + + # Multi value PTR check + for change in plan.changes: + record = change.new + if not record: + # Delete - doesn't need to be checked + continue + + if record._type == 'PTR' and len(record.values) > 1 and not \ + target.SUPPORTS_MUTLIVALUE_PTR: + self.log.warn('target=%s does not support multi-value PTR ' + 'record %s; using %s as its only answer', + target, record.fqdn, record.value) + # Make a new copy of the record so as to not change a + # potentially shared object + change.new = Record.new(record.zone, record.name, { + 'type': record._type, + 'ttl': record.ttl, + 'value': record.value, + }, source=record.source, lenient=lenient) + + plans.append((target, plan)) # Return the zone as it's the desired state return plans, zone diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index c75ef4b..20a9d7b 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -456,6 +456,7 @@ class AzureProvider(BaseProvider): ''' SUPPORTS_GEO = False SUPPORTS_DYNAMIC = True + SUPPORTS_MUTLIVALUE_PTR = True SUPPORTS = set(('A', 'AAAA', 'CAA', 'CNAME', 'MX', 'NS', 'PTR', 'SRV', 'TXT')) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 9209ede..3bd0b54 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -261,6 +261,7 @@ class Ns1Provider(BaseProvider): ''' SUPPORTS_GEO = True SUPPORTS_DYNAMIC = True + SUPPORTS_MUTLIVALUE_PTR = True SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CAA', 'CNAME', 'MX', 'NAPTR', 'NS', 'PTR', 'SPF', 'SRV', 'TXT', 'URLFWD')) diff --git a/octodns/record/__init__.py b/octodns/record/__init__.py index 2e17948..77663a5 100644 --- a/octodns/record/__init__.py +++ b/octodns/record/__init__.py @@ -1285,10 +1285,7 @@ class PtrRecord(_ValuesMixin, Record): # multi-value PTR records. @property def value(self): - if len(self.values) == 1: - return self.data['value'] - - raise AttributeError("Multi-value PTR record has no attribute 'value'") + return self.values[0] class SshfpValue(EqualityTupleMixin): diff --git a/octodns/source/base.py b/octodns/source/base.py index 79b5a2a..6094726 100644 --- a/octodns/source/base.py +++ b/octodns/source/base.py @@ -8,6 +8,8 @@ from __future__ import absolute_import, division, print_function, \ class BaseSource(object): + SUPPORTS_MUTLIVALUE_PTR = False + def __init__(self, id): self.id = id if not getattr(self, 'log', False): diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index 2d8e000..886dbfb 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -2755,16 +2755,6 @@ class TestRecordValidation(TestCase): self.assertEquals(['PTR value "foo.bar" missing trailing .'], ctx.exception.reasons) - # multi-value requesting single-value - with self.assertRaises(AttributeError) as ctx: - Record.new(self.zone, '', { - 'type': 'PTR', - 'ttl': 600, - 'values': ['foo.com.', 'bar.net.'], - }).value - self.assertEquals("Multi-value PTR record has no attribute 'value'", - text_type(ctx.exception)) - def test_SSHFP(self): # doesn't blow up Record.new(self.zone, '', { From fc39697bddd6149ca50a0f7f11b3e47ca5e235d3 Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Mon, 16 Aug 2021 20:33:22 -0700 Subject: [PATCH 324/358] consider only first PTR value for getting changes --- octodns/manager.py | 26 ++------------------------ octodns/provider/base.py | 32 +++++++++++++++++++++++++++++++- 2 files changed, 33 insertions(+), 25 deletions(-) diff --git a/octodns/manager.py b/octodns/manager.py index a991b4b..104e445 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -294,30 +294,8 @@ class Manager(object): for processor in processors: plan = processor.process_plan(plan, sources=sources, target=target) - if not plan: - continue - - # Multi value PTR check - for change in plan.changes: - record = change.new - if not record: - # Delete - doesn't need to be checked - continue - - if record._type == 'PTR' and len(record.values) > 1 and not \ - target.SUPPORTS_MUTLIVALUE_PTR: - self.log.warn('target=%s does not support multi-value PTR ' - 'record %s; using %s as its only answer', - target, record.fqdn, record.value) - # Make a new copy of the record so as to not change a - # potentially shared object - change.new = Record.new(record.zone, record.name, { - 'type': record._type, - 'ttl': record.ttl, - 'value': record.value, - }, source=record.source, lenient=lenient) - - plans.append((target, plan)) + if plan: + plans.append((target, plan)) # Return the zone as it's the desired state return plans, zone diff --git a/octodns/provider/base.py b/octodns/provider/base.py index 729c9ee..fcb557a 100644 --- a/octodns/provider/base.py +++ b/octodns/provider/base.py @@ -7,6 +7,7 @@ from __future__ import absolute_import, division, print_function, \ from six import text_type +from ..record import Record from ..source.base import BaseSource from ..zone import Zone from .plan import Plan @@ -44,6 +45,35 @@ class BaseProvider(BaseSource): ''' return [] + def _process_change(self, change): + ''' + Process/manipulate each change for feature-specific corner cases + ''' + if not self._include_change(change): + return False + + # Multi value PTR records + record = change.new + if record and record._type == 'PTR' and len(record.values) > 1 and \ + not self.SUPPORTS_MUTLIVALUE_PTR: + # replace with a single-value copy + change.new = Record.new(record.zone, record.name, { + 'type': 'PTR', + 'ttl': record.ttl, + 'values': [record.value], + }, source=record.source) + + existing = change.existing + if existing and not existing.changes(change.new, self): + # if new single-value replacement turns out to be the same, + # skip the change + return False + + self.log.warn('does not support multi-value PTR records; will ' + 'use only %s for %s', record.value, record.fqdn) + + return True + def plan(self, desired, processors=[]): self.log.info('plan: desired=%s', desired.name) @@ -63,7 +93,7 @@ class BaseProvider(BaseSource): # allow the provider to filter out false positives before = len(changes) - changes = [c for c in changes if self._include_change(c)] + changes = [c for c in changes if self._process_change(c)] after = len(changes) if before != after: self.log.info('plan: filtered out %s changes', before - after) From 536c0c68ecf4f6c22b8235cfad82467d48efaa3a Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Tue, 17 Aug 2021 19:15:34 -0700 Subject: [PATCH 325/358] no-op Provider.process_desired_zone --- octodns/provider/base.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/octodns/provider/base.py b/octodns/provider/base.py index 729c9ee..55b1cb4 100644 --- a/octodns/provider/base.py +++ b/octodns/provider/base.py @@ -44,9 +44,14 @@ class BaseProvider(BaseSource): ''' return [] + def process_desired_zone(self, desired): + return desired + def plan(self, desired, processors=[]): self.log.info('plan: desired=%s', desired.name) + desired = self.process_desired_zone(desired) + existing = Zone(desired.name, desired.sub_zones) exists = self.populate(existing, target=True, lenient=True) if exists is None: From c81450682cc33b6558b83cc0c40b99c0d0e5ed96 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Tue, 17 Aug 2021 20:26:51 -0700 Subject: [PATCH 326/358] Implement and test Route53Provider.process_desired_zone checking of NA-CA-* --- octodns/provider/base.py | 10 ++ octodns/provider/route53.py | 39 ++++++++ tests/test_octodns_provider_route53.py | 133 +++++++++++++++++++++++++ 3 files changed, 182 insertions(+) diff --git a/octodns/provider/base.py b/octodns/provider/base.py index 55b1cb4..238a68a 100644 --- a/octodns/provider/base.py +++ b/octodns/provider/base.py @@ -12,6 +12,10 @@ from ..zone import Zone from .plan import Plan +class ProviderException(Exception): + pass + + class BaseProvider(BaseSource): def __init__(self, id, apply_disabled=False, @@ -44,6 +48,12 @@ class BaseProvider(BaseSource): ''' return [] + def supports_warn_or_except(self, msg): + # TODO: base class param to control warn vs except + if False: + raise ProviderException(msg) + self.log.warning(msg) + def process_desired_zone(self, desired): return desired diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index f1b3b40..6b2fecb 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -19,6 +19,7 @@ from six import text_type from ..equality import EqualityTupleMixin from ..record import Record, Update from ..record.geo import GeoCodes +from ..zone import Zone from .base import BaseProvider octal_re = re.compile(r'\\(\d\d\d)') @@ -924,6 +925,44 @@ class Route53Provider(BaseProvider): return data + def process_desired_zone(self, desired): + ret = Zone(desired.name, desired.sub_zones) + for record in desired.records: + if getattr(record, 'dynamic', False): + # Make a copy of the record in case we have to muck with it + record = record.copy() + from pprint import pprint + pprint((record, record.data)) + dynamic = record.dynamic + rules = [] + for i, rule in enumerate(dynamic.rules): + geos = rule.data.get('geos', []) + if not geos: + rules.append(rule) + continue + filtered_geos = [g for g in geos + if not g.startswith('NA-CA-')] + if not filtered_geos: + # We've removed all geos, we'll have to skip this rule + msg = 'NA-CA-* not supported resulting in ' \ + 'empty geo target, skipping rule {}'.format(i) + self.supports_warn_or_except(msg) + continue + elif geos != filtered_geos: + msg = 'NA-CA-* not supported resulting in ' \ + 'empty geo target, skipping rule {}'.format(i) + self.supports_warn_or_except(msg) + rule.data['geos'] = filtered_geos + rules.append(rule) + + dynamic.rules = rules + + pprint((record, record.data)) + + ret.add_record(record) + + return super(Route53Provider, self).process_desired_zone(ret) + def populate(self, zone, target=False, lenient=False): self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, target, lenient) diff --git a/tests/test_octodns_provider_route53.py b/tests/test_octodns_provider_route53.py index 1bf3332..34124ee 100644 --- a/tests/test_octodns_provider_route53.py +++ b/tests/test_octodns_provider_route53.py @@ -394,6 +394,139 @@ class TestRoute53Provider(TestCase): return (provider, stubber) + def test_process_desired_zone(self): + provider, stubber = self._get_stubbed_fallback_auth_provider() + + # No records, essentially a no-op + desired = Zone('unit.tests.', []) + got = provider.process_desired_zone(desired) + self.assertEquals(desired.records, got.records) + + # Record without any geos + desired = Zone('unit.tests.', []) + record = Record.new(desired, 'a', { + 'ttl': 30, + 'type': 'A', + 'value': '1.2.3.4', + 'dynamic': { + 'pools': { + 'one': { + 'values': [{ + 'value': '2.2.3.4', + }], + }, + }, + 'rules': [{ + 'pool': 'one', + }], + }, + }) + desired.add_record(record) + got = provider.process_desired_zone(desired) + self.assertEquals(desired.records, got.records) + self.assertEquals(1, len(list(got.records)[0].dynamic.rules)) + self.assertFalse('geos' in list(got.records)[0].dynamic.rules[0].data) + + # Record where all geos are supported + desired = Zone('unit.tests.', []) + record = Record.new(desired, 'a', { + 'ttl': 30, + 'type': 'A', + 'value': '1.2.3.4', + 'dynamic': { + 'pools': { + 'one': { + 'values': [{ + 'value': '1.2.3.4', + }], + }, + 'two': { + 'values': [{ + 'value': '2.2.3.4', + }], + }, + }, + 'rules': [{ + 'geos': ['EU', 'NA-US-OR'], + 'pool': 'two', + }, { + 'pool': 'one', + }], + }, + }) + desired.add_record(record) + got = provider.process_desired_zone(desired) + self.assertEquals(2, len(list(got.records)[0].dynamic.rules)) + self.assertEquals(['EU', 'NA-US-OR'], + list(got.records)[0].dynamic.rules[0].data['geos']) + self.assertFalse('geos' in list(got.records)[0].dynamic.rules[1].data) + + # Record with NA-CA-* only rule which is removed + desired = Zone('unit.tests.', []) + record = Record.new(desired, 'a', { + 'ttl': 30, + 'type': 'A', + 'value': '1.2.3.4', + 'dynamic': { + 'pools': { + 'one': { + 'values': [{ + 'value': '1.2.3.4', + }], + }, + 'two': { + 'values': [{ + 'value': '2.2.3.4', + }], + }, + }, + 'rules': [{ + 'geos': ['NA-CA-BC'], + 'pool': 'two', + }, { + 'pool': 'one', + }], + }, + }) + desired.add_record(record) + got = provider.process_desired_zone(desired) + self.assertEquals(1, len(list(got.records)[0].dynamic.rules)) + self.assertFalse('geos' in list(got.records)[0].dynamic.rules[0].data) + + # Record with NA-CA-* rule combined with other geos, filtered + desired = Zone('unit.tests.', []) + record = Record.new(desired, 'a', { + 'ttl': 30, + 'type': 'A', + 'value': '1.2.3.4', + 'dynamic': { + 'pools': { + 'one': { + 'values': [{ + 'value': '1.2.3.4', + }], + }, + 'two': { + 'values': [{ + 'value': '2.2.3.4', + }], + }, + }, + 'rules': [{ + 'geos': ['EU', 'NA-CA-NB', 'NA-US-OR'], + 'pool': 'two', + }, { + 'pool': 'one', + }], + }, + }) + desired.add_record(record) + got = provider.process_desired_zone(desired) + self.assertEquals(2, len(list(got.records)[0].dynamic.rules)) + self.assertEquals(['EU', 'NA-US-OR'], + list(got.records)[0].dynamic.rules[0].data['geos']) + self.assertFalse('geos' in list(got.records)[0].dynamic.rules[1].data) + def test_populate_with_fallback(self): provider, stubber = self._get_stubbed_fallback_auth_provider() From d77e7d485b83664783ab03aa107c27d7397c95ba Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Tue, 17 Aug 2021 20:32:14 -0700 Subject: [PATCH 327/358] Remove some debugging pprints --- octodns/provider/route53.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index 6b2fecb..8841df4 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -931,8 +931,6 @@ class Route53Provider(BaseProvider): if getattr(record, 'dynamic', False): # Make a copy of the record in case we have to muck with it record = record.copy() - from pprint import pprint - pprint((record, record.data)) dynamic = record.dynamic rules = [] for i, rule in enumerate(dynamic.rules): @@ -957,8 +955,6 @@ class Route53Provider(BaseProvider): dynamic.rules = rules - pprint((record, record.data)) - ret.add_record(record) return super(Route53Provider, self).process_desired_zone(ret) From 5b0e47f31fc0c0e0c6e4d59ffcb1b4ec0a10032e Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Wed, 18 Aug 2021 10:07:21 -0700 Subject: [PATCH 328/358] Cleanup and test of _process_desired_zone and supports_warn_or_except --- octodns/provider/__init__.py | 4 ++++ octodns/provider/base.py | 33 ++++++++++++++++---------- octodns/provider/route53.py | 4 ++-- tests/test_octodns_provider_base.py | 27 ++++++++++++++++++++- tests/test_octodns_provider_route53.py | 10 ++++---- 5 files changed, 58 insertions(+), 20 deletions(-) diff --git a/octodns/provider/__init__.py b/octodns/provider/__init__.py index 14ccf18..dbbaaa8 100644 --- a/octodns/provider/__init__.py +++ b/octodns/provider/__init__.py @@ -4,3 +4,7 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals + + +class ProviderException(Exception): + pass diff --git a/octodns/provider/base.py b/octodns/provider/base.py index 238a68a..433eab8 100644 --- a/octodns/provider/base.py +++ b/octodns/provider/base.py @@ -10,17 +10,15 @@ from six import text_type from ..source.base import BaseSource from ..zone import Zone from .plan import Plan - - -class ProviderException(Exception): - pass +from . import ProviderException class BaseProvider(BaseSource): def __init__(self, id, apply_disabled=False, update_pcent_threshold=Plan.MAX_SAFE_UPDATE_PCENT, - delete_pcent_threshold=Plan.MAX_SAFE_DELETE_PCENT): + delete_pcent_threshold=Plan.MAX_SAFE_DELETE_PCENT, + strict_supports=False): super(BaseProvider, self).__init__(id) self.log.debug('__init__: id=%s, apply_disabled=%s, ' 'update_pcent_threshold=%.2f, ' @@ -32,6 +30,21 @@ class BaseProvider(BaseSource): self.apply_disabled = apply_disabled self.update_pcent_threshold = update_pcent_threshold self.delete_pcent_threshold = delete_pcent_threshold + self.strict_supports = strict_supports + + def _process_desired_zone(self, desired): + ''' + An opportunity for providers to modify that desired zone records before + planning. + + - Must do their work and then call super with the results of that work + - Must not modify the `desired` parameter or its records and should + make a copy of anything it's modifying + - Must call supports_warn_or_except with information about any changes + that are made to have them logged or throw errors depending on the + configuration + ''' + return desired def _include_change(self, change): ''' @@ -49,18 +62,14 @@ class BaseProvider(BaseSource): return [] def supports_warn_or_except(self, msg): - # TODO: base class param to control warn vs except - if False: - raise ProviderException(msg) + if self.strict_supports: + raise ProviderException('{}: {}'.format(self.id, msg)) self.log.warning(msg) - def process_desired_zone(self, desired): - return desired - def plan(self, desired, processors=[]): self.log.info('plan: desired=%s', desired.name) - desired = self.process_desired_zone(desired) + desired = self._process_desired_zone(desired) existing = Zone(desired.name, desired.sub_zones) exists = self.populate(existing, target=True, lenient=True) diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index 8841df4..8552a8c 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -925,7 +925,7 @@ class Route53Provider(BaseProvider): return data - def process_desired_zone(self, desired): + def _process_desired_zone(self, desired): ret = Zone(desired.name, desired.sub_zones) for record in desired.records: if getattr(record, 'dynamic', False): @@ -957,7 +957,7 @@ class Route53Provider(BaseProvider): ret.add_record(record) - return super(Route53Provider, self).process_desired_zone(ret) + return super(Route53Provider, self)._process_desired_zone(ret) def populate(self, zone, target=False, lenient=False): self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, diff --git a/tests/test_octodns_provider_base.py b/tests/test_octodns_provider_base.py index 4dfce48..b748762 100644 --- a/tests/test_octodns_provider_base.py +++ b/tests/test_octodns_provider_base.py @@ -6,11 +6,12 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals from logging import getLogger +from mock import MagicMock, call from six import text_type from unittest import TestCase from octodns.processor.base import BaseProcessor -from octodns.provider.base import BaseProvider +from octodns.provider.base import BaseProvider, ProviderException from octodns.provider.plan import Plan, UnsafePlan from octodns.record import Create, Delete, Record, Update from octodns.zone import Zone @@ -429,3 +430,27 @@ class TestBaseProvider(TestCase): delete_pcent_threshold=safe_pcent).raise_if_unsafe() self.assertTrue('Too many deletes' in text_type(ctx.exception)) + + def test_supports_warn_or_except(self): + class MinimalProvider(BaseProvider): + SUPPORTS = set() + SUPPORTS_GEO = False + + def __init__(self, **kwargs): + self.log = MagicMock() + super(MinimalProvider, self).__init__('minimal', **kwargs) + + normal = MinimalProvider(strict_supports=False) + # Should log and not expect + normal.supports_warn_or_except('Hello World!') + normal.log.warning.assert_called_once() + normal.log.warning.assert_has_calls([ + call('Hello World!') + ]) + + strict = MinimalProvider(strict_supports=True) + # Should log and not expect + with self.assertRaises(ProviderException) as ctx: + strict.supports_warn_or_except('Hello World!') + self.assertEquals('minimal: Hello World!', text_type(ctx.exception)) + strict.log.warning.assert_not_called() diff --git a/tests/test_octodns_provider_route53.py b/tests/test_octodns_provider_route53.py index 34124ee..b3e5ba4 100644 --- a/tests/test_octodns_provider_route53.py +++ b/tests/test_octodns_provider_route53.py @@ -399,7 +399,7 @@ class TestRoute53Provider(TestCase): # No records, essentially a no-op desired = Zone('unit.tests.', []) - got = provider.process_desired_zone(desired) + got = provider._process_desired_zone(desired) self.assertEquals(desired.records, got.records) # Record without any geos @@ -422,7 +422,7 @@ class TestRoute53Provider(TestCase): }, }) desired.add_record(record) - got = provider.process_desired_zone(desired) + got = provider._process_desired_zone(desired) self.assertEquals(desired.records, got.records) self.assertEquals(1, len(list(got.records)[0].dynamic.rules)) self.assertFalse('geos' in list(got.records)[0].dynamic.rules[0].data) @@ -455,7 +455,7 @@ class TestRoute53Provider(TestCase): }, }) desired.add_record(record) - got = provider.process_desired_zone(desired) + got = provider._process_desired_zone(desired) self.assertEquals(2, len(list(got.records)[0].dynamic.rules)) self.assertEquals(['EU', 'NA-US-OR'], list(got.records)[0].dynamic.rules[0].data['geos']) @@ -489,7 +489,7 @@ class TestRoute53Provider(TestCase): }, }) desired.add_record(record) - got = provider.process_desired_zone(desired) + got = provider._process_desired_zone(desired) self.assertEquals(1, len(list(got.records)[0].dynamic.rules)) self.assertFalse('geos' in list(got.records)[0].dynamic.rules[0].data) @@ -521,7 +521,7 @@ class TestRoute53Provider(TestCase): }, }) desired.add_record(record) - got = provider.process_desired_zone(desired) + got = provider._process_desired_zone(desired) self.assertEquals(2, len(list(got.records)[0].dynamic.rules)) self.assertEquals(['EU', 'NA-US-OR'], list(got.records)[0].dynamic.rules[0].data['geos']) From 8ddbb389aba255f5f8b5da993b0aff781136cbac Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Wed, 18 Aug 2021 11:22:12 -0700 Subject: [PATCH 329/358] method to custom-process desired zone --- octodns/provider/base.py | 50 ++++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 27 deletions(-) diff --git a/octodns/provider/base.py b/octodns/provider/base.py index fcb557a..166f681 100644 --- a/octodns/provider/base.py +++ b/octodns/provider/base.py @@ -7,7 +7,6 @@ from __future__ import absolute_import, division, print_function, \ from six import text_type -from ..record import Record from ..source.base import BaseSource from ..zone import Zone from .plan import Plan @@ -45,34 +44,28 @@ class BaseProvider(BaseSource): ''' return [] - def _process_change(self, change): + def _process_desired_zone(self, desired): ''' - Process/manipulate each change for feature-specific corner cases + Providers can use this method to make any custom changes to the + desired zone. ''' - if not self._include_change(change): - return False - - # Multi value PTR records - record = change.new - if record and record._type == 'PTR' and len(record.values) > 1 and \ - not self.SUPPORTS_MUTLIVALUE_PTR: - # replace with a single-value copy - change.new = Record.new(record.zone, record.name, { - 'type': 'PTR', - 'ttl': record.ttl, - 'values': [record.value], - }, source=record.source) - - existing = change.existing - if existing and not existing.changes(change.new, self): - # if new single-value replacement turns out to be the same, - # skip the change - return False - - self.log.warn('does not support multi-value PTR records; will ' - 'use only %s for %s', record.value, record.fqdn) + if self.SUPPORTS_MUTLIVALUE_PTR: + # nothing do here + return desired - return True + new_desired = Zone(desired.name, desired.sub_zones) + for record in desired.records: + if record._type == 'PTR' and len(record.values) > 1: + # replace with a single-value copy + self.log.warn('does not support multi-value PTR records; ' + 'will use only %s for %s', record.value, + record.fqdn) + record = record.copy() + record.values = [record.values[0]] + + new_desired.add_record(record) + + return new_desired def plan(self, desired, processors=[]): self.log.info('plan: desired=%s', desired.name) @@ -88,12 +81,15 @@ class BaseProvider(BaseSource): for processor in processors: existing = processor.process_target_zone(existing, target=self) + # process desired zone for any custom zone/record modification + desired = self._process_desired_zone(desired) + # compute the changes at the zone/record level changes = existing.changes(desired, self) # allow the provider to filter out false positives before = len(changes) - changes = [c for c in changes if self._process_change(c)] + changes = [c for c in changes if self._include_change(c)] after = len(changes) if before != after: self.log.info('plan: filtered out %s changes', before - after) From 2021fcc00cafe20b7e382fc2e3f9503d7859e4bb Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Wed, 18 Aug 2021 11:25:43 -0700 Subject: [PATCH 330/358] yaml supports multi value PTR --- octodns/provider/yaml.py | 1 + 1 file changed, 1 insertion(+) diff --git a/octodns/provider/yaml.py b/octodns/provider/yaml.py index b3dd2d9..803bbd4 100644 --- a/octodns/provider/yaml.py +++ b/octodns/provider/yaml.py @@ -104,6 +104,7 @@ class YamlProvider(BaseProvider): ''' SUPPORTS_GEO = True SUPPORTS_DYNAMIC = True + SUPPORTS_MUTLIVALUE_PTR = True SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CAA', 'CNAME', 'DNAME', 'LOC', 'MX', 'NAPTR', 'NS', 'PTR', 'SSHFP', 'SPF', 'SRV', 'TXT', 'URLFWD')) From 4517df555d69335e2e6afc8d3d3ee777398e6b1b Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Wed, 18 Aug 2021 11:38:38 -0700 Subject: [PATCH 331/358] add tests --- tests/test_octodns_provider_base.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/test_octodns_provider_base.py b/tests/test_octodns_provider_base.py index 4dfce48..28abfd3 100644 --- a/tests/test_octodns_provider_base.py +++ b/tests/test_octodns_provider_base.py @@ -230,6 +230,20 @@ class TestBaseProvider(TestCase): # We filtered out the only change self.assertFalse(plan) + def test_process_desired_zone(self): + zone1 = Zone('unit.tests.', []) + record1 = Record.new(zone1, 'ptr', { + 'type': 'PTR', + 'ttl': 3600, + 'values': ['foo.com.', 'bar.com.'], + }) + zone1.add_record(record1) + + zone2 = HelperProvider('hasptr')._process_desired_zone(zone1) + record2 = list(zone2.records)[0] + + self.assertEqual(len(record2.values), 1) + def test_safe_none(self): # No changes is safe Plan(None, None, [], True).raise_if_unsafe() From 7b748de2b3d0dd4b86f015b28bf2c47e3f5cd429 Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Wed, 18 Aug 2021 12:14:50 -0700 Subject: [PATCH 332/358] use the PTR value that is shown in logs --- octodns/provider/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/base.py b/octodns/provider/base.py index 166f681..c5a8dc9 100644 --- a/octodns/provider/base.py +++ b/octodns/provider/base.py @@ -61,7 +61,7 @@ class BaseProvider(BaseSource): 'will use only %s for %s', record.value, record.fqdn) record = record.copy() - record.values = [record.values[0]] + record.values = [record.value] new_desired.add_record(record) From 65f0bfc2435ffcdf7738692915d0cd42b6d66c7d Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Wed, 18 Aug 2021 12:35:49 -0700 Subject: [PATCH 333/358] Update multi-value PTR warn to supports_warn_or_except --- octodns/provider/base.py | 7 ++++--- tests/test_octodns_provider_base.py | 1 + 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/octodns/provider/base.py b/octodns/provider/base.py index e952f30..548662a 100644 --- a/octodns/provider/base.py +++ b/octodns/provider/base.py @@ -52,9 +52,10 @@ class BaseProvider(BaseSource): for record in desired.records: if record._type == 'PTR' and len(record.values) > 1: # replace with a single-value copy - self.log.warn('does not support multi-value PTR records; ' - 'will use only %s for %s', record.value, - record.fqdn) + self.supports_warn_or_except('does not support multi-value ' + 'PTR records; will use only {} ' + 'for {}'.format(record.value, + record.fqdn)) record = record.copy() record.values = [record.value] diff --git a/tests/test_octodns_provider_base.py b/tests/test_octodns_provider_base.py index 0927878..62e6587 100644 --- a/tests/test_octodns_provider_base.py +++ b/tests/test_octodns_provider_base.py @@ -22,6 +22,7 @@ class HelperProvider(BaseProvider): SUPPORTS = set(('A',)) id = 'test' + strict_supports = False def __init__(self, extra_changes=[], apply_disabled=False, include_change_callback=None): From 08f9ec56a3e898f1324fb9d9d679e3d7469da889 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Wed, 18 Aug 2021 16:25:13 -0700 Subject: [PATCH 334/358] Rework supports_warn_or_except to msg and fallback --- octodns/provider/base.py | 13 +++++++------ octodns/provider/route53.py | 16 ++++++++++------ tests/test_octodns_provider_base.py | 6 +++--- 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/octodns/provider/base.py b/octodns/provider/base.py index 548662a..16cad7a 100644 --- a/octodns/provider/base.py +++ b/octodns/provider/base.py @@ -52,10 +52,11 @@ class BaseProvider(BaseSource): for record in desired.records: if record._type == 'PTR' and len(record.values) > 1: # replace with a single-value copy - self.supports_warn_or_except('does not support multi-value ' - 'PTR records; will use only {} ' - 'for {}'.format(record.value, - record.fqdn)) + msg = 'multi-value PTR records not supported for {}' \ + .format(record.fqdn) + fallback = 'falling back to single value, {}' \ + .format(record.value) + self.supports_warn_or_except(msg, fallback) record = record.copy() record.values = [record.value] @@ -78,10 +79,10 @@ class BaseProvider(BaseSource): ''' return [] - def supports_warn_or_except(self, msg): + def supports_warn_or_except(self, msg, fallback): if self.strict_supports: raise ProviderException('{}: {}'.format(self.id, msg)) - self.log.warning(msg) + self.log.warning('{}; {}'.format(msg, fallback)) def plan(self, desired, processors=[]): self.log.info('plan: desired=%s', desired.name) diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index 8552a8c..477a53e 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -942,14 +942,18 @@ class Route53Provider(BaseProvider): if not g.startswith('NA-CA-')] if not filtered_geos: # We've removed all geos, we'll have to skip this rule - msg = 'NA-CA-* not supported resulting in ' \ - 'empty geo target, skipping rule {}'.format(i) - self.supports_warn_or_except(msg) + msg = 'NA-CA-* not supported for {}' \ + .format(record.fqdn) + fallback = 'skipping rule {}'.format(i) + self.supports_warn_or_except(msg, fallback) continue elif geos != filtered_geos: - msg = 'NA-CA-* not supported resulting in ' \ - 'empty geo target, skipping rule {}'.format(i) - self.supports_warn_or_except(msg) + msg = 'NA-CA-* not supported for {}' \ + .format(record.fqdn) + fallback = 'filtering rule {} from ({}) to ({})' \ + .format(i, ', '.join(geos), + ', '.join(filtered_geos)) + self.supports_warn_or_except(msg, fallback) rule.data['geos'] = filtered_geos rules.append(rule) diff --git a/tests/test_octodns_provider_base.py b/tests/test_octodns_provider_base.py index 62e6587..46afd89 100644 --- a/tests/test_octodns_provider_base.py +++ b/tests/test_octodns_provider_base.py @@ -457,15 +457,15 @@ class TestBaseProvider(TestCase): normal = MinimalProvider(strict_supports=False) # Should log and not expect - normal.supports_warn_or_except('Hello World!') + normal.supports_warn_or_except('Hello World!', 'Goodbye') normal.log.warning.assert_called_once() normal.log.warning.assert_has_calls([ - call('Hello World!') + call('Hello World!; Goodbye') ]) strict = MinimalProvider(strict_supports=True) # Should log and not expect with self.assertRaises(ProviderException) as ctx: - strict.supports_warn_or_except('Hello World!') + strict.supports_warn_or_except('Hello World!', 'Will not see') self.assertEquals('minimal: Hello World!', text_type(ctx.exception)) strict.log.warning.assert_not_called() From 074de669888582fc8e8857f0104949f84a56d52f Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Thu, 19 Aug 2021 18:18:07 -0700 Subject: [PATCH 335/358] Normalize IP addresses --- octodns/record/__init__.py | 9 +++++++-- tests/test_octodns_record.py | 16 +++++++++++++--- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/octodns/record/__init__.py b/octodns/record/__init__.py index 77663a5..a8dd834 100644 --- a/octodns/record/__init__.py +++ b/octodns/record/__init__.py @@ -749,8 +749,13 @@ class _IpList(object): @classmethod def process(cls, values): - # Translating None into '' so that the list will be sortable in python3 - return [v if v is not None else '' for v in values] + # Translating None into '' so that the list will be sortable in + # python3, get everything to str first + values = [text_type(v) if v is not None else '' for v in values] + # Now round trip all non-'' through the address type and back to a str + # to normalize the address representation. + return [text_type(cls._address_type(v)) if v != '' else '' + for v in values] class Ipv4List(_IpList): diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index 886dbfb..c848853 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -259,11 +259,21 @@ class TestRecord(TestCase): self.assertEquals(b_data, b.data) def test_aaaa(self): - a_values = ['2001:0db8:3c4d:0015:0000:0000:1a2f:1a2b', - '2001:0db8:3c4d:0015:0000:0000:1a2f:1a3b'] - b_value = '2001:0db8:3c4d:0015:0000:0000:1a2f:1a4b' + a_values = ['2001:db8:3c4d:15::1a2f:1a2b', + '2001:db8:3c4d:15::1a2f:1a3b'] + b_value = '2001:db8:3c4d:15::1a2f:1a4b' self.assertMultipleValues(AaaaRecord, a_values, b_value) + # Specifically validate that we normalize IPv6 addresses + values = ['2001:db8:3c4d:15:0000:0000:1a2f:1a2b', + '2001:0db8:3c4d:0015::1a2f:1a3b'] + data = { + 'ttl': 30, + 'values': values, + } + record = AaaaRecord(self.zone, 'aaaa', data) + self.assertEquals(a_values, record.values) + def assertSingleValue(self, _type, a_value, b_value): a_data = {'ttl': 30, 'value': a_value} a = _type(self.zone, 'a', a_data) From 5ab238c611c1d46efe9e8439e9910d4e78450e7e Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Fri, 20 Aug 2021 13:41:04 -0700 Subject: [PATCH 336/358] Cache NS1 zones and records for faster re-retrival --- octodns/provider/ns1.py | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index f87538d..bab7c7b 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -85,6 +85,8 @@ class Ns1Client(object): self._feeds_for_monitors = None self._monitors_cache = None self._notifylists_cache = None + self._zones_cache = {} + self._records_cache = {} @property def datasource_id(self): @@ -196,19 +198,43 @@ class Ns1Client(object): return self._try(self._records.create, zone, domain, _type, **params) def records_delete(self, zone, domain, _type): + try: + # remove record from cache + del self._records_cache[zone][domain][_type] + # remove record's zone from cache + del self._zones_cache[zone] + except KeyError: + # never mind if record is not found in cache + pass return self._try(self._records.delete, zone, domain, _type) def records_retrieve(self, zone, domain, _type): - return self._try(self._records.retrieve, zone, domain, _type) + cached = self._records_cache.setdefault(zone, {}) \ + .setdefault(domain, {}) + if _type not in cached: + cached[_type] = self._try(self._records.retrieve, zone, domain, + _type) + return cached[_type] def records_update(self, zone, domain, _type, **params): + try: + # remove record from cache + del self._records_cache[zone][domain][_type] + # remove record's zone from cache + del self._zones_cache[zone] + except KeyError: + # never mind if record is not found in cache + pass return self._try(self._records.update, zone, domain, _type, **params) def zones_create(self, name): return self._try(self._zones.create, name) def zones_retrieve(self, name): - return self._try(self._zones.retrieve, name) + if name not in self._zones_cache: + self._zones_cache[name] = self._try(self._zones.retrieve, name) + print(f'insert {name} to cache with val {self._zones_cache[name]}') + return self._zones_cache[name] def _try(self, method, *args, **kwargs): tries = self.retry_count From 8c04508a86739d434f2506ee9c46cac9548b2aa8 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 21 Aug 2021 10:11:23 -0700 Subject: [PATCH 337/358] Implement & test Zone.copy (shallow) and utilize it in processors --- octodns/processor/acme.py | 9 +++-- octodns/processor/base.py | 3 -- octodns/processor/filter.py | 12 +++---- octodns/processor/ownership.py | 4 +-- octodns/zone.py | 55 ++++++++++++++++++++++++++-- tests/test_octodns_manager.py | 6 ++-- tests/test_octodns_provider_base.py | 6 ++-- tests/test_octodns_zone.py | 56 +++++++++++++++++++++++++++++ 8 files changed, 126 insertions(+), 25 deletions(-) diff --git a/octodns/processor/acme.py b/octodns/processor/acme.py index 685130d..5aa10ec 100644 --- a/octodns/processor/acme.py +++ b/octodns/processor/acme.py @@ -33,7 +33,7 @@ class AcmeMangingProcessor(BaseProcessor): self._owned = set() def process_source_zone(self, zone, *args, **kwargs): - ret = self._clone_zone(zone) + ret = zone.copy() for record in zone.records: if record._type == 'TXT' and \ record.name.startswith('_acme-challenge'): @@ -45,11 +45,11 @@ class AcmeMangingProcessor(BaseProcessor): # This assumes we'll see things as sources before targets, # which is the case... self._owned.add(record) - ret.add_record(record) + ret.add_record(record, replace=True) return ret def process_target_zone(self, zone, *args, **kwargs): - ret = self._clone_zone(zone) + ret = zone.copy() for record in zone.records: # Uses a startswith rather than == to ignore subdomain challenges, # e.g. _acme-challenge.foo.domain.com when managing domain.com @@ -58,7 +58,6 @@ class AcmeMangingProcessor(BaseProcessor): '*octoDNS*' not in record.values and \ record not in self._owned: self.log.info('_process: ignoring %s', record.fqdn) - continue - ret.add_record(record) + ret.remove_record(record) return ret diff --git a/octodns/processor/base.py b/octodns/processor/base.py index 327b1c2..51f19dd 100644 --- a/octodns/processor/base.py +++ b/octodns/processor/base.py @@ -13,9 +13,6 @@ class BaseProcessor(object): def __init__(self, name): self.name = name - def _clone_zone(self, zone): - return Zone(zone.name, sub_zones=zone.sub_zones) - def process_source_zone(self, zone, sources): # sources may be empty, as will be the case for aliased zones return zone diff --git a/octodns/processor/filter.py b/octodns/processor/filter.py index 369e987..456048b 100644 --- a/octodns/processor/filter.py +++ b/octodns/processor/filter.py @@ -15,10 +15,10 @@ class TypeAllowlistFilter(BaseProcessor): self.allowlist = set(allowlist) def _process(self, zone, *args, **kwargs): - ret = self._clone_zone(zone) + ret = zone.copy() for record in zone.records: - if record._type in self.allowlist: - ret.add_record(record) + if record._type not in self.allowlist: + ret.remove_record(record) return ret @@ -33,10 +33,10 @@ class TypeRejectlistFilter(BaseProcessor): self.rejectlist = set(rejectlist) def _process(self, zone, *args, **kwargs): - ret = self._clone_zone(zone) + ret = zone.copy() for record in zone.records: - if record._type not in self.rejectlist: - ret.add_record(record) + if record._type in self.rejectlist: + ret.remove_record(record) return ret diff --git a/octodns/processor/ownership.py b/octodns/processor/ownership.py index 172f349..8281dd6 100644 --- a/octodns/processor/ownership.py +++ b/octodns/processor/ownership.py @@ -25,10 +25,8 @@ class OwnershipProcessor(BaseProcessor): self._txt_values = [txt_value] def process_source_zone(self, zone, *args, **kwargs): - ret = self._clone_zone(zone) + ret = zone.copy() for record in zone.records: - # Always copy over the source records - ret.add_record(record) # Then create and add an ownership TXT for each of them record_name = record.name.replace('*', '_wildcard') if record.name: diff --git a/octodns/zone.py b/octodns/zone.py index 5f099ac..dcc07c3 100644 --- a/octodns/zone.py +++ b/octodns/zone.py @@ -49,16 +49,26 @@ class Zone(object): # optional trailing . b/c some sources don't have it on their fqdn self._name_re = re.compile(r'\.?{}?$'.format(name)) + # Copy-on-write semantics support, when `not None` this property will + # point to a location with records for this `Zone`. Once `hydrated` + # this property will be set to None + self._origin = None + self.log.debug('__init__: zone=%s, sub_zones=%s', self, sub_zones) @property def records(self): + if self._origin: + return self._origin.records return set([r for _, node in self._records.items() for r in node]) def hostname_from_fqdn(self, fqdn): return self._name_re.sub('', fqdn) def add_record(self, record, replace=False, lenient=False): + if self._origin: + self.hydrate() + name = record.name last = name.split('.')[-1] @@ -94,10 +104,14 @@ class Zone(object): node.add(record) - def _remove_record(self, record): - 'Only for use in tests' + def remove_record(self, record): + if self._origin: + self.hydrate() self._records[record.name].discard(record) + # TODO: delete this + _remove_record = remove_record + def changes(self, desired, target): self.log.debug('changes: zone=%s, target=%s', self, target) @@ -184,5 +198,42 @@ class Zone(object): return changes + def hydrate(self): + ''' + Take a shallow copy Zone and make it a deeper copy holding its own + reference to records. These records will still be the originals and + they should not be modified. Changes should be made by calling + `add_record`, often with `replace=True`, and/or `remove_record`. + + Note: This method does not need to be called under normal circumstances + as `add_record` and `remove_record` will automatically call it when + appropriate. + ''' + origin = self._origin + if origin is None: + return False + # Need to clear this before the copy to prevent recursion + self._origin = None + for record in origin.records: + # Use lenient as we're copying origin and should take its records + # regardless + self.add_record(record, lenient=True) + return True + + def copy(self): + ''' + Copy-on-write semantics support. This method will create a shallow + clone of the zone which will be hydrated the first time `add_record` or + `remove_record` is called. + + This allows low-cost copies of things to be made in situations where + changes are unlikely and only incurs the "expense" of actually + copying the records when required. The actual record copy will not be + "deep" meaning that records should not be modified directly. + ''' + copy = Zone(self.name, self.sub_zones) + copy._origin = self + return copy + def __repr__(self): return 'Zone<{}>'.format(self.name) diff --git a/tests/test_octodns_manager.py b/tests/test_octodns_manager.py index 96f67fd..c9362d4 100644 --- a/tests/test_octodns_manager.py +++ b/tests/test_octodns_manager.py @@ -464,7 +464,7 @@ class TestManager(TestCase): class MockProcessor(BaseProcessor): def process_source_zone(self, zone, sources): - zone = self._clone_zone(zone) + zone = zone.copy() zone.add_record(record) return zone @@ -480,7 +480,7 @@ class TestManager(TestCase): class MockProcessor(BaseProcessor): def process_target_zone(self, zone, target): - zone = self._clone_zone(zone) + zone = zone.copy() zone.add_record(record) return zone @@ -496,7 +496,7 @@ class TestManager(TestCase): class MockProcessor(BaseProcessor): def process_target_zone(self, zone, target): - zone = self._clone_zone(zone) + zone = zone.copy() zone.add_record(record) return zone diff --git a/tests/test_octodns_provider_base.py b/tests/test_octodns_provider_base.py index 46afd89..501177e 100644 --- a/tests/test_octodns_provider_base.py +++ b/tests/test_octodns_provider_base.py @@ -61,11 +61,11 @@ class TrickyProcessor(BaseProcessor): self.existing = existing self.target = target - new = self._clone_zone(existing) + new = existing.copy() for record in existing.records: - new.add_record(record) + new.add_record(record, replace=True) for record in self.add_during_process_target_zone: - new.add_record(record) + new.add_record(record, replace=True) return new diff --git a/tests/test_octodns_zone.py b/tests/test_octodns_zone.py index 1d000f2..ddc2157 100644 --- a/tests/test_octodns_zone.py +++ b/tests/test_octodns_zone.py @@ -355,3 +355,59 @@ class TestZone(TestCase): self.assertTrue(zone_missing.changes(zone_normal, provider)) self.assertFalse(zone_missing.changes(zone_included, provider)) + + def assertEqualsNameAndValues(self, a, b): + a = dict([(r.name, r.values[0]) for r in a]) + b = dict([(r.name, r.values[0]) for r in b]) + self.assertEquals(a, b) + + def test_copy(self): + zone = Zone('unit.tests.', []) + + a = ARecord(zone, 'a', {'ttl': 42, 'value': '1.1.1.1'}) + zone.add_record(a) + b = ARecord(zone, 'b', {'ttl': 42, 'value': '1.1.1.2'}) + zone.add_record(b) + + # Sanity check + self.assertEqualsNameAndValues(set((a, b)), zone.records) + + copy = zone.copy() + # We have an origin set and it is the source/original zone + self.assertEquals(zone, copy._origin) + # Our records are zone's records to start (references) + self.assertEqualsNameAndValues(zone.records, copy.records) + + # If we try and change something that's already there we realize and + # then get an error about a duplicate + b_prime = ARecord(zone, 'b', {'ttl': 42, 'value': '1.1.1.3'}) + with self.assertRaises(DuplicateRecordException): + copy.add_record(b_prime) + self.assertIsNone(copy._origin) + # Unchanged, straight copies + self.assertEqualsNameAndValues(zone.records, copy.records) + + # If we add with replace things will be realized and the record will + # have changed + copy = zone.copy() + copy.add_record(b_prime, replace=True) + self.assertIsNone(copy._origin) + self.assertEqualsNameAndValues(set((a, b_prime)), copy.records) + + # If we add another record, things are reliazed and it has been added + copy = zone.copy() + c = ARecord(zone, 'c', {'ttl': 42, 'value': '1.1.1.3'}) + copy.add_record(c) + self.assertEqualsNameAndValues(set((a, b, c)), copy.records) + + # If we remove a record, things are reliazed and it has been removed + copy = zone.copy() + copy.remove_record(a) + self.assertEqualsNameAndValues(set((b,)), copy.records) + + # Re-realizing is a noop + copy = zone.copy() + # Happens the first time + self.assertTrue(copy.hydrate()) + # Doesn't the second + self.assertFalse(copy.hydrate()) From fe013b21e31ce3cae7023997124f4ad03386c829 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 21 Aug 2021 10:29:48 -0700 Subject: [PATCH 338/358] Utilize shallow copies in Provider._process_desired_zone and Route53Provider._process_desired_zone --- octodns/processor/base.py | 2 -- octodns/provider/base.py | 27 +++++++++++++++++---------- octodns/provider/route53.py | 11 +++++------ 3 files changed, 22 insertions(+), 18 deletions(-) diff --git a/octodns/processor/base.py b/octodns/processor/base.py index 51f19dd..7e8a63a 100644 --- a/octodns/processor/base.py +++ b/octodns/processor/base.py @@ -5,8 +5,6 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -from ..zone import Zone - class BaseProcessor(object): diff --git a/octodns/provider/base.py b/octodns/provider/base.py index 16cad7a..3d6c62e 100644 --- a/octodns/provider/base.py +++ b/octodns/provider/base.py @@ -34,21 +34,29 @@ class BaseProvider(BaseSource): def _process_desired_zone(self, desired): ''' - An opportunity for providers to modify that desired zone records before - planning. - - - Must do their work and then call super with the results of that work - - Must not modify the `desired` parameter or its records and should - make a copy of anything it's modifying + An opportunity for providers to modify the desired zone records before + planning. `desired` is a "shallow" copy, see `Zone.copy` for more + information + + - Must do their work and then call `super` with the results of that + work, returning the result of the `super` call. + - Must not modify `desired` directly, should call `desired.copy` and + modify the shallow copy returned from that. + - Must not modify records directly, `record.copy` should be called, + the results of which can be modified, and then `Zone.add_record` may + be used with `replace=True` + - Must call `Zone.remove_record` to remove records from the copy of + `desired` - Must call supports_warn_or_except with information about any changes that are made to have them logged or throw errors depending on the - configuration + provider configuration ''' if self.SUPPORTS_MUTLIVALUE_PTR: # nothing do here return desired - new_desired = Zone(desired.name, desired.sub_zones) + # Shallow copy + new_desired = desired.copy() for record in desired.records: if record._type == 'PTR' and len(record.values) > 1: # replace with a single-value copy @@ -59,8 +67,7 @@ class BaseProvider(BaseSource): self.supports_warn_or_except(msg, fallback) record = record.copy() record.values = [record.value] - - new_desired.add_record(record) + new_desired.add_record(record, replace=True) return new_desired diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index 477a53e..37ce413 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -19,7 +19,6 @@ from six import text_type from ..equality import EqualityTupleMixin from ..record import Record, Update from ..record.geo import GeoCodes -from ..zone import Zone from .base import BaseProvider octal_re = re.compile(r'\\(\d\d\d)') @@ -926,11 +925,10 @@ class Route53Provider(BaseProvider): return data def _process_desired_zone(self, desired): - ret = Zone(desired.name, desired.sub_zones) + ret = desired.copy() for record in desired.records: if getattr(record, 'dynamic', False): # Make a copy of the record in case we have to muck with it - record = record.copy() dynamic = record.dynamic rules = [] for i, rule in enumerate(dynamic.rules): @@ -957,9 +955,10 @@ class Route53Provider(BaseProvider): rule.data['geos'] = filtered_geos rules.append(rule) - dynamic.rules = rules - - ret.add_record(record) + if rules != dynamic.rules: + record = record.copy() + record.dynamic.rules = rules + ret.add_record(record, replace=True) return super(Route53Provider, self)._process_desired_zone(ret) From b84b933eb0b4612c66ba3847d8b481584277280f Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 21 Aug 2021 13:41:10 -0700 Subject: [PATCH 339/358] Copy zones early on and allow modifications after that. Doc requirements. --- octodns/processor/acme.py | 18 ++++---- octodns/processor/base.py | 54 ++++++++++++++++++++--- octodns/processor/filter.py | 10 ++--- octodns/processor/ownership.py | 11 +++-- octodns/provider/base.py | 27 ++++++------ octodns/provider/route53.py | 5 +-- tests/test_octodns_processor_filter.py | 16 +++---- tests/test_octodns_processor_ownership.py | 4 +- 8 files changed, 92 insertions(+), 53 deletions(-) diff --git a/octodns/processor/acme.py b/octodns/processor/acme.py index 5aa10ec..2d7c101 100644 --- a/octodns/processor/acme.py +++ b/octodns/processor/acme.py @@ -32,9 +32,8 @@ class AcmeMangingProcessor(BaseProcessor): self._owned = set() - def process_source_zone(self, zone, *args, **kwargs): - ret = zone.copy() - for record in zone.records: + def process_source_zone(self, desired, *args, **kwargs): + for record in desired.records: if record._type == 'TXT' and \ record.name.startswith('_acme-challenge'): # We have a managed acme challenge record (owned by octoDNS) so @@ -45,12 +44,11 @@ class AcmeMangingProcessor(BaseProcessor): # This assumes we'll see things as sources before targets, # which is the case... self._owned.add(record) - ret.add_record(record, replace=True) - return ret + desired.add_record(record, replace=True) + return desired - def process_target_zone(self, zone, *args, **kwargs): - ret = zone.copy() - for record in zone.records: + def process_target_zone(self, existing, *args, **kwargs): + for record in existing.records: # Uses a startswith rather than == to ignore subdomain challenges, # e.g. _acme-challenge.foo.domain.com when managing domain.com if record._type == 'TXT' and \ @@ -58,6 +56,6 @@ class AcmeMangingProcessor(BaseProcessor): '*octoDNS*' not in record.values and \ record not in self._owned: self.log.info('_process: ignoring %s', record.fqdn) - ret.remove_record(record) + existing.remove_record(record) - return ret + return existing diff --git a/octodns/processor/base.py b/octodns/processor/base.py index 7e8a63a..98f2baa 100644 --- a/octodns/processor/base.py +++ b/octodns/processor/base.py @@ -11,14 +11,58 @@ class BaseProcessor(object): def __init__(self, name): self.name = name - def process_source_zone(self, zone, sources): - # sources may be empty, as will be the case for aliased zones - return zone + def process_source_zone(self, desired, sources): + ''' + Called after all sources have completed populate. Provides an + opportunity for the processor to modify the desired `Zone` that targets + will recieve. + + - Will see `desired` after any modifications done by + `Provider._process_desired_zone` and processors configured to run + before this one. + - May modify `desired` directly. + - Must return `desired` which will normally be the `desired` param. + - Must not modify records directly, `record.copy` should be called, + the results of which can be modified, and then `Zone.add_record` may + be used with `replace=True`. + - May call `Zone.remove_record` to remove records from `desired`. + - Sources may be empty, as will be the case for aliased zones. + ''' + return desired - def process_target_zone(self, zone, target): - return zone + def process_target_zone(self, existing, target): + ''' + Called after a target has completed `populate`, before changes are + computed between `existing` and `desired`. This provides an opportunity + to modify the `existing` `Zone`. + + - Will see `existing` after any modifrications done by processors + configured to run before this one. + - May modify `existing` directly. + - Must return `existing` which will normally be the `existing` param. + - Must not modify records directly, `record.copy` should be called, + the results of which can be modified, and then `Zone.add_record` may + be used with `replace=True`. + - May call `Zone.remove_record` to remove records from `existing`. + ''' + return existing def process_plan(self, plan, sources, target): + ''' + Called after the planning phase has completed. Provides an opportunity + for the processors to modify the plan thus changing the actions that + will be displayed and potentially applied. + + - `plan` may be None if no changes were detected, if so a `Plan` may + still be created and returned. + - May modify `plan.changes` directly or create a new `Plan`. + - Does not have to modify `plan.desired` and/or `plan.existing` to line + up with any modifications made to `plan.changes`. + - Should copy over `plan.exists`, `plan.update_pcent_threshold`, and + `plan.delete_pcent_threshold` when creating a new `Plan`. + - Must return a `Plan` which may be `plan` or can be a newly created + one `plan.desired` and `plan.existing` copied over as-is or modified. + ''' # plan may be None if no changes were detected up until now, the # process may still create a plan. # sources may be empty, as will be the case for aliased zones diff --git a/octodns/processor/filter.py b/octodns/processor/filter.py index 456048b..d9b8ee3 100644 --- a/octodns/processor/filter.py +++ b/octodns/processor/filter.py @@ -15,12 +15,11 @@ class TypeAllowlistFilter(BaseProcessor): self.allowlist = set(allowlist) def _process(self, zone, *args, **kwargs): - ret = zone.copy() for record in zone.records: if record._type not in self.allowlist: - ret.remove_record(record) + zone.remove_record(record) - return ret + return zone process_source_zone = _process process_target_zone = _process @@ -33,12 +32,11 @@ class TypeRejectlistFilter(BaseProcessor): self.rejectlist = set(rejectlist) def _process(self, zone, *args, **kwargs): - ret = zone.copy() for record in zone.records: if record._type in self.rejectlist: - ret.remove_record(record) + zone.remove_record(record) - return ret + return zone process_source_zone = _process process_target_zone = _process diff --git a/octodns/processor/ownership.py b/octodns/processor/ownership.py index 8281dd6..42f041d 100644 --- a/octodns/processor/ownership.py +++ b/octodns/processor/ownership.py @@ -24,9 +24,8 @@ class OwnershipProcessor(BaseProcessor): self.txt_value = txt_value self._txt_values = [txt_value] - def process_source_zone(self, zone, *args, **kwargs): - ret = zone.copy() - for record in zone.records: + def process_source_zone(self, desired, *args, **kwargs): + for record in desired.records: # Then create and add an ownership TXT for each of them record_name = record.name.replace('*', '_wildcard') if record.name: @@ -34,14 +33,14 @@ class OwnershipProcessor(BaseProcessor): record_name) else: name = '{}.{}'.format(self.txt_name, record._type) - txt = Record.new(zone, name, { + txt = Record.new(desired, name, { 'type': 'TXT', 'ttl': 60, 'value': self.txt_value, }) - ret.add_record(txt) + desired.add_record(txt) - return ret + return desired def _is_ownership(self, record): return record._type == 'TXT' and \ diff --git a/octodns/provider/base.py b/octodns/provider/base.py index 3d6c62e..c862e2d 100644 --- a/octodns/provider/base.py +++ b/octodns/provider/base.py @@ -38,25 +38,22 @@ class BaseProvider(BaseSource): planning. `desired` is a "shallow" copy, see `Zone.copy` for more information - - Must do their work and then call `super` with the results of that - work, returning the result of the `super` call. - - Must not modify `desired` directly, should call `desired.copy` and - modify the shallow copy returned from that. + - Must call `super` at an appropriate point for their work, generally + that means as the final step of the method, returning the result of + the `super` call. + - May modify `desired` directly. - Must not modify records directly, `record.copy` should be called, the results of which can be modified, and then `Zone.add_record` may - be used with `replace=True` - - Must call `Zone.remove_record` to remove records from the copy of - `desired` + be used with `replace=True`. + - May call `Zone.remove_record` to remove records from `desired`. - Must call supports_warn_or_except with information about any changes that are made to have them logged or throw errors depending on the - provider configuration + provider configuration. ''' if self.SUPPORTS_MUTLIVALUE_PTR: # nothing do here return desired - # Shallow copy - new_desired = desired.copy() for record in desired.records: if record._type == 'PTR' and len(record.values) > 1: # replace with a single-value copy @@ -67,9 +64,9 @@ class BaseProvider(BaseSource): self.supports_warn_or_except(msg, fallback) record = record.copy() record.values = [record.value] - new_desired.add_record(record, replace=True) + desired.add_record(record, replace=True) - return new_desired + return desired def _include_change(self, change): ''' @@ -94,7 +91,11 @@ class BaseProvider(BaseSource): def plan(self, desired, processors=[]): self.log.info('plan: desired=%s', desired.name) - # process desired zone for any custom zone/record modification + # Make a (shallow) copy of the desired state so that everything from + # now on (in this target) can modify it as they see fit without + # worrying about impacting other targets. + desired = desired.copy() + desired = self._process_desired_zone(desired) existing = Zone(desired.name, desired.sub_zones) diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index 37ce413..06f7d8c 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -925,7 +925,6 @@ class Route53Provider(BaseProvider): return data def _process_desired_zone(self, desired): - ret = desired.copy() for record in desired.records: if getattr(record, 'dynamic', False): # Make a copy of the record in case we have to muck with it @@ -958,9 +957,9 @@ class Route53Provider(BaseProvider): if rules != dynamic.rules: record = record.copy() record.dynamic.rules = rules - ret.add_record(record, replace=True) + desired.add_record(record, replace=True) - return super(Route53Provider, self)._process_desired_zone(ret) + return super(Route53Provider, self)._process_desired_zone(desired) def populate(self, zone, target=False, lenient=False): self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, diff --git a/tests/test_octodns_processor_filter.py b/tests/test_octodns_processor_filter.py index f421676..176f7d1 100644 --- a/tests/test_octodns_processor_filter.py +++ b/tests/test_octodns_processor_filter.py @@ -47,20 +47,20 @@ class TestTypeAllowListFilter(TestCase): def test_basics(self): filter_a = TypeAllowlistFilter('only-a', set(('A'))) - got = filter_a.process_source_zone(zone) + got = filter_a.process_source_zone(zone.copy()) self.assertEquals(['a', 'a2'], sorted([r.name for r in got.records])) filter_aaaa = TypeAllowlistFilter('only-aaaa', ('AAAA',)) - got = filter_aaaa.process_source_zone(zone) + got = filter_aaaa.process_source_zone(zone.copy()) self.assertEquals(['aaaa'], sorted([r.name for r in got.records])) filter_txt = TypeAllowlistFilter('only-txt', ['TXT']) - got = filter_txt.process_target_zone(zone) + got = filter_txt.process_target_zone(zone.copy()) self.assertEquals(['txt', 'txt2'], sorted([r.name for r in got.records])) filter_a_aaaa = TypeAllowlistFilter('only-aaaa', set(('A', 'AAAA'))) - got = filter_a_aaaa.process_target_zone(zone) + got = filter_a_aaaa.process_target_zone(zone.copy()) self.assertEquals(['a', 'a2', 'aaaa'], sorted([r.name for r in got.records])) @@ -70,21 +70,21 @@ class TestTypeRejectListFilter(TestCase): def test_basics(self): filter_a = TypeRejectlistFilter('not-a', set(('A'))) - got = filter_a.process_source_zone(zone) + got = filter_a.process_source_zone(zone.copy()) self.assertEquals(['aaaa', 'txt', 'txt2'], sorted([r.name for r in got.records])) filter_aaaa = TypeRejectlistFilter('not-aaaa', ('AAAA',)) - got = filter_aaaa.process_source_zone(zone) + got = filter_aaaa.process_source_zone(zone.copy()) self.assertEquals(['a', 'a2', 'txt', 'txt2'], sorted([r.name for r in got.records])) filter_txt = TypeRejectlistFilter('not-txt', ['TXT']) - got = filter_txt.process_target_zone(zone) + got = filter_txt.process_target_zone(zone.copy()) self.assertEquals(['a', 'a2', 'aaaa'], sorted([r.name for r in got.records])) filter_a_aaaa = TypeRejectlistFilter('not-a-aaaa', set(('A', 'AAAA'))) - got = filter_a_aaaa.process_target_zone(zone) + got = filter_a_aaaa.process_target_zone(zone.copy()) self.assertEquals(['txt', 'txt2'], sorted([r.name for r in got.records])) diff --git a/tests/test_octodns_processor_ownership.py b/tests/test_octodns_processor_ownership.py index 959f4c2..e6b248b 100644 --- a/tests/test_octodns_processor_ownership.py +++ b/tests/test_octodns_processor_ownership.py @@ -55,7 +55,7 @@ class TestOwnershipProcessor(TestCase): def test_process_source_zone(self): ownership = OwnershipProcessor('ownership') - got = ownership.process_source_zone(zone) + got = ownership.process_source_zone(zone.copy()) self.assertEquals([ '', '*', @@ -88,7 +88,7 @@ class TestOwnershipProcessor(TestCase): self.assertFalse(ownership.process_plan(None)) # Nothing exists create both records and ownership - ownership_added = ownership.process_source_zone(zone) + ownership_added = ownership.process_source_zone(zone.copy()) plan = provider.plan(ownership_added) self.assertTrue(plan) # Double the number of records From e06c42c4fc6798688a2a8498ea46cdfb3031f0a8 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 23 Aug 2021 12:41:06 -0700 Subject: [PATCH 340/358] No need to ip_address normalize AAAA in constellix now --- octodns/provider/constellix.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/octodns/provider/constellix.py b/octodns/provider/constellix.py index 5ca89e1..cd0d85b 100644 --- a/octodns/provider/constellix.py +++ b/octodns/provider/constellix.py @@ -8,7 +8,6 @@ from __future__ import absolute_import, division, print_function, \ from collections import defaultdict from requests import Session from base64 import b64encode -from ipaddress import ip_address from six import string_types import hashlib import hmac @@ -138,11 +137,6 @@ class ConstellixClient(object): v['value'] = self._absolutize_value(v['value'], zone_name) - # compress IPv6 addresses - if record['type'] == 'AAAA': - for i, v in enumerate(value): - value[i] = str(ip_address(v)) - return resp def record_create(self, zone_name, record_type, params): From aa3bc8fb9e682bba46ef32b0728b917e6f9f8a2e Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 23 Aug 2021 13:33:06 -0700 Subject: [PATCH 341/358] Correct Route53 IPv6 IPAddress comment --- octodns/provider/route53.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index 477a53e..b4bd557 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -1090,10 +1090,11 @@ class Route53Provider(BaseProvider): health_check, value=None): config = health_check['HealthCheckConfig'] - # So interestingly Route53 normalizes IPAddress which will cause us to - # fail to find see things as equivalent. To work around this we'll - # ip_address's returned object for equivalence - # E.g 2001:4860:4860::8842 -> 2001:4860:4860:0:0:0:0:8842 + # So interestingly Route53 normalizes IPv6 addresses to a funky, but + # valid, form which will cause us to fail to find see things as + # equivalent. To work around this we'll ip_address's returned objects + # for equivalence. + # E.g 2001:4860:4860:0:0:0:0:8842 -> 2001:4860:4860::8842 if value: value = ip_address(text_type(value)) config_ip_address = ip_address(text_type(config['IPAddress'])) From e1c8e96e2e51055e58d8ebd6115a9e6f66e359bc Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 23 Aug 2021 13:34:14 -0700 Subject: [PATCH 342/358] No need to normalize IPv6 values in ultra now that they are by default --- octodns/provider/ultra.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/octodns/provider/ultra.py b/octodns/provider/ultra.py index bc855d4..afa4d0a 100644 --- a/octodns/provider/ultra.py +++ b/octodns/provider/ultra.py @@ -1,5 +1,4 @@ from collections import defaultdict -from ipaddress import ip_address from logging import getLogger from requests import Session @@ -196,8 +195,6 @@ class UltraProvider(BaseProvider): } def _data_for_AAAA(self, _type, records): - for i, v in enumerate(records['rdata']): - records['rdata'][i] = str(ip_address(v)) return { 'ttl': records['ttl'], 'type': _type, From 9522da210dad1b077f0596606d005411edebc867 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 23 Aug 2021 14:32:10 -0700 Subject: [PATCH 343/358] implement & use NS1Client.reset_caches --- octodns/provider/ns1.py | 4 +++- tests/test_octodns_provider_ns1.py | 25 +++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index bab7c7b..793a59f 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -80,8 +80,10 @@ class Ns1Client(object): self._datasource = client.datasource() self._datafeed = client.datafeed() - self._datasource_id = None + self.reset_caches() + def reset_caches(self): + self._datasource_id = None self._feeds_for_monitors = None self._monitors_cache = None self._notifylists_cache = None diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index 6243348..02e70d1 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -198,6 +198,7 @@ class TestNs1Provider(TestCase): provider = Ns1Provider('test', 'api-key') # Bad auth + provider._client.reset_caches() zone_retrieve_mock.side_effect = AuthException('unauthorized') zone = Zone('unit.tests.', []) with self.assertRaises(AuthException) as ctx: @@ -205,6 +206,7 @@ class TestNs1Provider(TestCase): self.assertEquals(zone_retrieve_mock.side_effect, ctx.exception) # General error + provider._client.reset_caches() zone_retrieve_mock.reset_mock() zone_retrieve_mock.side_effect = ResourceException('boom') zone = Zone('unit.tests.', []) @@ -214,6 +216,7 @@ class TestNs1Provider(TestCase): self.assertEquals(('unit.tests',), zone_retrieve_mock.call_args[0]) # Non-existent zone doesn't populate anything + provider._client.reset_caches() zone_retrieve_mock.reset_mock() zone_retrieve_mock.side_effect = \ ResourceException('server error: zone not found') @@ -224,6 +227,7 @@ class TestNs1Provider(TestCase): self.assertFalse(exists) # Existing zone w/o records + provider._client.reset_caches() zone_retrieve_mock.reset_mock() record_retrieve_mock.reset_mock() ns1_zone = { @@ -255,6 +259,7 @@ class TestNs1Provider(TestCase): 'geo.unit.tests', 'A')]) # Existing zone w/records + provider._client.reset_caches() zone_retrieve_mock.reset_mock() record_retrieve_mock.reset_mock() ns1_zone = { @@ -286,6 +291,7 @@ class TestNs1Provider(TestCase): 'geo.unit.tests', 'A')]) # Test skipping unsupported record type + provider._client.reset_caches() zone_retrieve_mock.reset_mock() record_retrieve_mock.reset_mock() ns1_zone = { @@ -341,6 +347,7 @@ class TestNs1Provider(TestCase): self.assertTrue(plan.exists) # Fails, general error + provider._client.reset_caches() zone_retrieve_mock.reset_mock() record_retrieve_mock.reset_mock() zone_create_mock.reset_mock() @@ -350,6 +357,7 @@ class TestNs1Provider(TestCase): self.assertEquals(zone_retrieve_mock.side_effect, ctx.exception) # Fails, bad auth + provider._client.reset_caches() zone_retrieve_mock.reset_mock() record_retrieve_mock.reset_mock() zone_create_mock.reset_mock() @@ -361,6 +369,7 @@ class TestNs1Provider(TestCase): self.assertEquals(zone_create_mock.side_effect, ctx.exception) # non-existent zone, create + provider._client.reset_caches() zone_retrieve_mock.reset_mock() record_retrieve_mock.reset_mock() zone_create_mock.reset_mock() @@ -395,6 +404,7 @@ class TestNs1Provider(TestCase): ]) # Update & delete + provider._client.reset_caches() zone_retrieve_mock.reset_mock() record_retrieve_mock.reset_mock() zone_create_mock.reset_mock() @@ -1304,6 +1314,7 @@ class TestNs1ProviderDynamic(TestCase): # provider._params_for_A() calls provider._monitors_for() and # provider._monitor_sync(). Mock their return values so that we don't # make NS1 API calls during tests + provider._client.reset_caches() monitors_for_mock.reset_mock() monitor_sync_mock.reset_mock() monitors_for_mock.side_effect = [{ @@ -1944,6 +1955,7 @@ class TestNs1ProviderDynamic(TestCase): monitors_for_mock.assert_not_called() # Non-existent zone. No changes + provider._client.reset_caches() monitors_for_mock.reset_mock() zones_retrieve_mock.side_effect = \ ResourceException('server error: zone not found') @@ -1952,6 +1964,7 @@ class TestNs1ProviderDynamic(TestCase): self.assertFalse(extra) # Unexpected exception message + provider._client.reset_caches() zones_retrieve_mock.reset_mock() zones_retrieve_mock.side_effect = ResourceException('boom') with self.assertRaises(ResourceException) as ctx: @@ -1959,6 +1972,7 @@ class TestNs1ProviderDynamic(TestCase): self.assertEquals(zones_retrieve_mock.side_effect, ctx.exception) # Simple record, ignored, filter update lookups ignored + provider._client.reset_caches() monitors_for_mock.reset_mock() zones_retrieve_mock.reset_mock() records_retrieve_mock.reset_mock() @@ -2006,6 +2020,7 @@ class TestNs1ProviderDynamic(TestCase): desired.add_record(dynamic) # untouched, but everything in sync so no change needed + provider._client.reset_caches() monitors_for_mock.reset_mock() zones_retrieve_mock.reset_mock() records_retrieve_mock.reset_mock() @@ -2026,6 +2041,7 @@ class TestNs1ProviderDynamic(TestCase): # If we don't have a notify list we're broken and we'll expect to see # an Update + provider._client.reset_caches() monitors_for_mock.reset_mock() zones_retrieve_mock.reset_mock() records_retrieve_mock.reset_mock() @@ -2042,6 +2058,7 @@ class TestNs1ProviderDynamic(TestCase): # Add notify_list back and change the healthcheck protocol, we'll still # expect to see an update + provider._client.reset_caches() monitors_for_mock.reset_mock() zones_retrieve_mock.reset_mock() records_retrieve_mock.reset_mock() @@ -2059,6 +2076,7 @@ class TestNs1ProviderDynamic(TestCase): monitors_for_mock.assert_has_calls([call(dynamic)]) # If it's in the changed list, it'll be ignored + provider._client.reset_caches() monitors_for_mock.reset_mock() zones_retrieve_mock.reset_mock() records_retrieve_mock.reset_mock() @@ -2069,6 +2087,7 @@ class TestNs1ProviderDynamic(TestCase): # Test changes in filters # No change in filters + provider._client.reset_caches() monitors_for_mock.reset_mock() zones_retrieve_mock.reset_mock() records_retrieve_mock.reset_mock() @@ -2088,6 +2107,7 @@ class TestNs1ProviderDynamic(TestCase): self.assertFalse(extra) # filters need an update + provider._client.reset_caches() monitors_for_mock.reset_mock() zones_retrieve_mock.reset_mock() records_retrieve_mock.reset_mock() @@ -2107,6 +2127,7 @@ class TestNs1ProviderDynamic(TestCase): self.assertTrue(extra) # Mixed disabled in filters. Raise Ns1Exception + provider._client.reset_caches() monitors_for_mock.reset_mock() zones_retrieve_mock.reset_mock() records_retrieve_mock.reset_mock() @@ -2234,12 +2255,14 @@ class TestNs1Client(TestCase): client = Ns1Client('dummy-key') # No retry required, just calls and is returned + client.reset_caches() zone_retrieve_mock.reset_mock() zone_retrieve_mock.side_effect = ['foo'] self.assertEquals('foo', client.zones_retrieve('unit.tests')) zone_retrieve_mock.assert_has_calls([call('unit.tests')]) # One retry required + client.reset_caches() zone_retrieve_mock.reset_mock() zone_retrieve_mock.side_effect = [ RateLimitException('boo', period=0), @@ -2249,6 +2272,7 @@ class TestNs1Client(TestCase): zone_retrieve_mock.assert_has_calls([call('unit.tests')]) # Two retries required + client.reset_caches() zone_retrieve_mock.reset_mock() zone_retrieve_mock.side_effect = [ RateLimitException('boo', period=0), @@ -2258,6 +2282,7 @@ class TestNs1Client(TestCase): zone_retrieve_mock.assert_has_calls([call('unit.tests')]) # Exhaust our retries + client.reset_caches() zone_retrieve_mock.reset_mock() zone_retrieve_mock.side_effect = [ RateLimitException('first', period=0), From 886ab89decd51249e03f2193e31ea10ac4228d6a Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 23 Aug 2021 14:43:38 -0700 Subject: [PATCH 344/358] Clean up NS1 mock resetting --- tests/test_octodns_provider_ns1.py | 192 +++++++++++------------------ 1 file changed, 70 insertions(+), 122 deletions(-) diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index 02e70d1..d220381 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -197,8 +197,14 @@ class TestNs1Provider(TestCase): def test_populate(self, zone_retrieve_mock, record_retrieve_mock): provider = Ns1Provider('test', 'api-key') + def reset(): + provider._client.reset_caches() + zone_retrieve_mock.reset_mock() + record_retrieve_mock.reset_mock() + + # Bad auth - provider._client.reset_caches() + reset() zone_retrieve_mock.side_effect = AuthException('unauthorized') zone = Zone('unit.tests.', []) with self.assertRaises(AuthException) as ctx: @@ -206,8 +212,7 @@ class TestNs1Provider(TestCase): self.assertEquals(zone_retrieve_mock.side_effect, ctx.exception) # General error - provider._client.reset_caches() - zone_retrieve_mock.reset_mock() + reset() zone_retrieve_mock.side_effect = ResourceException('boom') zone = Zone('unit.tests.', []) with self.assertRaises(ResourceException) as ctx: @@ -216,8 +221,7 @@ class TestNs1Provider(TestCase): self.assertEquals(('unit.tests',), zone_retrieve_mock.call_args[0]) # Non-existent zone doesn't populate anything - provider._client.reset_caches() - zone_retrieve_mock.reset_mock() + reset() zone_retrieve_mock.side_effect = \ ResourceException('server error: zone not found') zone = Zone('unit.tests.', []) @@ -227,9 +231,7 @@ class TestNs1Provider(TestCase): self.assertFalse(exists) # Existing zone w/o records - provider._client.reset_caches() - zone_retrieve_mock.reset_mock() - record_retrieve_mock.reset_mock() + reset() ns1_zone = { 'records': [{ "domain": "geo.unit.tests", @@ -259,9 +261,7 @@ class TestNs1Provider(TestCase): 'geo.unit.tests', 'A')]) # Existing zone w/records - provider._client.reset_caches() - zone_retrieve_mock.reset_mock() - record_retrieve_mock.reset_mock() + reset() ns1_zone = { 'records': self.ns1_records + [{ "domain": "geo.unit.tests", @@ -291,9 +291,7 @@ class TestNs1Provider(TestCase): 'geo.unit.tests', 'A')]) # Test skipping unsupported record type - provider._client.reset_caches() - zone_retrieve_mock.reset_mock() - record_retrieve_mock.reset_mock() + reset() ns1_zone = { 'records': self.ns1_records + [{ 'type': 'UNSUPPORTED', @@ -346,21 +344,21 @@ class TestNs1Provider(TestCase): self.assertEquals(expected_n, len(plan.changes)) self.assertTrue(plan.exists) + def reset(): + provider._client.reset_caches() + record_retrieve_mock.reset_mock() + zone_create_mock.reset_mock() + zone_retrieve_mock.reset_mock() + # Fails, general error - provider._client.reset_caches() - zone_retrieve_mock.reset_mock() - record_retrieve_mock.reset_mock() - zone_create_mock.reset_mock() + reset() zone_retrieve_mock.side_effect = ResourceException('boom') with self.assertRaises(ResourceException) as ctx: provider.apply(plan) self.assertEquals(zone_retrieve_mock.side_effect, ctx.exception) # Fails, bad auth - provider._client.reset_caches() - zone_retrieve_mock.reset_mock() - record_retrieve_mock.reset_mock() - zone_create_mock.reset_mock() + reset() zone_retrieve_mock.side_effect = \ ResourceException('server error: zone not found') zone_create_mock.side_effect = AuthException('unauthorized') @@ -369,10 +367,7 @@ class TestNs1Provider(TestCase): self.assertEquals(zone_create_mock.side_effect, ctx.exception) # non-existent zone, create - provider._client.reset_caches() - zone_retrieve_mock.reset_mock() - record_retrieve_mock.reset_mock() - zone_create_mock.reset_mock() + reset() zone_retrieve_mock.side_effect = \ ResourceException('server error: zone not found') @@ -404,10 +399,7 @@ class TestNs1Provider(TestCase): ]) # Update & delete - provider._client.reset_caches() - zone_retrieve_mock.reset_mock() - record_retrieve_mock.reset_mock() - zone_create_mock.reset_mock() + reset() ns1_zone = { 'records': self.ns1_records + [{ @@ -947,11 +939,14 @@ class TestNs1ProviderDynamic(TestCase): 'mon-id': 'feed-id', } + def reset(): + feed_create_mock.reset_mock() + monitor_create_mock.reset_mock() + monitor_gen_mock.reset_mock() + monitors_update_mock.reset_mock() + # No existing monitor - monitor_gen_mock.reset_mock() - monitor_create_mock.reset_mock() - monitors_update_mock.reset_mock() - feed_create_mock.reset_mock() + reset() monitor_gen_mock.side_effect = [{'key': 'value'}] monitor_create_mock.side_effect = [('mon-id', 'feed-id')] value = '1.2.3.4' @@ -965,10 +960,7 @@ class TestNs1ProviderDynamic(TestCase): feed_create_mock.assert_not_called() # Existing monitor that doesn't need updates - monitor_gen_mock.reset_mock() - monitor_create_mock.reset_mock() - monitors_update_mock.reset_mock() - feed_create_mock.reset_mock() + reset() monitor = { 'id': 'mon-id', 'key': 'value', @@ -985,10 +977,7 @@ class TestNs1ProviderDynamic(TestCase): feed_create_mock.assert_not_called() # Existing monitor that doesn't need updates, but is missing its feed - monitor_gen_mock.reset_mock() - monitor_create_mock.reset_mock() - monitors_update_mock.reset_mock() - feed_create_mock.reset_mock() + reset() monitor = { 'id': 'mon-id2', 'key': 'value', @@ -1006,10 +995,7 @@ class TestNs1ProviderDynamic(TestCase): feed_create_mock.assert_has_calls([call(monitor)]) # Existing monitor that needs updates - monitor_gen_mock.reset_mock() - monitor_create_mock.reset_mock() - monitors_update_mock.reset_mock() - feed_create_mock.reset_mock() + reset() monitor = { 'id': 'mon-id', 'key': 'value', @@ -1043,11 +1029,14 @@ class TestNs1ProviderDynamic(TestCase): 'mon-id': 'feed-id', } + def reset(): + datafeed_delete_mock.reset_mock() + monitors_delete_mock.reset_mock() + monitors_for_mock.reset_mock() + notifylists_delete_mock.reset_mock() + # No active monitors and no existing, nothing will happen - monitors_for_mock.reset_mock() - datafeed_delete_mock.reset_mock() - monitors_delete_mock.reset_mock() - notifylists_delete_mock.reset_mock() + reset() monitors_for_mock.side_effect = [{}] record = self.record() provider._monitors_gc(record) @@ -1057,10 +1046,7 @@ class TestNs1ProviderDynamic(TestCase): notifylists_delete_mock.assert_not_called() # No active monitors and one existing, delete all the things - monitors_for_mock.reset_mock() - datafeed_delete_mock.reset_mock() - monitors_delete_mock.reset_mock() - notifylists_delete_mock.reset_mock() + reset() monitors_for_mock.side_effect = [{ 'x': { 'id': 'mon-id', @@ -1080,10 +1066,7 @@ class TestNs1ProviderDynamic(TestCase): notifylists_delete_mock.assert_has_calls([call('nl-id')]) # Same existing, this time in active list, should be noop - monitors_for_mock.reset_mock() - datafeed_delete_mock.reset_mock() - monitors_delete_mock.reset_mock() - notifylists_delete_mock.reset_mock() + reset() monitors_for_mock.side_effect = [{ 'x': { 'id': 'mon-id', @@ -1098,10 +1081,7 @@ class TestNs1ProviderDynamic(TestCase): # Non-active monitor w/o a feed, and another monitor that's left alone # b/c it's active - monitors_for_mock.reset_mock() - datafeed_delete_mock.reset_mock() - monitors_delete_mock.reset_mock() - notifylists_delete_mock.reset_mock() + reset() monitors_for_mock.side_effect = [{ 'x': { 'id': 'mon-id', @@ -1130,10 +1110,7 @@ class TestNs1ProviderDynamic(TestCase): # Non-active monitor w/o a notifylist, generally shouldn't happen, but # code should handle it just in case someone gets clicky in the UI - monitors_for_mock.reset_mock() - datafeed_delete_mock.reset_mock() - monitors_delete_mock.reset_mock() - notifylists_delete_mock.reset_mock() + reset() monitors_for_mock.side_effect = [{ 'y': { 'id': 'mon-id2', @@ -1158,11 +1135,8 @@ class TestNs1ProviderDynamic(TestCase): # Non-active monitor with a shared notifylist, monitor deleted, but # notifylist is left alone + reset() provider.shared_notifylist = True - monitors_for_mock.reset_mock() - datafeed_delete_mock.reset_mock() - monitors_delete_mock.reset_mock() - notifylists_delete_mock.reset_mock() monitors_for_mock.side_effect = [{ 'y': { 'id': 'mon-id2', @@ -1945,37 +1919,35 @@ class TestNs1ProviderDynamic(TestCase): desired = Zone('unit.tests.', []) + def reset(): + monitors_for_mock.reset_mock() + provider._client.reset_caches() + records_retrieve_mock.reset_mock() + zones_retrieve_mock.reset_mock() + # Empty zone and no changes - monitors_for_mock.reset_mock() - zones_retrieve_mock.reset_mock() - records_retrieve_mock.reset_mock() + reset() extra = provider._extra_changes(desired, []) self.assertFalse(extra) monitors_for_mock.assert_not_called() # Non-existent zone. No changes - provider._client.reset_caches() - monitors_for_mock.reset_mock() + reset() zones_retrieve_mock.side_effect = \ ResourceException('server error: zone not found') - records_retrieve_mock.reset_mock() extra = provider._extra_changes(desired, []) self.assertFalse(extra) # Unexpected exception message - provider._client.reset_caches() - zones_retrieve_mock.reset_mock() + reset() zones_retrieve_mock.side_effect = ResourceException('boom') with self.assertRaises(ResourceException) as ctx: extra = provider._extra_changes(desired, []) self.assertEquals(zones_retrieve_mock.side_effect, ctx.exception) # Simple record, ignored, filter update lookups ignored - provider._client.reset_caches() - monitors_for_mock.reset_mock() - zones_retrieve_mock.reset_mock() - records_retrieve_mock.reset_mock() + reset() zones_retrieve_mock.side_effect = \ ResourceException('server error: zone not found') @@ -2020,10 +1992,7 @@ class TestNs1ProviderDynamic(TestCase): desired.add_record(dynamic) # untouched, but everything in sync so no change needed - provider._client.reset_caches() - monitors_for_mock.reset_mock() - zones_retrieve_mock.reset_mock() - records_retrieve_mock.reset_mock() + reset() # Generate what we expect to have gend = provider._monitor_gen(dynamic, '1.2.3.4') gend.update({ @@ -2041,10 +2010,7 @@ class TestNs1ProviderDynamic(TestCase): # If we don't have a notify list we're broken and we'll expect to see # an Update - provider._client.reset_caches() - monitors_for_mock.reset_mock() - zones_retrieve_mock.reset_mock() - records_retrieve_mock.reset_mock() + reset() del gend['notify_list'] monitors_for_mock.side_effect = [{ '1.2.3.4': gend, @@ -2058,10 +2024,7 @@ class TestNs1ProviderDynamic(TestCase): # Add notify_list back and change the healthcheck protocol, we'll still # expect to see an update - provider._client.reset_caches() - monitors_for_mock.reset_mock() - zones_retrieve_mock.reset_mock() - records_retrieve_mock.reset_mock() + reset() gend['notify_list'] = 'xyz' dynamic._octodns['healthcheck']['protocol'] = 'HTTPS' del gend['notify_list'] @@ -2076,10 +2039,7 @@ class TestNs1ProviderDynamic(TestCase): monitors_for_mock.assert_has_calls([call(dynamic)]) # If it's in the changed list, it'll be ignored - provider._client.reset_caches() - monitors_for_mock.reset_mock() - zones_retrieve_mock.reset_mock() - records_retrieve_mock.reset_mock() + reset() extra = provider._extra_changes(desired, [update]) self.assertFalse(extra) monitors_for_mock.assert_not_called() @@ -2087,10 +2047,7 @@ class TestNs1ProviderDynamic(TestCase): # Test changes in filters # No change in filters - provider._client.reset_caches() - monitors_for_mock.reset_mock() - zones_retrieve_mock.reset_mock() - records_retrieve_mock.reset_mock() + reset() ns1_zone = { 'records': [{ "domain": "dyn.unit.tests", @@ -2107,10 +2064,7 @@ class TestNs1ProviderDynamic(TestCase): self.assertFalse(extra) # filters need an update - provider._client.reset_caches() - monitors_for_mock.reset_mock() - zones_retrieve_mock.reset_mock() - records_retrieve_mock.reset_mock() + reset() ns1_zone = { 'records': [{ "domain": "dyn.unit.tests", @@ -2127,10 +2081,7 @@ class TestNs1ProviderDynamic(TestCase): self.assertTrue(extra) # Mixed disabled in filters. Raise Ns1Exception - provider._client.reset_caches() - monitors_for_mock.reset_mock() - zones_retrieve_mock.reset_mock() - records_retrieve_mock.reset_mock() + reset() ns1_zone = { 'records': [{ "domain": "dyn.unit.tests", @@ -2494,9 +2445,12 @@ class TestNs1Client(TestCase): notifylists_delete_mock): client = Ns1Client('dummy-key') - notifylists_list_mock.reset_mock() - notifylists_create_mock.reset_mock() - notifylists_delete_mock.reset_mock() + def reset(): + notifylists_create_mock.reset_mock() + notifylists_delete_mock.reset_mock() + notifylists_list_mock.reset_mock() + + reset() notifylists_list_mock.side_effect = [{}] expected = { 'id': 'nl-id', @@ -2518,9 +2472,7 @@ class TestNs1Client(TestCase): ]) notifylists_delete_mock.assert_not_called() - notifylists_list_mock.reset_mock() - notifylists_create_mock.reset_mock() - notifylists_delete_mock.reset_mock() + reset() client.notifylists_delete('nlid') notifylists_list_mock.assert_not_called() notifylists_create_mock.assert_not_called() @@ -2528,9 +2480,7 @@ class TestNs1Client(TestCase): # Delete again, this time with a cache item that needs cleaned out and # another that needs to be ignored - notifylists_list_mock.reset_mock() - notifylists_create_mock.reset_mock() - notifylists_delete_mock.reset_mock() + reset() client._notifylists_cache = { 'another': { 'id': 'notid', @@ -2549,9 +2499,7 @@ class TestNs1Client(TestCase): # Only another left self.assertEquals(['another'], list(client._notifylists_cache.keys())) - notifylists_list_mock.reset_mock() - notifylists_create_mock.reset_mock() - notifylists_delete_mock.reset_mock() + reset() expected = ['one', 'two', 'three'] notifylists_list_mock.side_effect = [expected] nls = client.notifylists_list() From 64072f9f43df9da95b6dcf9223ac0aa6d47bb938 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 23 Aug 2021 15:36:57 -0700 Subject: [PATCH 345/358] Coverage test for NS1 client caching behaviors --- octodns/provider/ns1.py | 9 ++- tests/test_octodns_provider_ns1.py | 109 ++++++++++++++++++++++++++++- 2 files changed, 115 insertions(+), 3 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 793a59f..f46bca9 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -197,7 +197,11 @@ class Ns1Client(object): return self._try(self._notifylists.list) def records_create(self, zone, domain, _type, **params): - return self._try(self._records.create, zone, domain, _type, **params) + cached = self._records_cache.setdefault(zone, {}) \ + .setdefault(domain, {}) + cached[_type] = self._try(self._records.create, zone, domain, _type, + **params) + return cached[_type] def records_delete(self, zone, domain, _type): try: @@ -230,7 +234,8 @@ class Ns1Client(object): return self._try(self._records.update, zone, domain, _type, **params) def zones_create(self, name): - return self._try(self._zones.create, name) + self._zones_cache[name] = self._try(self._zones.create, name) + return self._zones_cache[name] def zones_retrieve(self, name): if name not in self._zones_cache: diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index d220381..3b4fc6b 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -202,7 +202,6 @@ class TestNs1Provider(TestCase): zone_retrieve_mock.reset_mock() record_retrieve_mock.reset_mock() - # Bad auth reset() zone_retrieve_mock.side_effect = AuthException('unauthorized') @@ -2507,3 +2506,111 @@ class TestNs1Client(TestCase): notifylists_list_mock.assert_has_calls([call()]) notifylists_create_mock.assert_not_called() notifylists_delete_mock.assert_not_called() + + @patch('ns1.rest.records.Records.delete') + @patch('ns1.rest.records.Records.update') + @patch('ns1.rest.records.Records.create') + @patch('ns1.rest.records.Records.retrieve') + @patch('ns1.rest.zones.Zones.create') + @patch('ns1.rest.zones.Zones.delete') + @patch('ns1.rest.zones.Zones.retrieve') + def test_client_caching(self, zone_retrieve_mock, zone_delete_mock, + zone_create_mock, record_retrieve_mock, + record_create_mock, record_update_mock, + record_delete_mock): + client = Ns1Client('dummy-key') + + def reset(): + zone_retrieve_mock.reset_mock() + zone_delete_mock.reset_mock() + zone_create_mock.reset_mock() + record_retrieve_mock.reset_mock() + record_create_mock.reset_mock() + record_update_mock.reset_mock() + record_delete_mock.reset_mock() + # Testing caches so we don't reset those + + # Initial zone get fetches and caches + reset() + zone_retrieve_mock.side_effect = ['foo'] + self.assertEquals('foo', client.zones_retrieve('unit.tests')) + zone_retrieve_mock.assert_has_calls([call('unit.tests')]) + self.assertEquals({ + 'unit.tests': 'foo', + }, client._zones_cache) + + # Subsequent zone get does not fetch and returns from cache + reset() + self.assertEquals('foo', client.zones_retrieve('unit.tests')) + zone_retrieve_mock.assert_not_called() + + # Zone create stores in cache + reset() + zone_create_mock.side_effect = ['bar'] + self.assertEquals('bar', client.zones_create('sub.unit.tests')) + zone_create_mock.assert_has_calls([call('sub.unit.tests')]) + self.assertEquals({ + 'sub.unit.tests': 'bar', + 'unit.tests': 'foo', + }, client._zones_cache) + + # Initial record get fetches and caches + reset() + record_retrieve_mock.side_effect = ['baz'] + self.assertEquals('baz', client.records_retrieve('unit.tests', + 'a.unit.tests', 'A')) + record_retrieve_mock.assert_has_calls([call('unit.tests', + 'a.unit.tests', 'A')]) + self.assertEquals({ + 'unit.tests': { + 'a.unit.tests': { + 'A': 'baz' + } + } + }, client._records_cache) + + # Subsequent record get does not fetch and returns from cache + reset() + self.assertEquals('baz', client.records_retrieve('unit.tests', + 'a.unit.tests', 'A')) + record_retrieve_mock.assert_not_called() + + # Record create stores in cache + reset() + record_create_mock.side_effect = ['boo'] + self.assertEquals('boo', client.records_create('unit.tests', + 'aaaa.unit.tests', + 'AAAA', key='val')) + record_create_mock.assert_has_calls([call('unit.tests', + 'aaaa.unit.tests', 'AAAA', + key='val')]) + self.assertEquals({ + 'unit.tests': { + 'a.unit.tests': { + 'A': 'baz' + }, + 'aaaa.unit.tests': { + 'AAAA': 'boo' + }, + } + }, client._records_cache) + + # Record delete removes from cache and removes zone + reset() + record_delete_mock.side_effect = ['hoo'] + self.assertEquals('hoo', client.records_delete('unit.tests', + 'aaaa.unit.tests', + 'AAAA')) + record_delete_mock.assert_has_calls([call('unit.tests', + 'aaaa.unit.tests', 'AAAA')]) + self.assertEquals({ + 'unit.tests': { + 'a.unit.tests': { + 'A': 'baz' + }, + 'aaaa.unit.tests': {}, + } + }, client._records_cache) + self.assertEquals({ + 'sub.unit.tests': 'bar', + }, client._zones_cache) From efdb4866c019c6b35dddba06f1f3c7e2355f16cb Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 23 Aug 2021 15:37:02 -0700 Subject: [PATCH 346/358] Remove a couple of stray prints --- octodns/provider/ns1.py | 1 - tests/test_octodns_processor_acme.py | 1 - 2 files changed, 2 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index f46bca9..32a6b3e 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -240,7 +240,6 @@ class Ns1Client(object): def zones_retrieve(self, name): if name not in self._zones_cache: self._zones_cache[name] = self._try(self._zones.retrieve, name) - print(f'insert {name} to cache with val {self._zones_cache[name]}') return self._zones_cache[name] def _try(self, method, *args, **kwargs): diff --git a/tests/test_octodns_processor_acme.py b/tests/test_octodns_processor_acme.py index c927608..02177f7 100644 --- a/tests/test_octodns_processor_acme.py +++ b/tests/test_octodns_processor_acme.py @@ -71,7 +71,6 @@ class TestAcmeMangingProcessor(TestCase): ], sorted([r.name for r in got.records])) managed = None for record in got.records: - print(record.name) if record.name.endswith('managed'): managed = record break From aa88b877c4c4e08d0336a01b532d9d5b21dbedb3 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 23 Aug 2021 17:53:07 -0700 Subject: [PATCH 347/358] Clear NS1 zone cache before record cache --- octodns/provider/ns1.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 32a6b3e..1072245 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -205,10 +205,11 @@ class Ns1Client(object): def records_delete(self, zone, domain, _type): try: - # remove record from cache - del self._records_cache[zone][domain][_type] # remove record's zone from cache del self._zones_cache[zone] + # remove record from cache, after zone since we may not have + # fetched the record details + del self._records_cache[zone][domain][_type] except KeyError: # never mind if record is not found in cache pass @@ -224,10 +225,11 @@ class Ns1Client(object): def records_update(self, zone, domain, _type, **params): try: - # remove record from cache - del self._records_cache[zone][domain][_type] # remove record's zone from cache del self._zones_cache[zone] + # remove record from cache, after zone since we may not have + # fetched the record details + del self._records_cache[zone][domain][_type] except KeyError: # never mind if record is not found in cache pass From 025180ac3f80e358e233e0ee52d7632cfc3b0609 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 23 Aug 2021 18:00:24 -0700 Subject: [PATCH 348/358] NS1Client.records_update result caching & tests --- octodns/provider/ns1.py | 8 ++++++-- tests/test_octodns_provider_ns1.py | 23 +++++++++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 1072245..b93d3b6 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -224,16 +224,20 @@ class Ns1Client(object): return cached[_type] def records_update(self, zone, domain, _type, **params): + cached = self._records_cache.setdefault(zone, {}) \ + .setdefault(domain, {}) try: # remove record's zone from cache del self._zones_cache[zone] # remove record from cache, after zone since we may not have # fetched the record details - del self._records_cache[zone][domain][_type] + del cached[_type] except KeyError: # never mind if record is not found in cache pass - return self._try(self._records.update, zone, domain, _type, **params) + cached[_type] = self._try(self._records.update, zone, domain, _type, + **params) + return cached[_type] def zones_create(self, name): self._zones_cache[name] = self._try(self._zones.create, name) diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index 3b4fc6b..7a36fb9 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -2614,3 +2614,26 @@ class TestNs1Client(TestCase): self.assertEquals({ 'sub.unit.tests': 'bar', }, client._zones_cache) + + # Record update removes zone and caches result + record_update_mock.side_effect = ['done'] + self.assertEquals('done', client.records_update('sub.unit.tests', + 'aaaa.sub.unit.tests', + 'AAAA', key='val')) + record_update_mock.assert_has_calls([call('sub.unit.tests', + 'aaaa.sub.unit.tests', + 'AAAA', key='val')]) + self.assertEquals({ + 'unit.tests': { + 'a.unit.tests': { + 'A': 'baz' + }, + 'aaaa.unit.tests': {}, + }, + 'sub.unit.tests': { + 'aaaa.sub.unit.tests': { + 'AAAA': 'done', + }, + } + }, client._records_cache) + self.assertEquals({}, client._zones_cache) From c9fc8feae2101f19aa46896f552acecfc666a2a2 Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Tue, 24 Aug 2021 00:01:49 -0700 Subject: [PATCH 349/358] Centralized NS1 record cache management with decorator --- octodns/provider/ns1.py | 61 ++++++++++++++++-------------- tests/test_octodns_provider_ns1.py | 8 ++-- 2 files changed, 37 insertions(+), 32 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index b93d3b6..242ebfb 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -90,6 +90,34 @@ class Ns1Client(object): self._zones_cache = {} self._records_cache = {} + def reset_record_cache(self, zone, domain, _type): + try: + # remove record's zone from cache + del self._zones_cache[zone] + except KeyError: + # never mind if zone is not found in cache + pass + + try: + # remove record from cache + del self._records_cache[zone][domain][_type] + except KeyError: + # never mind if record is not found in cache + pass + + def update_record_cache(func): + def call(self, zone, domain, _type, **params): + self.reset_record_cache(zone, domain, _type) + new_record = func(self, zone, domain, _type, **params) + if new_record: + cached = self._records_cache.setdefault(zone, {}) \ + .setdefault(domain, {}) + cached[_type] = new_record + + return new_record + + return call + @property def datasource_id(self): if self._datasource_id is None: @@ -196,23 +224,12 @@ class Ns1Client(object): def notifylists_list(self): return self._try(self._notifylists.list) + @update_record_cache def records_create(self, zone, domain, _type, **params): - cached = self._records_cache.setdefault(zone, {}) \ - .setdefault(domain, {}) - cached[_type] = self._try(self._records.create, zone, domain, _type, - **params) - return cached[_type] + return self._try(self._records.create, zone, domain, _type, **params) + @update_record_cache def records_delete(self, zone, domain, _type): - try: - # remove record's zone from cache - del self._zones_cache[zone] - # remove record from cache, after zone since we may not have - # fetched the record details - del self._records_cache[zone][domain][_type] - except KeyError: - # never mind if record is not found in cache - pass return self._try(self._records.delete, zone, domain, _type) def records_retrieve(self, zone, domain, _type): @@ -223,21 +240,9 @@ class Ns1Client(object): _type) return cached[_type] + @update_record_cache def records_update(self, zone, domain, _type, **params): - cached = self._records_cache.setdefault(zone, {}) \ - .setdefault(domain, {}) - try: - # remove record's zone from cache - del self._zones_cache[zone] - # remove record from cache, after zone since we may not have - # fetched the record details - del cached[_type] - except KeyError: - # never mind if record is not found in cache - pass - cached[_type] = self._try(self._records.update, zone, domain, _type, - **params) - return cached[_type] + return self._try(self._records.update, zone, domain, _type, **params) def zones_create(self, name): self._zones_cache[name] = self._try(self._zones.create, name) diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index 7a36fb9..bd13e97 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -2597,10 +2597,10 @@ class TestNs1Client(TestCase): # Record delete removes from cache and removes zone reset() - record_delete_mock.side_effect = ['hoo'] - self.assertEquals('hoo', client.records_delete('unit.tests', - 'aaaa.unit.tests', - 'AAAA')) + record_delete_mock.side_effect = [{}] + self.assertEquals({}, client.records_delete('unit.tests', + 'aaaa.unit.tests', + 'AAAA')) record_delete_mock.assert_has_calls([call('unit.tests', 'aaaa.unit.tests', 'AAAA')]) self.assertEquals({ From 3754c1677497e299ed4391038c5f7b3e3dd2ea5a Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Tue, 24 Aug 2021 00:19:53 -0700 Subject: [PATCH 350/358] Cleaner and more compact NS1 record cache management --- octodns/provider/ns1.py | 47 ++++++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 242ebfb..3044c85 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -90,34 +90,37 @@ class Ns1Client(object): self._zones_cache = {} self._records_cache = {} - def reset_record_cache(self, zone, domain, _type): - try: - # remove record's zone from cache - del self._zones_cache[zone] - except KeyError: - # never mind if zone is not found in cache - pass - - try: - # remove record from cache - del self._records_cache[zone][domain][_type] - except KeyError: - # never mind if record is not found in cache - pass - def update_record_cache(func): def call(self, zone, domain, _type, **params): - self.reset_record_cache(zone, domain, _type) + if zone in self._zones_cache: + # remove record's zone from cache + del self._zones_cache[zone] + + cached = self._records_cache.setdefault(zone, {}) \ + .setdefault(domain, {}) new_record = func(self, zone, domain, _type, **params) if new_record: - cached = self._records_cache.setdefault(zone, {}) \ - .setdefault(domain, {}) + # record is created/updated cached[_type] = new_record + elif _type in cached: + # record is deleted + del cached[_type] return new_record return call + def read_or_set_record_cache(func): + def call(self, zone, domain, _type): + cached = self._records_cache.setdefault(zone, {}) \ + .setdefault(domain, {}) + if _type not in cached: + cached[_type] = func(self, zone, domain, _type) + + return cached[_type] + + return call + @property def datasource_id(self): if self._datasource_id is None: @@ -232,13 +235,9 @@ class Ns1Client(object): def records_delete(self, zone, domain, _type): return self._try(self._records.delete, zone, domain, _type) + @read_or_set_record_cache def records_retrieve(self, zone, domain, _type): - cached = self._records_cache.setdefault(zone, {}) \ - .setdefault(domain, {}) - if _type not in cached: - cached[_type] = self._try(self._records.retrieve, zone, domain, - _type) - return cached[_type] + return self._try(self._records.retrieve, zone, domain, _type) @update_record_cache def records_update(self, zone, domain, _type, **params): From 106971853cf7d98036f2f333e788cb606c31d4f9 Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Tue, 24 Aug 2021 00:23:35 -0700 Subject: [PATCH 351/358] add comment --- octodns/provider/ns1.py | 1 + 1 file changed, 1 insertion(+) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 3044c85..fec5c47 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -96,6 +96,7 @@ class Ns1Client(object): # remove record's zone from cache del self._zones_cache[zone] + # write to (or delete) record cache cached = self._records_cache.setdefault(zone, {}) \ .setdefault(domain, {}) new_record = func(self, zone, domain, _type, **params) From 2914f52ff3612741177c4078b3160253c9689e42 Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Tue, 24 Aug 2021 00:29:56 -0700 Subject: [PATCH 352/358] evict NS1 record from cache before an operation --- octodns/provider/ns1.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index fec5c47..80e9603 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -96,16 +96,17 @@ class Ns1Client(object): # remove record's zone from cache del self._zones_cache[zone] - # write to (or delete) record cache cached = self._records_cache.setdefault(zone, {}) \ .setdefault(domain, {}) + + if _type in cached: + # remove record from cache + del cached[_type] + + # write record to cache if its not a delete new_record = func(self, zone, domain, _type, **params) if new_record: - # record is created/updated cached[_type] = new_record - elif _type in cached: - # record is deleted - del cached[_type] return new_record From 56b8b23391cd59a47da3cd751b67bb8b176d3dc0 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Tue, 24 Aug 2021 09:18:14 -0700 Subject: [PATCH 353/358] Delete second ns1 record to make sure cache clears w/o zone --- tests/test_octodns_provider_ns1.py | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index bd13e97..1d3f49b 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -2615,6 +2615,24 @@ class TestNs1Client(TestCase): 'sub.unit.tests': 'bar', }, client._zones_cache) + # Delete the other record, no zone this time, record should still go + # away + reset() + record_delete_mock.side_effect = [{}] + self.assertEquals({}, client.records_delete('unit.tests', + 'a.unit.tests', 'A')) + record_delete_mock.assert_has_calls([call('unit.tests', 'a.unit.tests', + 'A')]) + self.assertEquals({ + 'unit.tests': { + 'a.unit.tests': {}, + 'aaaa.unit.tests': {}, + } + }, client._records_cache) + self.assertEquals({ + 'sub.unit.tests': 'bar', + }, client._zones_cache) + # Record update removes zone and caches result record_update_mock.side_effect = ['done'] self.assertEquals('done', client.records_update('sub.unit.tests', @@ -2625,9 +2643,7 @@ class TestNs1Client(TestCase): 'AAAA', key='val')]) self.assertEquals({ 'unit.tests': { - 'a.unit.tests': { - 'A': 'baz' - }, + 'a.unit.tests': {}, 'aaaa.unit.tests': {}, }, 'sub.unit.tests': { From b19c68fe0ee68b8e3c85b320211762a4add06249 Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Wed, 25 Aug 2021 10:01:10 -0700 Subject: [PATCH 354/358] Enable IPv6 for AAAA NS1 monitors --- octodns/provider/ns1.py | 3 ++ tests/test_octodns_provider_ns1.py | 56 ++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 80e9603..dd7b354 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -1047,6 +1047,9 @@ class Ns1Provider(BaseProvider): 'regions': self.monitor_regions, } + if _type == 'AAAA': + ret['config']['v6'] = True + if record.healthcheck_protocol != 'TCP': # IF it's HTTP we need to send the request string path = record.healthcheck_path diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index 1d3f49b..5652202 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -608,6 +608,54 @@ class TestNs1ProviderDynamic(TestCase): 'meta': {}, }) + def aaaa_record(self): + return Record.new(self.zone, '', { + 'dynamic': { + 'pools': { + 'lhr': { + 'fallback': 'iad', + 'values': [{ + 'value': '::ffff:3.4.5.6', + }], + }, + 'iad': { + 'values': [{ + 'value': '::ffff:1.2.3.4', + }, { + 'value': '::ffff:2.3.4.5', + }], + }, + }, + 'rules': [{ + 'geos': [ + 'AF', + 'EU-GB', + 'NA-US-FL' + ], + 'pool': 'lhr', + }, { + 'geos': [ + 'AF-ZW', + ], + 'pool': 'iad', + }, { + 'pool': 'iad', + }], + }, + 'octodns': { + 'healthcheck': { + 'host': 'send.me', + 'path': '/_ping', + 'port': 80, + 'protocol': 'HTTP', + } + }, + 'ttl': 32, + 'type': 'AAAA', + 'value': '::ffff:1.2.3.4', + 'meta': {}, + }) + def cname_record(self): return Record.new(self.zone, 'foo', { 'dynamic': { @@ -872,6 +920,14 @@ class TestNs1ProviderDynamic(TestCase): # No http response expected self.assertFalse('rules' in monitor) + def test_monitor_gen_AAAA(self): + provider = Ns1Provider('test', 'api-key') + + value = '::ffff:3.4.5.6' + record = self.aaaa_record() + monitor = provider._monitor_gen(record, value) + self.assertTrue(monitor['config']['v6']) + def test_monitor_gen_CNAME(self): provider = Ns1Provider('test', 'api-key') From ecb1753a075ddd35f80810d90c62c33f52e40882 Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Wed, 25 Aug 2021 11:09:31 -0700 Subject: [PATCH 355/358] Fix NS1 IPv6 monitor field --- octodns/provider/ns1.py | 2 +- tests/test_octodns_provider_ns1.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index dd7b354..38b9240 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -1048,7 +1048,7 @@ class Ns1Provider(BaseProvider): } if _type == 'AAAA': - ret['config']['v6'] = True + ret['config']['ipv6'] = True if record.healthcheck_protocol != 'TCP': # IF it's HTTP we need to send the request string diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index 5652202..de6bdc9 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -926,7 +926,7 @@ class TestNs1ProviderDynamic(TestCase): value = '::ffff:3.4.5.6' record = self.aaaa_record() monitor = provider._monitor_gen(record, value) - self.assertTrue(monitor['config']['v6']) + self.assertTrue(monitor['config']['ipv6']) def test_monitor_gen_CNAME(self): provider = Ns1Provider('test', 'api-key') From fd148d1803002a79d196678423375b7ae771dcad Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Thu, 26 Aug 2021 15:56:27 -0700 Subject: [PATCH 356/358] Documentation of strict_supports and lenient, changelog info --- CHANGELOG.md | 11 ++++++++++- README.md | 26 ++++++++++++++++++++++++++ octodns/provider/__init__.py | 4 ++++ octodns/provider/base.py | 4 ++-- tests/test_octodns_provider_base.py | 5 +++-- 5 files changed, 45 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ca42a68..3111716 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,16 @@ -## v0.9.14 - 2021-??-?? - ... +## v0.9.14 - 2021-??-?? - A new supports system #### Noteworthy changes +* Provider `strict_supports` param added, currently defaults to `false`, along + with Provider._process_desired_zone this forms the foundations of a new + "supports" system where providers will warn or error (depending on the value + of `strict_supports` during planning about their inability to do what they're + being asked. When `false` they will warn and "adjust" the desired records. + When true they will abort with an error indicating the problem. Over time it + is expected that all "supports" checking/handling will move into this + paradigm and `strict_supports` will likely be changed to default to `true`. +* Zone shallow copy support, reworking of Processors (alpha) semantics * NS1 NA target now includes `SX` and `UM`. If `NA` continent is in use in dynamic records care must be taken to upgrade/downgrade to v0.9.13. * Ns1Provider now supports a new parameter, shared_notifylist, which results in diff --git a/README.md b/README.md index 1f0e9e3..b6530b0 100644 --- a/README.md +++ b/README.md @@ -228,6 +228,32 @@ The above command pulled the existing data out of Route53 and placed the results * Dnsimple's uses the configured TTL when serving things through the ALIAS, there's also a secondary TXT record created alongside the ALIAS that octoDNS ignores * octoDNS itself supports non-ASCII character sets, but in testing Cloudflare is the only provider where that is currently functional end-to-end. Others have failures either in the client libraries or API calls +## Compatibilty & Compliance + +### `lenient` + +`lenient` mostly focuses on the details of `Record`s and standards compliance. When set to `true` will octoDNS will allow allow non-compliant configurations & values when possible. For example CNAME values that don't end with a `.`, label length restrictions, or invalid geo codes on `dynamic` records. When in lenient mode octoDNS will log validation problems at `WARNING` and try and continue with the configuration or source data as it exists to the degree possible. See [Lenience](/docs/records.md#lenience) for more information on the concept and how it can be configured. + +### `strict_supports` (Work In Progress) + +`strict_supports` is a `Provider` level parameter that comes into play when a provider has been asked to create a record that it is unable to support. The simplest case of this would be record type, e.g. `SSHFP` not being supported by `AzureProvider`. If such a record is passed to an `AzureProvider` as a target the provider will take action based on the `strict_supports`. When `true` it will throw an exception saying that it's unable to create the record, when set to `false` it will log at `WARNING` with information about what it's unable to do and how it is attempting to working around it. Other examples of things that cannot be supported would be `dynamic` records on a provider that only supports simple or the lack of support for specific geos in a provider, e.g. Route53Provider does not support `NA-CA-*`. + +It is worth noting that these errors will happen during the plan phase of things so that problems will be indicated without having to make changes. + +This concept is currently a work in progress and partially implemented. While work is on-going `strict_supports` will defaults to `false`. Once the work is considered complete & ready the default will change to `true` as it's a much safer and less surprising default as what you configure is what you'll get unless an error is throw telling you why it cannot be done. You will then have the choice to explicitly request that things continue with a work-around. In the meantime it is encouraged that you manually configure the parameter to `true` in your provider configs. + +### Configuring `strict_supports` + +The `strict_supports` parameter is available on all providers and can be configured in YAML as follows: + +```yaml +providers: + someprovider: + class: whatever.TheProvider + ... + strict_supports: true +``` + ## Custom Sources and Providers You can check out the [source](/octodns/source/) and [provider](/octodns/provider/) directory to see what's currently supported. Sources act as a source of record information. AxfrSource and TinyDnsFileSource are currently the only OSS sources, though we have several others internally that are specific to our environment. These include something to pull host data from [gPanel](https://githubengineering.com/githubs-metal-cloud/) and a similar provider that sources information about our network gear to create both `A` & `PTR` records for their interfaces. Things that might make good OSS sources might include an `ElbSource` that pulls information about [AWS Elastic Load Balancers](https://aws.amazon.com/elasticloadbalancing/) and dynamically creates `CNAME`s for them, or `Ec2Source` that pulls instance information so that records can be created for hosts similar to how our `GPanelProvider` works. diff --git a/octodns/provider/__init__.py b/octodns/provider/__init__.py index dbbaaa8..7e18783 100644 --- a/octodns/provider/__init__.py +++ b/octodns/provider/__init__.py @@ -8,3 +8,7 @@ from __future__ import absolute_import, division, print_function, \ class ProviderException(Exception): pass + + +class SupportsException(ProviderException): + pass diff --git a/octodns/provider/base.py b/octodns/provider/base.py index c862e2d..b636d65 100644 --- a/octodns/provider/base.py +++ b/octodns/provider/base.py @@ -10,7 +10,7 @@ from six import text_type from ..source.base import BaseSource from ..zone import Zone from .plan import Plan -from . import ProviderException +from . import SupportsException class BaseProvider(BaseSource): @@ -85,7 +85,7 @@ class BaseProvider(BaseSource): def supports_warn_or_except(self, msg, fallback): if self.strict_supports: - raise ProviderException('{}: {}'.format(self.id, msg)) + raise SupportsException('{}: {}'.format(self.id, msg)) self.log.warning('{}; {}'.format(msg, fallback)) def plan(self, desired, processors=[]): diff --git a/tests/test_octodns_provider_base.py b/tests/test_octodns_provider_base.py index 501177e..cee7c2c 100644 --- a/tests/test_octodns_provider_base.py +++ b/tests/test_octodns_provider_base.py @@ -11,7 +11,8 @@ from six import text_type from unittest import TestCase from octodns.processor.base import BaseProcessor -from octodns.provider.base import BaseProvider, ProviderException +from octodns.provider import SupportsException +from octodns.provider.base import BaseProvider from octodns.provider.plan import Plan, UnsafePlan from octodns.record import Create, Delete, Record, Update from octodns.zone import Zone @@ -465,7 +466,7 @@ class TestBaseProvider(TestCase): strict = MinimalProvider(strict_supports=True) # Should log and not expect - with self.assertRaises(ProviderException) as ctx: + with self.assertRaises(SupportsException) as ctx: strict.supports_warn_or_except('Hello World!', 'Will not see') self.assertEquals('minimal: Hello World!', text_type(ctx.exception)) strict.log.warning.assert_not_called() From 816d54f3d0005c20adf89d8c09108f6286e30c0f Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Tue, 31 Aug 2021 15:07:59 -0700 Subject: [PATCH 357/358] Pass at copy editing the new strict_supports documentation --- CHANGELOG.md | 8 ++++---- README.md | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3111716..caf5e00 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,10 +5,10 @@ * Provider `strict_supports` param added, currently defaults to `false`, along with Provider._process_desired_zone this forms the foundations of a new "supports" system where providers will warn or error (depending on the value - of `strict_supports` during planning about their inability to do what they're - being asked. When `false` they will warn and "adjust" the desired records. - When true they will abort with an error indicating the problem. Over time it - is expected that all "supports" checking/handling will move into this + of `strict_supports`) during planning about their inability to do what + they're being asked. When `false` they will warn and "adjust" the desired + records. When true they will abort with an error indicating the problem. Over + time it is expected that all "supports" checking/handling will move into this paradigm and `strict_supports` will likely be changed to default to `true`. * Zone shallow copy support, reworking of Processors (alpha) semantics * NS1 NA target now includes `SX` and `UM`. If `NA` continent is in use in diff --git a/README.md b/README.md index b6530b0..28d9e7f 100644 --- a/README.md +++ b/README.md @@ -232,15 +232,15 @@ The above command pulled the existing data out of Route53 and placed the results ### `lenient` -`lenient` mostly focuses on the details of `Record`s and standards compliance. When set to `true` will octoDNS will allow allow non-compliant configurations & values when possible. For example CNAME values that don't end with a `.`, label length restrictions, or invalid geo codes on `dynamic` records. When in lenient mode octoDNS will log validation problems at `WARNING` and try and continue with the configuration or source data as it exists to the degree possible. See [Lenience](/docs/records.md#lenience) for more information on the concept and how it can be configured. +`lenient` mostly focuses on the details of `Record`s and standards compliance. When set to `true` octoDNS will allow allow non-compliant configurations & values where possible. For example CNAME values that don't end with a `.`, label length restrictions, and invalid geo codes on `dynamic` records. When in lenient mode octoDNS will log validation problems at `WARNING` and try and continue with the configuration or source data as it exists. See [Lenience](/docs/records.md#lenience) for more information on the concept and how it can be configured. ### `strict_supports` (Work In Progress) `strict_supports` is a `Provider` level parameter that comes into play when a provider has been asked to create a record that it is unable to support. The simplest case of this would be record type, e.g. `SSHFP` not being supported by `AzureProvider`. If such a record is passed to an `AzureProvider` as a target the provider will take action based on the `strict_supports`. When `true` it will throw an exception saying that it's unable to create the record, when set to `false` it will log at `WARNING` with information about what it's unable to do and how it is attempting to working around it. Other examples of things that cannot be supported would be `dynamic` records on a provider that only supports simple or the lack of support for specific geos in a provider, e.g. Route53Provider does not support `NA-CA-*`. -It is worth noting that these errors will happen during the plan phase of things so that problems will be indicated without having to make changes. +It is worth noting that these errors will happen during the plan phase of things so that problems will be visible without having to make changes. -This concept is currently a work in progress and partially implemented. While work is on-going `strict_supports` will defaults to `false`. Once the work is considered complete & ready the default will change to `true` as it's a much safer and less surprising default as what you configure is what you'll get unless an error is throw telling you why it cannot be done. You will then have the choice to explicitly request that things continue with a work-around. In the meantime it is encouraged that you manually configure the parameter to `true` in your provider configs. +This concept is currently a work in progress and only partially implemented. While work is on-going `strict_supports` will default to `false`. Once the work is considered complete & ready the default will change to `true` as it's a much safer and less surprising default as what you configure is what you'll get unless an error is throw telling you why it cannot be done. You will then have the choice to explicitly request that things continue with work-arounds with `strict_supports` set to false`. In the meantime it is encouraged that you manually configure the parameter to `true` in your provider configs. ### Configuring `strict_supports` From af22e8c9c7323605737768ff14ae8a276099c327 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Tue, 31 Aug 2021 19:39:04 -0700 Subject: [PATCH 358/358] Use ProviderException as the base for all provider exception classes --- octodns/provider/azuredns.py | 3 ++- octodns/provider/cloudflare.py | 3 ++- octodns/provider/constellix.py | 3 ++- octodns/provider/digitalocean.py | 3 ++- octodns/provider/dnsimple.py | 3 ++- octodns/provider/dnsmadeeasy.py | 3 ++- octodns/provider/easydns.py | 3 ++- octodns/provider/edgedns.py | 3 ++- octodns/provider/gandi.py | 3 ++- octodns/provider/gcore.py | 3 ++- octodns/provider/hetzner.py | 3 ++- octodns/provider/mythicbeasts.py | 5 +++-- octodns/provider/ns1.py | 3 ++- octodns/provider/route53.py | 3 ++- octodns/provider/selectel.py | 3 ++- octodns/provider/transip.py | 3 ++- octodns/provider/ultra.py | 3 ++- 17 files changed, 35 insertions(+), 18 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 20a9d7b..0b4e1c9 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -20,10 +20,11 @@ from azure.mgmt.trafficmanager.models import Profile, DnsConfig, \ import logging from functools import reduce from ..record import Record, Update, GeoCodes +from . import ProviderException from .base import BaseProvider -class AzureException(Exception): +class AzureException(ProviderException): pass diff --git a/octodns/provider/cloudflare.py b/octodns/provider/cloudflare.py index ad057eb..a774acc 100644 --- a/octodns/provider/cloudflare.py +++ b/octodns/provider/cloudflare.py @@ -13,10 +13,11 @@ from time import sleep from urllib.parse import urlsplit from ..record import Record, Update +from . import ProviderException from .base import BaseProvider -class CloudflareError(Exception): +class CloudflareError(ProviderException): def __init__(self, data): try: message = data['errors'][0]['message'] diff --git a/octodns/provider/constellix.py b/octodns/provider/constellix.py index cd0d85b..14a7e49 100644 --- a/octodns/provider/constellix.py +++ b/octodns/provider/constellix.py @@ -15,10 +15,11 @@ import logging import time from ..record import Record +from . import ProviderException from .base import BaseProvider -class ConstellixClientException(Exception): +class ConstellixClientException(ProviderException): pass diff --git a/octodns/provider/digitalocean.py b/octodns/provider/digitalocean.py index 6ccee1d..fe31754 100644 --- a/octodns/provider/digitalocean.py +++ b/octodns/provider/digitalocean.py @@ -10,10 +10,11 @@ from requests import Session import logging from ..record import Record +from . import ProviderException from .base import BaseProvider -class DigitalOceanClientException(Exception): +class DigitalOceanClientException(ProviderException): pass diff --git a/octodns/provider/dnsimple.py b/octodns/provider/dnsimple.py index a4b9350..009e829 100644 --- a/octodns/provider/dnsimple.py +++ b/octodns/provider/dnsimple.py @@ -10,10 +10,11 @@ from requests import Session import logging from ..record import Record +from . import ProviderException from .base import BaseProvider -class DnsimpleClientException(Exception): +class DnsimpleClientException(ProviderException): pass diff --git a/octodns/provider/dnsmadeeasy.py b/octodns/provider/dnsmadeeasy.py index b222b5c..ddd40d2 100644 --- a/octodns/provider/dnsmadeeasy.py +++ b/octodns/provider/dnsmadeeasy.py @@ -13,10 +13,11 @@ import hmac import logging from ..record import Record +from . import ProviderException from .base import BaseProvider -class DnsMadeEasyClientException(Exception): +class DnsMadeEasyClientException(ProviderException): pass diff --git a/octodns/provider/easydns.py b/octodns/provider/easydns.py index d7a75a4..67c846a 100644 --- a/octodns/provider/easydns.py +++ b/octodns/provider/easydns.py @@ -12,10 +12,11 @@ import logging import base64 from ..record import Record +from . import ProviderException from .base import BaseProvider -class EasyDNSClientException(Exception): +class EasyDNSClientException(ProviderException): pass diff --git a/octodns/provider/edgedns.py b/octodns/provider/edgedns.py index 26f0917..1dde770 100644 --- a/octodns/provider/edgedns.py +++ b/octodns/provider/edgedns.py @@ -12,10 +12,11 @@ from collections import defaultdict from logging import getLogger from ..record import Record +from . import ProviderException from .base import BaseProvider -class AkamaiClientNotFound(Exception): +class AkamaiClientNotFound(ProviderException): def __init__(self, resp): message = "404: Resource not found" diff --git a/octodns/provider/gandi.py b/octodns/provider/gandi.py index 8401ea4..0938ac9 100644 --- a/octodns/provider/gandi.py +++ b/octodns/provider/gandi.py @@ -10,10 +10,11 @@ from requests import Session import logging from ..record import Record +from . import ProviderException from .base import BaseProvider -class GandiClientException(Exception): +class GandiClientException(ProviderException): pass diff --git a/octodns/provider/gcore.py b/octodns/provider/gcore.py index bbbf81f..b76f90e 100644 --- a/octodns/provider/gcore.py +++ b/octodns/provider/gcore.py @@ -17,10 +17,11 @@ import urllib.parse from ..record import GeoCodes from ..record import Record +from . import ProviderException from .base import BaseProvider -class GCoreClientException(Exception): +class GCoreClientException(ProviderException): def __init__(self, r): super(GCoreClientException, self).__init__(r.text) diff --git a/octodns/provider/hetzner.py b/octodns/provider/hetzner.py index 53914f5..b1b2884 100644 --- a/octodns/provider/hetzner.py +++ b/octodns/provider/hetzner.py @@ -10,10 +10,11 @@ from requests import Session import logging from ..record import Record +from . import ProviderException from .base import BaseProvider -class HetznerClientException(Exception): +class HetznerClientException(ProviderException): pass diff --git a/octodns/provider/mythicbeasts.py b/octodns/provider/mythicbeasts.py index e1a2b04..683209d 100644 --- a/octodns/provider/mythicbeasts.py +++ b/octodns/provider/mythicbeasts.py @@ -11,6 +11,7 @@ from requests import Session from logging import getLogger from ..record import Record +from . import ProviderException from .base import BaseProvider from collections import defaultdict @@ -34,7 +35,7 @@ def remove_trailing_dot(value): return value[:-1] -class MythicBeastsUnauthorizedException(Exception): +class MythicBeastsUnauthorizedException(ProviderException): def __init__(self, zone, *args): self.zone = zone self.message = 'Mythic Beasts unauthorized for zone: {}'.format( @@ -45,7 +46,7 @@ class MythicBeastsUnauthorizedException(Exception): self.message, self.zone, *args) -class MythicBeastsRecordException(Exception): +class MythicBeastsRecordException(ProviderException): def __init__(self, zone, command, *args): self.zone = zone self.command = command diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 38b9240..8c09f39 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -17,6 +17,7 @@ from uuid import uuid4 from six import text_type from ..record import Record, Update +from . import ProviderException from .base import BaseProvider @@ -24,7 +25,7 @@ def _ensure_endswith_dot(string): return string if string.endswith('.') else '{}.'.format(string) -class Ns1Exception(Exception): +class Ns1Exception(ProviderException): pass diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index b4bd557..14d15ce 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -20,6 +20,7 @@ from ..equality import EqualityTupleMixin from ..record import Record, Update from ..record.geo import GeoCodes from ..zone import Zone +from . import ProviderException from .base import BaseProvider octal_re = re.compile(r'\\(\d\d\d)') @@ -513,7 +514,7 @@ class _Route53GeoRecord(_Route53Record): self.values) -class Route53ProviderException(Exception): +class Route53ProviderException(ProviderException): pass diff --git a/octodns/provider/selectel.py b/octodns/provider/selectel.py index b9a99aa..87f8d4f 100644 --- a/octodns/provider/selectel.py +++ b/octodns/provider/selectel.py @@ -12,6 +12,7 @@ from logging import getLogger from requests import Session from ..record import Record, Update +from . import ProviderException from .base import BaseProvider @@ -20,7 +21,7 @@ def escape_semicolon(s): return s.replace(';', '\\;') -class SelectelAuthenticationRequired(Exception): +class SelectelAuthenticationRequired(ProviderException): def __init__(self, msg): message = 'Authorization failed. Invalid or empty token.' super(SelectelAuthenticationRequired, self).__init__(message) diff --git a/octodns/provider/transip.py b/octodns/provider/transip.py index 6ccbe22..2bdedc4 100644 --- a/octodns/provider/transip.py +++ b/octodns/provider/transip.py @@ -8,6 +8,7 @@ from __future__ import absolute_import, division, print_function, \ from suds import WebFault from collections import defaultdict +from . import ProviderException from .base import BaseProvider from logging import getLogger from ..record import Record @@ -15,7 +16,7 @@ from transip.service.domain import DomainService from transip.service.objects import DnsEntry -class TransipException(Exception): +class TransipException(ProviderException): pass diff --git a/octodns/provider/ultra.py b/octodns/provider/ultra.py index afa4d0a..e382a33 100644 --- a/octodns/provider/ultra.py +++ b/octodns/provider/ultra.py @@ -3,10 +3,11 @@ from logging import getLogger from requests import Session from ..record import Record +from . import ProviderException from .base import BaseProvider -class UltraClientException(Exception): +class UltraClientException(ProviderException): ''' Base Ultra exception type '''