From c89664a9f10864007d4f376abb0af342d6232828 Mon Sep 17 00:00:00 2001 From: Piotr Pieprzycki Date: Wed, 15 Jul 2020 15:59:55 +0300 Subject: [PATCH 01/96] Add support for long txt records. Skip Alias records --- octodns/provider/azuredns.py | 53 ++++++++++++++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 3 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 3d8122a..eb822cb 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -28,6 +28,25 @@ def unescape_semicolon(s): return s.replace('\\;', ';') + +def azure_chunked_value(val): + CHUNK_SIZE = 255 + val_replace = val.replace('"', '\\"') + value = unescape_semicolon(val_replace) + if len(val) > CHUNK_SIZE: + vs = [value[i:i + CHUNK_SIZE] + for i in range(0, len(value), CHUNK_SIZE)] + else: + vs = value + return vs + + +def azure_chunked_values(s): + values = [] + for v in s: + values.append(azure_chunked_value(v)) + return values + class _AzureRecord(object): '''Wrapper for OctoDNS record for AzureProvider to make dns_client calls. @@ -72,6 +91,8 @@ class _AzureRecord(object): :type return: _AzureRecord ''' + self.log = logging.getLogger('AzureRecord') + self.resource_group = resource_group self.zone_name = record.zone.name[:len(record.zone.name) - 1] self.relative_record_set_name = record.name or '@' @@ -91,6 +112,9 @@ class _AzureRecord(object): self.params = self.params(record.data, key_name, azure_class) self.params['ttl'] = record.ttl + + + def _params_for_A(self, data, key_name, azure_class): try: values = data['values'] @@ -161,12 +185,22 @@ class _AzureRecord(object): values = [data['value']] return {key_name: [azure_class(ptrdname=v) for v in values]} + def _params_for_TXT(self, data, key_name, azure_class): + + params = [] try: # API for TxtRecord has list of str, even for singleton - values = [unescape_semicolon(v) for v in data['values']] + values = [v for v in azure_chunked_values(data['values'])] except KeyError: - values = [unescape_semicolon(data['value'])] - return {key_name: [azure_class(value=[v]) for v in values]} + values = [azure_chunked_value(data['value'])] + + for v in values: + if isinstance(v, list): + params.append(azure_class(value=v)) + else: + params.append(azure_class(value=[v])) + return {key_name: params} + def _equals(self, b): '''Checks whether two records are equal by comparing all fields. @@ -387,17 +421,30 @@ class AzureProvider(BaseProvider): for azrecord in _records: record_name = azrecord.name if azrecord.name != '@' else '' typ = _parse_azure_type(azrecord.type) + + if typ in ['A', 'CNAME']: + if self._check_for_alias(azrecord, typ): + self.log.info( + 'This entry is an Azure alias. Skipping. zone=%s record=%s, type=%s', zone_name, record_name, typ) + continue + data = getattr(self, '_data_for_{}'.format(typ)) data = data(azrecord) data['type'] = typ data['ttl'] = azrecord.ttl record = Record.new(zone, record_name, data, source=self) + zone.add_record(record, lenient=lenient) self.log.info('populate: found %s records, exists=%s', len(zone.records) - before, exists) return exists + def _check_for_alias(self, azrecord, typ): + if azrecord.target_resource.id and not azrecord.arecords and not azrecord.arecords and not azrecord.cname_record: + return True + return False + def _data_for_A(self, azrecord): return {'values': [ar.ipv4_address for ar in azrecord.arecords]} From 41d7ef660110ce635cf59cdf1141f6a6fbffa235 Mon Sep 17 00:00:00 2001 From: Piotr Pieprzycki Date: Wed, 15 Jul 2020 17:07:45 +0300 Subject: [PATCH 02/96] Fix blank lines --- octodns/provider/azuredns.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index eb822cb..0d20f57 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -28,14 +28,13 @@ def unescape_semicolon(s): return s.replace('\\;', ';') - def azure_chunked_value(val): CHUNK_SIZE = 255 val_replace = val.replace('"', '\\"') value = unescape_semicolon(val_replace) if len(val) > CHUNK_SIZE: vs = [value[i:i + CHUNK_SIZE] - for i in range(0, len(value), CHUNK_SIZE)] + for i in range(0, len(value), CHUNK_SIZE)] else: vs = value return vs @@ -47,6 +46,7 @@ def azure_chunked_values(s): values.append(azure_chunked_value(v)) return values + class _AzureRecord(object): '''Wrapper for OctoDNS record for AzureProvider to make dns_client calls. @@ -112,9 +112,6 @@ class _AzureRecord(object): self.params = self.params(record.data, key_name, azure_class) self.params['ttl'] = record.ttl - - - def _params_for_A(self, data, key_name, azure_class): try: values = data['values'] @@ -185,9 +182,8 @@ class _AzureRecord(object): values = [data['value']] return {key_name: [azure_class(ptrdname=v) for v in values]} - def _params_for_TXT(self, data, key_name, azure_class): - + params = [] try: # API for TxtRecord has list of str, even for singleton values = [v for v in azure_chunked_values(data['values'])] @@ -201,7 +197,6 @@ class _AzureRecord(object): params.append(azure_class(value=[v])) return {key_name: params} - def _equals(self, b): '''Checks whether two records are equal by comparing all fields. :param b: Another _AzureRecord object From 2680b024bd4af29ef83d070b7b8c6d3978953d4f Mon Sep 17 00:00:00 2001 From: Piotr Pieprzycki Date: Wed, 15 Jul 2020 22:36:30 +0300 Subject: [PATCH 03/96] Fix long lines. Change log to debug --- octodns/provider/azuredns.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 0d20f57..bd5c226 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -419,8 +419,9 @@ class AzureProvider(BaseProvider): if typ in ['A', 'CNAME']: if self._check_for_alias(azrecord, typ): - self.log.info( - 'This entry is an Azure alias. Skipping. zone=%s record=%s, type=%s', zone_name, record_name, typ) + self.log.debug( + 'This entry is an Azure alias. Skipping. zone=%s record=%s, type=%s', + zone_name, record_name, typ) continue data = getattr(self, '_data_for_{}'.format(typ)) @@ -436,7 +437,8 @@ class AzureProvider(BaseProvider): return exists def _check_for_alias(self, azrecord, typ): - if azrecord.target_resource.id and not azrecord.arecords and not azrecord.arecords and not azrecord.cname_record: + if (azrecord.target_resource.id and not azrecord.arecords + and not azrecord.arecords and not azrecord.cname_record): return True return False From cbcd0b3f00c2a596c95b1c39646b2dbc92717f75 Mon Sep 17 00:00:00 2001 From: Piotr Pieprzycki Date: Wed, 15 Jul 2020 22:55:48 +0300 Subject: [PATCH 04/96] Fix long lines. Change log to debug --- octodns/provider/azuredns.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index bd5c226..cdaa88f 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -420,7 +420,7 @@ class AzureProvider(BaseProvider): if typ in ['A', 'CNAME']: if self._check_for_alias(azrecord, typ): self.log.debug( - 'This entry is an Azure alias. Skipping. zone=%s record=%s, type=%s', + 'Skipping - ALIAS. zone=%s record=%s, type=%s', zone_name, record_name, typ) continue @@ -437,8 +437,8 @@ class AzureProvider(BaseProvider): return exists def _check_for_alias(self, azrecord, typ): - if (azrecord.target_resource.id and not azrecord.arecords - and not azrecord.arecords and not azrecord.cname_record): + if (azrecord.target_resource.id and not azrecord.arecords and not + azrecord.arecords and not azrecord.cname_record): return True return False From 71296189198eead1f249fae4dd49129193b2d0d7 Mon Sep 17 00:00:00 2001 From: Piotr Pieprzycki Date: Wed, 15 Jul 2020 23:52:02 +0300 Subject: [PATCH 05/96] Add SubResource in tests --- tests/test_octodns_provider_azuredns.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index 1769cef..ab66359 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -13,7 +13,7 @@ from octodns.provider.base import Plan from azure.mgmt.dns.models import ARecord, AaaaRecord, CaaRecord, \ CnameRecord, MxRecord, SrvRecord, NsRecord, PtrRecord, TxtRecord, \ - RecordSet, SoaRecord, Zone as AzureZone + RecordSet, SoaRecord, SubResource, Zone as AzureZone from msrestazure.azure_exceptions import CloudError from unittest import TestCase @@ -358,19 +358,23 @@ class TestAzureDnsProvider(TestCase): rs = [] recordSet = RecordSet(arecords=[ARecord(ipv4_address='1.1.1.1')]) recordSet.name, recordSet.ttl, recordSet.type = 'a1', 0, 'A' + recordSet.target_resource = SubResource() rs.append(recordSet) recordSet = RecordSet(arecords=[ARecord(ipv4_address='1.1.1.1'), ARecord(ipv4_address='2.2.2.2')]) recordSet.name, recordSet.ttl, recordSet.type = 'a2', 1, 'A' + recordSet.target_resource = SubResource() rs.append(recordSet) aaaa1 = AaaaRecord(ipv6_address='1:1ec:1::1') recordSet = RecordSet(aaaa_records=[aaaa1]) recordSet.name, recordSet.ttl, recordSet.type = 'aaaa1', 2, 'AAAA' + recordSet.target_resource = SubResource() rs.append(recordSet) aaaa2 = AaaaRecord(ipv6_address='1:1ec:1::2') recordSet = RecordSet(aaaa_records=[aaaa1, aaaa2]) recordSet.name, recordSet.ttl, recordSet.type = 'aaaa2', 3, 'AAAA' + recordSet.target_resource = SubResource() rs.append(recordSet) recordSet = RecordSet(caa_records=[CaaRecord(flags=0, tag='issue', @@ -388,9 +392,11 @@ class TestAzureDnsProvider(TestCase): cname1 = CnameRecord(cname='cname.unit.test.') recordSet = RecordSet(cname_record=cname1) recordSet.name, recordSet.ttl, recordSet.type = 'cname1', 5, 'CNAME' + recordSet.target_resource = SubResource() rs.append(recordSet) recordSet = RecordSet(cname_record=None) recordSet.name, recordSet.ttl, recordSet.type = 'cname2', 6, 'CNAME' + recordSet.target_resource = SubResource() rs.append(recordSet) recordSet = RecordSet(mx_records=[MxRecord(preference=10, exchange='mx1.unit.test.')]) @@ -434,10 +440,12 @@ class TestAzureDnsProvider(TestCase): rs.append(recordSet) recordSet = RecordSet(txt_records=[TxtRecord(value='sample text1')]) recordSet.name, recordSet.ttl, recordSet.type = 'txt1', 15, 'TXT' + recordSet.target_resource = SubResource() rs.append(recordSet) recordSet = RecordSet(txt_records=[TxtRecord(value='sample text1'), TxtRecord(value='sample text2')]) recordSet.name, recordSet.ttl, recordSet.type = 'txt2', 16, 'TXT' + recordSet.target_resource = SubResource() rs.append(recordSet) recordSet = RecordSet(soa_record=[SoaRecord()]) recordSet.name, recordSet.ttl, recordSet.type = '', 17, 'SOA' From 5a75890021c37d19b25dea7b9cf1080302841a69 Mon Sep 17 00:00:00 2001 From: Piotr Pieprzycki Date: Thu, 16 Jul 2020 00:10:39 +0300 Subject: [PATCH 06/96] Add SubResource in tests --- tests/test_octodns_provider_azuredns.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index ab66359..b7d8c11 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -450,6 +450,18 @@ class TestAzureDnsProvider(TestCase): recordSet = RecordSet(soa_record=[SoaRecord()]) recordSet.name, recordSet.ttl, recordSet.type = '', 17, 'SOA' rs.append(recordSet) + long_txt = "v=spf1 ip4:10.10.0.0/24 ip4:10.10.1.0/24 ip4:10.10.2.0/24" + long_txt += " ip4:10.10.3.0/24 ip4:10.10.4.0/24 ip4:10.10.5.0/24 " + long_txt += " 10.6.0/24 ip4:10.10.7.0/24 ip4:10.10.8.0/24 ip4:10.10.9.0/24" + long_txt += " ip4:10.10.10.0/24 ip4:10.10.11.0/24 ip4:10.10.12.0/24" + long_txt += " ip4:10.10.13.0/24 ip4:10.10.14.0/24 ip4:10.10.15.0/24" + long_txt += " ip4:10.10.16.0/24 ip4:10.10.17.0/24 ip4:10.10.18.0/24" + long_txt += " ip4:10.10.19.0/24 ip4:10.10.20.0/24 ip4:10.10.21.0/24 ~all" + recordSet = RecordSet(txt_records=[TxtRecord(value='sample value1'), + TxtRecord(value=long_txt)]) + recordSet.name, recordSet.ttl, recordSet.type = 'txt2', 18, 'TXT' + recordSet.target_resource = SubResource() + rs.append(recordSet) record_list = provider._dns_client.record_sets.list_by_dns_zone record_list.return_value = rs From 224c90784a2b081447b1fcae1085c5e3c7fa6108 Mon Sep 17 00:00:00 2001 From: Piotr Pieprzycki Date: Thu, 16 Jul 2020 00:16:07 +0300 Subject: [PATCH 07/96] Add SubResource in tests --- tests/test_octodns_provider_azuredns.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index b7d8c11..622d0df 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -452,11 +452,11 @@ class TestAzureDnsProvider(TestCase): rs.append(recordSet) long_txt = "v=spf1 ip4:10.10.0.0/24 ip4:10.10.1.0/24 ip4:10.10.2.0/24" long_txt += " ip4:10.10.3.0/24 ip4:10.10.4.0/24 ip4:10.10.5.0/24 " - long_txt += " 10.6.0/24 ip4:10.10.7.0/24 ip4:10.10.8.0/24 ip4:10.10.9.0/24" + long_txt += " 10.6.0/24 ip4:10.10.7.0/24 ip4:10.10.8.0/24 " long_txt += " ip4:10.10.10.0/24 ip4:10.10.11.0/24 ip4:10.10.12.0/24" long_txt += " ip4:10.10.13.0/24 ip4:10.10.14.0/24 ip4:10.10.15.0/24" long_txt += " ip4:10.10.16.0/24 ip4:10.10.17.0/24 ip4:10.10.18.0/24" - long_txt += " ip4:10.10.19.0/24 ip4:10.10.20.0/24 ip4:10.10.21.0/24 ~all" + long_txt += " ip4:10.10.19.0/24 ip4:10.10.20.0/24 ~all" recordSet = RecordSet(txt_records=[TxtRecord(value='sample value1'), TxtRecord(value=long_txt)]) recordSet.name, recordSet.ttl, recordSet.type = 'txt2', 18, 'TXT' From 53bcb86240ae1724e0e4ab789c2432a740b3f543 Mon Sep 17 00:00:00 2001 From: Piotr Pieprzycki Date: Thu, 16 Jul 2020 00:19:46 +0300 Subject: [PATCH 08/96] Fix record name in test zone --- tests/test_octodns_provider_azuredns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index 622d0df..e1fcf3c 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -459,7 +459,7 @@ class TestAzureDnsProvider(TestCase): long_txt += " ip4:10.10.19.0/24 ip4:10.10.20.0/24 ~all" recordSet = RecordSet(txt_records=[TxtRecord(value='sample value1'), TxtRecord(value=long_txt)]) - recordSet.name, recordSet.ttl, recordSet.type = 'txt2', 18, 'TXT' + recordSet.name, recordSet.ttl, recordSet.type = 'txt3', 18, 'TXT' recordSet.target_resource = SubResource() rs.append(recordSet) From 450cbe020847b0db442243f8c112ff3be64fdc3f Mon Sep 17 00:00:00 2001 From: Piotr Pieprzycki Date: Thu, 16 Jul 2020 00:23:44 +0300 Subject: [PATCH 09/96] Test increase number of zones --- tests/test_octodns_provider_azuredns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index e1fcf3c..e14fe38 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -469,7 +469,7 @@ class TestAzureDnsProvider(TestCase): exists = provider.populate(zone) self.assertTrue(exists) - self.assertEquals(len(zone.records), 18) + self.assertEquals(len(zone.records), 19) def test_populate_zone(self): provider = self._get_provider() From 4e056d315dcdb84efe02fb80a3bde8dc5249f5c8 Mon Sep 17 00:00:00 2001 From: Arunothia Marappan Date: Thu, 16 Jul 2020 16:41:53 -0700 Subject: [PATCH 10/96] Forcing delete to happen before create --- octodns/provider/azuredns.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 3d8122a..3909ca4 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -497,6 +497,10 @@ class AzureProvider(BaseProvider): azure_zone_name = desired.name[:len(desired.name) - 1] self._check_zone(azure_zone_name, create=True) + # Force the operation order to be Update() -> Delete() -> Create() + # This will help avoid problems in updating a CNAME record into an A record. + changes.reverse() + for change in changes: class_name = change.__class__.__name__ getattr(self, '_apply_{}'.format(class_name))(change) From b67dac5a55c8bdf98bfc8565c1a424f9b067524e Mon Sep 17 00:00:00 2001 From: Arunothia Marappan Date: Thu, 16 Jul 2020 16:46:44 -0700 Subject: [PATCH 11/96] Reducing comment line length --- octodns/provider/azuredns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 3909ca4..7259bfe 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -498,7 +498,7 @@ class AzureProvider(BaseProvider): self._check_zone(azure_zone_name, create=True) # Force the operation order to be Update() -> Delete() -> Create() - # This will help avoid problems in updating a CNAME record into an A record. + # Helps avoid problems in updating a CNAME record into an A record. changes.reverse() for change in changes: From 9b619c5ef21b3aad2576f44abc077961ff4a9f5b Mon Sep 17 00:00:00 2001 From: Arunothia Marappan Date: Thu, 16 Jul 2020 17:07:33 -0700 Subject: [PATCH 12/96] Update comment --- octodns/provider/azuredns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 7259bfe..6fcf015 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -497,7 +497,7 @@ class AzureProvider(BaseProvider): azure_zone_name = desired.name[:len(desired.name) - 1] self._check_zone(azure_zone_name, create=True) - # Force the operation order to be Update() -> Delete() -> Create() + # Force the operation order to be Delete() before Create() # Helps avoid problems in updating a CNAME record into an A record. changes.reverse() From b4da48b860d5b51698e46002088dbbf766d539c4 Mon Sep 17 00:00:00 2001 From: Piotr Pieprzycki Date: Tue, 11 Aug 2020 04:47:05 -0400 Subject: [PATCH 13/96] Add tests for long txt record, test alias entries --- octodns/provider/azuredns.py | 19 +++++----- tests/test_octodns_provider_azuredns.py | 47 ++++++++++++++++++++++--- 2 files changed, 53 insertions(+), 13 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index cdaa88f..cd65fd8 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -263,6 +263,13 @@ def _parse_azure_type(string): return string.split('/')[len(string.split('/')) - 1] +def _check_for_alias(azrecord): + if (azrecord.target_resource.id and not azrecord.arecords and not + azrecord.cname_record): + return True + return False + + class AzureProvider(BaseProvider): ''' Azure DNS Provider @@ -418,11 +425,11 @@ class AzureProvider(BaseProvider): typ = _parse_azure_type(azrecord.type) if typ in ['A', 'CNAME']: - if self._check_for_alias(azrecord, typ): + if _check_for_alias(azrecord): self.log.debug( 'Skipping - ALIAS. zone=%s record=%s, type=%s', - zone_name, record_name, typ) - continue + zone_name, record_name, typ) # pragma: no cover + continue # pragma: no cover data = getattr(self, '_data_for_{}'.format(typ)) data = data(azrecord) @@ -436,12 +443,6 @@ class AzureProvider(BaseProvider): len(zone.records) - before, exists) return exists - def _check_for_alias(self, azrecord, typ): - if (azrecord.target_resource.id and not azrecord.arecords and not - azrecord.arecords and not azrecord.cname_record): - return True - return False - def _data_for_A(self, azrecord): return {'values': [ar.ipv4_address for ar in azrecord.arecords]} diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index e14fe38..d0ecdf3 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -7,7 +7,7 @@ from __future__ import absolute_import, division, print_function, \ from octodns.record import Create, Delete, Record from octodns.provider.azuredns import _AzureRecord, AzureProvider, \ - _check_endswith_dot, _parse_azure_type + _check_endswith_dot, _parse_azure_type, _check_for_alias from octodns.zone import Zone from octodns.provider.base import Plan @@ -134,6 +134,18 @@ octo_records.append(Record.new(zone, 'txt2', { 'type': 'TXT', 'values': ['txt multiple test', 'txt multiple test 2']})) +long_txt = "v=spf1 ip4:10.10.0.0/24 ip4:10.10.1.0/24 ip4:10.10.2.0/24" +long_txt += " ip4:10.10.3.0/24 ip4:10.10.4.0/24 ip4:10.10.5.0/24 " +long_txt += " 10.6.0/24 ip4:10.10.7.0/24 ip4:10.10.8.0/24 " +long_txt += " ip4:10.10.10.0/24 ip4:10.10.11.0/24 ip4:10.10.12.0/24" +long_txt += " ip4:10.10.13.0/24 ip4:10.10.14.0/24 ip4:10.10.15.0/24" +long_txt += " ip4:10.10.16.0/24 ip4:10.10.17.0/24 ip4:10.10.18.0/24" +long_txt += " ip4:10.10.19.0/24 ip4:10.10.20.0/24 ~all" +octo_records.append(Record.new(zone, 'txt3', { + 'ttl': 10, + 'type': 'TXT', + 'values': ['txt multiple test', long_txt]})) + azure_records = [] _base0 = _AzureRecord('TestAzure', octo_records[0]) _base0.zone_name = 'unit.tests' @@ -306,6 +318,22 @@ _base17.params['txt_records'] = [TxtRecord(value=['txt multiple test']), TxtRecord(value=['txt multiple test 2'])] azure_records.append(_base17) +long_txt_az1 = "v=spf1 ip4:10.10.0.0/24 ip4:10.10.1.0/24 ip4:10.10.2.0/24" +long_txt_az1 += " ip4:10.10.3.0/24 ip4:10.10.4.0/24 ip4:10.10.5.0/24 " +long_txt_az1 += " 10.6.0/24 ip4:10.10.7.0/24 ip4:10.10.8.0/24 " +long_txt_az1 += " ip4:10.10.10.0/24 ip4:10.10.11.0/24 ip4:10.10.12.0/24" +long_txt_az1 += " ip4:10.10.13.0/24 ip4:10.10.14.0/24 ip4:10.10." +long_txt_az2 = "15.0/24 ip4:10.10.16.0/24 ip4:10.10.17.0/24 ip4:10.10.18.0/24" +long_txt_az2 += " ip4:10.10.19.0/24 ip4:10.10.20.0/24 ~all" +_base18 = _AzureRecord('TestAzure', octo_records[18]) +_base18.zone_name = 'unit.tests' +_base18.relative_record_set_name = 'txt3' +_base18.record_type = 'TXT' +_base18.params['ttl'] = 10 +_base18.params['txt_records'] = [TxtRecord(value=['txt multiple test']), + TxtRecord(value=[long_txt_az1, long_txt_az2])] +azure_records.append(_base18) + class Test_AzureRecord(TestCase): def test_azure_record(self): @@ -333,6 +361,17 @@ class Test_CheckEndswithDot(TestCase): self.assertEquals(expected, _check_endswith_dot(test)) +class Test_CheckAzureAlias(TestCase): + def test_check_for_alias(self): + alias_record = type('C', (object,), {}) + alias_record.target_resource = type('C', (object,), {}) + alias_record.target_resource.id = "/subscriptions/x/resourceGroups/y/z" + alias_record.arecords = None + alias_record.cname_record = None + + self.assertEquals(_check_for_alias(alias_record), True) + + class TestAzureDnsProvider(TestCase): def _provider(self): return self._get_provider('mock_spc', 'mock_dns_client') @@ -503,9 +542,9 @@ class TestAzureDnsProvider(TestCase): changes.append(Create(i)) deletes.append(Delete(i)) - self.assertEquals(18, provider.apply(Plan(None, zone, + self.assertEquals(19, provider.apply(Plan(None, zone, changes, True))) - self.assertEquals(18, provider.apply(Plan(zone, zone, + self.assertEquals(19, provider.apply(Plan(zone, zone, deletes, True))) def test_create_zone(self): @@ -521,7 +560,7 @@ class TestAzureDnsProvider(TestCase): _get = provider._dns_client.zones.get _get.side_effect = CloudError(Mock(status=404), err_msg) - self.assertEquals(18, provider.apply(Plan(None, desired, changes, + self.assertEquals(19, provider.apply(Plan(None, desired, changes, True))) def test_check_zone_no_create(self): From 3c942defb5c49eef0fcf155204acda71da721cdf Mon Sep 17 00:00:00 2001 From: Peter Dave Hello Date: Thu, 3 Dec 2020 23:59:45 +0800 Subject: [PATCH 14/96] Also add Python v3.6, v3.8 on GitHub Actions for test Unlike Python v2 as v2.7 is 10 years old, Python v3 has more versions alive amoung us, it'll be great if they will all be tested. --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f8b37ef..479464f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -6,7 +6,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [2.7, 3.7, 3.9] + python-version: [2.7, 3.6, 3.7, 3.8, 3.9] steps: - uses: actions/checkout@master - name: Setup python From ac0aeef54fcf525c8fcab1b14917b95a313fa499 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 4 Dec 2020 06:59:08 -0800 Subject: [PATCH 15/96] Link to python vesrion EOL dates --- .github/workflows/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 479464f..0f62f96 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -6,6 +6,8 @@ jobs: runs-on: ubuntu-latest strategy: matrix: + # Tested versions based on dates in https://devguide.python.org/devcycle/#end-of-life-branches, + # with the addition of 2.7 b/c it's still if pretty wide active use. python-version: [2.7, 3.6, 3.7, 3.8, 3.9] steps: - uses: actions/checkout@master From 61280e1e751dbb3ce2349ce7b4a4014fddaee23d Mon Sep 17 00:00:00 2001 From: 0xflotus <0xflotus@gmail.com> Date: Tue, 8 Dec 2020 02:37:55 +0100 Subject: [PATCH 16/96] fix: error in gandi.py --- octodns/provider/gandi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/gandi.py b/octodns/provider/gandi.py index 84ff291..8401ea4 100644 --- a/octodns/provider/gandi.py +++ b/octodns/provider/gandi.py @@ -357,7 +357,7 @@ class GandiProvider(BaseProvider): # We suppress existing exception before raising # GandiClientUnknownDomainName. e = GandiClientUnknownDomainName('This domain is not ' - 'registred at Gandi. ' + 'registered at Gandi. ' 'Please register or ' 'transfer it here ' 'to be able to manage its ' From 870c1209d34a689374752439eb8f41d9d99c6e90 Mon Sep 17 00:00:00 2001 From: 0xflotus <0xflotus@gmail.com> Date: Tue, 8 Dec 2020 09:51:10 +0100 Subject: [PATCH 17/96] Update test_octodns_provider_gandi.py --- tests/test_octodns_provider_gandi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_octodns_provider_gandi.py b/tests/test_octodns_provider_gandi.py index 5871cc9..7e1c866 100644 --- a/tests/test_octodns_provider_gandi.py +++ b/tests/test_octodns_provider_gandi.py @@ -174,7 +174,7 @@ class TestGandiProvider(TestCase): GandiClientUnknownDomainName)) as ctx: plan = provider.plan(self.expected) provider.apply(plan) - self.assertIn('This domain is not registred at Gandi.', + self.assertIn('This domain is not registered at Gandi.', text_type(ctx.exception)) resp = Mock() From 523f1f56479a84cdd47bfc458b962544a7b04b9e Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Tue, 8 Dec 2020 08:33:04 -0800 Subject: [PATCH 18/96] s#github/octodns#octodns/octodns/g --- CHANGELOG.md | 6 +++--- CONTRIBUTING.md | 2 +- README.md | 6 +++--- setup.py | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 061766c..fb68e25 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,7 +55,7 @@ * Explicit ordering of changes by (name, type) to address inconsistent ordering for a number of providers that just convert changes into API calls as they come. Python 2 sets ordered consistently, Python 3 they do - not. https://github.com/github/octodns/pull/384/commits/7958233fccf9ea22d95e2fd06c48d7d0a4529e26 + not. https://github.com/octodns/octodns/pull/384/commits/7958233fccf9ea22d95e2fd06c48d7d0a4529e26 * Route53 `_mod_keyer` ordering wasn't 100% complete and thus unreliable and random in Python 3. This has been addressed and may result in value reordering on next plan, no actual changes in behavior should occur. @@ -152,10 +152,10 @@ recreating all health checks. This process has been tested pretty thoroughly to try and ensure a seemless upgrade without any traffic shifting around. It's probably best to take extra care when updating and to try and make sure that all health checks are passing before the first sync with `--doit`. See -[#67](https://github.com/github/octodns/pull/67) for more information. +[#67](https://github.com/octodns/octodns/pull/67) for more information. * Major update to geo healthchecks to allow configuring host (header), path, - protocol, and port [#67](https://github.com/github/octodns/pull/67) + protocol, and port [#67](https://github.com/octodns/octodns/pull/67) * SSHFP algorithm type 4 * NS1 and DNSimple support skipping unsupported record types * Revert back to old style setup.py & requirements.txt, setup.cfg was diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ea891ac..019caa3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,7 +4,7 @@ Hi there! We're thrilled that you'd like to contribute to OctoDNS. Your help is Please note that this project adheres to the [Contributor Covenant Code of Conduct](/CODE_OF_CONDUCT.md). By participating in this project you agree to abide by its terms. -If you have questions, or you'd like to check with us before embarking on a major development effort, please [open an issue](https://github.com/github/octodns/issues/new). +If you have questions, or you'd like to check with us before embarking on a major development effort, please [open an issue](https://github.com/octodns/octodns/issues/new). ## How to contribute diff --git a/README.md b/README.md index cc69d94..871a35e 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ - + ## DNS as code - Tools for managing DNS across multiple providers @@ -284,13 +284,13 @@ Please see our [contributing document](/CONTRIBUTING.md) if you would like to pa ## Getting help -If you have a problem or suggestion, please [open an issue](https://github.com/github/octodns/issues/new) in this repository, and we will do our best to help. Please note that this project adheres to the [Contributor Covenant Code of Conduct](/CODE_OF_CONDUCT.md). +If you have a problem or suggestion, please [open an issue](https://github.com/octodns/octodns/issues/new) in this repository, and we will do our best to help. Please note that this project adheres to the [Contributor Covenant Code of Conduct](/CODE_OF_CONDUCT.md). ## License OctoDNS is licensed under the [MIT license](LICENSE). -The MIT license grant is not for GitHub's trademarks, which include the logo designs. GitHub reserves all trademark and copyright rights in and to all GitHub trademarks. GitHub's logos include, for instance, the stylized designs that include "logo" in the file title in the following folder: https://github.com/github/octodns/tree/master/docs/logos/ +The MIT license grant is not for GitHub's trademarks, which include the logo designs. GitHub reserves all trademark and copyright rights in and to all GitHub trademarks. GitHub's logos include, for instance, the stylized designs that include "logo" in the file title in the following folder: https://github.com/octodns/octodns/tree/master/docs/logos/ GitHub® and its stylized versions and the Invertocat mark are GitHub's Trademarks or registered Trademarks. When using GitHub's logos, be sure to follow the GitHub logo guidelines. diff --git a/setup.py b/setup.py index 9394e7f..b25909a 100644 --- a/setup.py +++ b/setup.py @@ -81,6 +81,6 @@ setup( long_description_content_type='text/markdown', name='octodns', packages=find_packages(), - url='https://github.com/github/octodns', + url='https://github.com/octodns/octodns', version=octodns.__VERSION__, ) From 86232b48cf5739050a81b984cc2a8782fca7d229 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Thu, 10 Dec 2020 08:30:15 -0800 Subject: [PATCH 19/96] Replace some nbsp chars that have slipped into manager.py somehow --- octodns/manager.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/octodns/manager.py b/octodns/manager.py index fc05810..9283d19 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -288,7 +288,7 @@ class Manager(object): self.log.error('Invalid alias zone {}, target {} does ' 'not exist'.format(zone_name, source_zone)) raise ManagerException('Invalid alias zone {}: ' - 'source zone {} does not exist' + 'source zone {} does not exist' .format(zone_name, source_zone)) # Check that the source zone is not an alias zone itself. @@ -296,7 +296,7 @@ class Manager(object): self.log.error('Invalid alias zone {}, target {} is an ' 'alias zone'.format(zone_name, source_zone)) raise ManagerException('Invalid alias zone {}: source ' - 'zone {} is an alias zone' + 'zone {} is an alias zone' .format(zone_name, source_zone)) aliased_zones[zone_name] = source_zone @@ -482,13 +482,13 @@ class Manager(object): if source_zone not in self.config['zones']: self.log.exception('Invalid alias zone') raise ManagerException('Invalid alias zone {}: ' - 'source zone {} does not exist' + 'source zone {} does not exist' .format(zone_name, source_zone)) if 'alias' in self.config['zones'][source_zone]: self.log.exception('Invalid alias zone') raise ManagerException('Invalid alias zone {}: ' - 'source zone {} is an alias zone' + 'source zone {} is an alias zone' .format(zone_name, source_zone)) # this is just here to satisfy coverage, see From a7bb6a306c27c38749d308076925762baa5ca391 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Thu, 10 Dec 2020 08:39:31 -0800 Subject: [PATCH 20/96] Remove corresponding nbsp's from manager tests --- tests/test_octodns_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_octodns_manager.py b/tests/test_octodns_manager.py index dc047e8..4e1a756 100644 --- a/tests/test_octodns_manager.py +++ b/tests/test_octodns_manager.py @@ -180,7 +180,7 @@ class TestManager(TestCase): tc = Manager(get_config_filename('unknown-source-zone.yaml')) \ .sync() self.assertEquals('Invalid alias zone alias.tests.: source zone ' - 'does-not-exists.tests. does not exist', + 'does-not-exists.tests. does not exist', text_type(ctx.exception)) # Alias zone that points to another alias zone. @@ -188,7 +188,7 @@ class TestManager(TestCase): tc = Manager(get_config_filename('alias-zone-loop.yaml')) \ .sync() self.assertEquals('Invalid alias zone alias-loop.tests.: source ' - 'zone alias.tests. is an alias zone', + 'zone alias.tests. is an alias zone', text_type(ctx.exception)) def test_compare(self): From 9549a0dec9429c90bd6b693c08a5b193ac2e4920 Mon Sep 17 00:00:00 2001 From: Nikolay Denev Date: Thu, 10 Dec 2020 22:12:53 +0000 Subject: [PATCH 21/96] Ignore records with unsupported rrtypes and log warning. --- octodns/provider/ultra.py | 7 ++++++- tests/fixtures/ultra-records-page-1.json | 2 +- tests/fixtures/ultra-records-page-2.json | 12 ++++++++++-- tests/fixtures/ultra-zones-page-1.json | 2 +- 4 files changed, 18 insertions(+), 5 deletions(-) diff --git a/octodns/provider/ultra.py b/octodns/provider/ultra.py index eb10e0d..a6ecd23 100644 --- a/octodns/provider/ultra.py +++ b/octodns/provider/ultra.py @@ -287,7 +287,12 @@ class UltraProvider(BaseProvider): name = zone.hostname_from_fqdn(record['ownerName']) if record['rrtype'] == 'SOA (6)': continue - _type = self.RECORDS_TO_TYPE[record['rrtype']] + try: + _type = self.RECORDS_TO_TYPE[record['rrtype']] + except KeyError: + self.log.warning('populate: ignoring record with ' + 'unsupported rrtype=%s', record) + continue values[name][_type] = record for name, types in values.items(): diff --git a/tests/fixtures/ultra-records-page-1.json b/tests/fixtures/ultra-records-page-1.json index 2f5f836..8614427 100644 --- a/tests/fixtures/ultra-records-page-1.json +++ b/tests/fixtures/ultra-records-page-1.json @@ -87,7 +87,7 @@ } ], "resultInfo": { - "totalCount": 12, + "totalCount": 13, "offset": 0, "returnedCount": 10 } diff --git a/tests/fixtures/ultra-records-page-2.json b/tests/fixtures/ultra-records-page-2.json index db51828..abdc44f 100644 --- a/tests/fixtures/ultra-records-page-2.json +++ b/tests/fixtures/ultra-records-page-2.json @@ -24,11 +24,19 @@ "order": "FIXED", "description": "octodns1.test." } + }, + { + "ownerName": "octodns1.test.", + "rrtype": "APEXALIAS (65282)", + "ttl": 3600, + "rdata": [ + "www.octodns1.test." + ] } ], "resultInfo": { - "totalCount": 12, + "totalCount": 13, "offset": 10, - "returnedCount": 2 + "returnedCount": 3 } } \ No newline at end of file diff --git a/tests/fixtures/ultra-zones-page-1.json b/tests/fixtures/ultra-zones-page-1.json index ad98d48..f748d08 100644 --- a/tests/fixtures/ultra-zones-page-1.json +++ b/tests/fixtures/ultra-zones-page-1.json @@ -19,7 +19,7 @@ "dnssecStatus": "UNSIGNED", "status": "ACTIVE", "owner": "phelpstest", - "resourceRecordCount": 5, + "resourceRecordCount": 6, "lastModifiedDateTime": "2020-06-19T01:05Z" } }, From 20dc4dc6a7cbeb5502e6053a92d0f2bc092af8af Mon Sep 17 00:00:00 2001 From: Nikolay Denev <46969469+nikolay-te@users.noreply.github.com> Date: Fri, 11 Dec 2020 16:54:09 +0000 Subject: [PATCH 22/96] Update octodns/provider/ultra.py Co-authored-by: Ross McFarland --- octodns/provider/ultra.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/ultra.py b/octodns/provider/ultra.py index a6ecd23..3a4c4bf 100644 --- a/octodns/provider/ultra.py +++ b/octodns/provider/ultra.py @@ -291,7 +291,7 @@ class UltraProvider(BaseProvider): _type = self.RECORDS_TO_TYPE[record['rrtype']] except KeyError: self.log.warning('populate: ignoring record with ' - 'unsupported rrtype=%s', record) + 'unsupported rrtype, %s %s', name, record['rrtype']) continue values[name][_type] = record From 049bdb55af645292efb0344e29ee0536b9ca6660 Mon Sep 17 00:00:00 2001 From: Nikolay Denev Date: Fri, 11 Dec 2020 17:47:22 +0000 Subject: [PATCH 23/96] Shorten line --- octodns/provider/ultra.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/octodns/provider/ultra.py b/octodns/provider/ultra.py index 3a4c4bf..03b70de 100644 --- a/octodns/provider/ultra.py +++ b/octodns/provider/ultra.py @@ -291,7 +291,8 @@ class UltraProvider(BaseProvider): _type = self.RECORDS_TO_TYPE[record['rrtype']] except KeyError: self.log.warning('populate: ignoring record with ' - 'unsupported rrtype, %s %s', name, record['rrtype']) + 'unsupported rrtype, %s %s', + name, record['rrtype']) continue values[name][_type] = record From 75e75a8846720af4741adb5f9ea57d9518efe004 Mon Sep 17 00:00:00 2001 From: root Date: Sat, 12 Dec 2020 08:09:19 +0000 Subject: [PATCH 24/96] Updated doc for unsafe thresholds --- docs/records.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/records.md b/docs/records.md index d287d8a..56e2493 100644 --- a/docs/records.md +++ b/docs/records.md @@ -120,3 +120,18 @@ If you'd like to enable lenience for a whole zone you can do so with the followi targets: - ns1 ``` + +#### Restrict Record manipulations + +OctoDNS currently provides us the ability to limit the frequency of update/deletes on +DNS records by allowing us to configure a percentage of the allowed operations as a +threshold parameter. If left unconfigured, suitable defaults take over instead. +In the below example, the Dynamic provider configured accomodates only 40% of both +update and delete operations over all the records present. + +````yaml +dyn: + class: octodns.provider.dyn.DynProvider + update_pcent_threshold: 0.4 + delete_pcent_threshold: 0.4 +```` \ No newline at end of file From 77c65b042ec63564bfff0334177c99abb2f34fda Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 14 Dec 2020 06:24:17 -0800 Subject: [PATCH 25/96] Wording tweaks to threshold params doc --- docs/records.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/records.md b/docs/records.md index 56e2493..4cf1e4b 100644 --- a/docs/records.md +++ b/docs/records.md @@ -123,15 +123,15 @@ If you'd like to enable lenience for a whole zone you can do so with the followi #### Restrict Record manipulations -OctoDNS currently provides us the ability to limit the frequency of update/deletes on -DNS records by allowing us to configure a percentage of the allowed operations as a -threshold parameter. If left unconfigured, suitable defaults take over instead. -In the below example, the Dynamic provider configured accomodates only 40% of both -update and delete operations over all the records present. +OctoDNS currently provides the ability to limit the number of updates/deletes on +DNS records by configuring a percentage of allowed operations as a threshold. +If left unconfigured, suitable defaults take over instead. In the below example, +the Dyn provider is configured with limits of 40% on both update and +delete operations over all the records present. ````yaml dyn: class: octodns.provider.dyn.DynProvider update_pcent_threshold: 0.4 delete_pcent_threshold: 0.4 -```` \ No newline at end of file +```` From 2b454ccc229f14fccf2246a152a41721230a88f7 Mon Sep 17 00:00:00 2001 From: Marc 'risson' Schmitt Date: Fri, 27 Nov 2020 21:36:50 +0100 Subject: [PATCH 26/96] manager: error when an alias zone is synced without its source Signed-off-by: Marc 'risson' Schmitt --- octodns/manager.py | 9 ++++++++- tests/test_octodns_manager.py | 8 ++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/octodns/manager.py b/octodns/manager.py index 9283d19..6c05a55 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -375,12 +375,19 @@ class Manager(object): futures = [] for zone_name, zone_source in aliased_zones.items(): source_config = self.config['zones'][zone_source] + print(source_config) + try: + desired_config = desired[zone_source] + except KeyError: + raise ManagerException('Zone {} cannot be sync without zone ' + '{} sinced it is aliased' + .format(zone_name, zone_source)) futures.append(self._executor.submit( self._populate_and_plan, zone_name, [], [self.providers[t] for t in source_config['targets']], - desired=desired[zone_source], + desired=desired_config, lenient=lenient )) diff --git a/tests/test_octodns_manager.py b/tests/test_octodns_manager.py index 4e1a756..f757466 100644 --- a/tests/test_octodns_manager.py +++ b/tests/test_octodns_manager.py @@ -191,6 +191,14 @@ class TestManager(TestCase): 'zone alias.tests. is an alias zone', text_type(ctx.exception)) + # Sync an alias without the zone it refers to + with self.assertRaises(ManagerException) as ctx: + tc = Manager(get_config_filename('simple-alias-zone.yaml')) \ + .sync(eligible_zones=["alias.tests."]) + self.assertEquals('Zone alias.tests. cannot be sync without zone ' + 'unit.tests. sinced it is aliased', + text_type(ctx.exception)) + def test_compare(self): with TemporaryDirectory() as tmpdir: environ['YAML_TMP_DIR'] = tmpdir.dirname From 3e09451fd7cabe1fcdd22cec4d9ab636c7b7a56e Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 18 Dec 2020 08:50:59 -0800 Subject: [PATCH 27/96] Remove debug print. --- octodns/manager.py | 1 - 1 file changed, 1 deletion(-) diff --git a/octodns/manager.py b/octodns/manager.py index 6c05a55..9ce10ff 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -375,7 +375,6 @@ class Manager(object): futures = [] for zone_name, zone_source in aliased_zones.items(): source_config = self.config['zones'][zone_source] - print(source_config) try: desired_config = desired[zone_source] except KeyError: From bdf4a6f425e9b1f4cdcc70c4de7238159f19bbf6 Mon Sep 17 00:00:00 2001 From: Peter Dave Hello Date: Sat, 19 Dec 2020 16:45:17 +0800 Subject: [PATCH 28/96] Update development dependency twine to v3.2.0 Twine version < 2.0 was reported has a security vulnerability From Twine version 2.0+, it requires Python 3.6, but doesn't seem to have other breaking changes. Reference: - https://github.com/pypa/twine/issues/491 - https://twine.readthedocs.io/en/latest/changelog.html --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 485a33f..146d673 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,4 +5,4 @@ pycodestyle==2.6.0 pyflakes==2.2.0 readme_renderer[md]==26.0 requests_mock -twine==1.15.0 +twine==3.2.0 From 8095f4560a374724739422eb37fe52a70a806896 Mon Sep 17 00:00:00 2001 From: Peter Dave Hello Date: Mon, 21 Dec 2020 03:29:09 +0800 Subject: [PATCH 29/96] Replace legacy `...` w/ $(...) in .git_hooks_pre-commit --- .git_hooks_pre-commit | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.git_hooks_pre-commit b/.git_hooks_pre-commit index 6b3b02e..e73b892 100755 --- a/.git_hooks_pre-commit +++ b/.git_hooks_pre-commit @@ -2,9 +2,9 @@ set -e -HOOKS=`dirname $0` -GIT=`dirname $HOOKS` -ROOT=`dirname $GIT` +HOOKS=$(dirname $0) +GIT=$(dirname $HOOKS) +ROOT=$(dirname $GIT) . $ROOT/env/bin/activate $ROOT/script/lint From 0fad8fa0d976387878fcd2475be51309717ae0bc Mon Sep 17 00:00:00 2001 From: Peter Dave Hello Date: Mon, 21 Dec 2020 03:32:25 +0800 Subject: [PATCH 30/96] Quote variable to prevent globbing, word splitting in .git_hooks_pre-commit --- .git_hooks_pre-commit | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.git_hooks_pre-commit b/.git_hooks_pre-commit index e73b892..1fb621f 100755 --- a/.git_hooks_pre-commit +++ b/.git_hooks_pre-commit @@ -2,10 +2,10 @@ set -e -HOOKS=$(dirname $0) -GIT=$(dirname $HOOKS) -ROOT=$(dirname $GIT) +HOOKS=$(dirname "$0") +GIT=$(dirname "$HOOKS") +ROOT=$(dirname "$GIT") -. $ROOT/env/bin/activate -$ROOT/script/lint -$ROOT/script/coverage +. "$ROOT/env/bin/activate" +"$ROOT/script/lint" +"$ROOT/script/coverage" From 949a136f533f1ca4642edbc076a3c5fcf4599329 Mon Sep 17 00:00:00 2001 From: Arunothia Marappan Date: Fri, 25 Dec 2020 21:07:23 -0800 Subject: [PATCH 31/96] Enforcing Delete to happen before all other operations in _apply --- octodns/provider/azuredns.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 8ad6dd9..f36c25e 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -488,10 +488,15 @@ class AzureProvider(BaseProvider): azure_zone_name = desired.name[:len(desired.name) - 1] self._check_zone(azure_zone_name, create=True) - # Force the operation order to be Delete() before Create() - # Helps avoid problems in updating a CNAME record into an A record. - changes.reverse() + # Force the operation order to be Delete() before all other operations. + # Helps avoid problems in updating a CNAME record into an A record and vice-versa. for change in changes: class_name = change.__class__.__name__ - getattr(self, '_apply_{}'.format(class_name))(change) + if class_name == 'Delete': + getattr(self, '_apply_{}'.format(class_name))(change) + + for change in changes: + class_name = change.__class__.__name__ + if class_name != 'Delete': + getattr(self, '_apply_{}'.format(class_name))(change) From d28d51290b011418b240dbcba60f69616dee5f5d Mon Sep 17 00:00:00 2001 From: Arunothia Marappan Date: Fri, 25 Dec 2020 21:11:02 -0800 Subject: [PATCH 32/96] Removing space from blank line --- octodns/provider/azuredns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index f36c25e..3bbe1ea 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -495,7 +495,7 @@ class AzureProvider(BaseProvider): class_name = change.__class__.__name__ if class_name == 'Delete': getattr(self, '_apply_{}'.format(class_name))(change) - + for change in changes: class_name = change.__class__.__name__ if class_name != 'Delete': From cad48ea4e8371fb1a7f1303495d7c59a888c0fdb Mon Sep 17 00:00:00 2001 From: Arunothia Marappan Date: Fri, 25 Dec 2020 21:50:46 -0800 Subject: [PATCH 33/96] Updating lengthy comment --- octodns/provider/azuredns.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 3bbe1ea..a86d671 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -488,8 +488,12 @@ class AzureProvider(BaseProvider): azure_zone_name = desired.name[:len(desired.name) - 1] self._check_zone(azure_zone_name, create=True) - # Force the operation order to be Delete() before all other operations. - # Helps avoid problems in updating a CNAME record into an A record and vice-versa. + ''' + Force the operation order to be Delete() before all other operations. + Helps avoid problems in updating + - a CNAME record into an A record. + - an A record into a CNAME record. + ''' for change in changes: class_name = change.__class__.__name__ From cffc90607161fb1785aac5311414c3ce49c9904b Mon Sep 17 00:00:00 2001 From: Arunothia Marappan Date: Fri, 25 Dec 2020 21:53:47 -0800 Subject: [PATCH 34/96] Removing trailing whitespace in comment --- octodns/provider/azuredns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index a86d671..4be0d4d 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -490,7 +490,7 @@ class AzureProvider(BaseProvider): ''' Force the operation order to be Delete() before all other operations. - Helps avoid problems in updating + Helps avoid problems in updating - a CNAME record into an A record. - an A record into a CNAME record. ''' From d801a4005733f050163d335e4059b8d99d227540 Mon Sep 17 00:00:00 2001 From: Peter Dave Hello Date: Sun, 27 Dec 2020 04:49:11 +0800 Subject: [PATCH 35/96] Fix indent of config/example.com.yaml in README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 871a35e..a65e28f 100644 --- a/README.md +++ b/README.md @@ -102,8 +102,8 @@ Now that we have something to tell OctoDNS about our providers & zones we need t ttl: 60 type: A values: - - 1.2.3.4 - - 1.2.3.5 + - 1.2.3.4 + - 1.2.3.5 ``` Further information can be found in [Records Documentation](/docs/records.md). From 32811ed5c1906aa93afa0c83307811febfa0b531 Mon Sep 17 00:00:00 2001 From: Arunothia Marappan Date: Sat, 26 Dec 2020 15:01:14 -0800 Subject: [PATCH 36/96] Update octodns/provider/azuredns.py Co-authored-by: Ross McFarland --- octodns/provider/azuredns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 4be0d4d..d1ec333 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -498,7 +498,7 @@ class AzureProvider(BaseProvider): for change in changes: class_name = change.__class__.__name__ if class_name == 'Delete': - getattr(self, '_apply_{}'.format(class_name))(change) + self._apply_Delete(change) for change in changes: class_name = change.__class__.__name__ From 7fe72f00612186c3bbc32ce39b7e71557d831cf5 Mon Sep 17 00:00:00 2001 From: Piotr Pieprzycki Date: Mon, 4 Jan 2021 09:14:58 +0100 Subject: [PATCH 37/96] Remove blank lines --- tests/test_octodns_provider_azuredns.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index 0b94dfe..2b2c1e7 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -500,8 +500,6 @@ class TestAzureDnsProvider(TestCase): exists = provider.populate(zone) self.assertTrue(exists) - - self.assertEquals(len(zone.records), 17) def test_populate_zone(self): From 8d1f6c69e7c4c011d86ebbc51bd38b20a78bb02d Mon Sep 17 00:00:00 2001 From: Adam Smith Date: Fri, 8 Jan 2021 15:08:38 -0800 Subject: [PATCH 38/96] Add fqdn module to setup.py as dependency --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index b25909a..0b15571 100644 --- a/setup.py +++ b/setup.py @@ -69,6 +69,7 @@ setup( 'PyYaml>=4.2b1', 'dnspython>=1.15.0', 'futures>=3.2.0; python_version<"3.2"', + 'fqdn>=1.5.0', 'ipaddress>=1.0.22; python_version<"3.3"', 'natsort>=5.5.0', 'pycountry>=19.8.18', From b2eab63d54533b7dfb469b4cea8d0fcd72eaa3eb Mon Sep 17 00:00:00 2001 From: Adam Smith Date: Fri, 8 Jan 2021 19:36:58 -0500 Subject: [PATCH 39/96] ZoneFileSource: allow users to specify file extension --- octodns/source/axfr.py | 19 +++++++++++++++---- tests/test_octodns_source_axfr.py | 7 +++++++ tests/zones/unit.tests.extension | 12 ++++++++++++ 3 files changed, 34 insertions(+), 4 deletions(-) create mode 100644 tests/zones/unit.tests.extension diff --git a/octodns/source/axfr.py b/octodns/source/axfr.py index 2e18ef0..1ca2c67 100644 --- a/octodns/source/axfr.py +++ b/octodns/source/axfr.py @@ -206,17 +206,24 @@ class ZoneFileSource(AxfrBaseSource): class: octodns.source.axfr.ZoneFileSource # The directory holding the zone files # Filenames should match zone name (eg. example.com.) + # with optional extension specified with file_extension directory: ./zonefiles + # File extension on zone files + # Appended to zone name to locate file + # (optional, default None) + file_extension: zone # Should sanity checks of the origin node be done # (optional, default true) check_origin: false ''' - def __init__(self, id, directory, check_origin=True): + def __init__(self, id, directory, file_extension=None, check_origin=True): self.log = logging.getLogger('ZoneFileSource[{}]'.format(id)) - self.log.debug('__init__: id=%s, directory=%s, check_origin=%s', id, - directory, check_origin) + self.log.debug('__init__: id=%s, directory=%s, file_extension=%s, ' + 'check_origin=%s', id, + directory, file_extension, check_origin) super(ZoneFileSource, self).__init__(id) self.directory = directory + self.file_extension = file_extension self.check_origin = check_origin self._zone_records = {} @@ -225,7 +232,11 @@ class ZoneFileSource(AxfrBaseSource): zonefiles = listdir(self.directory) if zone_name in zonefiles: try: - z = dns.zone.from_file(join(self.directory, zone_name), + filename = zone_name + if self.file_extension: + filename = '{}{}'.format(zone_name, + self.file_extension.lstrip('.')) + z = dns.zone.from_file(join(self.directory, filename), zone_name, relativize=False, check_origin=self.check_origin) except DNSException as error: diff --git a/tests/test_octodns_source_axfr.py b/tests/test_octodns_source_axfr.py index 1bf3f22..f808166 100644 --- a/tests/test_octodns_source_axfr.py +++ b/tests/test_octodns_source_axfr.py @@ -45,6 +45,13 @@ class TestAxfrSource(TestCase): class TestZoneFileSource(TestCase): source = ZoneFileSource('test', './tests/zones') + source_extension = ZoneFileSource('test', './tests/zones', 'extension') + + def test_zonefiles_with_extension(self): + # Load zonefiles with a specified file extension + valid = Zone('unit.tests.', []) + self.source_extension.populate(valid) + self.assertEquals(1, len(valid.records)) def test_populate(self): # Valid zone file in directory diff --git a/tests/zones/unit.tests.extension b/tests/zones/unit.tests.extension new file mode 100644 index 0000000..2821d9a --- /dev/null +++ b/tests/zones/unit.tests.extension @@ -0,0 +1,12 @@ +$ORIGIN unit.tests. +@ 3600 IN SOA ns1.unit.tests. root.unit.tests. ( + 2018071501 ; Serial + 3600 ; Refresh (1 hour) + 600 ; Retry (10 minutes) + 604800 ; Expire (1 week) + 3600 ; NXDOMAIN ttl (1 hour) + ) + +; NS Records +@ 3600 IN NS ns1.unit.tests. +@ 3600 IN NS ns2.unit.tests. From 97feaa7823878a4f87e7caa18301eb710528e2cc Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 25 Jan 2021 15:32:30 -0800 Subject: [PATCH 40/96] Rename extention zonefile test to avoid existing unit.tests. --- tests/test_octodns_source_axfr.py | 6 +++--- .../{unit.tests.extension => ext.unit.tests.extension} | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) rename tests/zones/{unit.tests.extension => ext.unit.tests.extension} (61%) diff --git a/tests/test_octodns_source_axfr.py b/tests/test_octodns_source_axfr.py index f808166..a1d2e1c 100644 --- a/tests/test_octodns_source_axfr.py +++ b/tests/test_octodns_source_axfr.py @@ -45,12 +45,12 @@ class TestAxfrSource(TestCase): class TestZoneFileSource(TestCase): source = ZoneFileSource('test', './tests/zones') - source_extension = ZoneFileSource('test', './tests/zones', 'extension') def test_zonefiles_with_extension(self): + source = ZoneFileSource('test', './tests/zones', 'extension') # Load zonefiles with a specified file extension - valid = Zone('unit.tests.', []) - self.source_extension.populate(valid) + valid = Zone('ext.unit.tests.', []) + source.populate(valid) self.assertEquals(1, len(valid.records)) def test_populate(self): diff --git a/tests/zones/unit.tests.extension b/tests/zones/ext.unit.tests.extension similarity index 61% rename from tests/zones/unit.tests.extension rename to tests/zones/ext.unit.tests.extension index 2821d9a..2ed7ac6 100644 --- a/tests/zones/unit.tests.extension +++ b/tests/zones/ext.unit.tests.extension @@ -1,5 +1,5 @@ -$ORIGIN unit.tests. -@ 3600 IN SOA ns1.unit.tests. root.unit.tests. ( +$ORIGIN ext.unit.tests. +@ 3600 IN SOA ns1.ext.unit.tests. root.ext.unit.tests. ( 2018071501 ; Serial 3600 ; Refresh (1 hour) 600 ; Retry (10 minutes) @@ -8,5 +8,5 @@ $ORIGIN unit.tests. ) ; NS Records -@ 3600 IN NS ns1.unit.tests. -@ 3600 IN NS ns2.unit.tests. +@ 3600 IN NS ns1.ext.unit.tests. +@ 3600 IN NS ns2.ext.unit.tests. From c08d4ac88f9e8d0e02435e9ec27a57706ca62cc5 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 25 Jan 2021 15:35:37 -0800 Subject: [PATCH 41/96] Look for zone filename not zone_name in axfr directory listing --- octodns/source/axfr.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/octodns/source/axfr.py b/octodns/source/axfr.py index 1ca2c67..ed3f98f 100644 --- a/octodns/source/axfr.py +++ b/octodns/source/axfr.py @@ -229,14 +229,16 @@ class ZoneFileSource(AxfrBaseSource): self._zone_records = {} def _load_zone_file(self, zone_name): + + zone_filename = zone_name + if self.file_extension: + zone_filename = '{}{}'.format(zone_name, + self.file_extension.lstrip('.')) + zonefiles = listdir(self.directory) - if zone_name in zonefiles: + if zone_filename in zonefiles: try: - filename = zone_name - if self.file_extension: - filename = '{}{}'.format(zone_name, - self.file_extension.lstrip('.')) - z = dns.zone.from_file(join(self.directory, filename), + z = dns.zone.from_file(join(self.directory, zone_filename), zone_name, relativize=False, check_origin=self.check_origin) except DNSException as error: From ea943e606ed358cc23cd91bd0ba7f2508ab81fcf Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 25 Jan 2021 15:45:23 -0800 Subject: [PATCH 42/96] Avoid . on the end of files, but still test axfr default --- .gitignore | 3 ++- tests/test_octodns_source_axfr.py | 13 +++++++++++-- .../zones/{invalid.records. => invalid.records.tst} | 0 tests/zones/{invalid.zone. => invalid.zone.tst} | 0 tests/zones/{unit.tests. => unit.tests.tst} | 0 5 files changed, 13 insertions(+), 3 deletions(-) rename tests/zones/{invalid.records. => invalid.records.tst} (100%) rename tests/zones/{invalid.zone. => invalid.zone.tst} (100%) rename tests/zones/{unit.tests. => unit.tests.tst} (100%) diff --git a/.gitignore b/.gitignore index 715b687..5192821 100644 --- a/.gitignore +++ b/.gitignore @@ -5,8 +5,8 @@ *.pyc .coverage .env -/config/ /build/ +/config/ coverage.xml dist/ env/ @@ -14,4 +14,5 @@ htmlcov/ nosetests.xml octodns.egg-info/ output/ +tests/zones/unit.tests. tmp/ diff --git a/tests/test_octodns_source_axfr.py b/tests/test_octodns_source_axfr.py index a1d2e1c..8d0a527 100644 --- a/tests/test_octodns_source_axfr.py +++ b/tests/test_octodns_source_axfr.py @@ -9,6 +9,7 @@ import dns.zone from dns.exception import DNSException from mock import patch +from shutil import copyfile from six import text_type from unittest import TestCase @@ -21,7 +22,7 @@ from octodns.record import ValidationError class TestAxfrSource(TestCase): source = AxfrSource('test', 'localhost') - forward_zonefile = dns.zone.from_file('./tests/zones/unit.tests.', + forward_zonefile = dns.zone.from_file('./tests/zones/unit.tests.tst', 'unit.tests', relativize=False) @patch('dns.zone.from_xfr') @@ -44,7 +45,7 @@ class TestAxfrSource(TestCase): class TestZoneFileSource(TestCase): - source = ZoneFileSource('test', './tests/zones') + source = ZoneFileSource('test', './tests/zones', file_extension='tst') def test_zonefiles_with_extension(self): source = ZoneFileSource('test', './tests/zones', 'extension') @@ -53,6 +54,14 @@ class TestZoneFileSource(TestCase): source.populate(valid) self.assertEquals(1, len(valid.records)) + def test_zonefiles_without_extension(self): + copyfile('./tests/zones/unit.tests.tst', './tests/zones/unit.tests.') + source = ZoneFileSource('test', './tests/zones') + # Load zonefiles without a specified file extension + valid = Zone('unit.tests.', []) + source.populate(valid) + self.assertEquals(12, len(valid.records)) + def test_populate(self): # Valid zone file in directory valid = Zone('unit.tests.', []) diff --git a/tests/zones/invalid.records. b/tests/zones/invalid.records.tst similarity index 100% rename from tests/zones/invalid.records. rename to tests/zones/invalid.records.tst diff --git a/tests/zones/invalid.zone. b/tests/zones/invalid.zone.tst similarity index 100% rename from tests/zones/invalid.zone. rename to tests/zones/invalid.zone.tst diff --git a/tests/zones/unit.tests. b/tests/zones/unit.tests.tst similarity index 100% rename from tests/zones/unit.tests. rename to tests/zones/unit.tests.tst From 37381bd2740a899c1944c5a5a274814ee807d1ff Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 25 Jan 2021 15:50:34 -0800 Subject: [PATCH 43/96] Skip the axfr default name test if we can't create the needed tests file --- tests/test_octodns_source_axfr.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_octodns_source_axfr.py b/tests/test_octodns_source_axfr.py index 8d0a527..f732060 100644 --- a/tests/test_octodns_source_axfr.py +++ b/tests/test_octodns_source_axfr.py @@ -55,7 +55,12 @@ class TestZoneFileSource(TestCase): self.assertEquals(1, len(valid.records)) def test_zonefiles_without_extension(self): - copyfile('./tests/zones/unit.tests.tst', './tests/zones/unit.tests.') + try: + copyfile('./tests/zones/unit.tests.tst', + './tests/zones/unit.tests.') + except: + self.skipTest('Unable to create unit.tests. (ending with .) so ' + 'skipping default filename testing.') source = ZoneFileSource('test', './tests/zones') # Load zonefiles without a specified file extension valid = Zone('unit.tests.', []) From dd1dbfbfdd1d66ee4c62487a6394fa5d10f2e443 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Thu, 28 Jan 2021 12:23:13 -0800 Subject: [PATCH 44/96] Rework copyfile and skip based on feedback from windows test --- tests/test_octodns_source_axfr.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/test_octodns_source_axfr.py b/tests/test_octodns_source_axfr.py index f732060..e0871ee 100644 --- a/tests/test_octodns_source_axfr.py +++ b/tests/test_octodns_source_axfr.py @@ -9,6 +9,7 @@ import dns.zone from dns.exception import DNSException from mock import patch +from os.path import exists from shutil import copyfile from six import text_type from unittest import TestCase @@ -55,12 +56,19 @@ class TestZoneFileSource(TestCase): self.assertEquals(1, len(valid.records)) def test_zonefiles_without_extension(self): - try: - copyfile('./tests/zones/unit.tests.tst', - './tests/zones/unit.tests.') - except: + # Windows doesn't let files end with a `.` so we add a .tst to them in + # the repo and then try and create the `.` version we need for the + # default case (no extension.) + copyfile('./tests/zones/unit.tests.tst', './tests/zones/unit.tests.') + # Unfortunately copyfile silently works and create the file without + # the `.` so we have to check to see if it did that + if exists('./tests/zones/unit.tests'): + # It did so we need to skip this test, that means windows won't + # have full code coverage, but skipping the test is going out of + # our way enough for a os-specific/oddball case. self.skipTest('Unable to create unit.tests. (ending with .) so ' 'skipping default filename testing.') + source = ZoneFileSource('test', './tests/zones') # Load zonefiles without a specified file extension valid = Zone('unit.tests.', []) From 4ce2563d2ec1fdd5b6e3cfbaf4e938f17b2a26e2 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Thu, 28 Jan 2021 13:24:35 -0800 Subject: [PATCH 45/96] Remove the rest of the . ending files, clean up code and tests for better coverage --- octodns/provider/yaml.py | 6 ++-- octodns/source/axfr.py | 9 ++---- tests/config/simple-split.yaml | 3 ++ .../a.yaml | 0 .../aaaa.yaml | 0 .../cname.yaml | 0 .../real-ish-a.yaml | 0 .../simple-weighted.yaml | 0 .../split/{empty. => empty.tst}/.gitkeep | 0 .../12.yaml | 0 .../2.yaml | 0 .../test.yaml | 0 .../$unit.tests.yaml | 0 .../_srv._tcp.yaml | 0 .../{unit.tests. => unit.tests.tst}/aaaa.yaml | 0 .../cname.yaml | 0 .../dname.yaml | 0 .../excluded.yaml | 0 .../ignored.yaml | 0 .../included.yaml | 0 .../{unit.tests. => unit.tests.tst}/mx.yaml | 0 .../naptr.yaml | 0 .../{unit.tests. => unit.tests.tst}/ptr.yaml | 0 .../{unit.tests. => unit.tests.tst}/spf.yaml | 0 .../{unit.tests. => unit.tests.tst}/sub.yaml | 0 .../{unit.tests. => unit.tests.tst}/txt.yaml | 0 .../www.sub.yaml | 0 .../{unit.tests. => unit.tests.tst}/www.yaml | 0 .../{unordered. => unordered.tst}/abc.yaml | 0 .../{unordered. => unordered.tst}/xyz.yaml | 0 tests/test_octodns_provider_yaml.py | 31 ++++++++++++------- tests/test_octodns_source_axfr.py | 4 +-- 32 files changed, 30 insertions(+), 23 deletions(-) rename tests/config/split/{dynamic.tests. => dynamic.tests.tst}/a.yaml (100%) rename tests/config/split/{dynamic.tests. => dynamic.tests.tst}/aaaa.yaml (100%) rename tests/config/split/{dynamic.tests. => dynamic.tests.tst}/cname.yaml (100%) rename tests/config/split/{dynamic.tests. => dynamic.tests.tst}/real-ish-a.yaml (100%) rename tests/config/split/{dynamic.tests. => dynamic.tests.tst}/simple-weighted.yaml (100%) rename tests/config/split/{empty. => empty.tst}/.gitkeep (100%) rename tests/config/split/{subzone.unit.tests. => subzone.unit.tests.tst}/12.yaml (100%) rename tests/config/split/{subzone.unit.tests. => subzone.unit.tests.tst}/2.yaml (100%) rename tests/config/split/{subzone.unit.tests. => subzone.unit.tests.tst}/test.yaml (100%) rename tests/config/split/{unit.tests. => unit.tests.tst}/$unit.tests.yaml (100%) rename tests/config/split/{unit.tests. => unit.tests.tst}/_srv._tcp.yaml (100%) rename tests/config/split/{unit.tests. => unit.tests.tst}/aaaa.yaml (100%) rename tests/config/split/{unit.tests. => unit.tests.tst}/cname.yaml (100%) rename tests/config/split/{unit.tests. => unit.tests.tst}/dname.yaml (100%) rename tests/config/split/{unit.tests. => unit.tests.tst}/excluded.yaml (100%) rename tests/config/split/{unit.tests. => unit.tests.tst}/ignored.yaml (100%) rename tests/config/split/{unit.tests. => unit.tests.tst}/included.yaml (100%) rename tests/config/split/{unit.tests. => unit.tests.tst}/mx.yaml (100%) rename tests/config/split/{unit.tests. => unit.tests.tst}/naptr.yaml (100%) rename tests/config/split/{unit.tests. => unit.tests.tst}/ptr.yaml (100%) rename tests/config/split/{unit.tests. => unit.tests.tst}/spf.yaml (100%) rename tests/config/split/{unit.tests. => unit.tests.tst}/sub.yaml (100%) rename tests/config/split/{unit.tests. => unit.tests.tst}/txt.yaml (100%) rename tests/config/split/{unit.tests. => unit.tests.tst}/www.sub.yaml (100%) rename tests/config/split/{unit.tests. => unit.tests.tst}/www.yaml (100%) rename tests/config/split/{unordered. => unordered.tst}/abc.yaml (100%) rename tests/config/split/{unordered. => unordered.tst}/xyz.yaml (100%) diff --git a/octodns/provider/yaml.py b/octodns/provider/yaml.py index 55a1632..3deca01 100644 --- a/octodns/provider/yaml.py +++ b/octodns/provider/yaml.py @@ -239,11 +239,13 @@ class SplitYamlProvider(YamlProvider): # instead of a file matching the record name. CATCHALL_RECORD_NAMES = ('*', '') - def __init__(self, id, directory, *args, **kwargs): + def __init__(self, id, directory, extension='.', *args, **kwargs): super(SplitYamlProvider, self).__init__(id, directory, *args, **kwargs) + self.extension = extension def _zone_directory(self, zone): - return join(self.directory, zone.name) + filename = '{}{}'.format(zone.name[:-1], self.extension) + return join(self.directory, filename) def populate(self, zone, target=False, lenient=False): self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, diff --git a/octodns/source/axfr.py b/octodns/source/axfr.py index ed3f98f..e21f29f 100644 --- a/octodns/source/axfr.py +++ b/octodns/source/axfr.py @@ -216,7 +216,7 @@ class ZoneFileSource(AxfrBaseSource): # (optional, default true) check_origin: false ''' - def __init__(self, id, directory, file_extension=None, check_origin=True): + def __init__(self, id, directory, file_extension='.', check_origin=True): self.log = logging.getLogger('ZoneFileSource[{}]'.format(id)) self.log.debug('__init__: id=%s, directory=%s, file_extension=%s, ' 'check_origin=%s', id, @@ -229,12 +229,7 @@ class ZoneFileSource(AxfrBaseSource): self._zone_records = {} def _load_zone_file(self, zone_name): - - zone_filename = zone_name - if self.file_extension: - zone_filename = '{}{}'.format(zone_name, - self.file_extension.lstrip('.')) - + zone_filename = '{}{}'.format(zone_name[:-1], self.file_extension) zonefiles = listdir(self.directory) if zone_filename in zonefiles: try: diff --git a/tests/config/simple-split.yaml b/tests/config/simple-split.yaml index d106506..a798258 100644 --- a/tests/config/simple-split.yaml +++ b/tests/config/simple-split.yaml @@ -4,14 +4,17 @@ providers: in: class: octodns.provider.yaml.SplitYamlProvider directory: tests/config/split + extension: .tst dump: class: octodns.provider.yaml.SplitYamlProvider directory: env/YAML_TMP_DIR + extension: .tst # This is sort of ugly, but it shouldn't hurt anything. It'll just write out # the target file twice where it and dump are both used dump2: class: octodns.provider.yaml.SplitYamlProvider directory: env/YAML_TMP_DIR + extension: .tst simple: class: helpers.SimpleProvider geo: diff --git a/tests/config/split/dynamic.tests./a.yaml b/tests/config/split/dynamic.tests.tst/a.yaml similarity index 100% rename from tests/config/split/dynamic.tests./a.yaml rename to tests/config/split/dynamic.tests.tst/a.yaml diff --git a/tests/config/split/dynamic.tests./aaaa.yaml b/tests/config/split/dynamic.tests.tst/aaaa.yaml similarity index 100% rename from tests/config/split/dynamic.tests./aaaa.yaml rename to tests/config/split/dynamic.tests.tst/aaaa.yaml diff --git a/tests/config/split/dynamic.tests./cname.yaml b/tests/config/split/dynamic.tests.tst/cname.yaml similarity index 100% rename from tests/config/split/dynamic.tests./cname.yaml rename to tests/config/split/dynamic.tests.tst/cname.yaml diff --git a/tests/config/split/dynamic.tests./real-ish-a.yaml b/tests/config/split/dynamic.tests.tst/real-ish-a.yaml similarity index 100% rename from tests/config/split/dynamic.tests./real-ish-a.yaml rename to tests/config/split/dynamic.tests.tst/real-ish-a.yaml diff --git a/tests/config/split/dynamic.tests./simple-weighted.yaml b/tests/config/split/dynamic.tests.tst/simple-weighted.yaml similarity index 100% rename from tests/config/split/dynamic.tests./simple-weighted.yaml rename to tests/config/split/dynamic.tests.tst/simple-weighted.yaml diff --git a/tests/config/split/empty./.gitkeep b/tests/config/split/empty.tst/.gitkeep similarity index 100% rename from tests/config/split/empty./.gitkeep rename to tests/config/split/empty.tst/.gitkeep diff --git a/tests/config/split/subzone.unit.tests./12.yaml b/tests/config/split/subzone.unit.tests.tst/12.yaml similarity index 100% rename from tests/config/split/subzone.unit.tests./12.yaml rename to tests/config/split/subzone.unit.tests.tst/12.yaml diff --git a/tests/config/split/subzone.unit.tests./2.yaml b/tests/config/split/subzone.unit.tests.tst/2.yaml similarity index 100% rename from tests/config/split/subzone.unit.tests./2.yaml rename to tests/config/split/subzone.unit.tests.tst/2.yaml diff --git a/tests/config/split/subzone.unit.tests./test.yaml b/tests/config/split/subzone.unit.tests.tst/test.yaml similarity index 100% rename from tests/config/split/subzone.unit.tests./test.yaml rename to tests/config/split/subzone.unit.tests.tst/test.yaml diff --git a/tests/config/split/unit.tests./$unit.tests.yaml b/tests/config/split/unit.tests.tst/$unit.tests.yaml similarity index 100% rename from tests/config/split/unit.tests./$unit.tests.yaml rename to tests/config/split/unit.tests.tst/$unit.tests.yaml diff --git a/tests/config/split/unit.tests./_srv._tcp.yaml b/tests/config/split/unit.tests.tst/_srv._tcp.yaml similarity index 100% rename from tests/config/split/unit.tests./_srv._tcp.yaml rename to tests/config/split/unit.tests.tst/_srv._tcp.yaml diff --git a/tests/config/split/unit.tests./aaaa.yaml b/tests/config/split/unit.tests.tst/aaaa.yaml similarity index 100% rename from tests/config/split/unit.tests./aaaa.yaml rename to tests/config/split/unit.tests.tst/aaaa.yaml diff --git a/tests/config/split/unit.tests./cname.yaml b/tests/config/split/unit.tests.tst/cname.yaml similarity index 100% rename from tests/config/split/unit.tests./cname.yaml rename to tests/config/split/unit.tests.tst/cname.yaml diff --git a/tests/config/split/unit.tests./dname.yaml b/tests/config/split/unit.tests.tst/dname.yaml similarity index 100% rename from tests/config/split/unit.tests./dname.yaml rename to tests/config/split/unit.tests.tst/dname.yaml diff --git a/tests/config/split/unit.tests./excluded.yaml b/tests/config/split/unit.tests.tst/excluded.yaml similarity index 100% rename from tests/config/split/unit.tests./excluded.yaml rename to tests/config/split/unit.tests.tst/excluded.yaml diff --git a/tests/config/split/unit.tests./ignored.yaml b/tests/config/split/unit.tests.tst/ignored.yaml similarity index 100% rename from tests/config/split/unit.tests./ignored.yaml rename to tests/config/split/unit.tests.tst/ignored.yaml diff --git a/tests/config/split/unit.tests./included.yaml b/tests/config/split/unit.tests.tst/included.yaml similarity index 100% rename from tests/config/split/unit.tests./included.yaml rename to tests/config/split/unit.tests.tst/included.yaml diff --git a/tests/config/split/unit.tests./mx.yaml b/tests/config/split/unit.tests.tst/mx.yaml similarity index 100% rename from tests/config/split/unit.tests./mx.yaml rename to tests/config/split/unit.tests.tst/mx.yaml diff --git a/tests/config/split/unit.tests./naptr.yaml b/tests/config/split/unit.tests.tst/naptr.yaml similarity index 100% rename from tests/config/split/unit.tests./naptr.yaml rename to tests/config/split/unit.tests.tst/naptr.yaml diff --git a/tests/config/split/unit.tests./ptr.yaml b/tests/config/split/unit.tests.tst/ptr.yaml similarity index 100% rename from tests/config/split/unit.tests./ptr.yaml rename to tests/config/split/unit.tests.tst/ptr.yaml diff --git a/tests/config/split/unit.tests./spf.yaml b/tests/config/split/unit.tests.tst/spf.yaml similarity index 100% rename from tests/config/split/unit.tests./spf.yaml rename to tests/config/split/unit.tests.tst/spf.yaml diff --git a/tests/config/split/unit.tests./sub.yaml b/tests/config/split/unit.tests.tst/sub.yaml similarity index 100% rename from tests/config/split/unit.tests./sub.yaml rename to tests/config/split/unit.tests.tst/sub.yaml diff --git a/tests/config/split/unit.tests./txt.yaml b/tests/config/split/unit.tests.tst/txt.yaml similarity index 100% rename from tests/config/split/unit.tests./txt.yaml rename to tests/config/split/unit.tests.tst/txt.yaml diff --git a/tests/config/split/unit.tests./www.sub.yaml b/tests/config/split/unit.tests.tst/www.sub.yaml similarity index 100% rename from tests/config/split/unit.tests./www.sub.yaml rename to tests/config/split/unit.tests.tst/www.sub.yaml diff --git a/tests/config/split/unit.tests./www.yaml b/tests/config/split/unit.tests.tst/www.yaml similarity index 100% rename from tests/config/split/unit.tests./www.yaml rename to tests/config/split/unit.tests.tst/www.yaml diff --git a/tests/config/split/unordered./abc.yaml b/tests/config/split/unordered.tst/abc.yaml similarity index 100% rename from tests/config/split/unordered./abc.yaml rename to tests/config/split/unordered.tst/abc.yaml diff --git a/tests/config/split/unordered./xyz.yaml b/tests/config/split/unordered.tst/xyz.yaml similarity index 100% rename from tests/config/split/unordered./xyz.yaml rename to tests/config/split/unordered.tst/xyz.yaml diff --git a/tests/test_octodns_provider_yaml.py b/tests/test_octodns_provider_yaml.py index 15e90da..38dfc11 100644 --- a/tests/test_octodns_provider_yaml.py +++ b/tests/test_octodns_provider_yaml.py @@ -207,18 +207,20 @@ class TestSplitYamlProvider(TestCase): def test_zone_directory(self): source = SplitYamlProvider( - 'test', join(dirname(__file__), 'config/split')) + 'test', join(dirname(__file__), 'config/split'), + extension='.tst') zone = Zone('unit.tests.', []) self.assertEqual( - join(dirname(__file__), 'config/split/unit.tests.'), + join(dirname(__file__), 'config/split/unit.tests.tst'), source._zone_directory(zone)) def test_apply_handles_existing_zone_directory(self): with TemporaryDirectory() as td: - provider = SplitYamlProvider('test', join(td.dirname, 'config')) - makedirs(join(td.dirname, 'config', 'does.exist.')) + provider = SplitYamlProvider('test', join(td.dirname, 'config'), + extension='.tst') + makedirs(join(td.dirname, 'config', 'does.exist.tst')) zone = Zone('does.exist.', []) self.assertTrue(isdir(provider._zone_directory(zone))) @@ -227,7 +229,8 @@ class TestSplitYamlProvider(TestCase): def test_provider(self): source = SplitYamlProvider( - 'test', join(dirname(__file__), 'config/split')) + 'test', join(dirname(__file__), 'config/split'), + extension='.tst') zone = Zone('unit.tests.', []) dynamic_zone = Zone('dynamic.tests.', []) @@ -246,9 +249,10 @@ class TestSplitYamlProvider(TestCase): with TemporaryDirectory() as td: # Add some subdirs to make sure that it can create them directory = join(td.dirname, 'sub', 'dir') - zone_dir = join(directory, 'unit.tests.') - dynamic_zone_dir = join(directory, 'dynamic.tests.') - target = SplitYamlProvider('test', directory) + zone_dir = join(directory, 'unit.tests.tst') + dynamic_zone_dir = join(directory, 'dynamic.tests.tst') + target = SplitYamlProvider('test', directory, + extension='.tst') # We add everything plan = target.plan(zone) @@ -335,7 +339,8 @@ class TestSplitYamlProvider(TestCase): def test_empty(self): source = SplitYamlProvider( - 'test', join(dirname(__file__), 'config/split')) + 'test', join(dirname(__file__), 'config/split'), + extension='.tst') zone = Zone('empty.', []) @@ -345,7 +350,8 @@ class TestSplitYamlProvider(TestCase): def test_unsorted(self): source = SplitYamlProvider( - 'test', join(dirname(__file__), 'config/split')) + 'test', join(dirname(__file__), 'config/split'), + extension='.tst') zone = Zone('unordered.', []) @@ -356,14 +362,15 @@ class TestSplitYamlProvider(TestCase): source = SplitYamlProvider( 'test', join(dirname(__file__), 'config/split'), - enforce_order=False) + extension='.tst', enforce_order=False) # no exception source.populate(zone) self.assertEqual(2, len(zone.records)) def test_subzone_handling(self): source = SplitYamlProvider( - 'test', join(dirname(__file__), 'config/split')) + 'test', join(dirname(__file__), 'config/split'), + extension='.tst') # If we add `sub` as a sub-zone we'll reject `www.sub` zone = Zone('unit.tests.', ['sub']) diff --git a/tests/test_octodns_source_axfr.py b/tests/test_octodns_source_axfr.py index e0871ee..44e04d0 100644 --- a/tests/test_octodns_source_axfr.py +++ b/tests/test_octodns_source_axfr.py @@ -46,10 +46,10 @@ class TestAxfrSource(TestCase): class TestZoneFileSource(TestCase): - source = ZoneFileSource('test', './tests/zones', file_extension='tst') + source = ZoneFileSource('test', './tests/zones', file_extension='.tst') def test_zonefiles_with_extension(self): - source = ZoneFileSource('test', './tests/zones', 'extension') + source = ZoneFileSource('test', './tests/zones', '.extension') # Load zonefiles with a specified file extension valid = Zone('ext.unit.tests.', []) source.populate(valid) From a8505d66f13421f1c67242f4ecbf1fbeae85aef5 Mon Sep 17 00:00:00 2001 From: Robert Reichel Date: Fri, 29 Jan 2021 15:11:27 -0500 Subject: [PATCH 46/96] Improve checking and creating Azure DNS Zones This change improves the process for checking for AzureDNS zones by using the known set and not relying upon custom error handling. Since the provider already fetches the zones, octodns doesn't need to make a second call to check for the existence of the zone - _populate_zones already does that for us. --- octodns/provider/azuredns.py | 36 ++++++++++--------------- tests/test_octodns_provider_azuredns.py | 30 ++++++++++++++------- 2 files changed, 34 insertions(+), 32 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index a1ef6fe..0224f06 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -341,7 +341,8 @@ class AzureProvider(BaseProvider): self.log.debug('azure_zones: loading') list_zones = self._dns_client.zones.list_by_resource_group for zone in list_zones(self._resource_group): - self._azure_zones.add(zone.name) + if zone.name: + self._azure_zones.add(zone.name.rstrip('.')) def _check_zone(self, name, create=False): '''Checks whether a zone specified in a source exist in Azure server. @@ -356,29 +357,20 @@ class AzureProvider(BaseProvider): :type return: str or None ''' - self.log.debug('_check_zone: name=%s', name) - try: - if name in self._azure_zones: - return name - self._dns_client.zones.get(self._resource_group, name) + self.log.debug('_check_zone: name=%s create=%s', name, create) + # Check if the zone already exists in our set + if name in self._azure_zones: + return name + # If not, and its time to create, lets do it. + if create: + self.log.debug('_check_zone:no matching zone; creating %s', name) + create_zone = self._dns_client.zones.create_or_update + create_zone(self._resource_group, name, Zone(location='global')) self._azure_zones.add(name) return name - except CloudError as err: - msg = 'The Resource \'Microsoft.Network/dnszones/{}\''.format(name) - msg += ' under resource group \'{}\''.format(self._resource_group) - msg += ' was not found.' - if msg == err.message: - # Then the only error is that the zone doesn't currently exist - if create: - self.log.debug('_check_zone:no matching zone; creating %s', - name) - create_zone = self._dns_client.zones.create_or_update - create_zone(self._resource_group, name, - Zone(location='global')) - return name - else: - return - raise + else: + # Else return nothing (aka false) + return def populate(self, zone, target=False, lenient=False): '''Required function of manager.py to collect records from zone. diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index 2b2c1e7..3c93a27 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -498,32 +498,42 @@ class TestAzureDnsProvider(TestCase): record_list = provider._dns_client.record_sets.list_by_dns_zone record_list.return_value = rs + zone_list = provider._dns_client.zones.list_by_resource_group + zone_list.return_value = [zone] + exists = provider.populate(zone) - self.assertTrue(exists) + self.assertEquals(len(zone.records), 17) + self.assertTrue(exists) def test_populate_zone(self): provider = self._get_provider() zone_list = provider._dns_client.zones.list_by_resource_group - zone_list.return_value = [AzureZone(location='global'), - AzureZone(location='global')] + zone_1 = AzureZone(location='global') + # This is far from ideal but the zone constructor doesn't let me set it on creation + zone_1.name = "zone-1" + zone_2 = AzureZone(location='global') + # This is far from ideal but the zone constructor doesn't let me set it on creation + zone_2.name = "zone-2" + zone_list.return_value = [zone_1, + zone_2, + zone_1] provider._populate_zones() - self.assertEquals(len(provider._azure_zones), 1) + # This should be returning two zones since two zones are the same + self.assertEquals(len(provider._azure_zones), 2) def test_bad_zone_response(self): provider = self._get_provider() _get = provider._dns_client.zones.get _get.side_effect = CloudError(Mock(status=404), 'Azure Error') - trip = False - try: - provider._check_zone('unit.test', create=False) - except CloudError: - trip = True - self.assertEquals(trip, True) + self.assertEquals( + provider._check_zone('unit.test', create=False), + None + ) def test_apply(self): provider = self._get_provider() From 83cc7fbe1a5cca6d1f2ae2910378859ca39411a5 Mon Sep 17 00:00:00 2001 From: Robert Reichel Date: Fri, 29 Jan 2021 15:19:19 -0500 Subject: [PATCH 47/96] Appease the linter --- tests/test_octodns_provider_azuredns.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index 3c93a27..cb790ea 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -502,7 +502,7 @@ class TestAzureDnsProvider(TestCase): zone_list.return_value = [zone] exists = provider.populate(zone) - + self.assertEquals(len(zone.records), 17) self.assertTrue(exists) @@ -511,10 +511,12 @@ class TestAzureDnsProvider(TestCase): zone_list = provider._dns_client.zones.list_by_resource_group zone_1 = AzureZone(location='global') - # This is far from ideal but the zone constructor doesn't let me set it on creation + # This is far from ideal but the + # zone constructor doesn't let me set it on creation zone_1.name = "zone-1" zone_2 = AzureZone(location='global') - # This is far from ideal but the zone constructor doesn't let me set it on creation + # This is far from ideal but the + # zone constructor doesn't let me set it on creation zone_2.name = "zone-2" zone_list.return_value = [zone_1, zone_2, From f79ad89e82a04af07a5a15ec0c88932cbf32ddc5 Mon Sep 17 00:00:00 2001 From: Robert Reichel Date: Fri, 29 Jan 2021 15:22:13 -0500 Subject: [PATCH 48/96] Continue linter appeasement --- tests/test_octodns_provider_azuredns.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index cb790ea..b62762e 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -511,7 +511,7 @@ class TestAzureDnsProvider(TestCase): zone_list = provider._dns_client.zones.list_by_resource_group zone_1 = AzureZone(location='global') - # This is far from ideal but the + # This is far from ideal but the # zone constructor doesn't let me set it on creation zone_1.name = "zone-1" zone_2 = AzureZone(location='global') @@ -533,7 +533,7 @@ class TestAzureDnsProvider(TestCase): _get = provider._dns_client.zones.get _get.side_effect = CloudError(Mock(status=404), 'Azure Error') self.assertEquals( - provider._check_zone('unit.test', create=False), + provider._check_zone('unit.test', create=False), None ) From f92fdfce172bb3748e50e0cd2a5a01f2e41e0df1 Mon Sep 17 00:00:00 2001 From: Robert Reichel Date: Fri, 29 Jan 2021 15:24:42 -0500 Subject: [PATCH 49/96] Even more desperate attempts to appease linter --- octodns/provider/azuredns.py | 1 - 1 file changed, 1 deletion(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 0224f06..6a65354 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -7,7 +7,6 @@ from __future__ import absolute_import, division, print_function, \ from azure.common.credentials import ServicePrincipalCredentials from azure.mgmt.dns import DnsManagementClient -from msrestazure.azure_exceptions import CloudError from azure.mgmt.dns.models import ARecord, AaaaRecord, CaaRecord, \ CnameRecord, MxRecord, SrvRecord, NsRecord, PtrRecord, TxtRecord, Zone From 3ab532c5af1c71c5c35a2d6450ba0ab17a55cf31 Mon Sep 17 00:00:00 2001 From: Robert Reichel Date: Fri, 29 Jan 2021 15:30:17 -0500 Subject: [PATCH 50/96] Fix test coverage --- octodns/provider/azuredns.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 6a65354..30e9af5 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -340,8 +340,7 @@ class AzureProvider(BaseProvider): self.log.debug('azure_zones: loading') list_zones = self._dns_client.zones.list_by_resource_group for zone in list_zones(self._resource_group): - if zone.name: - self._azure_zones.add(zone.name.rstrip('.')) + self._azure_zones.add(zone.name.rstrip('.')) def _check_zone(self, name, create=False): '''Checks whether a zone specified in a source exist in Azure server. From 0b116a57b96c66e4e051e980d372820618937260 Mon Sep 17 00:00:00 2001 From: Robert Reichel Date: Tue, 2 Feb 2021 12:18:40 -0500 Subject: [PATCH 51/96] Modify Azure DNS Client logic to lazy load Lazy loading the Azure DNS client ensures that the necessary network calls only occur when it is time to actually do something with the client. --- octodns/provider/azuredns.py | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index a1ef6fe..888f5e3 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -323,6 +323,22 @@ class AzureProvider(BaseProvider): SUPPORTS = set(('A', 'AAAA', 'CAA', 'CNAME', 'MX', 'NS', 'PTR', 'SRV', 'TXT')) + def _get_dns_client(self): + if not self._dns_client_handle: + # Not initialized yet, we need to do that. + credentials = ServicePrincipalCredentials( + self._dns_client_client_id, + secret=self._dns_client_key, + tenant=self._dns_client_directory_id + ) + self._dns_client_handle = DnsManagementClient( + credentials, + self._dns_client_subscription_id + ) + + return self._dns_client_handle + + def __init__(self, id, client_id, key, directory_id, sub_id, resource_group, *args, **kwargs): self.log = logging.getLogger('AzureProvider[{}]'.format(id)) @@ -330,10 +346,13 @@ class AzureProvider(BaseProvider): 'key=***, directory_id:%s', id, client_id, directory_id) super(AzureProvider, self).__init__(id, *args, **kwargs) - credentials = ServicePrincipalCredentials( - client_id, secret=key, tenant=directory_id - ) - self._dns_client = DnsManagementClient(credentials, sub_id) + # Store necessary initialization params + self._dns_client_handle = None + self._dns_client_client_id = client_id + self._dns_client_key = key + self._dns_client_directory_id = directory_id + self._dns_client_subscription_id = sub_id + self._dns_client = property(_get_dns_client, _set_dns_client) self._resource_group = resource_group self._azure_zones = set() From 6146be8ec3ad861320f50f1d8542f3cd020a468b Mon Sep 17 00:00:00 2001 From: Robert Reichel Date: Tue, 2 Feb 2021 12:21:00 -0500 Subject: [PATCH 52/96] Remove unused set_dns_client property --- octodns/provider/azuredns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 888f5e3..9561385 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -352,7 +352,7 @@ class AzureProvider(BaseProvider): self._dns_client_key = key self._dns_client_directory_id = directory_id self._dns_client_subscription_id = sub_id - self._dns_client = property(_get_dns_client, _set_dns_client) + self._dns_client = property(_get_dns_client) self._resource_group = resource_group self._azure_zones = set() From 6fb77c08109c5a4d4537280fe291fb921ad7b28d Mon Sep 17 00:00:00 2001 From: Robert Reichel Date: Tue, 2 Feb 2021 12:21:48 -0500 Subject: [PATCH 53/96] Add set DNS client logic if needed for testing --- octodns/provider/azuredns.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 9561385..2431e7d 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -337,6 +337,8 @@ class AzureProvider(BaseProvider): ) return self._dns_client_handle + def _set_dns_client(self, client) + self.dns_client_handle = client def __init__(self, id, client_id, key, directory_id, sub_id, @@ -352,7 +354,7 @@ class AzureProvider(BaseProvider): self._dns_client_key = key self._dns_client_directory_id = directory_id self._dns_client_subscription_id = sub_id - self._dns_client = property(_get_dns_client) + self._dns_client = property(_get_dns_client, _set_dns_client) self._resource_group = resource_group self._azure_zones = set() From 975376d09dd58ef9f02aa522ef3842443cb595a4 Mon Sep 17 00:00:00 2001 From: Robert Reichel Date: Tue, 2 Feb 2021 12:26:04 -0500 Subject: [PATCH 54/96] Remove trailing whitespace --- octodns/provider/azuredns.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 2431e7d..0162e18 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -327,19 +327,19 @@ class AzureProvider(BaseProvider): if not self._dns_client_handle: # Not initialized yet, we need to do that. credentials = ServicePrincipalCredentials( - self._dns_client_client_id, - secret=self._dns_client_key, + self._dns_client_client_id, + secret=self._dns_client_key, tenant=self._dns_client_directory_id ) self._dns_client_handle = DnsManagementClient( - credentials, + credentials, self._dns_client_subscription_id ) - + return self._dns_client_handle + def _set_dns_client(self, client) self.dns_client_handle = client - def __init__(self, id, client_id, key, directory_id, sub_id, resource_group, *args, **kwargs): From 5e78d07a97c2634bf41cb67b2510240ca502726b Mon Sep 17 00:00:00 2001 From: Robert Reichel Date: Tue, 2 Feb 2021 12:31:04 -0500 Subject: [PATCH 55/96] Use @property in lieu of property() --- octodns/provider/azuredns.py | 35 ++++++++++++++++------------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 0162e18..53b5e48 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -323,24 +323,6 @@ class AzureProvider(BaseProvider): SUPPORTS = set(('A', 'AAAA', 'CAA', 'CNAME', 'MX', 'NS', 'PTR', 'SRV', 'TXT')) - def _get_dns_client(self): - if not self._dns_client_handle: - # Not initialized yet, we need to do that. - credentials = ServicePrincipalCredentials( - self._dns_client_client_id, - secret=self._dns_client_key, - tenant=self._dns_client_directory_id - ) - self._dns_client_handle = DnsManagementClient( - credentials, - self._dns_client_subscription_id - ) - - return self._dns_client_handle - - def _set_dns_client(self, client) - self.dns_client_handle = client - def __init__(self, id, client_id, key, directory_id, sub_id, resource_group, *args, **kwargs): self.log = logging.getLogger('AzureProvider[{}]'.format(id)) @@ -354,10 +336,25 @@ class AzureProvider(BaseProvider): self._dns_client_key = key self._dns_client_directory_id = directory_id self._dns_client_subscription_id = sub_id - self._dns_client = property(_get_dns_client, _set_dns_client) + self._dns_client = None + self._resource_group = resource_group self._azure_zones = set() + @property + def _dns_client(self) + if self._dns_client is None: + credentials = ServicePrincipalCredentials( + self._dns_client_client_id, + secret=self._dns_client_key, + tenant=self._dns_client_directory_id + ) + self._dns_client = DnsManagementClient( + credentials, + self._dns_client_subscription_id + ) + return self._dns_client + def _populate_zones(self): self.log.debug('azure_zones: loading') list_zones = self._dns_client.zones.list_by_resource_group From 831d1cc30b003f5e8f9f444c5a805777183f8007 Mon Sep 17 00:00:00 2001 From: Robert Reichel Date: Tue, 2 Feb 2021 12:34:44 -0500 Subject: [PATCH 56/96] Add missing colon --- octodns/provider/azuredns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 53b5e48..f41f892 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -342,7 +342,7 @@ class AzureProvider(BaseProvider): self._azure_zones = set() @property - def _dns_client(self) + def _dns_client(self): if self._dns_client is None: credentials = ServicePrincipalCredentials( self._dns_client_client_id, From a58371e3bbf896f8c5bd1edb91f0cfaccd7ae921 Mon Sep 17 00:00:00 2001 From: Robert Reichel Date: Tue, 2 Feb 2021 12:40:57 -0500 Subject: [PATCH 57/96] Apply suggestions from code review Co-authored-by: Ross McFarland --- octodns/provider/azuredns.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index f41f892..046ddfe 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -336,24 +336,24 @@ class AzureProvider(BaseProvider): self._dns_client_key = key self._dns_client_directory_id = directory_id self._dns_client_subscription_id = sub_id - self._dns_client = None + self.__dns_client = None self._resource_group = resource_group self._azure_zones = set() @property def _dns_client(self): - if self._dns_client is None: + if self.__dns_client is None: credentials = ServicePrincipalCredentials( self._dns_client_client_id, secret=self._dns_client_key, tenant=self._dns_client_directory_id ) - self._dns_client = DnsManagementClient( + self.__dns_client = DnsManagementClient( credentials, self._dns_client_subscription_id ) - return self._dns_client + return self.__dns_client def _populate_zones(self): self.log.debug('azure_zones: loading') From c2c05a761e6e6f83495f4bfda6f4ea5dadc47d48 Mon Sep 17 00:00:00 2001 From: Robert Reichel Date: Tue, 2 Feb 2021 13:14:29 -0500 Subject: [PATCH 58/96] Fix patching --- tests/test_octodns_provider_azuredns.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index 2b2c1e7..19cc826 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -376,8 +376,6 @@ class TestAzureDnsProvider(TestCase): def _provider(self): return self._get_provider('mock_spc', 'mock_dns_client') - @patch('octodns.provider.azuredns.DnsManagementClient') - @patch('octodns.provider.azuredns.ServicePrincipalCredentials') def _get_provider(self, mock_spc, mock_dns_client): '''Returns a mock AzureProvider object to use in testing. @@ -390,7 +388,9 @@ class TestAzureDnsProvider(TestCase): ''' return AzureProvider('mock_id', 'mock_client', 'mock_key', 'mock_directory', 'mock_sub', 'mock_rg') - + + @patch('octodns.provider.azuredns.DnsManagementClient') + @patch('octodns.provider.azuredns.ServicePrincipalCredentials') def test_populate_records(self): provider = self._get_provider() @@ -501,7 +501,9 @@ class TestAzureDnsProvider(TestCase): exists = provider.populate(zone) self.assertTrue(exists) self.assertEquals(len(zone.records), 17) - + + @patch('octodns.provider.azuredns.DnsManagementClient') + @patch('octodns.provider.azuredns.ServicePrincipalCredentials') def test_populate_zone(self): provider = self._get_provider() @@ -512,7 +514,9 @@ class TestAzureDnsProvider(TestCase): provider._populate_zones() self.assertEquals(len(provider._azure_zones), 1) - + + @patch('octodns.provider.azuredns.DnsManagementClient') + @patch('octodns.provider.azuredns.ServicePrincipalCredentials') def test_bad_zone_response(self): provider = self._get_provider() @@ -539,6 +543,8 @@ class TestAzureDnsProvider(TestCase): self.assertEquals(19, provider.apply(Plan(zone, zone, deletes, True))) + @patch('octodns.provider.azuredns.DnsManagementClient') + @patch('octodns.provider.azuredns.ServicePrincipalCredentials') def test_create_zone(self): provider = self._get_provider() @@ -555,6 +561,8 @@ class TestAzureDnsProvider(TestCase): self.assertEquals(19, provider.apply(Plan(None, desired, changes, True))) + @patch('octodns.provider.azuredns.DnsManagementClient') + @patch('octodns.provider.azuredns.ServicePrincipalCredentials') def test_check_zone_no_create(self): provider = self._get_provider() From 1982fbddac3533e87a3627e6fe2a3996dd0ab0b9 Mon Sep 17 00:00:00 2001 From: Robert Reichel Date: Tue, 2 Feb 2021 13:18:04 -0500 Subject: [PATCH 59/96] Update patching --- tests/test_octodns_provider_azuredns.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index 19cc826..030bf9f 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -375,7 +375,9 @@ class Test_CheckAzureAlias(TestCase): class TestAzureDnsProvider(TestCase): def _provider(self): return self._get_provider('mock_spc', 'mock_dns_client') - + + @patch('octodns.provider.azuredns.DnsManagementClient') + @patch('octodns.provider.azuredns.ServicePrincipalCredentials') def _get_provider(self, mock_spc, mock_dns_client): '''Returns a mock AzureProvider object to use in testing. @@ -386,8 +388,11 @@ class TestAzureDnsProvider(TestCase): :type return: AzureProvider ''' - return AzureProvider('mock_id', 'mock_client', 'mock_key', + provider = AzureProvider('mock_id', 'mock_client', 'mock_key', 'mock_directory', 'mock_sub', 'mock_rg') + # Fetch the client to force it to load the creds + client = provider._dns_client + return provider @patch('octodns.provider.azuredns.DnsManagementClient') @patch('octodns.provider.azuredns.ServicePrincipalCredentials') From 950090bb547aa58591d3535c2986a8b46b5ceba6 Mon Sep 17 00:00:00 2001 From: Robert Reichel Date: Tue, 2 Feb 2021 13:23:02 -0500 Subject: [PATCH 60/96] Update test_octodns_provider_azuredns.py --- tests/test_octodns_provider_azuredns.py | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index 030bf9f..8f840f0 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -375,7 +375,7 @@ class Test_CheckAzureAlias(TestCase): class TestAzureDnsProvider(TestCase): def _provider(self): return self._get_provider('mock_spc', 'mock_dns_client') - + @patch('octodns.provider.azuredns.DnsManagementClient') @patch('octodns.provider.azuredns.ServicePrincipalCredentials') def _get_provider(self, mock_spc, mock_dns_client): @@ -393,9 +393,7 @@ class TestAzureDnsProvider(TestCase): # Fetch the client to force it to load the creds client = provider._dns_client return provider - - @patch('octodns.provider.azuredns.DnsManagementClient') - @patch('octodns.provider.azuredns.ServicePrincipalCredentials') + def test_populate_records(self): provider = self._get_provider() @@ -506,9 +504,7 @@ class TestAzureDnsProvider(TestCase): exists = provider.populate(zone) self.assertTrue(exists) self.assertEquals(len(zone.records), 17) - - @patch('octodns.provider.azuredns.DnsManagementClient') - @patch('octodns.provider.azuredns.ServicePrincipalCredentials') + def test_populate_zone(self): provider = self._get_provider() @@ -519,9 +515,7 @@ class TestAzureDnsProvider(TestCase): provider._populate_zones() self.assertEquals(len(provider._azure_zones), 1) - - @patch('octodns.provider.azuredns.DnsManagementClient') - @patch('octodns.provider.azuredns.ServicePrincipalCredentials') + def test_bad_zone_response(self): provider = self._get_provider() @@ -548,8 +542,6 @@ class TestAzureDnsProvider(TestCase): self.assertEquals(19, provider.apply(Plan(zone, zone, deletes, True))) - @patch('octodns.provider.azuredns.DnsManagementClient') - @patch('octodns.provider.azuredns.ServicePrincipalCredentials') def test_create_zone(self): provider = self._get_provider() @@ -566,8 +558,6 @@ class TestAzureDnsProvider(TestCase): self.assertEquals(19, provider.apply(Plan(None, desired, changes, True))) - @patch('octodns.provider.azuredns.DnsManagementClient') - @patch('octodns.provider.azuredns.ServicePrincipalCredentials') def test_check_zone_no_create(self): provider = self._get_provider() From d94db03f5b32dc95d8f63cfe7c6a3a8eec8542fd Mon Sep 17 00:00:00 2001 From: Robert Reichel Date: Tue, 2 Feb 2021 13:26:28 -0500 Subject: [PATCH 61/96] Fix lint --- tests/test_octodns_provider_azuredns.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index 8f840f0..2b048c0 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -389,7 +389,8 @@ class TestAzureDnsProvider(TestCase): :type return: AzureProvider ''' provider = AzureProvider('mock_id', 'mock_client', 'mock_key', - 'mock_directory', 'mock_sub', 'mock_rg') + 'mock_directory', 'mock_sub', 'mock_rg' + ) # Fetch the client to force it to load the creds client = provider._dns_client return provider From 290a6303037a817fa7a3da80562946ef7b8c69b5 Mon Sep 17 00:00:00 2001 From: Robert Reichel Date: Tue, 2 Feb 2021 13:29:52 -0500 Subject: [PATCH 62/96] Update test_octodns_provider_azuredns.py --- tests/test_octodns_provider_azuredns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index 2d6f08d..b7721b7 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -390,7 +390,7 @@ class TestAzureDnsProvider(TestCase): ''' provider = AzureProvider('mock_id', 'mock_client', 'mock_key', 'mock_directory', 'mock_sub', 'mock_rg' - ) + ) # Fetch the client to force it to load the creds client = provider._dns_client return provider From ec0b309437369cefdda372aa9723e550201072bc Mon Sep 17 00:00:00 2001 From: Robert Reichel Date: Tue, 2 Feb 2021 13:34:15 -0500 Subject: [PATCH 63/96] Remove unused client ref --- tests/test_octodns_provider_azuredns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index b7721b7..9523b51 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -392,7 +392,7 @@ class TestAzureDnsProvider(TestCase): 'mock_directory', 'mock_sub', 'mock_rg' ) # Fetch the client to force it to load the creds - client = provider._dns_client + provider._dns_client return provider def test_populate_records(self): From cda56a3ca7ffed9b62a5f960682a9c25b9173c05 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Thu, 4 Feb 2021 10:48:45 -0800 Subject: [PATCH 64/96] Force the value passed to FQDN to be a str --- octodns/record/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/octodns/record/__init__.py b/octodns/record/__init__.py index f22eebf..7beb570 100644 --- a/octodns/record/__init__.py +++ b/octodns/record/__init__.py @@ -758,7 +758,9 @@ class _TargetValue(object): reasons.append('empty value') elif not data: reasons.append('missing value') - elif not FQDN(data, allow_underscores=True).is_valid: + # NOTE: FQDN complains if the data it receives isn't a str, it doesn't + # allow unicode... This is likely specific to 2.7 + elif not FQDN(str(data), allow_underscores=True).is_valid: reasons.append('{} value "{}" is not a valid FQDN' .format(_type, data)) elif not data.endswith('.'): From 4081c7b31b1ea31855b94c1572cc18e5a46410b7 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 5 Feb 2021 11:55:37 -0800 Subject: [PATCH 65/96] Add the number of changes and zone name to "making changes" --- octodns/provider/base.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/octodns/provider/base.py b/octodns/provider/base.py index ae87844..eb097a2 100644 --- a/octodns/provider/base.py +++ b/octodns/provider/base.py @@ -91,7 +91,10 @@ class BaseProvider(BaseSource): self.log.info('apply: disabled') return 0 - self.log.info('apply: making changes') + zone_name = plan.desired.name + num_changes = len(plan.changes) + self.log.info('apply: making %d changes to %s', num_changes, + zone_name) self._apply(plan) return len(plan.changes) From 858628a867218e566013cde9d5a5fa920ecce382 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 5 Feb 2021 12:06:46 -0800 Subject: [PATCH 66/96] Update yaml test path to work on windows --- tests/test_octodns_provider_yaml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_octodns_provider_yaml.py b/tests/test_octodns_provider_yaml.py index 38dfc11..f255238 100644 --- a/tests/test_octodns_provider_yaml.py +++ b/tests/test_octodns_provider_yaml.py @@ -213,7 +213,7 @@ class TestSplitYamlProvider(TestCase): zone = Zone('unit.tests.', []) self.assertEqual( - join(dirname(__file__), 'config/split/unit.tests.tst'), + join(dirname(__file__), 'config/split', 'unit.tests.tst'), source._zone_directory(zone)) def test_apply_handles_existing_zone_directory(self): From 2346ebc1ce72edca8d066218cafb29e46d1f7ee9 Mon Sep 17 00:00:00 2001 From: Patrick Connolly Date: Mon, 8 Feb 2021 21:24:41 -0500 Subject: [PATCH 67/96] Added list of projects & resources. --- README.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/README.md b/README.md index a65e28f..84dc033 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,7 @@ It is similar to [Netflix/denominator](https://github.com/Netflix/denominator). - [Dynamic sources](#dynamic-sources) - [Contributing](#contributing) - [Getting help](#getting-help) +- [Related Projects & Resources](#related-projects--resources) - [License](#license) - [Authors](#authors) @@ -225,6 +226,8 @@ Most of the things included in OctoDNS are providers, the obvious difference bei The `class` key in the providers config section can be used to point to arbitrary classes in the python path so internal or 3rd party providers can easily be included with no coordination beyond getting them into PYTHONPATH, most likely installed into the virtualenv with OctoDNS. +For examples of building third-party sources and providers, see [Related Projects & Resources](#related-projects--resources). + ## Other Uses ### Syncing between providers @@ -286,6 +289,29 @@ Please see our [contributing document](/CONTRIBUTING.md) if you would like to pa If you have a problem or suggestion, please [open an issue](https://github.com/octodns/octodns/issues/new) in this repository, and we will do our best to help. Please note that this project adheres to the [Contributor Covenant Code of Conduct](/CODE_OF_CONDUCT.md). +## Related Projects & Resources + +- **GitHub Action:** [OctoDNS-Sync](https://github.com/marketplace/actions/octodns-sync) +- **Sample Implementations.** See how others are using it + - [`hackclub/dns`](https://github.com/hackclub/dns) + - [`kubernetes/k8s.io:/dns`](https://github.com/kubernetes/k8s.io/tree/master/dns) + - [`g0v-network/domains`](https://github.com/g0v-network/domains) + - [`jekyll/dns`](https://github.com/jekyll/dns) + - [`parkr/dns`](https://github.com/parkr/dns) +- **Custom Sources & Providers.** + - [`octodns/octodns-ddns`](https://github.com/octodns/octodns-ddns): A simple Dynamic DNS source. + - [`doddo/octodns-lexicon`](https://github.com/doddo/octodns-lexicon): Use [Lexicon](https://github.com/AnalogJ/lexicon) providers as octoDNS providers. + - [`asyncon/octoblox`](https://github.com/asyncon/octoblox): [Infoblox](https://www.infoblox.com/) provider. + - [`sukiyaki/octodns-netbox`](https://github.com/sukiyaki/octodns-netbox): [NetBox](https://github.com/netbox-community/netbox) source. + - [`kompetenzbolzen/octodns-custom-provider`](https://github.com/kompetenzbolzen/octodns-custom-provider): zonefile provider & phpIPAM source. +- **Resources.** + - Article: [Visualising DNS records with Neo4j](https://medium.com/@costask/querying-and-visualising-octodns-records-with-neo4j-f4f72ab2d474) + code + - Video: [FOSDEM 2019 - DNS as code with octodns](https://archive.fosdem.org/2019/schedule/event/dns_octodns/) + - GitHub Blog: [Enabling DNS split authority with OctoDNS](https://github.blog/2017-04-27-enabling-split-authority-dns-with-octodns/) + - Tutorial: [How To Deploy and Manage Your DNS using OctoDNS on Ubuntu 18.04](https://www.digitalocean.com/community/tutorials/how-to-deploy-and-manage-your-dns-using-octodns-on-ubuntu-18-04) + +If you know of any other resources, please do let us know! + ## License OctoDNS is licensed under the [MIT license](LICENSE). From 9d4bd0aaec43764077a20ba23a2065cd1011c92a Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Sun, 29 Nov 2020 23:42:51 +0800 Subject: [PATCH 68/96] Add support for LOC records --- docs/records.md | 1 + octodns/provider/yaml.py | 2 +- octodns/record/__init__.py | 190 ++++++++ tests/config/unit.tests.yaml | 28 ++ tests/test_octodns_manager.py | 14 +- tests/test_octodns_provider_constellix.py | 2 +- tests/test_octodns_provider_digitalocean.py | 2 +- tests/test_octodns_provider_dnsimple.py | 4 +- tests/test_octodns_provider_dnsmadeeasy.py | 2 +- tests/test_octodns_provider_easydns.py | 2 +- tests/test_octodns_provider_gandi.py | 4 +- tests/test_octodns_provider_powerdns.py | 4 +- tests/test_octodns_provider_yaml.py | 9 +- tests/test_octodns_record.py | 488 +++++++++++++++++++- tests/zones/unit.tests.tst | 4 + 15 files changed, 730 insertions(+), 26 deletions(-) diff --git a/docs/records.md b/docs/records.md index 4cf1e4b..e39a85d 100644 --- a/docs/records.md +++ b/docs/records.md @@ -10,6 +10,7 @@ OctoDNS supports the following record types: * `CAA` * `CNAME` * `DNAME` +* `LOC` * `MX` * `NAPTR` * `NS` diff --git a/octodns/provider/yaml.py b/octodns/provider/yaml.py index 3deca01..8314f38 100644 --- a/octodns/provider/yaml.py +++ b/octodns/provider/yaml.py @@ -104,7 +104,7 @@ class YamlProvider(BaseProvider): ''' SUPPORTS_GEO = True SUPPORTS_DYNAMIC = True - SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CAA', 'CNAME', 'DNAME', 'MX', + SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CAA', 'CNAME', 'DNAME', 'LOC', 'MX', 'NAPTR', 'NS', 'PTR', 'SSHFP', 'SPF', 'SRV', 'TXT')) def __init__(self, id, directory, default_ttl=3600, enforce_order=True, diff --git a/octodns/record/__init__.py b/octodns/record/__init__.py index 7beb570..8ee2eaa 100644 --- a/octodns/record/__init__.py +++ b/octodns/record/__init__.py @@ -97,6 +97,7 @@ class Record(EqualityTupleMixin): 'CAA': CaaRecord, 'CNAME': CnameRecord, 'DNAME': DnameRecord, + 'LOC': LocRecord, 'MX': MxRecord, 'NAPTR': NaptrRecord, 'NS': NsRecord, @@ -879,6 +880,195 @@ class DnameRecord(_DynamicMixin, _ValueMixin, Record): _value_type = DnameValue +class LocValue(EqualityTupleMixin): + # TODO: work out how to do defaults per RFC + + @classmethod + def validate(cls, data, _type): + int_keys = [ + 'lat_degrees', + 'lat_minutes', + 'long_degrees', + 'long_minutes', + ] + + float_keys = [ + 'lat_seconds', + 'long_seconds', + 'altitude', + 'size', + 'precision_horz', + 'precision_vert', + ] + + direction_keys = [ + 'lat_direction', + 'long_direction', + ] + + if not isinstance(data, (list, tuple)): + data = (data,) + reasons = [] + for value in data: + for key in int_keys: + try: + int(value[key]) + if ( + ( + key == 'lat_degrees' and + not 0 <= int(value[key]) <= 90 + ) or ( + key == 'long_degrees' and + not 0 <= int(value[key]) <= 180 + ) or ( + key in ['lat_minutes', 'long_minutes'] and + not 0 <= int(value[key]) <= 59 + ) + ): + reasons.append('invalid value for {} "{}"' + .format(key, value[key])) + except KeyError: + reasons.append('missing {}'.format(key)) + except ValueError: + reasons.append('invalid {} "{}"' + .format(key, value[key])) + + for key in float_keys: + try: + float(value[key]) + if ( + ( + key in ['lat_seconds', 'long_seconds'] and + not 0 <= float(value[key]) <= 59.999 + ) or ( + key == 'altitude' and + not -100000.00 <= float(value[key]) <= 42849672.95 + ) or ( + key in ['size', + 'precision_horz', + 'precision_vert'] and + not 0 <= float(value[key]) <= 90000000.00 + ) + ): + reasons.append('invalid value for {} "{}"' + .format(key, value[key])) + except KeyError: + reasons.append('missing {}'.format(key)) + except ValueError: + reasons.append('invalid {} "{}"' + .format(key, value[key])) + + for key in direction_keys: + try: + str(value[key]) + if ( + key == 'lat_direction' and + value[key] not in ['N', 'S'] + ): + reasons.append('invalid direction for {} "{}"' + .format(key, value[key])) + if ( + key == 'long_direction' and + value[key] not in ['E', 'W'] + ): + reasons.append('invalid direction for {} "{}"' + .format(key, value[key])) + except KeyError: + reasons.append('missing {}'.format(key)) + return reasons + + @classmethod + def process(cls, values): + return [LocValue(v) for v in values] + + def __init__(self, value): + self.lat_degrees = int(value['lat_degrees']) + self.lat_minutes = int(value['lat_minutes']) + self.lat_seconds = float(value['lat_seconds']) + self.lat_direction = value['lat_direction'].upper() + self.long_degrees = int(value['long_degrees']) + self.long_minutes = int(value['long_minutes']) + self.long_seconds = float(value['long_seconds']) + self.long_direction = value['long_direction'].upper() + self.altitude = float(value['altitude']) + self.size = float(value['size']) + self.precision_horz = float(value['precision_horz']) + self.precision_vert = float(value['precision_vert']) + + @property + def data(self): + return { + 'lat_degrees': self.lat_degrees, + 'lat_minutes': self.lat_minutes, + 'lat_seconds': self.lat_seconds, + 'lat_direction': self.lat_direction, + 'long_degrees': self.long_degrees, + 'long_minutes': self.long_minutes, + 'long_seconds': self.long_seconds, + 'long_direction': self.long_direction, + 'altitude': self.altitude, + 'size': self.size, + 'precision_horz': self.precision_horz, + 'precision_vert': self.precision_vert, + } + + def __hash__(self): + return hash(( + self.lat_degrees, + self.lat_minutes, + self.lat_seconds, + self.lat_direction, + self.long_degrees, + self.long_minutes, + self.long_seconds, + self.long_direction, + self.altitude, + self.size, + self.precision_horz, + self.precision_vert, + )) + + def _equality_tuple(self): + return ( + self.lat_degrees, + self.lat_minutes, + self.lat_seconds, + self.lat_direction, + self.long_degrees, + self.long_minutes, + self.long_seconds, + self.long_direction, + self.altitude, + self.size, + self.precision_horz, + self.precision_vert, + ) + + def __repr__(self): + loc_format = "'{0} {1} {2:.3f} {3} " + \ + "{4} {5} {6:.3f} {7} " + \ + "{8:.2f}m {9:.2f}m {10:.2f}m {11:.2f}m'" + return loc_format.format( + self.lat_degrees, + self.lat_minutes, + self.lat_seconds, + self.lat_direction, + self.long_degrees, + self.long_minutes, + self.long_seconds, + self.long_direction, + self.altitude, + self.size, + self.precision_horz, + self.precision_vert, + ) + + +class LocRecord(_ValuesMixin, Record): + _type = 'LOC' + _value_type = LocValue + + class MxValue(EqualityTupleMixin): @classmethod diff --git a/tests/config/unit.tests.yaml b/tests/config/unit.tests.yaml index 7b84ac9..f5cf648 100644 --- a/tests/config/unit.tests.yaml +++ b/tests/config/unit.tests.yaml @@ -77,6 +77,34 @@ included: - test type: CNAME value: unit.tests. +loc: + ttl: 300 + type: LOC + values: + - altitude: 20 + lat_degrees: 31 + lat_direction: S + lat_minutes: 58 + lat_seconds: 52.1 + long_degrees: 115 + long_direction: E + long_minutes: 49 + long_seconds: 11.7 + precision_horz: 10 + precision_vert: 2 + size: 10 + - altitude: 20 + lat_degrees: 53 + lat_direction: N + lat_minutes: 13 + lat_seconds: 10 + long_degrees: 2 + long_direction: W + long_minutes: 18 + long_seconds: 26 + precision_horz: 1000 + precision_vert: 2 + size: 10 mx: ttl: 300 type: MX diff --git a/tests/test_octodns_manager.py b/tests/test_octodns_manager.py index f757466..3e0b122 100644 --- a/tests/test_octodns_manager.py +++ b/tests/test_octodns_manager.py @@ -118,12 +118,12 @@ class TestManager(TestCase): environ['YAML_TMP_DIR'] = tmpdir.dirname tc = Manager(get_config_filename('simple.yaml')) \ .sync(dry_run=False) - self.assertEquals(22, tc) + self.assertEquals(23, tc) # try with just one of the zones tc = Manager(get_config_filename('simple.yaml')) \ .sync(dry_run=False, eligible_zones=['unit.tests.']) - self.assertEquals(16, tc) + self.assertEquals(17, tc) # the subzone, with 2 targets tc = Manager(get_config_filename('simple.yaml')) \ @@ -138,18 +138,18 @@ class TestManager(TestCase): # Again with force tc = Manager(get_config_filename('simple.yaml')) \ .sync(dry_run=False, force=True) - self.assertEquals(22, tc) + self.assertEquals(23, tc) # Again with max_workers = 1 tc = Manager(get_config_filename('simple.yaml'), max_workers=1) \ .sync(dry_run=False, force=True) - self.assertEquals(22, tc) + self.assertEquals(23, tc) # Include meta tc = Manager(get_config_filename('simple.yaml'), max_workers=1, include_meta=True) \ .sync(dry_run=False, force=True) - self.assertEquals(26, tc) + self.assertEquals(27, tc) def test_eligible_sources(self): with TemporaryDirectory() as tmpdir: @@ -215,13 +215,13 @@ class TestManager(TestCase): fh.write('---\n{}') changes = manager.compare(['in'], ['dump'], 'unit.tests.') - self.assertEquals(16, len(changes)) + self.assertEquals(17, len(changes)) # Compound sources with varying support changes = manager.compare(['in', 'nosshfp'], ['dump'], 'unit.tests.') - self.assertEquals(15, len(changes)) + self.assertEquals(16, len(changes)) with self.assertRaises(ManagerException) as ctx: manager.compare(['nope'], ['dump'], 'unit.tests.') diff --git a/tests/test_octodns_provider_constellix.py b/tests/test_octodns_provider_constellix.py index bc17b50..7392271 100644 --- a/tests/test_octodns_provider_constellix.py +++ b/tests/test_octodns_provider_constellix.py @@ -132,7 +132,7 @@ class TestConstellixProvider(TestCase): plan = provider.plan(self.expected) # No root NS, no ignored, no excluded, no unsupported - n = len(self.expected.records) - 6 + n = len(self.expected.records) - 7 self.assertEquals(n, len(plan.changes)) self.assertEquals(n, provider.apply(plan)) diff --git a/tests/test_octodns_provider_digitalocean.py b/tests/test_octodns_provider_digitalocean.py index 0ad8f72..d1fa208 100644 --- a/tests/test_octodns_provider_digitalocean.py +++ b/tests/test_octodns_provider_digitalocean.py @@ -163,7 +163,7 @@ class TestDigitalOceanProvider(TestCase): plan = provider.plan(self.expected) # No root NS, no ignored, no excluded, no unsupported - n = len(self.expected.records) - 8 + n = len(self.expected.records) - 9 self.assertEquals(n, len(plan.changes)) self.assertEquals(n, provider.apply(plan)) self.assertFalse(plan.exists) diff --git a/tests/test_octodns_provider_dnsimple.py b/tests/test_octodns_provider_dnsimple.py index 92f32b1..e97751f 100644 --- a/tests/test_octodns_provider_dnsimple.py +++ b/tests/test_octodns_provider_dnsimple.py @@ -136,8 +136,8 @@ class TestDnsimpleProvider(TestCase): ] plan = provider.plan(self.expected) - # No root NS, no ignored, no excluded - n = len(self.expected.records) - 4 + # No root NS, no ignored, no excluded, no unsupported + n = len(self.expected.records) - 5 self.assertEquals(n, len(plan.changes)) self.assertEquals(n, provider.apply(plan)) self.assertFalse(plan.exists) diff --git a/tests/test_octodns_provider_dnsmadeeasy.py b/tests/test_octodns_provider_dnsmadeeasy.py index 0ad059d..3c709cf 100644 --- a/tests/test_octodns_provider_dnsmadeeasy.py +++ b/tests/test_octodns_provider_dnsmadeeasy.py @@ -134,7 +134,7 @@ class TestDnsMadeEasyProvider(TestCase): plan = provider.plan(self.expected) # No root NS, no ignored, no excluded, no unsupported - n = len(self.expected.records) - 6 + n = len(self.expected.records) - 7 self.assertEquals(n, len(plan.changes)) self.assertEquals(n, provider.apply(plan)) diff --git a/tests/test_octodns_provider_easydns.py b/tests/test_octodns_provider_easydns.py index 8df0e22..a0f03f9 100644 --- a/tests/test_octodns_provider_easydns.py +++ b/tests/test_octodns_provider_easydns.py @@ -374,7 +374,7 @@ class TestEasyDNSProvider(TestCase): plan = provider.plan(self.expected) # No root NS, no ignored, no excluded, no unsupported - n = len(self.expected.records) - 7 + n = len(self.expected.records) - 8 self.assertEquals(n, len(plan.changes)) self.assertEquals(n, provider.apply(plan)) self.assertFalse(plan.exists) diff --git a/tests/test_octodns_provider_gandi.py b/tests/test_octodns_provider_gandi.py index 7e1c866..41b0109 100644 --- a/tests/test_octodns_provider_gandi.py +++ b/tests/test_octodns_provider_gandi.py @@ -192,8 +192,8 @@ class TestGandiProvider(TestCase): ] plan = provider.plan(self.expected) - # No root NS, no ignored, no excluded - n = len(self.expected.records) - 4 + # No root NS, no ignored, no excluded, no LOC + n = len(self.expected.records) - 5 self.assertEquals(n, len(plan.changes)) self.assertEquals(n, provider.apply(plan)) self.assertFalse(plan.exists) diff --git a/tests/test_octodns_provider_powerdns.py b/tests/test_octodns_provider_powerdns.py index 33b5e44..5605c5b 100644 --- a/tests/test_octodns_provider_powerdns.py +++ b/tests/test_octodns_provider_powerdns.py @@ -185,7 +185,7 @@ class TestPowerDnsProvider(TestCase): expected = Zone('unit.tests.', []) source = YamlProvider('test', join(dirname(__file__), 'config')) source.populate(expected) - expected_n = len(expected.records) - 3 + expected_n = len(expected.records) - 4 self.assertEquals(16, expected_n) # No diffs == no changes @@ -291,7 +291,7 @@ class TestPowerDnsProvider(TestCase): expected = Zone('unit.tests.', []) source = YamlProvider('test', join(dirname(__file__), 'config')) source.populate(expected) - self.assertEquals(19, len(expected.records)) + self.assertEquals(20, len(expected.records)) # A small change to a single record with requests_mock() as mock: diff --git a/tests/test_octodns_provider_yaml.py b/tests/test_octodns_provider_yaml.py index f255238..08d1df0 100644 --- a/tests/test_octodns_provider_yaml.py +++ b/tests/test_octodns_provider_yaml.py @@ -35,7 +35,7 @@ class TestYamlProvider(TestCase): # without it we see everything source.populate(zone) - self.assertEquals(19, len(zone.records)) + self.assertEquals(20, len(zone.records)) source.populate(dynamic_zone) self.assertEquals(5, len(dynamic_zone.records)) @@ -58,12 +58,12 @@ class TestYamlProvider(TestCase): # We add everything plan = target.plan(zone) - self.assertEquals(16, len([c for c in plan.changes + self.assertEquals(17, len([c for c in plan.changes if isinstance(c, Create)])) self.assertFalse(isfile(yaml_file)) # Now actually do it - self.assertEquals(16, target.apply(plan)) + self.assertEquals(17, target.apply(plan)) self.assertTrue(isfile(yaml_file)) # Dynamic plan @@ -87,7 +87,7 @@ class TestYamlProvider(TestCase): # A 2nd sync should still create everything plan = target.plan(zone) - self.assertEquals(16, len([c for c in plan.changes + self.assertEquals(17, len([c for c in plan.changes if isinstance(c, Create)])) with open(yaml_file) as fh: @@ -106,6 +106,7 @@ class TestYamlProvider(TestCase): self.assertTrue('values' in data.pop('naptr')) self.assertTrue('values' in data.pop('sub')) self.assertTrue('values' in data.pop('txt')) + self.assertTrue('values' in data.pop('loc')) # these are stored as singular 'value' self.assertTrue('value' in data.pop('aaaa')) self.assertTrue('value' in data.pop('cname')) diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index d55b3b8..ce40b9b 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -9,10 +9,11 @@ from six import text_type from unittest import TestCase from octodns.record import ARecord, AaaaRecord, AliasRecord, CaaRecord, \ - CaaValue, CnameRecord, DnameRecord, Create, Delete, GeoValue, MxRecord, \ - MxValue, NaptrRecord, NaptrValue, NsRecord, PtrRecord, Record, \ - SshfpRecord, SshfpValue, SpfRecord, SrvRecord, SrvValue, TxtRecord, \ - Update, ValidationError, _Dynamic, _DynamicPool, _DynamicRule + CaaValue, CnameRecord, DnameRecord, Create, Delete, GeoValue, LocRecord, \ + LocValue, MxRecord, MxValue, NaptrRecord, NaptrValue, NsRecord, \ + PtrRecord, Record, SshfpRecord, SshfpValue, SpfRecord, SrvRecord, \ + SrvValue, TxtRecord, Update, ValidationError, _Dynamic, _DynamicPool, \ + _DynamicRule from octodns.zone import Zone from helpers import DynamicProvider, GeoProvider, SimpleProvider @@ -379,6 +380,98 @@ class TestRecord(TestCase): self.assertSingleValue(DnameRecord, 'target.foo.com.', 'other.foo.com.') + def test_loc(self): + a_values = [{ + 'lat_degrees': 31, + 'lat_minutes': 58, + 'lat_seconds': 52.1, + 'lat_direction': 'S', + 'long_degrees': 115, + 'long_minutes': 49, + 'long_seconds': 11.7, + 'long_direction': 'E', + 'altitude': 20, + 'size': 10, + 'precision_horz': 10, + 'precision_vert': 2, + }] + a_data = {'ttl': 30, 'values': a_values} + a = LocRecord(self.zone, 'a', a_data) + self.assertEquals('a', a.name) + self.assertEquals('a.unit.tests.', a.fqdn) + self.assertEquals(30, a.ttl) + self.assertEquals(a_values[0]['lat_degrees'], a.values[0].lat_degrees) + self.assertEquals(a_values[0]['lat_minutes'], a.values[0].lat_minutes) + self.assertEquals(a_values[0]['lat_seconds'], a.values[0].lat_seconds) + self.assertEquals(a_values[0]['lat_direction'], + a.values[0].lat_direction) + self.assertEquals(a_values[0]['long_degrees'], + a.values[0].long_degrees) + self.assertEquals(a_values[0]['long_minutes'], + a.values[0].long_minutes) + self.assertEquals(a_values[0]['long_seconds'], + a.values[0].long_seconds) + self.assertEquals(a_values[0]['long_direction'], + a.values[0].long_direction) + self.assertEquals(a_values[0]['altitude'], a.values[0].altitude) + self.assertEquals(a_values[0]['size'], a.values[0].size) + self.assertEquals(a_values[0]['precision_horz'], + a.values[0].precision_horz) + self.assertEquals(a_values[0]['precision_vert'], + a.values[0].precision_vert) + + b_value = { + 'lat_degrees': 32, + 'lat_minutes': 7, + 'lat_seconds': 19, + 'lat_direction': 'S', + 'long_degrees': 116, + 'long_minutes': 2, + 'long_seconds': 25, + 'long_direction': 'E', + 'altitude': 10, + 'size': 1, + 'precision_horz': 10000, + 'precision_vert': 10, + } + b_data = {'ttl': 30, 'value': b_value} + b = LocRecord(self.zone, 'b', b_data) + self.assertEquals(b_value['lat_degrees'], b.values[0].lat_degrees) + self.assertEquals(b_value['lat_minutes'], b.values[0].lat_minutes) + self.assertEquals(b_value['lat_seconds'], b.values[0].lat_seconds) + self.assertEquals(b_value['lat_direction'], b.values[0].lat_direction) + self.assertEquals(b_value['long_degrees'], b.values[0].long_degrees) + self.assertEquals(b_value['long_minutes'], b.values[0].long_minutes) + self.assertEquals(b_value['long_seconds'], b.values[0].long_seconds) + self.assertEquals(b_value['long_direction'], + b.values[0].long_direction) + self.assertEquals(b_value['altitude'], b.values[0].altitude) + self.assertEquals(b_value['size'], b.values[0].size) + self.assertEquals(b_value['precision_horz'], + b.values[0].precision_horz) + self.assertEquals(b_value['precision_vert'], + b.values[0].precision_vert) + self.assertEquals(b_data, b.data) + + target = SimpleProvider() + # No changes with self + self.assertFalse(a.changes(a, target)) + # Diff in lat_direction causes change + other = LocRecord(self.zone, 'a', {'ttl': 30, 'values': a_values}) + other.values[0].lat_direction = 'N' + change = a.changes(other, target) + self.assertEqual(change.existing, a) + self.assertEqual(change.new, other) + # Diff in altitude causes change + other.values[0].altitude = a.values[0].altitude + other.values[0].altitude = -10 + change = a.changes(other, target) + self.assertEqual(change.existing, a) + self.assertEqual(change.new, other) + + # __repr__ doesn't blow up + a.__repr__() + def test_mx(self): a_values = [{ 'preference': 10, @@ -1127,6 +1220,93 @@ class TestRecord(TestCase): self.assertTrue(d >= d) self.assertTrue(d <= d) + def test_loc_value(self): + a = LocValue({ + 'lat_degrees': 31, + 'lat_minutes': 58, + 'lat_seconds': 52.1, + 'lat_direction': 'S', + 'long_degrees': 115, + 'long_minutes': 49, + 'long_seconds': 11.7, + 'long_direction': 'E', + 'altitude': 20, + 'size': 10, + 'precision_horz': 10, + 'precision_vert': 2, + }) + b = LocValue({ + 'lat_degrees': 32, + 'lat_minutes': 7, + 'lat_seconds': 19, + 'lat_direction': 'S', + 'long_degrees': 116, + 'long_minutes': 2, + 'long_seconds': 25, + 'long_direction': 'E', + 'altitude': 10, + 'size': 1, + 'precision_horz': 10000, + 'precision_vert': 10, + }) + c = LocValue({ + 'lat_degrees': 53, + 'lat_minutes': 14, + 'lat_seconds': 10, + 'lat_direction': 'N', + 'long_degrees': 2, + 'long_minutes': 18, + 'long_seconds': 26, + 'long_direction': 'W', + 'altitude': 10, + 'size': 1, + 'precision_horz': 1000, + 'precision_vert': 10, + }) + + self.assertEqual(a, a) + self.assertEqual(b, b) + self.assertEqual(c, c) + + self.assertNotEqual(a, b) + self.assertNotEqual(a, c) + self.assertNotEqual(b, a) + self.assertNotEqual(b, c) + self.assertNotEqual(c, a) + self.assertNotEqual(c, b) + + self.assertTrue(a < b) + self.assertTrue(a < c) + + self.assertTrue(b > a) + self.assertTrue(b < c) + + self.assertTrue(c > a) + self.assertTrue(c > b) + + self.assertTrue(a <= b) + self.assertTrue(a <= c) + self.assertTrue(a <= a) + self.assertTrue(a >= a) + + self.assertTrue(b >= a) + self.assertTrue(b <= c) + self.assertTrue(b >= b) + self.assertTrue(b <= b) + + self.assertTrue(c >= a) + self.assertTrue(c >= b) + self.assertTrue(c >= c) + self.assertTrue(c <= c) + + # Hash + values = set() + values.add(a) + self.assertTrue(a in values) + self.assertFalse(b in values) + values.add(b) + self.assertTrue(b in values) + def test_mx_value(self): a = MxValue({'preference': 0, 'priority': 'a', 'exchange': 'v', 'value': '1'}) @@ -1960,6 +2140,306 @@ class TestRecordValidation(TestCase): self.assertEquals(['DNAME value "foo.bar.com" missing trailing .'], ctx.exception.reasons) + def test_LOC(self): + # doesn't blow up + Record.new(self.zone, '', { + 'type': 'LOC', + 'ttl': 600, + 'value': { + 'lat_degrees': 31, + 'lat_minutes': 58, + 'lat_seconds': 52.1, + 'lat_direction': 'S', + 'long_degrees': 115, + 'long_minutes': 49, + 'long_seconds': 11.7, + 'long_direction': 'E', + 'altitude': 20, + 'size': 10, + 'precision_horz': 10, + 'precision_vert': 2, + } + }) + + # missing int key + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'LOC', + 'ttl': 600, + 'value': { + 'lat_minutes': 58, + 'lat_seconds': 52.1, + 'lat_direction': 'S', + 'long_degrees': 115, + 'long_minutes': 49, + 'long_seconds': 11.7, + 'long_direction': 'E', + 'altitude': 20, + 'size': 10, + 'precision_horz': 10, + 'precision_vert': 2, + } + }) + + self.assertEquals(['missing lat_degrees'], ctx.exception.reasons) + + # missing float key + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'LOC', + 'ttl': 600, + 'value': { + 'lat_degrees': 31, + 'lat_minutes': 58, + 'lat_direction': 'S', + 'long_degrees': 115, + 'long_minutes': 49, + 'long_seconds': 11.7, + 'long_direction': 'E', + 'altitude': 20, + 'size': 10, + 'precision_horz': 10, + 'precision_vert': 2, + } + }) + + self.assertEquals(['missing lat_seconds'], ctx.exception.reasons) + + # missing text key + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'LOC', + 'ttl': 600, + 'value': { + 'lat_degrees': 31, + 'lat_minutes': 58, + 'lat_seconds': 52.1, + 'long_degrees': 115, + 'long_minutes': 49, + 'long_seconds': 11.7, + 'long_direction': 'E', + 'altitude': 20, + 'size': 10, + 'precision_horz': 10, + 'precision_vert': 2, + } + }) + + self.assertEquals(['missing lat_direction'], ctx.exception.reasons) + + # invalid direction + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'LOC', + 'ttl': 600, + 'value': { + 'lat_degrees': 31, + 'lat_minutes': 58, + 'lat_seconds': 52.1, + 'lat_direction': 'U', + 'long_degrees': 115, + 'long_minutes': 49, + 'long_seconds': 11.7, + 'long_direction': 'E', + 'altitude': 20, + 'size': 10, + 'precision_horz': 10, + 'precision_vert': 2, + } + }) + + self.assertEquals(['invalid direction for lat_direction "U"'], + ctx.exception.reasons) + + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'LOC', + 'ttl': 600, + 'value': { + 'lat_degrees': 31, + 'lat_minutes': 58, + 'lat_seconds': 52.1, + 'lat_direction': 'S', + 'long_degrees': 115, + 'long_minutes': 49, + 'long_seconds': 11.7, + 'long_direction': 'N', + 'altitude': 20, + 'size': 10, + 'precision_horz': 10, + 'precision_vert': 2, + } + }) + + self.assertEquals(['invalid direction for long_direction "N"'], + ctx.exception.reasons) + + # invalid degrees + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'LOC', + 'ttl': 600, + 'value': { + 'lat_degrees': 360, + 'lat_minutes': 58, + 'lat_seconds': 52.1, + 'lat_direction': 'S', + 'long_degrees': 115, + 'long_minutes': 49, + 'long_seconds': 11.7, + 'long_direction': 'E', + 'altitude': 20, + 'size': 10, + 'precision_horz': 10, + 'precision_vert': 2, + } + }) + + self.assertEquals(['invalid value for lat_degrees "360"'], + ctx.exception.reasons) + + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'LOC', + 'ttl': 600, + 'value': { + 'lat_degrees': 'nope', + 'lat_minutes': 58, + 'lat_seconds': 52.1, + 'lat_direction': 'S', + 'long_degrees': 115, + 'long_minutes': 49, + 'long_seconds': 11.7, + 'long_direction': 'E', + 'altitude': 20, + 'size': 10, + 'precision_horz': 10, + 'precision_vert': 2, + } + }) + + self.assertEquals(['invalid lat_degrees "nope"'], + ctx.exception.reasons) + + # invalid minutes + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'LOC', + 'ttl': 600, + 'value': { + 'lat_degrees': 31, + 'lat_minutes': 60, + 'lat_seconds': 52.1, + 'lat_direction': 'S', + 'long_degrees': 115, + 'long_minutes': 49, + 'long_seconds': 11.7, + 'long_direction': 'E', + 'altitude': 20, + 'size': 10, + 'precision_horz': 10, + 'precision_vert': 2, + } + }) + + self.assertEquals(['invalid value for lat_minutes "60"'], + ctx.exception.reasons) + + # invalid seconds + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'LOC', + 'ttl': 600, + 'value': { + 'lat_degrees': 31, + 'lat_minutes': 58, + 'lat_seconds': 60, + 'lat_direction': 'S', + 'long_degrees': 115, + 'long_minutes': 49, + 'long_seconds': 11.7, + 'long_direction': 'E', + 'altitude': 20, + 'size': 10, + 'precision_horz': 10, + 'precision_vert': 2, + } + }) + + self.assertEquals(['invalid value for lat_seconds "60"'], + ctx.exception.reasons) + + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'LOC', + 'ttl': 600, + 'value': { + 'lat_degrees': 31, + 'lat_minutes': 58, + 'lat_seconds': 'nope', + 'lat_direction': 'S', + 'long_degrees': 115, + 'long_minutes': 49, + 'long_seconds': 11.7, + 'long_direction': 'E', + 'altitude': 20, + 'size': 10, + 'precision_horz': 10, + 'precision_vert': 2, + } + }) + + self.assertEquals(['invalid lat_seconds "nope"'], + ctx.exception.reasons) + + # invalid altitude + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'LOC', + 'ttl': 600, + 'value': { + 'lat_degrees': 31, + 'lat_minutes': 58, + 'lat_seconds': 52.1, + 'lat_direction': 'S', + 'long_degrees': 115, + 'long_minutes': 49, + 'long_seconds': 11.7, + 'long_direction': 'E', + 'altitude': -666666, + 'size': 10, + 'precision_horz': 10, + 'precision_vert': 2, + } + }) + + self.assertEquals(['invalid value for altitude "-666666"'], + ctx.exception.reasons) + + # invalid size + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'LOC', + 'ttl': 600, + 'value': { + 'lat_degrees': 31, + 'lat_minutes': 58, + 'lat_seconds': 52.1, + 'lat_direction': 'S', + 'long_degrees': 115, + 'long_minutes': 49, + 'long_seconds': 11.7, + 'long_direction': 'E', + 'altitude': 20, + 'size': 99999999.99, + 'precision_horz': 10, + 'precision_vert': 2, + } + }) + + self.assertEquals(['invalid value for size "99999999.99"'], + ctx.exception.reasons) + def test_MX(self): # doesn't blow up Record.new(self.zone, '', { diff --git a/tests/zones/unit.tests.tst b/tests/zones/unit.tests.tst index 838de88..3a25415 100644 --- a/tests/zones/unit.tests.tst +++ b/tests/zones/unit.tests.tst @@ -32,6 +32,10 @@ mx 300 IN MX 20 smtp-2.unit.tests. mx 300 IN MX 30 smtp-3.unit.tests. mx 300 IN MX 40 smtp-1.unit.tests. +; LOC Records +loc 300 IN LOC 31 58 52.1 S 115 49 11.7 E 20m 10m 10m 2m +loc 300 IN LOC 53 14 10 N 2 18 26 W 20m 10m 1000m 2m + ; A Records @ 300 IN A 1.2.3.4 @ 300 IN A 1.2.3.5 From 3ac8d0fa1c3d6e1a97afd681d1872183ce5aeeae Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Mon, 30 Nov 2020 23:44:55 +0800 Subject: [PATCH 69/96] Add LOC record support to AXFR source --- README.md | 2 +- octodns/source/axfr.py | 31 +++++++++++++++++++++++++++++-- tests/test_octodns_source_axfr.py | 6 +++--- 3 files changed, 33 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index a65e28f..08933aa 100644 --- a/README.md +++ b/README.md @@ -205,7 +205,7 @@ The above command pulled the existing data out of Route53 and placed the results | [Selectel](/octodns/provider/selectel.py) | | A, AAAA, CNAME, MX, NS, SPF, SRV, TXT | No | | | [Transip](/octodns/provider/transip.py) | transip | A, AAAA, CNAME, MX, SRV, SPF, TXT, SSHFP, CAA | No | | | [UltraDns](/octodns/provider/ultra.py) | | A, AAAA, CAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | | -| [AxfrSource](/octodns/source/axfr.py) | | A, AAAA, CAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | read-only | +| [AxfrSource](/octodns/source/axfr.py) | | A, AAAA, CAA, CNAME, LOC, MX, NS, PTR, SPF, SRV, TXT | No | read-only | | [ZoneFileSource](/octodns/source/axfr.py) | | A, AAAA, CAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | read-only | | [TinyDnsFileSource](/octodns/source/tinydns.py) | | A, CNAME, MX, NS, PTR | No | read-only | | [YamlProvider](/octodns/provider/yaml.py) | | All | Yes | config | diff --git a/octodns/source/axfr.py b/octodns/source/axfr.py index e21f29f..7a45155 100644 --- a/octodns/source/axfr.py +++ b/octodns/source/axfr.py @@ -26,8 +26,8 @@ class AxfrBaseSource(BaseSource): SUPPORTS_GEO = False SUPPORTS_DYNAMIC = False - SUPPORTS = set(('A', 'AAAA', 'CAA', 'CNAME', 'MX', 'NS', 'PTR', 'SPF', - 'SRV', 'TXT')) + SUPPORTS = set(('A', 'AAAA', 'CAA', 'CNAME', 'LOC', 'MX', 'NS', 'PTR', + 'SPF', 'SRV', 'TXT')) def __init__(self, id): super(AxfrBaseSource, self).__init__(id) @@ -58,6 +58,33 @@ class AxfrBaseSource(BaseSource): 'values': values } + def _data_for_LOC(self, _type, records): + values = [] + for record in records: + lat_degrees, lat_minutes, lat_seconds, lat_direction, \ + long_degrees, long_minutes, long_seconds, long_direction, \ + altitude, size, precision_horz, precision_vert = \ + record['value'].replace('m', '').split(' ', 11) + values.append({ + 'lat_degrees': lat_degrees, + 'lat_minutes': lat_minutes, + 'lat_seconds': lat_seconds, + 'lat_direction': lat_direction, + 'long_degrees': long_degrees, + 'long_minutes': long_minutes, + 'long_seconds': long_seconds, + 'long_direction': long_direction, + 'altitude': altitude, + 'size': size, + 'precision_horz': precision_horz, + 'precision_vert': precision_vert, + }) + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': values + } + def _data_for_MX(self, _type, records): values = [] for record in records: diff --git a/tests/test_octodns_source_axfr.py b/tests/test_octodns_source_axfr.py index 44e04d0..8cf6929 100644 --- a/tests/test_octodns_source_axfr.py +++ b/tests/test_octodns_source_axfr.py @@ -36,7 +36,7 @@ class TestAxfrSource(TestCase): ] self.source.populate(got) - self.assertEquals(12, len(got.records)) + self.assertEquals(13, len(got.records)) with self.assertRaises(AxfrSourceZoneTransferFailed) as ctx: zone = Zone('unit.tests.', []) @@ -79,12 +79,12 @@ class TestZoneFileSource(TestCase): # Valid zone file in directory valid = Zone('unit.tests.', []) self.source.populate(valid) - self.assertEquals(12, len(valid.records)) + self.assertEquals(13, len(valid.records)) # 2nd populate does not read file again again = Zone('unit.tests.', []) self.source.populate(again) - self.assertEquals(12, len(again.records)) + self.assertEquals(13, len(again.records)) # bust the cache del self.source._zone_records[valid.name] From 8338e8db5855ff91a40847a5c6a4aaf6dc3945b8 Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Mon, 30 Nov 2020 23:56:04 +0800 Subject: [PATCH 70/96] Add LOC record support to Cloudflare provider --- README.md | 2 +- octodns/provider/cloudflare.py | 48 ++++++- .../cloudflare-dns_records-page-2.json | 56 +------- .../cloudflare-dns_records-page-3.json | 128 ++++++++++++++++++ tests/test_octodns_provider_cloudflare.py | 62 ++++++++- 5 files changed, 234 insertions(+), 62 deletions(-) create mode 100644 tests/fixtures/cloudflare-dns_records-page-3.json diff --git a/README.md b/README.md index 08933aa..5d57821 100644 --- a/README.md +++ b/README.md @@ -185,7 +185,7 @@ The above command pulled the existing data out of Route53 and placed the results |--|--|--|--|--| | [AzureProvider](/octodns/provider/azuredns.py) | azure-mgmt-dns | A, AAAA, CAA, CNAME, MX, NS, PTR, SRV, TXT | No | | | [Akamai](/octodns/provider/edgedns.py) | edgegrid-python | A, AAAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, SSHFP, TXT | No | | -| [CloudflareProvider](/octodns/provider/cloudflare.py) | | A, AAAA, ALIAS, CAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | CAA tags restricted | +| [CloudflareProvider](/octodns/provider/cloudflare.py) | | A, AAAA, ALIAS, CAA, CNAME, LOC, MX, NS, PTR, SPF, SRV, TXT | No | CAA tags restricted | | [ConstellixProvider](/octodns/provider/constellix.py) | | A, AAAA, ALIAS (ANAME), CAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | CAA tags restricted | | [DigitalOceanProvider](/octodns/provider/digitalocean.py) | | A, AAAA, CAA, CNAME, MX, NS, TXT, SRV | No | CAA tags restricted | | [DnsMadeEasyProvider](/octodns/provider/dnsmadeeasy.py) | | A, AAAA, ALIAS (ANAME), CAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | CAA tags restricted | diff --git a/octodns/provider/cloudflare.py b/octodns/provider/cloudflare.py index db937e5..05fef20 100644 --- a/octodns/provider/cloudflare.py +++ b/octodns/provider/cloudflare.py @@ -75,8 +75,8 @@ class CloudflareProvider(BaseProvider): ''' SUPPORTS_GEO = False SUPPORTS_DYNAMIC = False - SUPPORTS = set(('ALIAS', 'A', 'AAAA', 'CAA', 'CNAME', 'MX', 'NS', 'PTR', - 'SRV', 'SPF', 'TXT')) + SUPPORTS = set(('ALIAS', 'A', 'AAAA', 'CAA', 'CNAME', 'LOC', 'MX', 'NS', + 'PTR', 'SRV', 'SPF', 'TXT')) MIN_TTL = 120 TIMEOUT = 15 @@ -133,6 +133,7 @@ class CloudflareProvider(BaseProvider): timeout=self.TIMEOUT) self.log.debug('_request: status=%d', resp.status_code) if resp.status_code == 400: + self.log.debug('_request: data=%s', data) raise CloudflareError(resp.json()) if resp.status_code == 403: raise CloudflareAuthenticationError(resp.json()) @@ -216,6 +217,30 @@ class CloudflareProvider(BaseProvider): _data_for_ALIAS = _data_for_CNAME _data_for_PTR = _data_for_CNAME + def _data_for_LOC(self, _type, records): + values = [] + for record in records: + r = record['data'] + values.append({ + 'lat_degrees': int(r['lat_degrees']), + 'lat_minutes': int(r['lat_minutes']), + 'lat_seconds': float(r['lat_seconds']), + 'lat_direction': r['lat_direction'], + 'long_degrees': int(r['long_degrees']), + 'long_minutes': int(r['long_minutes']), + 'long_seconds': float(r['long_seconds']), + 'long_direction': r['long_direction'], + 'altitude': float(r['altitude']), + 'size': float(r['size']), + 'precision_horz': float(r['precision_horz']), + 'precision_vert': float(r['precision_vert']), + }) + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': values + } + def _data_for_MX(self, _type, records): values = [] for r in records: @@ -384,6 +409,25 @@ class CloudflareProvider(BaseProvider): _contents_for_PTR = _contents_for_CNAME + def _contents_for_LOC(self, record): + for value in record.values: + yield { + 'data': { + 'lat_degrees': value.lat_degrees, + 'lat_minutes': value.lat_minutes, + 'lat_seconds': value.lat_seconds, + 'lat_direction': value.lat_direction, + 'long_degrees': value.long_degrees, + 'long_minutes': value.long_minutes, + 'long_seconds': value.long_seconds, + 'long_direction': value.long_direction, + 'altitude': value.altitude, + 'size': value.size, + 'precision_horz': value.precision_horz, + 'precision_vert': value.precision_vert, + } + } + def _contents_for_MX(self, record): for value in record.values: yield { diff --git a/tests/fixtures/cloudflare-dns_records-page-2.json b/tests/fixtures/cloudflare-dns_records-page-2.json index b0bbaef..8075ba5 100644 --- a/tests/fixtures/cloudflare-dns_records-page-2.json +++ b/tests/fixtures/cloudflare-dns_records-page-2.json @@ -173,64 +173,14 @@ "meta": { "auto_added": false } - }, - { - "id": "fc12ab34cd5611334422ab3322997656", - "type": "SRV", - "name": "_srv._tcp.unit.tests", - "data": { - "service": "_srv", - "proto": "_tcp", - "name": "unit.tests", - "priority": 12, - "weight": 20, - "port": 30, - "target": "foo-2.unit.tests" - }, - "proxiable": true, - "proxied": false, - "ttl": 600, - "locked": false, - "zone_id": "ff12ab34cd5611334422ab3322997650", - "zone_name": "unit.tests", - "modified_on": "2017-03-11T18:01:43.940682Z", - "created_on": "2017-03-11T18:01:43.940682Z", - "meta": { - "auto_added": false - } - }, - { - "id": "fc12ab34cd5611334422ab3322997656", - "type": "SRV", - "name": "_srv._tcp.unit.tests", - "data": { - "service": "_srv", - "proto": "_tcp", - "name": "unit.tests", - "priority": 10, - "weight": 20, - "port": 30, - "target": "foo-1.unit.tests" - }, - "proxiable": true, - "proxied": false, - "ttl": 600, - "locked": false, - "zone_id": "ff12ab34cd5611334422ab3322997650", - "zone_name": "unit.tests", - "modified_on": "2017-03-11T18:01:43.940682Z", - "created_on": "2017-03-11T18:01:43.940682Z", - "meta": { - "auto_added": false - } } ], "result_info": { "page": 2, - "per_page": 11, - "total_pages": 2, + "per_page": 10, + "total_pages": 3, "count": 10, - "total_count": 20 + "total_count": 24 }, "success": true, "errors": [], diff --git a/tests/fixtures/cloudflare-dns_records-page-3.json b/tests/fixtures/cloudflare-dns_records-page-3.json new file mode 100644 index 0000000..0f06ab4 --- /dev/null +++ b/tests/fixtures/cloudflare-dns_records-page-3.json @@ -0,0 +1,128 @@ +{ + "result": [ + { + "id": "fc12ab34cd5611334422ab3322997656", + "type": "SRV", + "name": "_srv._tcp.unit.tests", + "data": { + "service": "_srv", + "proto": "_tcp", + "name": "unit.tests", + "priority": 12, + "weight": 20, + "port": 30, + "target": "foo-2.unit.tests" + }, + "proxiable": true, + "proxied": false, + "ttl": 600, + "locked": false, + "zone_id": "ff12ab34cd5611334422ab3322997650", + "zone_name": "unit.tests", + "modified_on": "2017-03-11T18:01:43.940682Z", + "created_on": "2017-03-11T18:01:43.940682Z", + "meta": { + "auto_added": false + } + }, + { + "id": "fc12ab34cd5611334422ab3322997656", + "type": "SRV", + "name": "_srv._tcp.unit.tests", + "data": { + "service": "_srv", + "proto": "_tcp", + "name": "unit.tests", + "priority": 10, + "weight": 20, + "port": 30, + "target": "foo-1.unit.tests" + }, + "proxiable": true, + "proxied": false, + "ttl": 600, + "locked": false, + "zone_id": "ff12ab34cd5611334422ab3322997650", + "zone_name": "unit.tests", + "modified_on": "2017-03-11T18:01:43.940682Z", + "created_on": "2017-03-11T18:01:43.940682Z", + "meta": { + "auto_added": false + } + }, + { + "id": "372e67954025e0ba6aaa6d586b9e0b59", + "type": "LOC", + "name": "loc.unit.tests", + "content": "IN LOC 31 58 52.1 S 115 49 11.7 E 20m 10m 10m 2m", + "proxiable": true, + "proxied": false, + "ttl": 300, + "locked": false, + "zone_id": "ff12ab34cd5611334422ab3322997650", + "zone_name": "unit.tests", + "created_on": "2020-01-28T05:20:00.12345Z", + "modified_on": "2020-01-28T05:20:00.12345Z", + "data": { + "lat_degrees": 31, + "lat_minutes": 58, + "lat_seconds": 52.1, + "lat_direction": "S", + "long_degrees": 115, + "long_minutes": 49, + "long_seconds": 11.7, + "long_direction": "E", + "altitude": 20, + "size": 10, + "precision_horz": 10, + "precision_vert": 2 + }, + "meta": { + "auto_added": true, + "source": "primary" + } + }, + { + "id": "372e67954025e0ba6aaa6d586b9e0b59", + "type": "LOC", + "name": "loc.unit.tests", + "content": "IN LOC 53 14 10 N 2 18 26 W 20m 10m 1000m 2m", + "proxiable": true, + "proxied": false, + "ttl": 300, + "locked": false, + "zone_id": "ff12ab34cd5611334422ab3322997650", + "zone_name": "unit.tests", + "created_on": "2020-01-28T05:20:00.12345Z", + "modified_on": "2020-01-28T05:20:00.12345Z", + "data": { + "lat_degrees": 53, + "lat_minutes": 13, + "lat_seconds": 10, + "lat_direction": "N", + "long_degrees": 2, + "long_minutes": 18, + "long_seconds": 26, + "long_direction": "W", + "altitude": 20, + "size": 10, + "precision_horz": 1000, + "precision_vert": 2 + }, + "meta": { + "auto_added": true, + "source": "primary" + } + } + ], + "result_info": { + "page": 3, + "per_page": 10, + "total_pages": 3, + "count": 4, + "total_count": 24 + }, + "success": true, + "errors": [], + "messages": [] +} diff --git a/tests/test_octodns_provider_cloudflare.py b/tests/test_octodns_provider_cloudflare.py index 735d95c..3727e20 100644 --- a/tests/test_octodns_provider_cloudflare.py +++ b/tests/test_octodns_provider_cloudflare.py @@ -177,10 +177,14 @@ class TestCloudflareProvider(TestCase): 'page-2.json') as fh: mock.get('{}?page=2'.format(base), status_code=200, text=fh.read()) + with open('tests/fixtures/cloudflare-dns_records-' + 'page-3.json') as fh: + mock.get('{}?page=3'.format(base), status_code=200, + text=fh.read()) zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals(13, len(zone.records)) + self.assertEquals(14, len(zone.records)) changes = self.expected.changes(zone, provider) @@ -189,7 +193,7 @@ class TestCloudflareProvider(TestCase): # re-populating the same zone/records comes out of cache, no calls again = Zone('unit.tests.', []) provider.populate(again) - self.assertEquals(13, len(again.records)) + self.assertEquals(14, len(again.records)) def test_apply(self): provider = CloudflareProvider('test', 'email', 'token', retry_period=0) @@ -203,12 +207,12 @@ class TestCloudflareProvider(TestCase): 'id': 42, } }, # zone create - ] + [None] * 22 # individual record creates + ] + [None] * 24 # individual record creates # non-existent zone, create everything plan = provider.plan(self.expected) - self.assertEquals(13, len(plan.changes)) - self.assertEquals(13, provider.apply(plan)) + self.assertEquals(14, len(plan.changes)) + self.assertEquals(14, provider.apply(plan)) self.assertFalse(plan.exists) provider._request.assert_has_calls([ @@ -234,7 +238,7 @@ class TestCloudflareProvider(TestCase): }), ], True) # expected number of total calls - self.assertEquals(23, provider._request.call_count) + self.assertEquals(25, provider._request.call_count) provider._request.reset_mock() @@ -566,6 +570,52 @@ class TestCloudflareProvider(TestCase): 'content': 'foo.bar.com.' }, list(ptr_record_contents)[0]) + def test_loc(self): + self.maxDiff = None + provider = CloudflareProvider('test', 'email', 'token') + + zone = Zone('unit.tests.', []) + # LOC record + loc_record = Record.new(zone, 'example', { + 'ttl': 300, + 'type': 'LOC', + 'value': { + 'lat_degrees': 31, + 'lat_minutes': 58, + 'lat_seconds': 52.1, + 'lat_direction': 'S', + 'long_degrees': 115, + 'long_minutes': 49, + 'long_seconds': 11.7, + 'long_direction': 'E', + 'altitude': 20, + 'size': 10, + 'precision_horz': 10, + 'precision_vert': 2, + } + }) + + loc_record_contents = provider._gen_data(loc_record) + self.assertEquals({ + 'name': 'example.unit.tests', + 'ttl': 300, + 'type': 'LOC', + 'data': { + 'lat_degrees': 31, + 'lat_minutes': 58, + 'lat_seconds': 52.1, + 'lat_direction': 'S', + 'long_degrees': 115, + 'long_minutes': 49, + 'long_seconds': 11.7, + 'long_direction': 'E', + 'altitude': 20, + 'size': 10, + 'precision_horz': 10, + 'precision_vert': 2, + } + }, list(loc_record_contents)[0]) + def test_srv(self): provider = CloudflareProvider('test', 'email', 'token') From 5963c8b89453e51e01d2b983d0421cb707fa1568 Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Tue, 1 Dec 2020 00:02:49 +0800 Subject: [PATCH 71/96] Force order of Delete() -> Create() -> Update() in Cloudflare provider Addresses issues with changing between A, AAAA and CNAME records in both directions with the Cloudflare API See also: github/octodns#507 See also: github/octodns#586 See also: github/octodns#587 --- octodns/provider/cloudflare.py | 10 ++++++++++ tests/test_octodns_provider_cloudflare.py | 10 +++++----- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/octodns/provider/cloudflare.py b/octodns/provider/cloudflare.py index 05fef20..f9d9fd0 100644 --- a/octodns/provider/cloudflare.py +++ b/octodns/provider/cloudflare.py @@ -143,6 +143,11 @@ class CloudflareProvider(BaseProvider): resp.raise_for_status() return resp.json() + def _change_keyer(self, change): + key = change.__class__.__name__ + order = {'Delete': 0, 'Create': 1, 'Update': 2} + return order[key] + @property def zones(self): if self._zones is None: @@ -660,6 +665,11 @@ class CloudflareProvider(BaseProvider): self.zones[name] = zone_id self._zone_records[name] = {} + # Force the operation order to be Delete() -> Create() -> Update() + # This will help avoid problems in updating a CNAME record into an + # A record and vice-versa + changes.sort(key=self._change_keyer) + for change in changes: class_name = change.__class__.__name__ getattr(self, '_apply_{}'.format(class_name))(change) diff --git a/tests/test_octodns_provider_cloudflare.py b/tests/test_octodns_provider_cloudflare.py index 3727e20..127480b 100644 --- a/tests/test_octodns_provider_cloudflare.py +++ b/tests/test_octodns_provider_cloudflare.py @@ -340,6 +340,10 @@ class TestCloudflareProvider(TestCase): self.assertTrue(plan.exists) # creates a the new value and then deletes all the old provider._request.assert_has_calls([ + call('DELETE', '/zones/ff12ab34cd5611334422ab3322997650/' + 'dns_records/fc12ab34cd5611334422ab3322997653'), + call('DELETE', '/zones/ff12ab34cd5611334422ab3322997650/' + 'dns_records/fc12ab34cd5611334422ab3322997654'), call('PUT', '/zones/42/dns_records/' 'fc12ab34cd5611334422ab3322997655', data={ 'content': '3.2.3.4', @@ -347,11 +351,7 @@ class TestCloudflareProvider(TestCase): 'name': 'ttl.unit.tests', 'proxied': False, 'ttl': 300 - }), - call('DELETE', '/zones/ff12ab34cd5611334422ab3322997650/' - 'dns_records/fc12ab34cd5611334422ab3322997653'), - call('DELETE', '/zones/ff12ab34cd5611334422ab3322997650/' - 'dns_records/fc12ab34cd5611334422ab3322997654') + }) ]) def test_update_add_swap(self): From 5852ae7a2ffdac74e3bff07b44d2a2c0db147024 Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Wed, 13 Jan 2021 21:46:23 +0800 Subject: [PATCH 72/96] Detect changes to LOC record correctly --- octodns/provider/cloudflare.py | 18 +++++++++++++++++- tests/test_octodns_provider_cloudflare.py | 17 +++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/octodns/provider/cloudflare.py b/octodns/provider/cloudflare.py index f9d9fd0..cc1169d 100644 --- a/octodns/provider/cloudflare.py +++ b/octodns/provider/cloudflare.py @@ -505,7 +505,7 @@ class CloudflareProvider(BaseProvider): # new records cleanly. In general when there are multiple records for a # name & type each will have a distinct/consistent `content` that can # serve as a unique identifier. - # BUT... there are exceptions. MX, CAA, and SRV don't have a simple + # BUT... there are exceptions. MX, CAA, LOC and SRV don't have a simple # content as things are currently implemented so we need to handle # those explicitly and create unique/hashable strings for them. _type = data['type'] @@ -517,6 +517,22 @@ class CloudflareProvider(BaseProvider): elif _type == 'SRV': data = data['data'] return '{port} {priority} {target} {weight}'.format(**data) + elif _type == 'LOC': + data = data['data'] + loc = ( + '{lat_degrees}', + '{lat_minutes}', + '{lat_seconds}', + '{lat_direction}', + '{long_degrees}', + '{long_minutes}', + '{long_seconds}', + '{long_direction}', + '{altitude}', + '{size}', + '{precision_horz}', + '{precision_vert}') + return ' '.join(loc).format(**data) return data['content'] def _apply_Create(self, change): diff --git a/tests/test_octodns_provider_cloudflare.py b/tests/test_octodns_provider_cloudflare.py index 127480b..d4fa74e 100644 --- a/tests/test_octodns_provider_cloudflare.py +++ b/tests/test_octodns_provider_cloudflare.py @@ -747,6 +747,23 @@ class TestCloudflareProvider(TestCase): }, 'type': 'SRV', }), + ('31 58 52.1 S 115 49 11.7 E 20 10 10 2', { + 'data': { + 'lat_degrees': 31, + 'lat_minutes': 58, + 'lat_seconds': 52.1, + 'lat_direction': 'S', + 'long_degrees': 115, + 'long_minutes': 49, + 'long_seconds': 11.7, + 'long_direction': 'E', + 'altitude': 20, + 'size': 10, + 'precision_horz': 10, + 'precision_vert': 2, + }, + 'type': 'LOC', + }), ): self.assertEqual(expected, provider._gen_key(data)) From f5c2f3a141ada969b9ef05e74c133abe877a8c10 Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Sun, 31 Jan 2021 22:45:48 +0800 Subject: [PATCH 73/96] Add LOC record support to PowerDNS provider --- octodns/provider/powerdns.py | 52 ++++++++++++++++++++++++- tests/fixtures/powerdns-full-data.json | 16 ++++++++ tests/test_octodns_provider_powerdns.py | 6 +-- 3 files changed, 69 insertions(+), 5 deletions(-) diff --git a/octodns/provider/powerdns.py b/octodns/provider/powerdns.py index de7743c..ee24eab 100644 --- a/octodns/provider/powerdns.py +++ b/octodns/provider/powerdns.py @@ -15,8 +15,8 @@ from .base import BaseProvider class PowerDnsBaseProvider(BaseProvider): SUPPORTS_GEO = False SUPPORTS_DYNAMIC = False - SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CAA', 'CNAME', 'MX', 'NAPTR', 'NS', - 'PTR', 'SPF', 'SSHFP', 'SRV', 'TXT')) + SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CAA', 'CNAME', 'LOC', 'MX', 'NAPTR', + 'NS', 'PTR', 'SPF', 'SSHFP', 'SRV', 'TXT')) TIMEOUT = 5 def __init__(self, id, host, api_key, port=8081, @@ -102,6 +102,33 @@ class PowerDnsBaseProvider(BaseProvider): _data_for_SPF = _data_for_quoted _data_for_TXT = _data_for_quoted + def _data_for_LOC(self, rrset): + values = [] + for record in rrset['records']: + lat_degrees, lat_minutes, lat_seconds, lat_direction, \ + long_degrees, long_minutes, long_seconds, long_direction, \ + altitude, size, precision_horz, precision_vert = \ + record['content'].replace('m', '').split(' ', 11) + values.append({ + 'lat_degrees': int(lat_degrees), + 'lat_minutes': int(lat_minutes), + 'lat_seconds': float(lat_seconds), + 'lat_direction': lat_direction, + 'long_degrees': int(long_degrees), + 'long_minutes': int(long_minutes), + 'long_seconds': float(long_seconds), + 'long_direction': long_direction, + 'altitude': float(altitude), + 'size': float(size), + 'precision_horz': float(precision_horz), + 'precision_vert': float(precision_vert), + }) + return { + 'ttl': rrset['ttl'], + 'type': rrset['type'], + 'values': values + } + def _data_for_MX(self, rrset): values = [] for record in rrset['records']: @@ -285,6 +312,27 @@ class PowerDnsBaseProvider(BaseProvider): _records_for_SPF = _records_for_quoted _records_for_TXT = _records_for_quoted + def _records_for_LOC(self, record): + return [{ + 'content': + '%d %d %0.3f %s %d %d %.3f %s %0.2fm %0.2fm %0.2fm %0.2fm' % + ( + int(v.lat_degrees), + int(v.lat_minutes), + float(v.lat_seconds), + v.lat_direction, + int(v.long_degrees), + int(v.long_minutes), + float(v.long_seconds), + v.long_direction, + float(v.altitude), + float(v.size), + float(v.precision_horz), + float(v.precision_vert) + ), + 'disabled': False + } for v in record.values] + def _records_for_MX(self, record): return [{ 'content': '{} {}'.format(v.preference, v.exchange), diff --git a/tests/fixtures/powerdns-full-data.json b/tests/fixtures/powerdns-full-data.json index 3d445d4..7da8232 100644 --- a/tests/fixtures/powerdns-full-data.json +++ b/tests/fixtures/powerdns-full-data.json @@ -32,6 +32,22 @@ "ttl": 300, "type": "MX" }, + { + "comments": [], + "name": "loc.unit.tests.", + "records": [ + { + "content": "31 58 52.100 S 115 49 11.700 E 20.00m 10.00m 10.00m 2.00m", + "disabled": false + }, + { + "content": "53 13 10.000 N 2 18 26.000 W 20.00m 10.00m 1000.00m 2.00m", + "disabled": false + } + ], + "ttl": 300, + "type": "LOC" + }, { "comments": [], "name": "sub.unit.tests.", diff --git a/tests/test_octodns_provider_powerdns.py b/tests/test_octodns_provider_powerdns.py index 5605c5b..f3f99e2 100644 --- a/tests/test_octodns_provider_powerdns.py +++ b/tests/test_octodns_provider_powerdns.py @@ -185,8 +185,8 @@ class TestPowerDnsProvider(TestCase): expected = Zone('unit.tests.', []) source = YamlProvider('test', join(dirname(__file__), 'config')) source.populate(expected) - expected_n = len(expected.records) - 4 - self.assertEquals(16, expected_n) + expected_n = len(expected.records) - 3 + self.assertEquals(17, expected_n) # No diffs == no changes with requests_mock() as mock: @@ -194,7 +194,7 @@ class TestPowerDnsProvider(TestCase): zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals(16, len(zone.records)) + self.assertEquals(17, len(zone.records)) changes = expected.changes(zone, provider) self.assertEquals(0, len(changes)) From 9e70caf92c8a44e0328117c3500c1af518e350aa Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Tue, 9 Feb 2021 20:47:03 +0800 Subject: [PATCH 74/96] Update test_octodns_source_axfr.py to catch up with upstream changes/tests --- tests/test_octodns_source_axfr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_octodns_source_axfr.py b/tests/test_octodns_source_axfr.py index 8cf6929..cd493cb 100644 --- a/tests/test_octodns_source_axfr.py +++ b/tests/test_octodns_source_axfr.py @@ -73,7 +73,7 @@ class TestZoneFileSource(TestCase): # Load zonefiles without a specified file extension valid = Zone('unit.tests.', []) source.populate(valid) - self.assertEquals(12, len(valid.records)) + self.assertEquals(13, len(valid.records)) def test_populate(self): # Valid zone file in directory From e991d8dc10d7693e8d4242f14e604fcd6d9df1c5 Mon Sep 17 00:00:00 2001 From: Patrick Connolly Date: Tue, 9 Feb 2021 16:02:52 -0500 Subject: [PATCH 75/96] Removed implementation example. --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 84dc033..9f608a0 100644 --- a/README.md +++ b/README.md @@ -297,7 +297,6 @@ If you have a problem or suggestion, please [open an issue](https://github.com/o - [`kubernetes/k8s.io:/dns`](https://github.com/kubernetes/k8s.io/tree/master/dns) - [`g0v-network/domains`](https://github.com/g0v-network/domains) - [`jekyll/dns`](https://github.com/jekyll/dns) - - [`parkr/dns`](https://github.com/parkr/dns) - **Custom Sources & Providers.** - [`octodns/octodns-ddns`](https://github.com/octodns/octodns-ddns): A simple Dynamic DNS source. - [`doddo/octodns-lexicon`](https://github.com/doddo/octodns-lexicon): Use [Lexicon](https://github.com/AnalogJ/lexicon) providers as octoDNS providers. From a8366aa02db6fac3d090f0d9a12edf3f7916ab4a Mon Sep 17 00:00:00 2001 From: Patrick Connolly Date: Tue, 9 Feb 2021 16:06:25 -0500 Subject: [PATCH 76/96] Stopped running CI for doc changes. --- .github/workflows/main.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0f62f96..337ebb0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,5 +1,8 @@ name: OctoDNS -on: [pull_request] +on: + pull_request: + paths-ignore: + - '**.md' jobs: ci: From 39d86f023e411c0af869a14ce7f12eda41f4d649 Mon Sep 17 00:00:00 2001 From: Steven Honson Date: Fri, 12 Feb 2021 00:29:48 +1100 Subject: [PATCH 77/96] powerdns: deletes before replaces --- octodns/provider/powerdns.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/octodns/provider/powerdns.py b/octodns/provider/powerdns.py index de7743c..a7f150b 100644 --- a/octodns/provider/powerdns.py +++ b/octodns/provider/powerdns.py @@ -7,6 +7,7 @@ from __future__ import absolute_import, division, print_function, \ from requests import HTTPError, Session import logging +from operator import itemgetter from ..record import Create, Record from .base import BaseProvider @@ -381,6 +382,12 @@ class PowerDnsBaseProvider(BaseProvider): for change in changes: class_name = change.__class__.__name__ mods.append(getattr(self, '_mod_{}'.format(class_name))(change)) + + # Ensure that any DELETE modifications always occur before any REPLACE + # modifications. This ensures that an A record can be replaced by a + # CNAME record and vice-versa. + mods = sorted(mods, key=itemgetter('changetype')) + self.log.debug('_apply: sending change request') try: From fdf74f9dd36365674473d8ee8d110da92f99b2c8 Mon Sep 17 00:00:00 2001 From: Steven Honson Date: Fri, 12 Feb 2021 00:48:42 +1100 Subject: [PATCH 78/96] powerdns: sort in place --- octodns/provider/powerdns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/powerdns.py b/octodns/provider/powerdns.py index a7f150b..ec30559 100644 --- a/octodns/provider/powerdns.py +++ b/octodns/provider/powerdns.py @@ -386,7 +386,7 @@ class PowerDnsBaseProvider(BaseProvider): # Ensure that any DELETE modifications always occur before any REPLACE # modifications. This ensures that an A record can be replaced by a # CNAME record and vice-versa. - mods = sorted(mods, key=itemgetter('changetype')) + mods.sort(key=itemgetter('changetype')) self.log.debug('_apply: sending change request') From 6034e8022f24a83ca80d1c32e1fc0420ecaca606 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Thu, 11 Feb 2021 17:05:42 -0800 Subject: [PATCH 79/96] Swap import order --- octodns/provider/powerdns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/powerdns.py b/octodns/provider/powerdns.py index ec30559..8ffff46 100644 --- a/octodns/provider/powerdns.py +++ b/octodns/provider/powerdns.py @@ -6,8 +6,8 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals from requests import HTTPError, Session -import logging from operator import itemgetter +import logging from ..record import Create, Record from .base import BaseProvider From 29c8f3253ed3d17cc8490bfc6e4b8e6f108b1411 Mon Sep 17 00:00:00 2001 From: Patrick Connolly Date: Thu, 11 Feb 2021 20:35:32 -0500 Subject: [PATCH 80/96] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 9f608a0..c784c0a 100644 --- a/README.md +++ b/README.md @@ -308,6 +308,7 @@ If you have a problem or suggestion, please [open an issue](https://github.com/o - Video: [FOSDEM 2019 - DNS as code with octodns](https://archive.fosdem.org/2019/schedule/event/dns_octodns/) - GitHub Blog: [Enabling DNS split authority with OctoDNS](https://github.blog/2017-04-27-enabling-split-authority-dns-with-octodns/) - Tutorial: [How To Deploy and Manage Your DNS using OctoDNS on Ubuntu 18.04](https://www.digitalocean.com/community/tutorials/how-to-deploy-and-manage-your-dns-using-octodns-on-ubuntu-18-04) + - Cloudflare Blog: [Improving the Resiliency of Our Infrastructure DNS Zone](https://blog.cloudflare.com/improving-the-resiliency-of-our-infrastructure-dns-zone/) If you know of any other resources, please do let us know! From 45d5da23cfc0940a536dc24102994820ab67f032 Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Wed, 9 Dec 2020 19:02:36 +0800 Subject: [PATCH 81/96] Add NULL SRV record examples to unit tests --- tests/config/unit.tests.yaml | 16 ++++++++++++++++ tests/zones/unit.tests.tst | 3 +++ 2 files changed, 19 insertions(+) diff --git a/tests/config/unit.tests.yaml b/tests/config/unit.tests.yaml index 7b84ac9..03f13a1 100644 --- a/tests/config/unit.tests.yaml +++ b/tests/config/unit.tests.yaml @@ -36,6 +36,22 @@ - flags: 0 tag: issue value: ca.unit.tests +_imap._tcp: + ttl: 600 + type: SRV + values: + - port: 0 + priority: 0 + target: . + weight: 0 +_pop3._tcp: + ttl: 600 + type: SRV + values: + - port: 0 + priority: 0 + target: . + weight: 0 _srv._tcp: ttl: 600 type: SRV diff --git a/tests/zones/unit.tests.tst b/tests/zones/unit.tests.tst index 838de88..b48b749 100644 --- a/tests/zones/unit.tests.tst +++ b/tests/zones/unit.tests.tst @@ -20,6 +20,9 @@ caa 1800 IN CAA 0 iodef "mailto:admin@unit.tests" ; SRV Records _srv._tcp 600 IN SRV 10 20 30 foo-1.unit.tests. _srv._tcp 600 IN SRV 10 20 30 foo-2.unit.tests. +; NULL SRV Records +_pop3._tcp 600 IN SRV 0 0 0 . +_imap._tcp 600 IN SRV 0 0 0 . ; TXT Records txt 600 IN TXT "Bah bah black sheep" From 403be8bb838bb8431700e61df6a4f150fe8b8a07 Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Wed, 9 Dec 2020 19:13:17 +0800 Subject: [PATCH 82/96] Fix handling of NULL SRV records in Cloudflare provider --- octodns/provider/cloudflare.py | 8 ++- .../cloudflare-dns_records-page-2.json | 50 +++++++++++++++++++ tests/test_octodns_provider_cloudflare.py | 12 ++--- 3 files changed, 62 insertions(+), 8 deletions(-) diff --git a/octodns/provider/cloudflare.py b/octodns/provider/cloudflare.py index db937e5..0f5c6cf 100644 --- a/octodns/provider/cloudflare.py +++ b/octodns/provider/cloudflare.py @@ -239,11 +239,13 @@ class CloudflareProvider(BaseProvider): def _data_for_SRV(self, _type, records): values = [] for r in records: + target = ('{}.'.format(r['data']['target']) + if r['data']['target'] != "." else ".") values.append({ 'priority': r['data']['priority'], 'weight': r['data']['weight'], 'port': r['data']['port'], - 'target': '{}.'.format(r['data']['target']), + 'target': target, }) return { 'type': _type, @@ -405,6 +407,8 @@ class CloudflareProvider(BaseProvider): name = subdomain for value in record.values: + target = value.target[:-1] if value.target != "." else "." + yield { 'data': { 'service': service, @@ -413,7 +417,7 @@ class CloudflareProvider(BaseProvider): 'priority': value.priority, 'weight': value.weight, 'port': value.port, - 'target': value.target[:-1], + 'target': target, } } diff --git a/tests/fixtures/cloudflare-dns_records-page-2.json b/tests/fixtures/cloudflare-dns_records-page-2.json index b0bbaef..860b6c3 100644 --- a/tests/fixtures/cloudflare-dns_records-page-2.json +++ b/tests/fixtures/cloudflare-dns_records-page-2.json @@ -174,6 +174,56 @@ "auto_added": false } }, + { + "id": "fc12ab34cd5611334422ab3322997656", + "type": "SRV", + "name": "_imap._tcp.unit.tests", + "data": { + "service": "_imap", + "proto": "_tcp", + "name": "unit.tests", + "priority": 0, + "weight": 0, + "port": 0, + "target": "." + }, + "proxiable": true, + "proxied": false, + "ttl": 600, + "locked": false, + "zone_id": "ff12ab34cd5611334422ab3322997650", + "zone_name": "unit.tests", + "modified_on": "2017-03-11T18:01:43.940682Z", + "created_on": "2017-03-11T18:01:43.940682Z", + "meta": { + "auto_added": false + } + }, + { + "id": "fc12ab34cd5611334422ab3322997656", + "type": "SRV", + "name": "_pop3._tcp.unit.tests", + "data": { + "service": "_imap", + "proto": "_pop3", + "name": "unit.tests", + "priority": 0, + "weight": 0, + "port": 0, + "target": "." + }, + "proxiable": true, + "proxied": false, + "ttl": 600, + "locked": false, + "zone_id": "ff12ab34cd5611334422ab3322997650", + "zone_name": "unit.tests", + "modified_on": "2017-03-11T18:01:43.940682Z", + "created_on": "2017-03-11T18:01:43.940682Z", + "meta": { + "auto_added": false + } + }, { "id": "fc12ab34cd5611334422ab3322997656", "type": "SRV", diff --git a/tests/test_octodns_provider_cloudflare.py b/tests/test_octodns_provider_cloudflare.py index 735d95c..94a37f4 100644 --- a/tests/test_octodns_provider_cloudflare.py +++ b/tests/test_octodns_provider_cloudflare.py @@ -180,7 +180,7 @@ class TestCloudflareProvider(TestCase): zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals(13, len(zone.records)) + self.assertEquals(15, len(zone.records)) changes = self.expected.changes(zone, provider) @@ -189,7 +189,7 @@ class TestCloudflareProvider(TestCase): # re-populating the same zone/records comes out of cache, no calls again = Zone('unit.tests.', []) provider.populate(again) - self.assertEquals(13, len(again.records)) + self.assertEquals(15, len(again.records)) def test_apply(self): provider = CloudflareProvider('test', 'email', 'token', retry_period=0) @@ -203,12 +203,12 @@ class TestCloudflareProvider(TestCase): 'id': 42, } }, # zone create - ] + [None] * 22 # individual record creates + ] + [None] * 24 # individual record creates # non-existent zone, create everything plan = provider.plan(self.expected) - self.assertEquals(13, len(plan.changes)) - self.assertEquals(13, provider.apply(plan)) + self.assertEquals(15, len(plan.changes)) + self.assertEquals(15, provider.apply(plan)) self.assertFalse(plan.exists) provider._request.assert_has_calls([ @@ -234,7 +234,7 @@ class TestCloudflareProvider(TestCase): }), ], True) # expected number of total calls - self.assertEquals(23, provider._request.call_count) + self.assertEquals(25, provider._request.call_count) provider._request.reset_mock() From 39412924da534adde67e1c9b11d0d93fd8e67db8 Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Wed, 9 Dec 2020 21:23:14 +0800 Subject: [PATCH 83/96] Fix handling of NULL SRV records in DigitalOcean provider --- octodns/provider/digitalocean.py | 6 +++++- tests/fixtures/digitalocean-page-2.json | 22 +++++++++++++++++++ tests/test_octodns_provider_digitalocean.py | 24 ++++++++++++++++++--- 3 files changed, 48 insertions(+), 4 deletions(-) diff --git a/octodns/provider/digitalocean.py b/octodns/provider/digitalocean.py index e192543..6ccee1d 100644 --- a/octodns/provider/digitalocean.py +++ b/octodns/provider/digitalocean.py @@ -186,10 +186,14 @@ class DigitalOceanProvider(BaseProvider): def _data_for_SRV(self, _type, records): values = [] for record in records: + target = ( + '{}.'.format(record['data']) + if record['data'] != "." else "." + ) values.append({ 'port': record['port'], 'priority': record['priority'], - 'target': '{}.'.format(record['data']), + 'target': target, 'weight': record['weight'] }) return { diff --git a/tests/fixtures/digitalocean-page-2.json b/tests/fixtures/digitalocean-page-2.json index 50f17f9..1405527 100644 --- a/tests/fixtures/digitalocean-page-2.json +++ b/tests/fixtures/digitalocean-page-2.json @@ -76,6 +76,28 @@ "weight": null, "flags": null, "tag": null + }, { + "id": 11189896, + "type": "SRV", + "name": "_imap._tcp", + "data": ".", + "priority": 0, + "port": 0, + "ttl": 600, + "weight": 0, + "flags": null, + "tag": null + }, { + "id": 11189897, + "type": "SRV", + "name": "_pop3._tcp", + "data": ".", + "priority": 0, + "port": 0, + "ttl": 600, + "weight": 0, + "flags": null, + "tag": null }], "links": { "pages": { diff --git a/tests/test_octodns_provider_digitalocean.py b/tests/test_octodns_provider_digitalocean.py index 0ad8f72..83fb5c3 100644 --- a/tests/test_octodns_provider_digitalocean.py +++ b/tests/test_octodns_provider_digitalocean.py @@ -83,14 +83,14 @@ class TestDigitalOceanProvider(TestCase): zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals(12, len(zone.records)) + self.assertEquals(14, len(zone.records)) changes = self.expected.changes(zone, provider) self.assertEquals(0, len(changes)) # 2nd populate makes no network calls/all from cache again = Zone('unit.tests.', []) provider.populate(again) - self.assertEquals(12, len(again.records)) + self.assertEquals(14, len(again.records)) # bust the cache del provider._zone_records[zone.name] @@ -190,6 +190,24 @@ class TestDigitalOceanProvider(TestCase): 'flags': 0, 'name': '@', 'tag': 'issue', 'ttl': 3600, 'type': 'CAA'}), + call('POST', '/domains/unit.tests/records', data={ + 'name': '_imap._tcp', + 'weight': 0, + 'data': '.', + 'priority': 0, + 'ttl': 600, + 'type': 'SRV', + 'port': 0 + }), + call('POST', '/domains/unit.tests/records', data={ + 'name': '_pop3._tcp', + 'weight': 0, + 'data': '.', + 'priority': 0, + 'ttl': 600, + 'type': 'SRV', + 'port': 0 + }), call('POST', '/domains/unit.tests/records', data={ 'name': '_srv._tcp', 'weight': 20, @@ -200,7 +218,7 @@ class TestDigitalOceanProvider(TestCase): 'port': 30 }), ]) - self.assertEquals(24, provider._client._request.call_count) + self.assertEquals(26, provider._client._request.call_count) provider._client._request.reset_mock() From 876a5b46f34693d7161b773babc95d1a38d02103 Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Sat, 19 Dec 2020 22:32:23 +0800 Subject: [PATCH 84/96] Update PowerDNS tests and fixtures for NULL SRV records --- tests/fixtures/powerdns-full-data.json | 24 ++++++++++++++++++++++++ tests/test_octodns_provider_powerdns.py | 6 +++--- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/tests/fixtures/powerdns-full-data.json b/tests/fixtures/powerdns-full-data.json index 3d445d4..a08f028 100644 --- a/tests/fixtures/powerdns-full-data.json +++ b/tests/fixtures/powerdns-full-data.json @@ -59,6 +59,30 @@ "ttl": 300, "type": "A" }, + { + "comments": [], + "name": "_imap._tcp.unit.tests.", + "records": [ + { + "content": "0 0 0 .", + "disabled": false + } + ], + "ttl": 600, + "type": "SRV" + }, + { + "comments": [], + "name": "_pop3._tcp.unit.tests.", + "records": [ + { + "content": "0 0 0 .", + "disabled": false + } + ], + "ttl": 600, + "type": "SRV" + }, { "comments": [], "name": "_srv._tcp.unit.tests.", diff --git a/tests/test_octodns_provider_powerdns.py b/tests/test_octodns_provider_powerdns.py index 33b5e44..7c418ff 100644 --- a/tests/test_octodns_provider_powerdns.py +++ b/tests/test_octodns_provider_powerdns.py @@ -186,7 +186,7 @@ class TestPowerDnsProvider(TestCase): source = YamlProvider('test', join(dirname(__file__), 'config')) source.populate(expected) expected_n = len(expected.records) - 3 - self.assertEquals(16, expected_n) + self.assertEquals(18, expected_n) # No diffs == no changes with requests_mock() as mock: @@ -194,7 +194,7 @@ class TestPowerDnsProvider(TestCase): zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals(16, len(zone.records)) + self.assertEquals(18, len(zone.records)) changes = expected.changes(zone, provider) self.assertEquals(0, len(changes)) @@ -291,7 +291,7 @@ class TestPowerDnsProvider(TestCase): expected = Zone('unit.tests.', []) source = YamlProvider('test', join(dirname(__file__), 'config')) source.populate(expected) - self.assertEquals(19, len(expected.records)) + self.assertEquals(21, len(expected.records)) # A small change to a single record with requests_mock() as mock: From 4105fb7ee798c83bf894ef39150335460536e888 Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Tue, 29 Dec 2020 17:06:50 +0800 Subject: [PATCH 85/96] Update Gandi tests and fixtures for NULL SRV records Thanks to @yzguy for assisting with API tests to confirm support --- tests/fixtures/gandi-no-changes.json | 18 ++++++++++++++++++ tests/test_octodns_provider_gandi.py | 20 ++++++++++++++++++-- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/tests/fixtures/gandi-no-changes.json b/tests/fixtures/gandi-no-changes.json index b018785..a67dc93 100644 --- a/tests/fixtures/gandi-no-changes.json +++ b/tests/fixtures/gandi-no-changes.json @@ -123,6 +123,24 @@ "2.2.3.6" ] }, + { + "rrset_type": "SRV", + "rrset_ttl": 600, + "rrset_name": "_imap._tcp", + "rrset_href": "https://api.gandi.net/v5/livedns/domains/unit.tests/records/_imap._tcp/SRV", + "rrset_values": [ + "0 0 0 ." + ] + }, + { + "rrset_type": "SRV", + "rrset_ttl": 600, + "rrset_name": "_pop3._tcp", + "rrset_href": "https://api.gandi.net/v5/livedns/domains/unit.tests/records/_pop3._tcp/SRV", + "rrset_values": [ + "0 0 0 ." + ] + }, { "rrset_type": "SRV", "rrset_ttl": 600, diff --git a/tests/test_octodns_provider_gandi.py b/tests/test_octodns_provider_gandi.py index 7e1c866..1b0443b 100644 --- a/tests/test_octodns_provider_gandi.py +++ b/tests/test_octodns_provider_gandi.py @@ -117,7 +117,7 @@ class TestGandiProvider(TestCase): zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals(14, len(zone.records)) + self.assertEquals(16, len(zone.records)) changes = self.expected.changes(zone, provider) self.assertEquals(0, len(changes)) @@ -284,6 +284,22 @@ class TestGandiProvider(TestCase): '12 20 30 foo-2.unit.tests.' ] }), + call('POST', '/livedns/domains/unit.tests/records', data={ + 'rrset_name': '_pop3._tcp', + 'rrset_ttl': 600, + 'rrset_type': 'SRV', + 'rrset_values': [ + '0 0 0 .', + ] + }), + call('POST', '/livedns/domains/unit.tests/records', data={ + 'rrset_name': '_imap._tcp', + 'rrset_ttl': 600, + 'rrset_type': 'SRV', + 'rrset_values': [ + '0 0 0 .', + ] + }), call('POST', '/livedns/domains/unit.tests/records', data={ 'rrset_name': '@', 'rrset_ttl': 3600, @@ -307,7 +323,7 @@ class TestGandiProvider(TestCase): }) ]) # expected number of total calls - self.assertEquals(17, provider._client._request.call_count) + self.assertEquals(19, provider._client._request.call_count) provider._client._request.reset_mock() From ae65311a96e77efea6719027f31b0a435ef7b9d8 Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Tue, 29 Dec 2020 17:11:22 +0800 Subject: [PATCH 86/96] Update Constellix tests and fixtures for NULL SRV records Thanks to @yzguy for assisting with API tests to confirm support --- tests/fixtures/constellix-records.json | 56 +++++++++++++++++++++++ tests/test_octodns_provider_constellix.py | 6 +-- 2 files changed, 59 insertions(+), 3 deletions(-) diff --git a/tests/fixtures/constellix-records.json b/tests/fixtures/constellix-records.json index 689fd53..282ca62 100644 --- a/tests/fixtures/constellix-records.json +++ b/tests/fixtures/constellix-records.json @@ -64,6 +64,62 @@ "roundRobinFailover": [], "pools": [], "poolsDetail": [] +}, { + "id": 1898527, + "type": "SRV", + "recordType": "srv", + "name": "_imap._tcp", + "recordOption": "roundRobin", + "noAnswer": false, + "note": "", + "ttl": 600, + "gtdRegion": 1, + "parentId": 123123, + "parent": "domain", + "source": "Domain", + "modifiedTs": 1565149714387, + "value": [{ + "value": ".", + "priority": 0, + "weight": 0, + "port": 0, + "disableFlag": false + }], + "roundRobin": [{ + "value": ".", + "priority": 0, + "weight": 0, + "port": 0, + "disableFlag": false + }] +}, { + "id": 1898528, + "type": "SRV", + "recordType": "srv", + "name": "_pop3._tcp", + "recordOption": "roundRobin", + "noAnswer": false, + "note": "", + "ttl": 600, + "gtdRegion": 1, + "parentId": 123123, + "parent": "domain", + "source": "Domain", + "modifiedTs": 1565149714387, + "value": [{ + "value": ".", + "priority": 0, + "weight": 0, + "port": 0, + "disableFlag": false + }], + "roundRobin": [{ + "value": ".", + "priority": 0, + "weight": 0, + "port": 0, + "disableFlag": false + }] }, { "id": 1808527, "type": "SRV", diff --git a/tests/test_octodns_provider_constellix.py b/tests/test_octodns_provider_constellix.py index bc17b50..8ba4860 100644 --- a/tests/test_octodns_provider_constellix.py +++ b/tests/test_octodns_provider_constellix.py @@ -101,14 +101,14 @@ class TestConstellixProvider(TestCase): zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals(14, len(zone.records)) + self.assertEquals(16, len(zone.records)) changes = self.expected.changes(zone, provider) self.assertEquals(0, len(changes)) # 2nd populate makes no network calls/all from cache again = Zone('unit.tests.', []) provider.populate(again) - self.assertEquals(14, len(again.records)) + self.assertEquals(16, len(again.records)) # bust the cache del provider._zone_records[zone.name] @@ -163,7 +163,7 @@ class TestConstellixProvider(TestCase): }), ]) - self.assertEquals(17, provider._client._request.call_count) + self.assertEquals(19, provider._client._request.call_count) provider._client._request.reset_mock() From 909c7ad7e88d54f4adeadbf9d3ad3d189de06360 Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Fri, 8 Jan 2021 16:51:36 +0800 Subject: [PATCH 87/96] Update EasyDNS tests and fixtures for NULL SRV records Thanks to @actazen for assisting with API tests to confirm support --- tests/fixtures/easydns-records.json | 26 ++++++++++++++++++++++++-- tests/test_octodns_provider_easydns.py | 6 +++--- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/tests/fixtures/easydns-records.json b/tests/fixtures/easydns-records.json index c3718b5..73ea953 100644 --- a/tests/fixtures/easydns-records.json +++ b/tests/fixtures/easydns-records.json @@ -264,10 +264,32 @@ "rdata": "v=DKIM1;k=rsa;s=email;h=sha256;p=A\/kinda+of\/long\/string+with+numb3rs", "geozone_id": "0", "last_mod": "2020-01-01 01:01:01" + }, + { + "id": "12340025", + "domain": "unit.tests", + "host": "_imap._tcp", + "ttl": "600", + "prio": "0", + "type": "SRV", + "rdata": "0 0 0 .", + "geozone_id": "0", + "last_mod": "2020-01-01 01:01:01" + }, + { + "id": "12340026", + "domain": "unit.tests", + "host": "_pop3._tcp", + "ttl": "600", + "prio": "0", + "type": "SRV", + "rdata": "0 0 0 .", + "geozone_id": "0", + "last_mod": "2020-01-01 01:01:01" } ], - "count": 24, - "total": 24, + "count": 26, + "total": 26, "start": 0, "max": 1000, "status": 200 diff --git a/tests/test_octodns_provider_easydns.py b/tests/test_octodns_provider_easydns.py index 8df0e22..2b137a6 100644 --- a/tests/test_octodns_provider_easydns.py +++ b/tests/test_octodns_provider_easydns.py @@ -80,14 +80,14 @@ class TestEasyDNSProvider(TestCase): text=fh.read()) provider.populate(zone) - self.assertEquals(13, len(zone.records)) + self.assertEquals(15, len(zone.records)) changes = self.expected.changes(zone, provider) self.assertEquals(0, len(changes)) # 2nd populate makes no network calls/all from cache again = Zone('unit.tests.', []) provider.populate(again) - self.assertEquals(13, len(again.records)) + self.assertEquals(15, len(again.records)) # bust the cache del provider._zone_records[zone.name] @@ -379,7 +379,7 @@ class TestEasyDNSProvider(TestCase): self.assertEquals(n, provider.apply(plan)) self.assertFalse(plan.exists) - self.assertEquals(23, provider._client._request.call_count) + self.assertEquals(25, provider._client._request.call_count) provider._client._request.reset_mock() From fb197b890e78d6a652fa276bb4148afd9174245b Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Mon, 11 Jan 2021 22:07:58 +0800 Subject: [PATCH 88/96] Update Mythic Beasts tests and fixtures for NULL SRV records Thanks to @pwaring for assisting with API tests to confirm support, and reaching out to Mythic Beasts to obtain a fix --- tests/fixtures/mythicbeasts-list.txt | 2 ++ tests/test_octodns_provider_mythicbeasts.py | 8 ++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/fixtures/mythicbeasts-list.txt b/tests/fixtures/mythicbeasts-list.txt index ed4ea4c..006a8ff 100644 --- a/tests/fixtures/mythicbeasts-list.txt +++ b/tests/fixtures/mythicbeasts-list.txt @@ -5,6 +5,8 @@ @ 3600 SSHFP 1 1 bf6b6825d2977c511a475bbefb88aad54a92ac73 @ 3600 SSHFP 1 1 7491973e5f8b39d5327cd4e08bc81b05f7710b49 @ 3600 CAA 0 issue ca.unit.tests +_imap._tcp 600 SRV 0 0 0 . +_pop3._tcp 600 SRV 0 0 0 . _srv._tcp 600 SRV 10 20 30 foo-1.unit.tests. _srv._tcp 600 SRV 12 20 30 foo-2.unit.tests. aaaa 600 AAAA 2601:644:500:e210:62f8:1dff:feb8:947a diff --git a/tests/test_octodns_provider_mythicbeasts.py b/tests/test_octodns_provider_mythicbeasts.py index f78cb0b..26af8c1 100644 --- a/tests/test_octodns_provider_mythicbeasts.py +++ b/tests/test_octodns_provider_mythicbeasts.py @@ -378,8 +378,8 @@ class TestMythicBeastsProvider(TestCase): zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals(15, len(zone.records)) - self.assertEquals(15, len(self.expected.records)) + self.assertEquals(17, len(zone.records)) + self.assertEquals(17, len(self.expected.records)) changes = self.expected.changes(zone, provider) self.assertEquals(0, len(changes)) @@ -445,7 +445,7 @@ class TestMythicBeastsProvider(TestCase): if isinstance(c, Update)])) self.assertEquals(1, len([c for c in plan.changes if isinstance(c, Delete)])) - self.assertEquals(14, len([c for c in plan.changes + self.assertEquals(16, len([c for c in plan.changes if isinstance(c, Create)])) - self.assertEquals(16, provider.apply(plan)) + self.assertEquals(18, provider.apply(plan)) self.assertTrue(plan.exists) From e0d79f826f752dc792ad03836e27daebe2c64691 Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Fri, 15 Jan 2021 18:32:11 +0800 Subject: [PATCH 89/96] Update Edge DNS tests and fixtures for NULL SRV records Thanks to @jdgri for assisting with API tests to confirm support --- tests/fixtures/edgedns-records.json | 20 ++++++++++++++++++-- tests/test_octodns_provider_edgedns.py | 10 +++++----- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/tests/fixtures/edgedns-records.json b/tests/fixtures/edgedns-records.json index 4693eb1..a5ce14f 100644 --- a/tests/fixtures/edgedns-records.json +++ b/tests/fixtures/edgedns-records.json @@ -9,6 +9,22 @@ "name": "_srv._tcp.unit.tests", "ttl": 600 }, + { + "rdata": [ + "0 0 0 ." + ], + "type": "SRV", + "name": "_imap._tcp.unit.tests", + "ttl": 600 + }, + { + "rdata": [ + "0 0 0 ." + ], + "type": "SRV", + "name": "_pop3._tcp.unit.tests", + "ttl": 600 + }, { "rdata": [ "2601:644:500:e210:62f8:1dff:feb8:947a" @@ -151,7 +167,7 @@ } ], "metadata": { - "totalElements": 16, + "totalElements": 18, "showAll": true } -} \ No newline at end of file +} diff --git a/tests/test_octodns_provider_edgedns.py b/tests/test_octodns_provider_edgedns.py index 20a9a07..694c762 100644 --- a/tests/test_octodns_provider_edgedns.py +++ b/tests/test_octodns_provider_edgedns.py @@ -77,14 +77,14 @@ class TestEdgeDnsProvider(TestCase): zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals(16, len(zone.records)) + self.assertEquals(18, len(zone.records)) changes = self.expected.changes(zone, provider) self.assertEquals(0, len(changes)) # 2nd populate makes no network calls/all from cache again = Zone('unit.tests.', []) provider.populate(again) - self.assertEquals(16, len(again.records)) + self.assertEquals(18, len(again.records)) # bust the cache del provider._zone_records[zone.name] @@ -105,7 +105,7 @@ class TestEdgeDnsProvider(TestCase): mock.delete(ANY, status_code=204) changes = provider.apply(plan) - self.assertEquals(29, changes) + self.assertEquals(31, changes) # Test against a zone that doesn't exist yet with requests_mock() as mock: @@ -118,7 +118,7 @@ class TestEdgeDnsProvider(TestCase): mock.delete(ANY, status_code=204) changes = provider.apply(plan) - self.assertEquals(14, changes) + self.assertEquals(16, changes) # Test against a zone that doesn't exist yet, but gid not provided with requests_mock() as mock: @@ -132,7 +132,7 @@ class TestEdgeDnsProvider(TestCase): mock.delete(ANY, status_code=204) changes = provider.apply(plan) - self.assertEquals(14, changes) + self.assertEquals(16, changes) # Test against a zone that doesn't exist, but cid not provided From 2cd5511dc68ccd0fad97dbc665e9cc20f20813d0 Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Sat, 30 Jan 2021 00:08:24 +0800 Subject: [PATCH 90/96] Warn that NULL SRV records are unsupported in DNSimple provider DNSimple does not handle NULL SRV records correctly (either via their web interface or API). Flag to end user if attempted. Issue noted with DNSimple support 2020-12-09 --- octodns/provider/dnsimple.py | 41 +++++++++++++++++++++++-- tests/test_octodns_provider_dnsimple.py | 2 +- 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/octodns/provider/dnsimple.py b/octodns/provider/dnsimple.py index f83098e..647b89c 100644 --- a/octodns/provider/dnsimple.py +++ b/octodns/provider/dnsimple.py @@ -218,12 +218,23 @@ class DnsimpleProvider(BaseProvider): try: weight, port, target = record['content'].split(' ', 2) except ValueError: - # see _data_for_NAPTR's continue + # their api/website will let you create invalid records, this + # essentially handles that by ignoring them for values + # purposes. That will cause updates to happen to delete them if + # they shouldn't exist or update them if they're wrong + self.log.warning( + '_data_for_SRV: unsupported %s record (%s)', + _type, + record['content'] + ) continue + + target = '{}.'.format(target) if target != "." else "." + values.append({ 'port': port, 'priority': record['priority'], - 'target': '{}.'.format(target), + 'target': target, 'weight': weight }) return { @@ -269,7 +280,12 @@ class DnsimpleProvider(BaseProvider): values = defaultdict(lambda: defaultdict(list)) for record in self.zone_records(zone): _type = record['type'] + data_for = getattr(self, '_data_for_{}'.format(_type), None) if _type not in self.SUPPORTS: + self.log.warning( + 'populate: skipping unsupported %s record', + _type + ) continue elif _type == 'TXT' and record['content'].startswith('ALIAS for'): # ALIAS has a "ride along" TXT record with 'ALIAS for XXXX', @@ -290,6 +306,27 @@ class DnsimpleProvider(BaseProvider): len(zone.records) - before, exists) return exists + def supports(self, record): + # DNSimple does not support empty/NULL SRV records + # + # Fails silently and leaves a corrupt record + # + # Skip the record and continue + if record._type == "SRV": + if 'value' in record.data: + targets = (record.data['value']['target'],) + else: + targets = [value['target'] for value in record.data['values']] + + if "." in targets: + self.log.warning( + 'supports: unsupported %s record with target (%s)', + record._type, targets + ) + return False + + return record._type in self.SUPPORTS + def _params_for_multiple(self, record): for value in record.values: yield { diff --git a/tests/test_octodns_provider_dnsimple.py b/tests/test_octodns_provider_dnsimple.py index 92f32b1..9f1dab3 100644 --- a/tests/test_octodns_provider_dnsimple.py +++ b/tests/test_octodns_provider_dnsimple.py @@ -137,7 +137,7 @@ class TestDnsimpleProvider(TestCase): plan = provider.plan(self.expected) # No root NS, no ignored, no excluded - n = len(self.expected.records) - 4 + n = len(self.expected.records) - 6 self.assertEquals(n, len(plan.changes)) self.assertEquals(n, provider.apply(plan)) self.assertFalse(plan.exists) From 84a0c089d4c7c28c20fa382d15113bac87f4caa6 Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Fri, 25 Dec 2020 19:21:41 +0800 Subject: [PATCH 91/96] Warn that NULL SRV records are unsupported in DNS Made Easy provider --- octodns/provider/dnsmadeeasy.py | 24 ++++++++++++++++++++++ tests/test_octodns_provider_dnsmadeeasy.py | 2 +- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/octodns/provider/dnsmadeeasy.py b/octodns/provider/dnsmadeeasy.py index 0bf05a0..7880280 100644 --- a/octodns/provider/dnsmadeeasy.py +++ b/octodns/provider/dnsmadeeasy.py @@ -284,6 +284,30 @@ class DnsMadeEasyProvider(BaseProvider): len(zone.records) - before, exists) return exists + def supports(self, record): + # DNS Made Easy does not support empty/NULL SRV records + # + # Attempting to sync such a record would generate the following error + # + # octodns.provider.dnsmadeeasy.DnsMadeEasyClientBadRequest: + # - Record value may not be a standalone dot. + # + # Skip the record and continue + if record._type == "SRV": + if 'value' in record.data: + targets = (record.data['value']['target'],) + else: + targets = [value['target'] for value in record.data['values']] + + if "." in targets: + self.log.warning( + 'supports: unsupported %s record with target (%s)', + record._type, targets + ) + return False + + return record._type in self.SUPPORTS + def _params_for_multiple(self, record): for value in record.values: yield { diff --git a/tests/test_octodns_provider_dnsmadeeasy.py b/tests/test_octodns_provider_dnsmadeeasy.py index 0ad059d..92aa547 100644 --- a/tests/test_octodns_provider_dnsmadeeasy.py +++ b/tests/test_octodns_provider_dnsmadeeasy.py @@ -134,7 +134,7 @@ class TestDnsMadeEasyProvider(TestCase): plan = provider.plan(self.expected) # No root NS, no ignored, no excluded, no unsupported - n = len(self.expected.records) - 6 + n = len(self.expected.records) - 8 self.assertEquals(n, len(plan.changes)) self.assertEquals(n, provider.apply(plan)) From 5d23977bbd26d4b12a96f7187ffc27fda8057bdd Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Sat, 30 Jan 2021 15:44:32 +0800 Subject: [PATCH 92/96] Adjust remaining unit tests due to extra records in test zone --- tests/test_octodns_manager.py | 14 +++++++------- tests/test_octodns_provider_transip.py | 4 ++-- tests/test_octodns_provider_ultra.py | 8 ++++---- tests/test_octodns_provider_yaml.py | 10 ++++++---- tests/test_octodns_source_axfr.py | 8 ++++---- 5 files changed, 23 insertions(+), 21 deletions(-) diff --git a/tests/test_octodns_manager.py b/tests/test_octodns_manager.py index f757466..1657f04 100644 --- a/tests/test_octodns_manager.py +++ b/tests/test_octodns_manager.py @@ -118,12 +118,12 @@ class TestManager(TestCase): environ['YAML_TMP_DIR'] = tmpdir.dirname tc = Manager(get_config_filename('simple.yaml')) \ .sync(dry_run=False) - self.assertEquals(22, tc) + self.assertEquals(24, tc) # try with just one of the zones tc = Manager(get_config_filename('simple.yaml')) \ .sync(dry_run=False, eligible_zones=['unit.tests.']) - self.assertEquals(16, tc) + self.assertEquals(18, tc) # the subzone, with 2 targets tc = Manager(get_config_filename('simple.yaml')) \ @@ -138,18 +138,18 @@ class TestManager(TestCase): # Again with force tc = Manager(get_config_filename('simple.yaml')) \ .sync(dry_run=False, force=True) - self.assertEquals(22, tc) + self.assertEquals(24, tc) # Again with max_workers = 1 tc = Manager(get_config_filename('simple.yaml'), max_workers=1) \ .sync(dry_run=False, force=True) - self.assertEquals(22, tc) + self.assertEquals(24, tc) # Include meta tc = Manager(get_config_filename('simple.yaml'), max_workers=1, include_meta=True) \ .sync(dry_run=False, force=True) - self.assertEquals(26, tc) + self.assertEquals(28, tc) def test_eligible_sources(self): with TemporaryDirectory() as tmpdir: @@ -215,13 +215,13 @@ class TestManager(TestCase): fh.write('---\n{}') changes = manager.compare(['in'], ['dump'], 'unit.tests.') - self.assertEquals(16, len(changes)) + self.assertEquals(18, len(changes)) # Compound sources with varying support changes = manager.compare(['in', 'nosshfp'], ['dump'], 'unit.tests.') - self.assertEquals(15, len(changes)) + self.assertEquals(17, len(changes)) with self.assertRaises(ManagerException) as ctx: manager.compare(['nope'], ['dump'], 'unit.tests.') diff --git a/tests/test_octodns_provider_transip.py b/tests/test_octodns_provider_transip.py index f792085..84cfebc 100644 --- a/tests/test_octodns_provider_transip.py +++ b/tests/test_octodns_provider_transip.py @@ -222,7 +222,7 @@ N4OiVz1I3rbZGYa396lpxO6ku8yCglisL1yrSP6DdEUp66ntpKVd provider._client = MockDomainService('unittest', self.bogus_key) plan = provider.plan(_expected) - self.assertEqual(12, plan.change_counts['Create']) + self.assertEqual(14, plan.change_counts['Create']) self.assertEqual(0, plan.change_counts['Update']) self.assertEqual(0, plan.change_counts['Delete']) @@ -235,7 +235,7 @@ N4OiVz1I3rbZGYa396lpxO6ku8yCglisL1yrSP6DdEUp66ntpKVd provider = TransipProvider('test', 'unittest', self.bogus_key) provider._client = MockDomainService('unittest', self.bogus_key) plan = provider.plan(_expected) - self.assertEqual(12, len(plan.changes)) + self.assertEqual(14, len(plan.changes)) changes = provider.apply(plan) self.assertEqual(changes, len(plan.changes)) diff --git a/tests/test_octodns_provider_ultra.py b/tests/test_octodns_provider_ultra.py index 43eac3c..b6d1017 100644 --- a/tests/test_octodns_provider_ultra.py +++ b/tests/test_octodns_provider_ultra.py @@ -285,12 +285,12 @@ class TestUltraProvider(TestCase): provider._request.side_effect = [ UltraNoZonesExistException('No Zones'), None, # zone create - ] + [None] * 13 # individual record creates + ] + [None] * 15 # individual record creates # non-existent zone, create everything plan = provider.plan(self.expected) - self.assertEquals(13, len(plan.changes)) - self.assertEquals(13, provider.apply(plan)) + self.assertEquals(15, len(plan.changes)) + self.assertEquals(15, provider.apply(plan)) self.assertFalse(plan.exists) provider._request.assert_has_calls([ @@ -320,7 +320,7 @@ class TestUltraProvider(TestCase): 'p=A/kinda+of/long/string+with+numb3rs']}), ], True) # expected number of total calls - self.assertEquals(15, provider._request.call_count) + self.assertEquals(17, provider._request.call_count) # Create sample rrset payload to attempt to alter page1 = json_load(open('tests/fixtures/ultra-records-page-1.json')) diff --git a/tests/test_octodns_provider_yaml.py b/tests/test_octodns_provider_yaml.py index f255238..d527fb3 100644 --- a/tests/test_octodns_provider_yaml.py +++ b/tests/test_octodns_provider_yaml.py @@ -35,7 +35,7 @@ class TestYamlProvider(TestCase): # without it we see everything source.populate(zone) - self.assertEquals(19, len(zone.records)) + self.assertEquals(21, len(zone.records)) source.populate(dynamic_zone) self.assertEquals(5, len(dynamic_zone.records)) @@ -58,12 +58,12 @@ class TestYamlProvider(TestCase): # We add everything plan = target.plan(zone) - self.assertEquals(16, len([c for c in plan.changes + self.assertEquals(18, len([c for c in plan.changes if isinstance(c, Create)])) self.assertFalse(isfile(yaml_file)) # Now actually do it - self.assertEquals(16, target.apply(plan)) + self.assertEquals(18, target.apply(plan)) self.assertTrue(isfile(yaml_file)) # Dynamic plan @@ -87,7 +87,7 @@ class TestYamlProvider(TestCase): # A 2nd sync should still create everything plan = target.plan(zone) - self.assertEquals(16, len([c for c in plan.changes + self.assertEquals(18, len([c for c in plan.changes if isinstance(c, Create)])) with open(yaml_file) as fh: @@ -107,6 +107,8 @@ class TestYamlProvider(TestCase): self.assertTrue('values' in data.pop('sub')) self.assertTrue('values' in data.pop('txt')) # these are stored as singular 'value' + self.assertTrue('value' in data.pop('_imap._tcp')) + self.assertTrue('value' in data.pop('_pop3._tcp')) self.assertTrue('value' in data.pop('aaaa')) self.assertTrue('value' in data.pop('cname')) self.assertTrue('value' in data.pop('dname')) diff --git a/tests/test_octodns_source_axfr.py b/tests/test_octodns_source_axfr.py index 44e04d0..f1a8109 100644 --- a/tests/test_octodns_source_axfr.py +++ b/tests/test_octodns_source_axfr.py @@ -36,7 +36,7 @@ class TestAxfrSource(TestCase): ] self.source.populate(got) - self.assertEquals(12, len(got.records)) + self.assertEquals(14, len(got.records)) with self.assertRaises(AxfrSourceZoneTransferFailed) as ctx: zone = Zone('unit.tests.', []) @@ -73,18 +73,18 @@ class TestZoneFileSource(TestCase): # Load zonefiles without a specified file extension valid = Zone('unit.tests.', []) source.populate(valid) - self.assertEquals(12, len(valid.records)) + self.assertEquals(14, len(valid.records)) def test_populate(self): # Valid zone file in directory valid = Zone('unit.tests.', []) self.source.populate(valid) - self.assertEquals(12, len(valid.records)) + self.assertEquals(14, len(valid.records)) # 2nd populate does not read file again again = Zone('unit.tests.', []) self.source.populate(again) - self.assertEquals(12, len(again.records)) + self.assertEquals(14, len(again.records)) # bust the cache del self.source._zone_records[valid.name] From cec53b2180fc02e9c47c3b2d11b62e00da9a6650 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Tue, 16 Feb 2021 07:11:14 -0800 Subject: [PATCH 93/96] Require different twines based on python version --- requirements-dev.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 146d673..85a93f5 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,4 +5,5 @@ pycodestyle==2.6.0 pyflakes==2.2.0 readme_renderer[md]==26.0 requests_mock -twine==3.2.0 +twine==1.15.0; python_version < '3.2' +twine==3.2.0; python_version >= '3.2' From f64ee57b14ba03363c69405ccdb08aca5bb880d0 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Tue, 16 Feb 2021 07:11:46 -0800 Subject: [PATCH 94/96] Actually only install twine on 3.x --- requirements-dev.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 85a93f5..522f112 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,5 +5,4 @@ pycodestyle==2.6.0 pyflakes==2.2.0 readme_renderer[md]==26.0 requests_mock -twine==1.15.0; python_version < '3.2' twine==3.2.0; python_version >= '3.2' From e516e7647b756c9c6d8d71a23ff5127e4d634d20 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Tue, 16 Feb 2021 07:39:52 -0800 Subject: [PATCH 95/96] Remove 2.7 form the version matix --- .github/workflows/main.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 337ebb0..b2f48dd 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -10,8 +10,7 @@ jobs: strategy: matrix: # Tested versions based on dates in https://devguide.python.org/devcycle/#end-of-life-branches, - # with the addition of 2.7 b/c it's still if pretty wide active use. - python-version: [2.7, 3.6, 3.7, 3.8, 3.9] + python-version: [3.6, 3.7, 3.8, 3.9] steps: - uses: actions/checkout@master - name: Setup python From 55af09d73cf512b00a03c58786a4f16971e8e915 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Thu, 18 Feb 2021 07:11:18 -0800 Subject: [PATCH 96/96] use super for supports base case, remove unused `data_for`. --- octodns/provider/dnsimple.py | 3 +-- octodns/provider/dnsmadeeasy.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/octodns/provider/dnsimple.py b/octodns/provider/dnsimple.py index 647b89c..599eacb 100644 --- a/octodns/provider/dnsimple.py +++ b/octodns/provider/dnsimple.py @@ -280,7 +280,6 @@ class DnsimpleProvider(BaseProvider): values = defaultdict(lambda: defaultdict(list)) for record in self.zone_records(zone): _type = record['type'] - data_for = getattr(self, '_data_for_{}'.format(_type), None) if _type not in self.SUPPORTS: self.log.warning( 'populate: skipping unsupported %s record', @@ -325,7 +324,7 @@ class DnsimpleProvider(BaseProvider): ) return False - return record._type in self.SUPPORTS + return super(DnsimpleProvider, self).supports(record) def _params_for_multiple(self, record): for value in record.values: diff --git a/octodns/provider/dnsmadeeasy.py b/octodns/provider/dnsmadeeasy.py index 7880280..b222b5c 100644 --- a/octodns/provider/dnsmadeeasy.py +++ b/octodns/provider/dnsmadeeasy.py @@ -306,7 +306,7 @@ class DnsMadeEasyProvider(BaseProvider): ) return False - return record._type in self.SUPPORTS + return super(DnsMadeEasyProvider, self).supports(record) def _params_for_multiple(self, record): for value in record.values: