From 45d5da23cfc0940a536dc24102994820ab67f032 Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Wed, 9 Dec 2020 19:02:36 +0800 Subject: [PATCH 01/13] Add NULL SRV record examples to unit tests --- tests/config/unit.tests.yaml | 16 ++++++++++++++++ tests/zones/unit.tests.tst | 3 +++ 2 files changed, 19 insertions(+) diff --git a/tests/config/unit.tests.yaml b/tests/config/unit.tests.yaml index 7b84ac9..03f13a1 100644 --- a/tests/config/unit.tests.yaml +++ b/tests/config/unit.tests.yaml @@ -36,6 +36,22 @@ - flags: 0 tag: issue value: ca.unit.tests +_imap._tcp: + ttl: 600 + type: SRV + values: + - port: 0 + priority: 0 + target: . + weight: 0 +_pop3._tcp: + ttl: 600 + type: SRV + values: + - port: 0 + priority: 0 + target: . + weight: 0 _srv._tcp: ttl: 600 type: SRV diff --git a/tests/zones/unit.tests.tst b/tests/zones/unit.tests.tst index 838de88..b48b749 100644 --- a/tests/zones/unit.tests.tst +++ b/tests/zones/unit.tests.tst @@ -20,6 +20,9 @@ caa 1800 IN CAA 0 iodef "mailto:admin@unit.tests" ; SRV Records _srv._tcp 600 IN SRV 10 20 30 foo-1.unit.tests. _srv._tcp 600 IN SRV 10 20 30 foo-2.unit.tests. +; NULL SRV Records +_pop3._tcp 600 IN SRV 0 0 0 . +_imap._tcp 600 IN SRV 0 0 0 . ; TXT Records txt 600 IN TXT "Bah bah black sheep" From 403be8bb838bb8431700e61df6a4f150fe8b8a07 Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Wed, 9 Dec 2020 19:13:17 +0800 Subject: [PATCH 02/13] Fix handling of NULL SRV records in Cloudflare provider --- octodns/provider/cloudflare.py | 8 ++- .../cloudflare-dns_records-page-2.json | 50 +++++++++++++++++++ tests/test_octodns_provider_cloudflare.py | 12 ++--- 3 files changed, 62 insertions(+), 8 deletions(-) diff --git a/octodns/provider/cloudflare.py b/octodns/provider/cloudflare.py index db937e5..0f5c6cf 100644 --- a/octodns/provider/cloudflare.py +++ b/octodns/provider/cloudflare.py @@ -239,11 +239,13 @@ class CloudflareProvider(BaseProvider): def _data_for_SRV(self, _type, records): values = [] for r in records: + target = ('{}.'.format(r['data']['target']) + if r['data']['target'] != "." else ".") values.append({ 'priority': r['data']['priority'], 'weight': r['data']['weight'], 'port': r['data']['port'], - 'target': '{}.'.format(r['data']['target']), + 'target': target, }) return { 'type': _type, @@ -405,6 +407,8 @@ class CloudflareProvider(BaseProvider): name = subdomain for value in record.values: + target = value.target[:-1] if value.target != "." else "." + yield { 'data': { 'service': service, @@ -413,7 +417,7 @@ class CloudflareProvider(BaseProvider): 'priority': value.priority, 'weight': value.weight, 'port': value.port, - 'target': value.target[:-1], + 'target': target, } } diff --git a/tests/fixtures/cloudflare-dns_records-page-2.json b/tests/fixtures/cloudflare-dns_records-page-2.json index b0bbaef..860b6c3 100644 --- a/tests/fixtures/cloudflare-dns_records-page-2.json +++ b/tests/fixtures/cloudflare-dns_records-page-2.json @@ -174,6 +174,56 @@ "auto_added": false } }, + { + "id": "fc12ab34cd5611334422ab3322997656", + "type": "SRV", + "name": "_imap._tcp.unit.tests", + "data": { + "service": "_imap", + "proto": "_tcp", + "name": "unit.tests", + "priority": 0, + "weight": 0, + "port": 0, + "target": "." + }, + "proxiable": true, + "proxied": false, + "ttl": 600, + "locked": false, + "zone_id": "ff12ab34cd5611334422ab3322997650", + "zone_name": "unit.tests", + "modified_on": "2017-03-11T18:01:43.940682Z", + "created_on": "2017-03-11T18:01:43.940682Z", + "meta": { + "auto_added": false + } + }, + { + "id": "fc12ab34cd5611334422ab3322997656", + "type": "SRV", + "name": "_pop3._tcp.unit.tests", + "data": { + "service": "_imap", + "proto": "_pop3", + "name": "unit.tests", + "priority": 0, + "weight": 0, + "port": 0, + "target": "." + }, + "proxiable": true, + "proxied": false, + "ttl": 600, + "locked": false, + "zone_id": "ff12ab34cd5611334422ab3322997650", + "zone_name": "unit.tests", + "modified_on": "2017-03-11T18:01:43.940682Z", + "created_on": "2017-03-11T18:01:43.940682Z", + "meta": { + "auto_added": false + } + }, { "id": "fc12ab34cd5611334422ab3322997656", "type": "SRV", diff --git a/tests/test_octodns_provider_cloudflare.py b/tests/test_octodns_provider_cloudflare.py index 735d95c..94a37f4 100644 --- a/tests/test_octodns_provider_cloudflare.py +++ b/tests/test_octodns_provider_cloudflare.py @@ -180,7 +180,7 @@ class TestCloudflareProvider(TestCase): zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals(13, len(zone.records)) + self.assertEquals(15, len(zone.records)) changes = self.expected.changes(zone, provider) @@ -189,7 +189,7 @@ class TestCloudflareProvider(TestCase): # re-populating the same zone/records comes out of cache, no calls again = Zone('unit.tests.', []) provider.populate(again) - self.assertEquals(13, len(again.records)) + self.assertEquals(15, len(again.records)) def test_apply(self): provider = CloudflareProvider('test', 'email', 'token', retry_period=0) @@ -203,12 +203,12 @@ class TestCloudflareProvider(TestCase): 'id': 42, } }, # zone create - ] + [None] * 22 # individual record creates + ] + [None] * 24 # individual record creates # non-existent zone, create everything plan = provider.plan(self.expected) - self.assertEquals(13, len(plan.changes)) - self.assertEquals(13, provider.apply(plan)) + self.assertEquals(15, len(plan.changes)) + self.assertEquals(15, provider.apply(plan)) self.assertFalse(plan.exists) provider._request.assert_has_calls([ @@ -234,7 +234,7 @@ class TestCloudflareProvider(TestCase): }), ], True) # expected number of total calls - self.assertEquals(23, provider._request.call_count) + self.assertEquals(25, provider._request.call_count) provider._request.reset_mock() From 39412924da534adde67e1c9b11d0d93fd8e67db8 Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Wed, 9 Dec 2020 21:23:14 +0800 Subject: [PATCH 03/13] Fix handling of NULL SRV records in DigitalOcean provider --- octodns/provider/digitalocean.py | 6 +++++- tests/fixtures/digitalocean-page-2.json | 22 +++++++++++++++++++ tests/test_octodns_provider_digitalocean.py | 24 ++++++++++++++++++--- 3 files changed, 48 insertions(+), 4 deletions(-) diff --git a/octodns/provider/digitalocean.py b/octodns/provider/digitalocean.py index e192543..6ccee1d 100644 --- a/octodns/provider/digitalocean.py +++ b/octodns/provider/digitalocean.py @@ -186,10 +186,14 @@ class DigitalOceanProvider(BaseProvider): def _data_for_SRV(self, _type, records): values = [] for record in records: + target = ( + '{}.'.format(record['data']) + if record['data'] != "." else "." + ) values.append({ 'port': record['port'], 'priority': record['priority'], - 'target': '{}.'.format(record['data']), + 'target': target, 'weight': record['weight'] }) return { diff --git a/tests/fixtures/digitalocean-page-2.json b/tests/fixtures/digitalocean-page-2.json index 50f17f9..1405527 100644 --- a/tests/fixtures/digitalocean-page-2.json +++ b/tests/fixtures/digitalocean-page-2.json @@ -76,6 +76,28 @@ "weight": null, "flags": null, "tag": null + }, { + "id": 11189896, + "type": "SRV", + "name": "_imap._tcp", + "data": ".", + "priority": 0, + "port": 0, + "ttl": 600, + "weight": 0, + "flags": null, + "tag": null + }, { + "id": 11189897, + "type": "SRV", + "name": "_pop3._tcp", + "data": ".", + "priority": 0, + "port": 0, + "ttl": 600, + "weight": 0, + "flags": null, + "tag": null }], "links": { "pages": { diff --git a/tests/test_octodns_provider_digitalocean.py b/tests/test_octodns_provider_digitalocean.py index 0ad8f72..83fb5c3 100644 --- a/tests/test_octodns_provider_digitalocean.py +++ b/tests/test_octodns_provider_digitalocean.py @@ -83,14 +83,14 @@ class TestDigitalOceanProvider(TestCase): zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals(12, len(zone.records)) + self.assertEquals(14, len(zone.records)) changes = self.expected.changes(zone, provider) self.assertEquals(0, len(changes)) # 2nd populate makes no network calls/all from cache again = Zone('unit.tests.', []) provider.populate(again) - self.assertEquals(12, len(again.records)) + self.assertEquals(14, len(again.records)) # bust the cache del provider._zone_records[zone.name] @@ -190,6 +190,24 @@ class TestDigitalOceanProvider(TestCase): 'flags': 0, 'name': '@', 'tag': 'issue', 'ttl': 3600, 'type': 'CAA'}), + call('POST', '/domains/unit.tests/records', data={ + 'name': '_imap._tcp', + 'weight': 0, + 'data': '.', + 'priority': 0, + 'ttl': 600, + 'type': 'SRV', + 'port': 0 + }), + call('POST', '/domains/unit.tests/records', data={ + 'name': '_pop3._tcp', + 'weight': 0, + 'data': '.', + 'priority': 0, + 'ttl': 600, + 'type': 'SRV', + 'port': 0 + }), call('POST', '/domains/unit.tests/records', data={ 'name': '_srv._tcp', 'weight': 20, @@ -200,7 +218,7 @@ class TestDigitalOceanProvider(TestCase): 'port': 30 }), ]) - self.assertEquals(24, provider._client._request.call_count) + self.assertEquals(26, provider._client._request.call_count) provider._client._request.reset_mock() From 876a5b46f34693d7161b773babc95d1a38d02103 Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Sat, 19 Dec 2020 22:32:23 +0800 Subject: [PATCH 04/13] Update PowerDNS tests and fixtures for NULL SRV records --- tests/fixtures/powerdns-full-data.json | 24 ++++++++++++++++++++++++ tests/test_octodns_provider_powerdns.py | 6 +++--- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/tests/fixtures/powerdns-full-data.json b/tests/fixtures/powerdns-full-data.json index 3d445d4..a08f028 100644 --- a/tests/fixtures/powerdns-full-data.json +++ b/tests/fixtures/powerdns-full-data.json @@ -59,6 +59,30 @@ "ttl": 300, "type": "A" }, + { + "comments": [], + "name": "_imap._tcp.unit.tests.", + "records": [ + { + "content": "0 0 0 .", + "disabled": false + } + ], + "ttl": 600, + "type": "SRV" + }, + { + "comments": [], + "name": "_pop3._tcp.unit.tests.", + "records": [ + { + "content": "0 0 0 .", + "disabled": false + } + ], + "ttl": 600, + "type": "SRV" + }, { "comments": [], "name": "_srv._tcp.unit.tests.", diff --git a/tests/test_octodns_provider_powerdns.py b/tests/test_octodns_provider_powerdns.py index 33b5e44..7c418ff 100644 --- a/tests/test_octodns_provider_powerdns.py +++ b/tests/test_octodns_provider_powerdns.py @@ -186,7 +186,7 @@ class TestPowerDnsProvider(TestCase): source = YamlProvider('test', join(dirname(__file__), 'config')) source.populate(expected) expected_n = len(expected.records) - 3 - self.assertEquals(16, expected_n) + self.assertEquals(18, expected_n) # No diffs == no changes with requests_mock() as mock: @@ -194,7 +194,7 @@ class TestPowerDnsProvider(TestCase): zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals(16, len(zone.records)) + self.assertEquals(18, len(zone.records)) changes = expected.changes(zone, provider) self.assertEquals(0, len(changes)) @@ -291,7 +291,7 @@ class TestPowerDnsProvider(TestCase): expected = Zone('unit.tests.', []) source = YamlProvider('test', join(dirname(__file__), 'config')) source.populate(expected) - self.assertEquals(19, len(expected.records)) + self.assertEquals(21, len(expected.records)) # A small change to a single record with requests_mock() as mock: From 4105fb7ee798c83bf894ef39150335460536e888 Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Tue, 29 Dec 2020 17:06:50 +0800 Subject: [PATCH 05/13] Update Gandi tests and fixtures for NULL SRV records Thanks to @yzguy for assisting with API tests to confirm support --- tests/fixtures/gandi-no-changes.json | 18 ++++++++++++++++++ tests/test_octodns_provider_gandi.py | 20 ++++++++++++++++++-- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/tests/fixtures/gandi-no-changes.json b/tests/fixtures/gandi-no-changes.json index b018785..a67dc93 100644 --- a/tests/fixtures/gandi-no-changes.json +++ b/tests/fixtures/gandi-no-changes.json @@ -123,6 +123,24 @@ "2.2.3.6" ] }, + { + "rrset_type": "SRV", + "rrset_ttl": 600, + "rrset_name": "_imap._tcp", + "rrset_href": "https://api.gandi.net/v5/livedns/domains/unit.tests/records/_imap._tcp/SRV", + "rrset_values": [ + "0 0 0 ." + ] + }, + { + "rrset_type": "SRV", + "rrset_ttl": 600, + "rrset_name": "_pop3._tcp", + "rrset_href": "https://api.gandi.net/v5/livedns/domains/unit.tests/records/_pop3._tcp/SRV", + "rrset_values": [ + "0 0 0 ." + ] + }, { "rrset_type": "SRV", "rrset_ttl": 600, diff --git a/tests/test_octodns_provider_gandi.py b/tests/test_octodns_provider_gandi.py index 7e1c866..1b0443b 100644 --- a/tests/test_octodns_provider_gandi.py +++ b/tests/test_octodns_provider_gandi.py @@ -117,7 +117,7 @@ class TestGandiProvider(TestCase): zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals(14, len(zone.records)) + self.assertEquals(16, len(zone.records)) changes = self.expected.changes(zone, provider) self.assertEquals(0, len(changes)) @@ -284,6 +284,22 @@ class TestGandiProvider(TestCase): '12 20 30 foo-2.unit.tests.' ] }), + call('POST', '/livedns/domains/unit.tests/records', data={ + 'rrset_name': '_pop3._tcp', + 'rrset_ttl': 600, + 'rrset_type': 'SRV', + 'rrset_values': [ + '0 0 0 .', + ] + }), + call('POST', '/livedns/domains/unit.tests/records', data={ + 'rrset_name': '_imap._tcp', + 'rrset_ttl': 600, + 'rrset_type': 'SRV', + 'rrset_values': [ + '0 0 0 .', + ] + }), call('POST', '/livedns/domains/unit.tests/records', data={ 'rrset_name': '@', 'rrset_ttl': 3600, @@ -307,7 +323,7 @@ class TestGandiProvider(TestCase): }) ]) # expected number of total calls - self.assertEquals(17, provider._client._request.call_count) + self.assertEquals(19, provider._client._request.call_count) provider._client._request.reset_mock() From ae65311a96e77efea6719027f31b0a435ef7b9d8 Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Tue, 29 Dec 2020 17:11:22 +0800 Subject: [PATCH 06/13] Update Constellix tests and fixtures for NULL SRV records Thanks to @yzguy for assisting with API tests to confirm support --- tests/fixtures/constellix-records.json | 56 +++++++++++++++++++++++ tests/test_octodns_provider_constellix.py | 6 +-- 2 files changed, 59 insertions(+), 3 deletions(-) diff --git a/tests/fixtures/constellix-records.json b/tests/fixtures/constellix-records.json index 689fd53..282ca62 100644 --- a/tests/fixtures/constellix-records.json +++ b/tests/fixtures/constellix-records.json @@ -64,6 +64,62 @@ "roundRobinFailover": [], "pools": [], "poolsDetail": [] +}, { + "id": 1898527, + "type": "SRV", + "recordType": "srv", + "name": "_imap._tcp", + "recordOption": "roundRobin", + "noAnswer": false, + "note": "", + "ttl": 600, + "gtdRegion": 1, + "parentId": 123123, + "parent": "domain", + "source": "Domain", + "modifiedTs": 1565149714387, + "value": [{ + "value": ".", + "priority": 0, + "weight": 0, + "port": 0, + "disableFlag": false + }], + "roundRobin": [{ + "value": ".", + "priority": 0, + "weight": 0, + "port": 0, + "disableFlag": false + }] +}, { + "id": 1898528, + "type": "SRV", + "recordType": "srv", + "name": "_pop3._tcp", + "recordOption": "roundRobin", + "noAnswer": false, + "note": "", + "ttl": 600, + "gtdRegion": 1, + "parentId": 123123, + "parent": "domain", + "source": "Domain", + "modifiedTs": 1565149714387, + "value": [{ + "value": ".", + "priority": 0, + "weight": 0, + "port": 0, + "disableFlag": false + }], + "roundRobin": [{ + "value": ".", + "priority": 0, + "weight": 0, + "port": 0, + "disableFlag": false + }] }, { "id": 1808527, "type": "SRV", diff --git a/tests/test_octodns_provider_constellix.py b/tests/test_octodns_provider_constellix.py index bc17b50..8ba4860 100644 --- a/tests/test_octodns_provider_constellix.py +++ b/tests/test_octodns_provider_constellix.py @@ -101,14 +101,14 @@ class TestConstellixProvider(TestCase): zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals(14, len(zone.records)) + self.assertEquals(16, len(zone.records)) changes = self.expected.changes(zone, provider) self.assertEquals(0, len(changes)) # 2nd populate makes no network calls/all from cache again = Zone('unit.tests.', []) provider.populate(again) - self.assertEquals(14, len(again.records)) + self.assertEquals(16, len(again.records)) # bust the cache del provider._zone_records[zone.name] @@ -163,7 +163,7 @@ class TestConstellixProvider(TestCase): }), ]) - self.assertEquals(17, provider._client._request.call_count) + self.assertEquals(19, provider._client._request.call_count) provider._client._request.reset_mock() From 909c7ad7e88d54f4adeadbf9d3ad3d189de06360 Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Fri, 8 Jan 2021 16:51:36 +0800 Subject: [PATCH 07/13] Update EasyDNS tests and fixtures for NULL SRV records Thanks to @actazen for assisting with API tests to confirm support --- tests/fixtures/easydns-records.json | 26 ++++++++++++++++++++++++-- tests/test_octodns_provider_easydns.py | 6 +++--- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/tests/fixtures/easydns-records.json b/tests/fixtures/easydns-records.json index c3718b5..73ea953 100644 --- a/tests/fixtures/easydns-records.json +++ b/tests/fixtures/easydns-records.json @@ -264,10 +264,32 @@ "rdata": "v=DKIM1;k=rsa;s=email;h=sha256;p=A\/kinda+of\/long\/string+with+numb3rs", "geozone_id": "0", "last_mod": "2020-01-01 01:01:01" + }, + { + "id": "12340025", + "domain": "unit.tests", + "host": "_imap._tcp", + "ttl": "600", + "prio": "0", + "type": "SRV", + "rdata": "0 0 0 .", + "geozone_id": "0", + "last_mod": "2020-01-01 01:01:01" + }, + { + "id": "12340026", + "domain": "unit.tests", + "host": "_pop3._tcp", + "ttl": "600", + "prio": "0", + "type": "SRV", + "rdata": "0 0 0 .", + "geozone_id": "0", + "last_mod": "2020-01-01 01:01:01" } ], - "count": 24, - "total": 24, + "count": 26, + "total": 26, "start": 0, "max": 1000, "status": 200 diff --git a/tests/test_octodns_provider_easydns.py b/tests/test_octodns_provider_easydns.py index 8df0e22..2b137a6 100644 --- a/tests/test_octodns_provider_easydns.py +++ b/tests/test_octodns_provider_easydns.py @@ -80,14 +80,14 @@ class TestEasyDNSProvider(TestCase): text=fh.read()) provider.populate(zone) - self.assertEquals(13, len(zone.records)) + self.assertEquals(15, len(zone.records)) changes = self.expected.changes(zone, provider) self.assertEquals(0, len(changes)) # 2nd populate makes no network calls/all from cache again = Zone('unit.tests.', []) provider.populate(again) - self.assertEquals(13, len(again.records)) + self.assertEquals(15, len(again.records)) # bust the cache del provider._zone_records[zone.name] @@ -379,7 +379,7 @@ class TestEasyDNSProvider(TestCase): self.assertEquals(n, provider.apply(plan)) self.assertFalse(plan.exists) - self.assertEquals(23, provider._client._request.call_count) + self.assertEquals(25, provider._client._request.call_count) provider._client._request.reset_mock() From fb197b890e78d6a652fa276bb4148afd9174245b Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Mon, 11 Jan 2021 22:07:58 +0800 Subject: [PATCH 08/13] Update Mythic Beasts tests and fixtures for NULL SRV records Thanks to @pwaring for assisting with API tests to confirm support, and reaching out to Mythic Beasts to obtain a fix --- tests/fixtures/mythicbeasts-list.txt | 2 ++ tests/test_octodns_provider_mythicbeasts.py | 8 ++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/fixtures/mythicbeasts-list.txt b/tests/fixtures/mythicbeasts-list.txt index ed4ea4c..006a8ff 100644 --- a/tests/fixtures/mythicbeasts-list.txt +++ b/tests/fixtures/mythicbeasts-list.txt @@ -5,6 +5,8 @@ @ 3600 SSHFP 1 1 bf6b6825d2977c511a475bbefb88aad54a92ac73 @ 3600 SSHFP 1 1 7491973e5f8b39d5327cd4e08bc81b05f7710b49 @ 3600 CAA 0 issue ca.unit.tests +_imap._tcp 600 SRV 0 0 0 . +_pop3._tcp 600 SRV 0 0 0 . _srv._tcp 600 SRV 10 20 30 foo-1.unit.tests. _srv._tcp 600 SRV 12 20 30 foo-2.unit.tests. aaaa 600 AAAA 2601:644:500:e210:62f8:1dff:feb8:947a diff --git a/tests/test_octodns_provider_mythicbeasts.py b/tests/test_octodns_provider_mythicbeasts.py index f78cb0b..26af8c1 100644 --- a/tests/test_octodns_provider_mythicbeasts.py +++ b/tests/test_octodns_provider_mythicbeasts.py @@ -378,8 +378,8 @@ class TestMythicBeastsProvider(TestCase): zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals(15, len(zone.records)) - self.assertEquals(15, len(self.expected.records)) + self.assertEquals(17, len(zone.records)) + self.assertEquals(17, len(self.expected.records)) changes = self.expected.changes(zone, provider) self.assertEquals(0, len(changes)) @@ -445,7 +445,7 @@ class TestMythicBeastsProvider(TestCase): if isinstance(c, Update)])) self.assertEquals(1, len([c for c in plan.changes if isinstance(c, Delete)])) - self.assertEquals(14, len([c for c in plan.changes + self.assertEquals(16, len([c for c in plan.changes if isinstance(c, Create)])) - self.assertEquals(16, provider.apply(plan)) + self.assertEquals(18, provider.apply(plan)) self.assertTrue(plan.exists) From e0d79f826f752dc792ad03836e27daebe2c64691 Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Fri, 15 Jan 2021 18:32:11 +0800 Subject: [PATCH 09/13] Update Edge DNS tests and fixtures for NULL SRV records Thanks to @jdgri for assisting with API tests to confirm support --- tests/fixtures/edgedns-records.json | 20 ++++++++++++++++++-- tests/test_octodns_provider_edgedns.py | 10 +++++----- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/tests/fixtures/edgedns-records.json b/tests/fixtures/edgedns-records.json index 4693eb1..a5ce14f 100644 --- a/tests/fixtures/edgedns-records.json +++ b/tests/fixtures/edgedns-records.json @@ -9,6 +9,22 @@ "name": "_srv._tcp.unit.tests", "ttl": 600 }, + { + "rdata": [ + "0 0 0 ." + ], + "type": "SRV", + "name": "_imap._tcp.unit.tests", + "ttl": 600 + }, + { + "rdata": [ + "0 0 0 ." + ], + "type": "SRV", + "name": "_pop3._tcp.unit.tests", + "ttl": 600 + }, { "rdata": [ "2601:644:500:e210:62f8:1dff:feb8:947a" @@ -151,7 +167,7 @@ } ], "metadata": { - "totalElements": 16, + "totalElements": 18, "showAll": true } -} \ No newline at end of file +} diff --git a/tests/test_octodns_provider_edgedns.py b/tests/test_octodns_provider_edgedns.py index 20a9a07..694c762 100644 --- a/tests/test_octodns_provider_edgedns.py +++ b/tests/test_octodns_provider_edgedns.py @@ -77,14 +77,14 @@ class TestEdgeDnsProvider(TestCase): zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals(16, len(zone.records)) + self.assertEquals(18, len(zone.records)) changes = self.expected.changes(zone, provider) self.assertEquals(0, len(changes)) # 2nd populate makes no network calls/all from cache again = Zone('unit.tests.', []) provider.populate(again) - self.assertEquals(16, len(again.records)) + self.assertEquals(18, len(again.records)) # bust the cache del provider._zone_records[zone.name] @@ -105,7 +105,7 @@ class TestEdgeDnsProvider(TestCase): mock.delete(ANY, status_code=204) changes = provider.apply(plan) - self.assertEquals(29, changes) + self.assertEquals(31, changes) # Test against a zone that doesn't exist yet with requests_mock() as mock: @@ -118,7 +118,7 @@ class TestEdgeDnsProvider(TestCase): mock.delete(ANY, status_code=204) changes = provider.apply(plan) - self.assertEquals(14, changes) + self.assertEquals(16, changes) # Test against a zone that doesn't exist yet, but gid not provided with requests_mock() as mock: @@ -132,7 +132,7 @@ class TestEdgeDnsProvider(TestCase): mock.delete(ANY, status_code=204) changes = provider.apply(plan) - self.assertEquals(14, changes) + self.assertEquals(16, changes) # Test against a zone that doesn't exist, but cid not provided From 2cd5511dc68ccd0fad97dbc665e9cc20f20813d0 Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Sat, 30 Jan 2021 00:08:24 +0800 Subject: [PATCH 10/13] Warn that NULL SRV records are unsupported in DNSimple provider DNSimple does not handle NULL SRV records correctly (either via their web interface or API). Flag to end user if attempted. Issue noted with DNSimple support 2020-12-09 --- octodns/provider/dnsimple.py | 41 +++++++++++++++++++++++-- tests/test_octodns_provider_dnsimple.py | 2 +- 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/octodns/provider/dnsimple.py b/octodns/provider/dnsimple.py index f83098e..647b89c 100644 --- a/octodns/provider/dnsimple.py +++ b/octodns/provider/dnsimple.py @@ -218,12 +218,23 @@ class DnsimpleProvider(BaseProvider): try: weight, port, target = record['content'].split(' ', 2) except ValueError: - # see _data_for_NAPTR's continue + # their api/website will let you create invalid records, this + # essentially handles that by ignoring them for values + # purposes. That will cause updates to happen to delete them if + # they shouldn't exist or update them if they're wrong + self.log.warning( + '_data_for_SRV: unsupported %s record (%s)', + _type, + record['content'] + ) continue + + target = '{}.'.format(target) if target != "." else "." + values.append({ 'port': port, 'priority': record['priority'], - 'target': '{}.'.format(target), + 'target': target, 'weight': weight }) return { @@ -269,7 +280,12 @@ class DnsimpleProvider(BaseProvider): values = defaultdict(lambda: defaultdict(list)) for record in self.zone_records(zone): _type = record['type'] + data_for = getattr(self, '_data_for_{}'.format(_type), None) if _type not in self.SUPPORTS: + self.log.warning( + 'populate: skipping unsupported %s record', + _type + ) continue elif _type == 'TXT' and record['content'].startswith('ALIAS for'): # ALIAS has a "ride along" TXT record with 'ALIAS for XXXX', @@ -290,6 +306,27 @@ class DnsimpleProvider(BaseProvider): len(zone.records) - before, exists) return exists + def supports(self, record): + # DNSimple does not support empty/NULL SRV records + # + # Fails silently and leaves a corrupt record + # + # Skip the record and continue + if record._type == "SRV": + if 'value' in record.data: + targets = (record.data['value']['target'],) + else: + targets = [value['target'] for value in record.data['values']] + + if "." in targets: + self.log.warning( + 'supports: unsupported %s record with target (%s)', + record._type, targets + ) + return False + + return record._type in self.SUPPORTS + def _params_for_multiple(self, record): for value in record.values: yield { diff --git a/tests/test_octodns_provider_dnsimple.py b/tests/test_octodns_provider_dnsimple.py index 92f32b1..9f1dab3 100644 --- a/tests/test_octodns_provider_dnsimple.py +++ b/tests/test_octodns_provider_dnsimple.py @@ -137,7 +137,7 @@ class TestDnsimpleProvider(TestCase): plan = provider.plan(self.expected) # No root NS, no ignored, no excluded - n = len(self.expected.records) - 4 + n = len(self.expected.records) - 6 self.assertEquals(n, len(plan.changes)) self.assertEquals(n, provider.apply(plan)) self.assertFalse(plan.exists) From 84a0c089d4c7c28c20fa382d15113bac87f4caa6 Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Fri, 25 Dec 2020 19:21:41 +0800 Subject: [PATCH 11/13] Warn that NULL SRV records are unsupported in DNS Made Easy provider --- octodns/provider/dnsmadeeasy.py | 24 ++++++++++++++++++++++ tests/test_octodns_provider_dnsmadeeasy.py | 2 +- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/octodns/provider/dnsmadeeasy.py b/octodns/provider/dnsmadeeasy.py index 0bf05a0..7880280 100644 --- a/octodns/provider/dnsmadeeasy.py +++ b/octodns/provider/dnsmadeeasy.py @@ -284,6 +284,30 @@ class DnsMadeEasyProvider(BaseProvider): len(zone.records) - before, exists) return exists + def supports(self, record): + # DNS Made Easy does not support empty/NULL SRV records + # + # Attempting to sync such a record would generate the following error + # + # octodns.provider.dnsmadeeasy.DnsMadeEasyClientBadRequest: + # - Record value may not be a standalone dot. + # + # Skip the record and continue + if record._type == "SRV": + if 'value' in record.data: + targets = (record.data['value']['target'],) + else: + targets = [value['target'] for value in record.data['values']] + + if "." in targets: + self.log.warning( + 'supports: unsupported %s record with target (%s)', + record._type, targets + ) + return False + + return record._type in self.SUPPORTS + def _params_for_multiple(self, record): for value in record.values: yield { diff --git a/tests/test_octodns_provider_dnsmadeeasy.py b/tests/test_octodns_provider_dnsmadeeasy.py index 0ad059d..92aa547 100644 --- a/tests/test_octodns_provider_dnsmadeeasy.py +++ b/tests/test_octodns_provider_dnsmadeeasy.py @@ -134,7 +134,7 @@ class TestDnsMadeEasyProvider(TestCase): plan = provider.plan(self.expected) # No root NS, no ignored, no excluded, no unsupported - n = len(self.expected.records) - 6 + n = len(self.expected.records) - 8 self.assertEquals(n, len(plan.changes)) self.assertEquals(n, provider.apply(plan)) From 5d23977bbd26d4b12a96f7187ffc27fda8057bdd Mon Sep 17 00:00:00 2001 From: Mark Tearle Date: Sat, 30 Jan 2021 15:44:32 +0800 Subject: [PATCH 12/13] Adjust remaining unit tests due to extra records in test zone --- tests/test_octodns_manager.py | 14 +++++++------- tests/test_octodns_provider_transip.py | 4 ++-- tests/test_octodns_provider_ultra.py | 8 ++++---- tests/test_octodns_provider_yaml.py | 10 ++++++---- tests/test_octodns_source_axfr.py | 8 ++++---- 5 files changed, 23 insertions(+), 21 deletions(-) diff --git a/tests/test_octodns_manager.py b/tests/test_octodns_manager.py index f757466..1657f04 100644 --- a/tests/test_octodns_manager.py +++ b/tests/test_octodns_manager.py @@ -118,12 +118,12 @@ class TestManager(TestCase): environ['YAML_TMP_DIR'] = tmpdir.dirname tc = Manager(get_config_filename('simple.yaml')) \ .sync(dry_run=False) - self.assertEquals(22, tc) + self.assertEquals(24, tc) # try with just one of the zones tc = Manager(get_config_filename('simple.yaml')) \ .sync(dry_run=False, eligible_zones=['unit.tests.']) - self.assertEquals(16, tc) + self.assertEquals(18, tc) # the subzone, with 2 targets tc = Manager(get_config_filename('simple.yaml')) \ @@ -138,18 +138,18 @@ class TestManager(TestCase): # Again with force tc = Manager(get_config_filename('simple.yaml')) \ .sync(dry_run=False, force=True) - self.assertEquals(22, tc) + self.assertEquals(24, tc) # Again with max_workers = 1 tc = Manager(get_config_filename('simple.yaml'), max_workers=1) \ .sync(dry_run=False, force=True) - self.assertEquals(22, tc) + self.assertEquals(24, tc) # Include meta tc = Manager(get_config_filename('simple.yaml'), max_workers=1, include_meta=True) \ .sync(dry_run=False, force=True) - self.assertEquals(26, tc) + self.assertEquals(28, tc) def test_eligible_sources(self): with TemporaryDirectory() as tmpdir: @@ -215,13 +215,13 @@ class TestManager(TestCase): fh.write('---\n{}') changes = manager.compare(['in'], ['dump'], 'unit.tests.') - self.assertEquals(16, len(changes)) + self.assertEquals(18, len(changes)) # Compound sources with varying support changes = manager.compare(['in', 'nosshfp'], ['dump'], 'unit.tests.') - self.assertEquals(15, len(changes)) + self.assertEquals(17, len(changes)) with self.assertRaises(ManagerException) as ctx: manager.compare(['nope'], ['dump'], 'unit.tests.') diff --git a/tests/test_octodns_provider_transip.py b/tests/test_octodns_provider_transip.py index f792085..84cfebc 100644 --- a/tests/test_octodns_provider_transip.py +++ b/tests/test_octodns_provider_transip.py @@ -222,7 +222,7 @@ N4OiVz1I3rbZGYa396lpxO6ku8yCglisL1yrSP6DdEUp66ntpKVd provider._client = MockDomainService('unittest', self.bogus_key) plan = provider.plan(_expected) - self.assertEqual(12, plan.change_counts['Create']) + self.assertEqual(14, plan.change_counts['Create']) self.assertEqual(0, plan.change_counts['Update']) self.assertEqual(0, plan.change_counts['Delete']) @@ -235,7 +235,7 @@ N4OiVz1I3rbZGYa396lpxO6ku8yCglisL1yrSP6DdEUp66ntpKVd provider = TransipProvider('test', 'unittest', self.bogus_key) provider._client = MockDomainService('unittest', self.bogus_key) plan = provider.plan(_expected) - self.assertEqual(12, len(plan.changes)) + self.assertEqual(14, len(plan.changes)) changes = provider.apply(plan) self.assertEqual(changes, len(plan.changes)) diff --git a/tests/test_octodns_provider_ultra.py b/tests/test_octodns_provider_ultra.py index 43eac3c..b6d1017 100644 --- a/tests/test_octodns_provider_ultra.py +++ b/tests/test_octodns_provider_ultra.py @@ -285,12 +285,12 @@ class TestUltraProvider(TestCase): provider._request.side_effect = [ UltraNoZonesExistException('No Zones'), None, # zone create - ] + [None] * 13 # individual record creates + ] + [None] * 15 # individual record creates # non-existent zone, create everything plan = provider.plan(self.expected) - self.assertEquals(13, len(plan.changes)) - self.assertEquals(13, provider.apply(plan)) + self.assertEquals(15, len(plan.changes)) + self.assertEquals(15, provider.apply(plan)) self.assertFalse(plan.exists) provider._request.assert_has_calls([ @@ -320,7 +320,7 @@ class TestUltraProvider(TestCase): 'p=A/kinda+of/long/string+with+numb3rs']}), ], True) # expected number of total calls - self.assertEquals(15, provider._request.call_count) + self.assertEquals(17, provider._request.call_count) # Create sample rrset payload to attempt to alter page1 = json_load(open('tests/fixtures/ultra-records-page-1.json')) diff --git a/tests/test_octodns_provider_yaml.py b/tests/test_octodns_provider_yaml.py index f255238..d527fb3 100644 --- a/tests/test_octodns_provider_yaml.py +++ b/tests/test_octodns_provider_yaml.py @@ -35,7 +35,7 @@ class TestYamlProvider(TestCase): # without it we see everything source.populate(zone) - self.assertEquals(19, len(zone.records)) + self.assertEquals(21, len(zone.records)) source.populate(dynamic_zone) self.assertEquals(5, len(dynamic_zone.records)) @@ -58,12 +58,12 @@ class TestYamlProvider(TestCase): # We add everything plan = target.plan(zone) - self.assertEquals(16, len([c for c in plan.changes + self.assertEquals(18, len([c for c in plan.changes if isinstance(c, Create)])) self.assertFalse(isfile(yaml_file)) # Now actually do it - self.assertEquals(16, target.apply(plan)) + self.assertEquals(18, target.apply(plan)) self.assertTrue(isfile(yaml_file)) # Dynamic plan @@ -87,7 +87,7 @@ class TestYamlProvider(TestCase): # A 2nd sync should still create everything plan = target.plan(zone) - self.assertEquals(16, len([c for c in plan.changes + self.assertEquals(18, len([c for c in plan.changes if isinstance(c, Create)])) with open(yaml_file) as fh: @@ -107,6 +107,8 @@ class TestYamlProvider(TestCase): self.assertTrue('values' in data.pop('sub')) self.assertTrue('values' in data.pop('txt')) # these are stored as singular 'value' + self.assertTrue('value' in data.pop('_imap._tcp')) + self.assertTrue('value' in data.pop('_pop3._tcp')) self.assertTrue('value' in data.pop('aaaa')) self.assertTrue('value' in data.pop('cname')) self.assertTrue('value' in data.pop('dname')) diff --git a/tests/test_octodns_source_axfr.py b/tests/test_octodns_source_axfr.py index 44e04d0..f1a8109 100644 --- a/tests/test_octodns_source_axfr.py +++ b/tests/test_octodns_source_axfr.py @@ -36,7 +36,7 @@ class TestAxfrSource(TestCase): ] self.source.populate(got) - self.assertEquals(12, len(got.records)) + self.assertEquals(14, len(got.records)) with self.assertRaises(AxfrSourceZoneTransferFailed) as ctx: zone = Zone('unit.tests.', []) @@ -73,18 +73,18 @@ class TestZoneFileSource(TestCase): # Load zonefiles without a specified file extension valid = Zone('unit.tests.', []) source.populate(valid) - self.assertEquals(12, len(valid.records)) + self.assertEquals(14, len(valid.records)) def test_populate(self): # Valid zone file in directory valid = Zone('unit.tests.', []) self.source.populate(valid) - self.assertEquals(12, len(valid.records)) + self.assertEquals(14, len(valid.records)) # 2nd populate does not read file again again = Zone('unit.tests.', []) self.source.populate(again) - self.assertEquals(12, len(again.records)) + self.assertEquals(14, len(again.records)) # bust the cache del self.source._zone_records[valid.name] From 55af09d73cf512b00a03c58786a4f16971e8e915 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Thu, 18 Feb 2021 07:11:18 -0800 Subject: [PATCH 13/13] use super for supports base case, remove unused `data_for`. --- octodns/provider/dnsimple.py | 3 +-- octodns/provider/dnsmadeeasy.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/octodns/provider/dnsimple.py b/octodns/provider/dnsimple.py index 647b89c..599eacb 100644 --- a/octodns/provider/dnsimple.py +++ b/octodns/provider/dnsimple.py @@ -280,7 +280,6 @@ class DnsimpleProvider(BaseProvider): values = defaultdict(lambda: defaultdict(list)) for record in self.zone_records(zone): _type = record['type'] - data_for = getattr(self, '_data_for_{}'.format(_type), None) if _type not in self.SUPPORTS: self.log.warning( 'populate: skipping unsupported %s record', @@ -325,7 +324,7 @@ class DnsimpleProvider(BaseProvider): ) return False - return record._type in self.SUPPORTS + return super(DnsimpleProvider, self).supports(record) def _params_for_multiple(self, record): for value in record.values: diff --git a/octodns/provider/dnsmadeeasy.py b/octodns/provider/dnsmadeeasy.py index 7880280..b222b5c 100644 --- a/octodns/provider/dnsmadeeasy.py +++ b/octodns/provider/dnsmadeeasy.py @@ -306,7 +306,7 @@ class DnsMadeEasyProvider(BaseProvider): ) return False - return record._type in self.SUPPORTS + return super(DnsMadeEasyProvider, self).supports(record) def _params_for_multiple(self, record): for value in record.values: