From 679c2be0e076f6d7f0ec8daf89385ebd7fad6564 Mon Sep 17 00:00:00 2001 From: Vietor Davis Date: Mon, 26 Jun 2017 17:03:15 -0700 Subject: [PATCH 001/141] Start sketchin of Rackspace provider, half rewritten from powerdns... --- octodns/provider/rackspace.py | 386 ++++++++++++++++++ tests/fixtures/rackspace-auth-response.json | 87 ++++ .../rackspace-list-domains-response.json | 68 +++ .../rackspace-sample-recordset-page1.json | 33 ++ .../rackspace-sample-recordset-page2.json | 35 ++ tests/test_octodns_source_rackspace.py | 294 +++++++++++++ 6 files changed, 903 insertions(+) create mode 100644 octodns/provider/rackspace.py create mode 100644 tests/fixtures/rackspace-auth-response.json create mode 100644 tests/fixtures/rackspace-list-domains-response.json create mode 100644 tests/fixtures/rackspace-sample-recordset-page1.json create mode 100644 tests/fixtures/rackspace-sample-recordset-page2.json create mode 100644 tests/test_octodns_source_rackspace.py diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py new file mode 100644 index 0000000..311d02d --- /dev/null +++ b/octodns/provider/rackspace.py @@ -0,0 +1,386 @@ +# +# +# + + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from requests import HTTPError, Session, post +import json +import logging + +from ..record import Create, Record +from .base import BaseProvider + + +class RackspaceProvider(BaseProvider): + SUPPORTS_GEO = False + TIMEOUT = 5 + + def __init__(self, username, api_key, *args, **kwargs): + ''' + Rackspace API v1 Provider + + rackspace: + class: octodns.provider.rackspace.RackspaceProvider + # The the username to authenticate with (required) + username: username + # The api key that grants access for that user (required) + api_key: api-key + ''' + self.log = logging.getLogger('RackspaceProvider[{}]'.format(username)) + super(RackspaceProvider, self).__init__(id, *args, **kwargs) + + auth_token, dns_endpoint = self._get_auth_token(username, api_key) + self.dns_endpoint = dns_endpoint + + sess = Session() + sess.headers.update({'X-Auth-Token': auth_token}) + self._sess = sess + + def _get_auth_token(self, username, api_key): + ret = post('https://identity.api.rackspacecloud.com/v2.0/tokens', + json={"auth": {"RAX-KSKEY:apiKeyCredentials": {"username": username, "apiKey": api_key}}}, + ) + cloud_dns_endpoint = [x for x in ret.json()['access']['serviceCatalog'] if x['name'] == 'cloudDNS'][0]['endpoints'][0]['publicURL'] + return ret.json()['access']['token']['id'], cloud_dns_endpoint + + def _get_zone_id_for(self, zone_name): + ret = self._request('GET', 'domains', pagination_key='domains') + if ret and 'name' in ret: + return [x for x in ret if x['name'] == zone_name][0]['id'] + else: + return None + + def _request(self, method, path, data=None, pagination_key=None): + self.log.debug('_request: method=%s, path=%s', method, path) + url = '{}/{}'.format(self.dns_endpoint, path) + + if pagination_key: + return self._paginated_request_for_url(method, url, data, pagination_key) + else: + return self._request_for_url(method, url, data) + + def _request_for_url(self, method, url, data): + resp = self._sess.request(method, url, json=data, timeout=self.TIMEOUT) + self.log.debug('_request: status=%d', resp.status_code) + resp.raise_for_status() + return resp + + def _paginated_request_for_url(self, method, url, data, pagination_key): + acc = [] + + resp = self._sess.request(method, url, json=data, timeout=self.TIMEOUT) + self.log.debug('_request: status=%d', resp.status_code) + resp.raise_for_status() + acc.extend(resp.json()[pagination_key]) + + next_page = [x for x in resp.json().get('links', []) if x['rel'] == 'next'] + if next_page: + url = next_page[0]['href'] + return acc.extend(self._paginated_request_for_url(method, url, data, pagination_key)) + else: + return acc + + def _get(self, path, data=None): + return self._request('GET', path, data=data) + + def _post(self, path, data=None): + return self._request('POST', path, data=data) + + def _patch(self, path, data=None): + return self._request('PATCH', path, data=data) + + def _data_for_multiple(self, rrset): + # TODO: geo not supported + return { + 'type': rrset['type'], + 'values': [r['content'] for r in rrset['records']], + 'ttl': rrset['ttl'] + } + + _data_for_A = _data_for_multiple + _data_for_AAAA = _data_for_multiple + _data_for_NS = _data_for_multiple + + def _data_for_single(self, record): + return { + 'type': record['type'], + 'value': record['data'], + 'ttl': record['ttl'] + } + + _data_for_ALIAS = _data_for_single + _data_for_CNAME = _data_for_single + _data_for_PTR = _data_for_single + + def _data_for_quoted(self, rrset): + return { + 'type': rrset['type'], + 'values': [r['content'][1:-1] for r in rrset['records']], + 'ttl': rrset['ttl'] + } + + _data_for_SPF = _data_for_quoted + _data_for_TXT = _data_for_quoted + + def _data_for_MX(self, rrset): + values = [] + for record in rrset['records']: + priority, value = record['content'].split(' ', 1) + values.append({ + 'priority': priority, + 'value': value, + }) + return { + 'type': rrset['type'], + 'values': values, + 'ttl': rrset['ttl'] + } + + def _data_for_NAPTR(self, rrset): + values = [] + for record in rrset['records']: + order, preference, flags, service, regexp, replacement = \ + record['content'].split(' ', 5) + values.append({ + 'order': order, + 'preference': preference, + 'flags': flags[1:-1], + 'service': service[1:-1], + 'regexp': regexp[1:-1], + 'replacement': replacement, + }) + return { + 'type': rrset['type'], + 'values': values, + 'ttl': rrset['ttl'] + } + + def _data_for_SSHFP(self, rrset): + values = [] + for record in rrset['records']: + algorithm, fingerprint_type, fingerprint = \ + record['content'].split(' ', 2) + values.append({ + 'algorithm': algorithm, + 'fingerprint_type': fingerprint_type, + 'fingerprint': fingerprint, + }) + return { + 'type': rrset['type'], + 'values': values, + 'ttl': rrset['ttl'] + } + + def _data_for_SRV(self, rrset): + values = [] + for record in rrset['records']: + priority, weight, port, target = \ + record['content'].split(' ', 3) + values.append({ + 'priority': priority, + 'weight': weight, + 'port': port, + 'target': target, + }) + return { + 'type': rrset['type'], + 'values': values, + 'ttl': rrset['ttl'] + } + + def populate(self, zone, target=False): + self.log.debug('populate: name=%s', zone.name) + resp = None + try: + domain_id = self._get_zone_id_for(zone.name) + resp = self._request('GET', '/domains/{}/records'.format(domain_id), pagination_key='records') + self.log.debug('populate: loaded') + except HTTPError as e: + if e.response.status_code == 401: + # Nicer error message for auth problems + raise Exception('Rackspace request unauthorized') + elif e.response.status_code == 422: + # 422 means powerdns doesn't know anything about the requsted + # domain. We'll just ignore it here and leave the zone + # untouched. + pass + else: + # just re-throw + raise + + before = len(zone.records) + + if resp: + for record in resp.json()['records']: + record_type = record['type'] + if record_type == 'SOA': + continue + data_for = getattr(self, '_data_for_{}'.format(record_type)) + record_name = zone.hostname_from_fqdn(record['name']) + record = Record.new(zone, record_name, data_for(record), + source=self) + zone.add_record(record) + + self.log.info('populate: found %s records', + len(zone.records) - before) + + def _records_for_multiple(self, record): + return [{'content': v, 'disabled': False} + for v in record.values] + + _records_for_A = _records_for_multiple + _records_for_AAAA = _records_for_multiple + _records_for_NS = _records_for_multiple + + def _records_for_single(self, record): + return [{'content': record.value, 'disabled': False}] + + _records_for_ALIAS = _records_for_single + _records_for_CNAME = _records_for_single + _records_for_PTR = _records_for_single + + def _records_for_quoted(self, record): + return [{'content': '"{}"'.format(v), 'disabled': False} + for v in record.values] + + _records_for_SPF = _records_for_quoted + _records_for_TXT = _records_for_quoted + + def _records_for_MX(self, record): + return [{ + 'content': '{} {}'.format(v.priority, v.value), + 'disabled': False + } for v in record.values] + + def _records_for_NAPTR(self, record): + return [{ + 'content': '{} {} "{}" "{}" "{}" {}'.format(v.order, v.preference, + v.flags, v.service, + v.regexp, + v.replacement), + 'disabled': False + } for v in record.values] + + def _records_for_SSHFP(self, record): + return [{ + 'content': '{} {} {}'.format(v.algorithm, v.fingerprint_type, + v.fingerprint), + 'disabled': False + } for v in record.values] + + def _records_for_SRV(self, record): + return [{ + 'content': '{} {} {} {}'.format(v.priority, v.weight, v.port, + v.target), + 'disabled': False + } for v in record.values] + + def _mod_Create(self, change): + new = change.new + records_for = getattr(self, '_records_for_{}'.format(new._type)) + return { + 'name': new.fqdn, + 'type': new._type, + 'ttl': new.ttl, + 'changetype': 'REPLACE', + 'records': records_for(new) + } + + _mod_Update = _mod_Create + + def _mod_Delete(self, change): + existing = change.existing + records_for = getattr(self, '_records_for_{}'.format(existing._type)) + return { + 'name': existing.fqdn, + 'type': existing._type, + 'ttl': existing.ttl, + 'changetype': 'DELETE', + 'records': records_for(existing) + } + + def _get_nameserver_record(self, existing): + return None + + def _extra_changes(self, existing, _): + self.log.debug('_extra_changes: zone=%s', existing.name) + + ns = self._get_nameserver_record(existing) + if not ns: + return [] + + # sorting mostly to make things deterministic for testing, but in + # theory it let us find what we're after quickier (though sorting would + # ve more exepensive.) + for record in sorted(existing.records): + if record == ns: + # We've found the top-level NS record, return any changes + change = record.changes(ns, self) + self.log.debug('_extra_changes: change=%s', change) + if change: + # We need to modify an existing record + return [change] + # No change is necessary + return [] + # No existing top-level NS + self.log.debug('_extra_changes: create') + return [Create(ns)] + + def _get_error(self, http_error): + try: + return http_error.response.json()['error'] + except Exception: + return '' + + def _apply(self, plan): + desired = plan.desired + changes = plan.changes + self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name, + len(changes)) + + mods = [] + for change in changes: + class_name = change.__class__.__name__ + mods.append(getattr(self, '_mod_{}'.format(class_name))(change)) + self.log.debug('_apply: sending change request') + + try: + self._patch('zones/{}'.format(desired.name), + data={'rrsets': mods}) + self.log.debug('_apply: patched') + except HTTPError as e: + error = self._get_error(e) + if e.response.status_code != 422 or \ + not error.startswith('Could not find domain '): + self.log.error('_apply: status=%d, text=%s', + e.response.status_code, + e.response.text) + raise + self.log.info('_apply: creating zone=%s', desired.name) + # 422 means powerdns doesn't know anything about the requsted + # domain. We'll try to create it with the correct records instead + # of update. Hopefully all the mods are creates :-) + data = { + 'name': desired.name, + 'kind': 'Master', + 'masters': [], + 'nameservers': [], + 'rrsets': mods, + 'soa_edit_api': 'INCEPTION-INCREMENT', + 'serial': 0, + } + try: + self._post('zones', data) + except HTTPError as e: + self.log.error('_apply: status=%d, text=%s', + e.response.status_code, + e.response.text) + raise + self.log.debug('_apply: created') + + self.log.debug('_apply: complete') + + diff --git a/tests/fixtures/rackspace-auth-response.json b/tests/fixtures/rackspace-auth-response.json new file mode 100644 index 0000000..cc811c7 --- /dev/null +++ b/tests/fixtures/rackspace-auth-response.json @@ -0,0 +1,87 @@ +{ + "access": { + "token": { + "id": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "expires": "2014-11-24T22:05:39.115Z", + "tenant": { + "id": "110011", + "name": "110011" + }, + "RAX-AUTH:authenticatedBy": [ + "APIKEY" + ] + }, + "serviceCatalog": [ + { + "name": "cloudDatabases", + "endpoints": [ + { + "publicURL": "https://syd.databases.api.rackspacecloud.com/v1.0/110011", + "region": "SYD", + "tenantId": "110011" + }, + { + "publicURL": "https://dfw.databases.api.rackspacecloud.com/v1.0/110011", + "region": "DFW", + "tenantId": "110011" + }, + { + "publicURL": "https://ord.databases.api.rackspacecloud.com/v1.0/110011", + "region": "ORD", + "tenantId": "110011" + }, + { + "publicURL": "https://iad.databases.api.rackspacecloud.com/v1.0/110011", + "region": "IAD", + "tenantId": "110011" + }, + { + "publicURL": "https://hkg.databases.api.rackspacecloud.com/v1.0/110011", + "region": "HKG", + "tenantId": "110011" + } + ], + "type": "rax:database" + }, + { + "name": "cloudDNS", + "endpoints": [ + { + "publicURL": "https://dns.api.rackspacecloud.com/v1.0/110011", + "tenantId": "110011" + } + ], + "type": "rax:dns" + }, + { + "name": "rackCDN", + "endpoints": [ + { + "internalURL": "https://global.cdn.api.rackspacecloud.com/v1.0/110011", + "publicURL": "https://global.cdn.api.rackspacecloud.com/v1.0/110011", + "tenantId": "110011" + } + ], + "type": "rax:cdn" + } + ], + "user": { + "id": "123456", + "roles": [ + { + "description": "A Role that allows a user access to keystone Service methods", + "id": "6", + "name": "compute:default", + "tenantId": "110011" + }, + { + "description": "User Admin Role.", + "id": "3", + "name": "identity:user-admin" + } + ], + "name": "jsmith", + "RAX-AUTH:defaultRegion": "ORD" + } + } +} \ No newline at end of file diff --git a/tests/fixtures/rackspace-list-domains-response.json b/tests/fixtures/rackspace-list-domains-response.json new file mode 100644 index 0000000..f124837 --- /dev/null +++ b/tests/fixtures/rackspace-list-domains-response.json @@ -0,0 +1,68 @@ +{ + "totalEntries" : 10, + "domains" : [ { + "name" : "example.com", + "id" : 2725233, + "comment" : "Optional domain comment...", + "updated" : "2011-06-24T01:23:15.000+0000", + "accountId" : 1234, + "emailAddress" : "sample@rackspace.com", + "created" : "2011-06-24T01:12:51.000+0000" + }, { + "name" : "sub1.example.com", + "id" : 2725257, + "comment" : "1st sample subdomain", + "updated" : "2011-06-23T03:09:34.000+0000", + "accountId" : 1234, + "emailAddress" : "sample@rackspace.com", + "created" : "2011-06-23T03:09:33.000+0000" + }, { + "name" : "sub2.example.com", + "id" : 2725258, + "comment" : "1st sample subdomain", + "updated" : "2011-06-23T03:52:55.000+0000", + "accountId" : 1234, + "emailAddress" : "sample@rackspace.com", + "created" : "2011-06-23T03:52:55.000+0000" + }, { + "name" : "north.example.com", + "id" : 2725260, + "updated" : "2011-06-23T03:53:10.000+0000", + "accountId" : 1234, + "emailAddress" : "sample@rackspace.com", + "created" : "2011-06-23T03:53:09.000+0000" + }, { + "name" : "south.example.com", + "id" : 2725261, + "comment" : "Final sample subdomain", + "updated" : "2011-06-23T03:53:14.000+0000", + "accountId" : 1234, + "emailAddress" : "sample@rackspace.com", + "created" : "2011-06-23T03:53:14.000+0000" + }, { + "name" : "region2.example.net", + "id" : 2725352, + "updated" : "2011-06-23T20:21:06.000+0000", + "accountId" : 1234, + "created" : "2011-06-23T19:24:27.000+0000" + }, { + "name" : "example.org", + "id" : 2718984, + "updated" : "2011-05-03T14:47:32.000+0000", + "accountId" : 1234, + "created" : "2011-05-03T14:47:30.000+0000" + }, { + "name" : "rackspace.example", + "id" : 2722346, + "updated" : "2011-06-21T15:54:31.000+0000", + "accountId" : 1234, + "created" : "2011-06-15T19:02:07.000+0000" + }, { + "name" : "dnsaas.example", + "id" : 2722347, + "comment" : "Sample comment", + "updated" : "2011-06-21T15:54:31.000+0000", + "accountId" : 1234, + "created" : "2011-06-15T19:02:07.000+0000" + } ] +} diff --git a/tests/fixtures/rackspace-sample-recordset-page1.json b/tests/fixtures/rackspace-sample-recordset-page1.json new file mode 100644 index 0000000..72dc7dd --- /dev/null +++ b/tests/fixtures/rackspace-sample-recordset-page1.json @@ -0,0 +1,33 @@ +{ + "totalEntries" : 6, + "records" : [ { + "name" : "ftp.example.com", + "id" : "A-6817754", + "type" : "A", + "data" : "192.0.2.8", + "updated" : "2011-05-19T13:07:08.000+0000", + "ttl" : 5771, + "created" : "2011-05-18T19:53:09.000+0000" + }, { + "name" : "example.com", + "id" : "A-6822994", + "type" : "A", + "data" : "192.0.2.17", + "updated" : "2011-06-24T01:12:52.000+0000", + "ttl" : 86400, + "created" : "2011-06-24T01:12:52.000+0000" + }, { + "name" : "example.com", + "id" : "NS-6251982", + "type" : "NS", + "data" : "ns.rackspace.com", + "updated" : "2011-06-24T01:12:51.000+0000", + "ttl" : 3600, + "created" : "2011-06-24T01:12:51.000+0000" + } ], + "links" : [ { + "content" : "", + "href" : "https://localhost/v1.0/1234/domains/domain_id/records?limit=3&offset=3", + "rel" : "next" + } ] +} diff --git a/tests/fixtures/rackspace-sample-recordset-page2.json b/tests/fixtures/rackspace-sample-recordset-page2.json new file mode 100644 index 0000000..dc3e39a --- /dev/null +++ b/tests/fixtures/rackspace-sample-recordset-page2.json @@ -0,0 +1,35 @@ +{ + "totalEntries" : 6, + "records" : [ { + "name" : "example.com", + "id" : "NS-6251983", + "type" : "NS", + "data" : "ns2.rackspace.com", + "updated" : "2011-06-24T01:12:51.000+0000", + "ttl" : 3600, + "created" : "2011-06-24T01:12:51.000+0000" + }, { + "name" : "example.com", + "priority" : 5, + "id" : "MX-3151218", + "type" : "MX", + "data" : "mail.example.com", + "updated" : "2011-06-24T01:12:53.000+0000", + "ttl" : 3600, + "created" : "2011-06-24T01:12:53.000+0000" + }, { + "name" : "www.example.com", + "id" : "CNAME-9778009", + "type" : "CNAME", + "comment" : "This is a comment on the CNAME record", + "data" : "example.com", + "updated" : "2011-06-24T01:12:54.000+0000", + "ttl" : 5400, + "created" : "2011-06-24T01:12:54.000+0000" + } ], + "links" : [ { + "content" : "", + "href" : "https://dns.api.rackspacecloud.com/v1.0/1234/domains/domain_id/records?limit=3&offset=0", + "rel" : "previous" + }] +} diff --git a/tests/test_octodns_source_rackspace.py b/tests/test_octodns_source_rackspace.py new file mode 100644 index 0000000..88ccb5f --- /dev/null +++ b/tests/test_octodns_source_rackspace.py @@ -0,0 +1,294 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +import re +from json import loads, dumps +from os.path import dirname, join +from unittest import TestCase + +from requests import HTTPError +from requests_mock import ANY, mock as requests_mock + +from octodns.provider.rackspace import RackspaceProvider +from octodns.provider.yaml import YamlProvider +from octodns.record import Record +from octodns.zone import Zone + +EMPTY_TEXT = ''' +{ + "totalEntries" : 6, + "records" : [] +} +''' + +with open('./tests/fixtures/rackspace-auth-response.json') as fh: + AUTH_RESPONSE = fh.read() + +with open('./tests/fixtures/rackspace-list-domains-response.json') as fh: + LIST_DOMAINS_RESPONSE = fh.read() + +with open('./tests/fixtures/rackspace-sample-recordset-page1.json') as fh: + RECORDS_PAGE_1 = fh.read() + +with open('./tests/fixtures/rackspace-sample-recordset-page2.json') as fh: + RECORDS_PAGE_2 = fh.read() + +def load_provider(): + with requests_mock() as mock: + mock.post(ANY, status_code=200, text=AUTH_RESPONSE) + return RackspaceProvider('test', 'api-key') + + +class TestRackspaceSource(TestCase): + + def test_provider(self): + provider = load_provider() + + # Bad auth + with requests_mock() as mock: + mock.get(ANY, status_code=401, text='Unauthorized') + + with self.assertRaises(Exception) as ctx: + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertTrue('unauthorized' in ctx.exception.message) + + # General error + with requests_mock() as mock: + mock.get(ANY, status_code=502, text='Things caught fire') + + with self.assertRaises(HTTPError) as ctx: + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals(502, ctx.exception.response.status_code) + + # Non-existant zone doesn't populate anything + with requests_mock() as mock: + mock.get(ANY, status_code=422, + json={'error': "Could not find domain 'unit.tests.'"}) + + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals(set(), zone.records) + + # The rest of this is messy/complicated b/c it's dealing with mocking + + expected = Zone('unit.tests.', []) + source = YamlProvider('test', join(dirname(__file__), 'config')) + source.populate(expected) + expected_n = len(expected.records) - 1 + self.assertEquals(14, expected_n) + + # No diffs == no changes + with requests_mock() as mock: + mock.get(re.compile('domains$'), status_code=200, text=LIST_DOMAINS_RESPONSE) + mock.get(re.compile('records'), status_code=200, text=RECORDS_PAGE_1) + mock.get(re.compile('records.*offset=3'), status_code=200, text=RECORDS_PAGE_2) + + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals(14, len(zone.records)) + changes = expected.changes(zone, provider) + self.assertEquals(0, len(changes)) + + # Used in a minute + def assert_rrsets_callback(request, context): + data = loads(request.body) + self.assertEquals(expected_n, len(data['rrsets'])) + return '' + + # No existing records -> creates for every record in expected + with requests_mock() as mock: + mock.get(ANY, status_code=200, text=EMPTY_TEXT) + # post 201, is reponse to the create with data + mock.patch(ANY, status_code=201, text=assert_rrsets_callback) + + plan = provider.plan(expected) + self.assertEquals(expected_n, len(plan.changes)) + self.assertEquals(expected_n, provider.apply(plan)) + + # Non-existent zone -> creates for every record in expected + # OMG this is fucking ugly, probably better to ditch requests_mocks and + # just mock things for real as it doesn't seem to provide a way to get + # at the request params or verify that things were called from what I + # can tell + not_found = {'error': "Could not find domain 'unit.tests.'"} + with requests_mock() as mock: + # get 422's, unknown zone + mock.get(ANY, status_code=422, text='') + # patch 422's, unknown zone + mock.patch(ANY, status_code=422, text=dumps(not_found)) + # post 201, is reponse to the create with data + mock.post(ANY, status_code=201, text=assert_rrsets_callback) + + plan = provider.plan(expected) + self.assertEquals(expected_n, len(plan.changes)) + self.assertEquals(expected_n, provider.apply(plan)) + + with requests_mock() as mock: + # get 422's, unknown zone + mock.get(ANY, status_code=422, text='') + # patch 422's, + data = {'error': "Key 'name' not present or not a String"} + mock.patch(ANY, status_code=422, text=dumps(data)) + + with self.assertRaises(HTTPError) as ctx: + plan = provider.plan(expected) + provider.apply(plan) + response = ctx.exception.response + self.assertEquals(422, response.status_code) + self.assertTrue('error' in response.json()) + + with requests_mock() as mock: + # get 422's, unknown zone + mock.get(ANY, status_code=422, text='') + # patch 500's, things just blew up + mock.patch(ANY, status_code=500, text='') + + with self.assertRaises(HTTPError): + plan = provider.plan(expected) + provider.apply(plan) + + with requests_mock() as mock: + # get 422's, unknown zone + mock.get(ANY, status_code=422, text='') + # patch 500's, things just blew up + mock.patch(ANY, status_code=422, text=dumps(not_found)) + # post 422's, something wrong with create + mock.post(ANY, status_code=422, text='Hello Word!') + + with self.assertRaises(HTTPError): + plan = provider.plan(expected) + provider.apply(plan) + + def test_small_change(self): + provider = load_provider() + + expected = Zone('unit.tests.', []) + source = YamlProvider('test', join(dirname(__file__), 'config')) + source.populate(expected) + self.assertEquals(15, len(expected.records)) + + # A small change to a single record + with requests_mock() as mock: + mock.get(ANY, status_code=200, text=FULL_TEXT) + + missing = Zone(expected.name, []) + # Find and delete the SPF record + for record in expected.records: + if record._type != 'SPF': + missing.add_record(record) + + def assert_delete_callback(request, context): + self.assertEquals({ + 'rrsets': [{ + 'records': [ + {'content': '"v=spf1 ip4:192.168.0.1/16-all"', + 'disabled': False} + ], + 'changetype': 'DELETE', + 'type': 'SPF', + 'name': 'spf.unit.tests.', + 'ttl': 600 + }] + }, loads(request.body)) + return '' + + mock.patch(ANY, status_code=201, text=assert_delete_callback) + + plan = provider.plan(missing) + self.assertEquals(1, len(plan.changes)) + self.assertEquals(1, provider.apply(plan)) + + def test_existing_nameservers(self): + ns_values = ['8.8.8.8.', '9.9.9.9.'] + provider = load_provider() + + expected = Zone('unit.tests.', []) + ns_record = Record.new(expected, '', { + 'type': 'NS', + 'ttl': 600, + 'values': ns_values + }) + expected.add_record(ns_record) + + # no changes + with requests_mock() as mock: + data = { + 'rrsets': [{ + 'comments': [], + 'name': 'unit.tests.', + 'records': [ + { + 'content': '8.8.8.8.', + 'disabled': False + }, + { + 'content': '9.9.9.9.', + 'disabled': False + } + ], + 'ttl': 600, + 'type': 'NS' + }, { + 'comments': [], + 'name': 'unit.tests.', + 'records': [{ + 'content': '1.2.3.4', + 'disabled': False, + }], + 'ttl': 60, + 'type': 'A' + }] + } + mock.get(ANY, status_code=200, json=data) + + unrelated_record = Record.new(expected, '', { + 'type': 'A', + 'ttl': 60, + 'value': '1.2.3.4' + }) + expected.add_record(unrelated_record) + plan = provider.plan(expected) + self.assertFalse(plan) + # remove it now that we don't need the unrelated change any longer + expected.records.remove(unrelated_record) + + # ttl diff + with requests_mock() as mock: + data = { + 'rrsets': [{ + 'comments': [], + 'name': 'unit.tests.', + 'records': [ + { + 'content': '8.8.8.8.', + 'disabled': False + }, + { + 'content': '9.9.9.9.', + 'disabled': False + }, + ], + 'ttl': 3600, + 'type': 'NS' + }] + } + mock.get(ANY, status_code=200, json=data) + + plan = provider.plan(expected) + self.assertEquals(1, len(plan.changes)) + + # create + with requests_mock() as mock: + data = { + 'rrsets': [] + } + mock.get(ANY, status_code=200, json=data) + + plan = provider.plan(expected) + self.assertEquals(1, len(plan.changes)) From c19ec41b6bb82c2b64cf39cec09cafd7807561c5 Mon Sep 17 00:00:00 2001 From: Vietor Davis Date: Fri, 7 Jul 2017 18:21:59 -0700 Subject: [PATCH 002/141] Parse all data in the sample return set --- octodns/provider/rackspace.py | 75 +++++++++++-------- .../rackspace-list-domains-response.json | 2 +- 2 files changed, 46 insertions(+), 31 deletions(-) diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py index 311d02d..f6817ae 100644 --- a/octodns/provider/rackspace.py +++ b/octodns/provider/rackspace.py @@ -8,6 +8,7 @@ from __future__ import absolute_import, division, print_function, \ from requests import HTTPError, Session, post import json +from collections import defaultdict import logging from ..record import Create, Record @@ -46,10 +47,10 @@ class RackspaceProvider(BaseProvider): cloud_dns_endpoint = [x for x in ret.json()['access']['serviceCatalog'] if x['name'] == 'cloudDNS'][0]['endpoints'][0]['publicURL'] return ret.json()['access']['token']['id'], cloud_dns_endpoint - def _get_zone_id_for(self, zone_name): + def _get_zone_id_for(self, zone): ret = self._request('GET', 'domains', pagination_key='domains') - if ret and 'name' in ret: - return [x for x in ret if x['name'] == zone_name][0]['id'] + if ret: + return [x for x in ret if x['name'] == zone.name[:-1]][0]['id'] else: return None @@ -79,7 +80,8 @@ class RackspaceProvider(BaseProvider): next_page = [x for x in resp.json().get('links', []) if x['rel'] == 'next'] if next_page: url = next_page[0]['href'] - return acc.extend(self._paginated_request_for_url(method, url, data, pagination_key)) + acc.extend(self._paginated_request_for_url(method, url, data, pagination_key)) + return acc else: return acc @@ -95,20 +97,27 @@ class RackspaceProvider(BaseProvider): def _data_for_multiple(self, rrset): # TODO: geo not supported return { - 'type': rrset['type'], - 'values': [r['content'] for r in rrset['records']], - 'ttl': rrset['ttl'] + 'type': rrset[0]['type'], + 'values': [r['data'] for r in rrset], + 'ttl': rrset[0]['ttl'] } _data_for_A = _data_for_multiple _data_for_AAAA = _data_for_multiple - _data_for_NS = _data_for_multiple + + def _data_for_NS(self, rrset): + # TODO: geo not supported + return { + 'type': rrset[0]['type'], + 'values': ["{}.".format(r['data']) for r in rrset], + 'ttl': rrset[0]['ttl'] + } def _data_for_single(self, record): return { - 'type': record['type'], - 'value': record['data'], - 'ttl': record['ttl'] + 'type': record[0]['type'], + 'value': "{}.".format(record[0]['data']), + 'ttl': record[0]['ttl'] } _data_for_ALIAS = _data_for_single @@ -127,16 +136,15 @@ class RackspaceProvider(BaseProvider): def _data_for_MX(self, rrset): values = [] - for record in rrset['records']: - priority, value = record['content'].split(' ', 1) + for record in rrset: values.append({ - 'priority': priority, - 'value': value, + 'priority': record['priority'], + 'value': record['data'], }) return { - 'type': rrset['type'], + 'type': rrset[0]['type'], 'values': values, - 'ttl': rrset['ttl'] + 'ttl': rrset[0]['ttl'] } def _data_for_NAPTR(self, rrset): @@ -193,10 +201,10 @@ class RackspaceProvider(BaseProvider): def populate(self, zone, target=False): self.log.debug('populate: name=%s', zone.name) - resp = None + resp_data = None try: - domain_id = self._get_zone_id_for(zone.name) - resp = self._request('GET', '/domains/{}/records'.format(domain_id), pagination_key='records') + domain_id = self._get_zone_id_for(zone) + resp_data = self._request('GET', '/domains/{}/records'.format(domain_id), pagination_key='records') self.log.debug('populate: loaded') except HTTPError as e: if e.response.status_code == 401: @@ -213,20 +221,27 @@ class RackspaceProvider(BaseProvider): before = len(zone.records) - if resp: - for record in resp.json()['records']: - record_type = record['type'] - if record_type == 'SOA': - continue - data_for = getattr(self, '_data_for_{}'.format(record_type)) - record_name = zone.hostname_from_fqdn(record['name']) - record = Record.new(zone, record_name, data_for(record), - source=self) - zone.add_record(record) + if resp_data: + records = self._group_records(resp_data) + for record_type, records_of_type in records.items(): + for raw_record_name, record_set in records_of_type.items(): + if record_type == 'SOA': + continue + data_for = getattr(self, '_data_for_{}'.format(record_type)) + record_name = zone.hostname_from_fqdn(raw_record_name) + record = Record.new(zone, record_name, data_for(record_set), + source=self) + zone.add_record(record) self.log.info('populate: found %s records', len(zone.records) - before) + def _group_records(self, all_records): + records = defaultdict(lambda: defaultdict(list)) + for record in all_records: + records[record['type']][record['name']].append(record) + return records + def _records_for_multiple(self, record): return [{'content': v, 'disabled': False} for v in record.values] diff --git a/tests/fixtures/rackspace-list-domains-response.json b/tests/fixtures/rackspace-list-domains-response.json index f124837..725641a 100644 --- a/tests/fixtures/rackspace-list-domains-response.json +++ b/tests/fixtures/rackspace-list-domains-response.json @@ -58,7 +58,7 @@ "accountId" : 1234, "created" : "2011-06-15T19:02:07.000+0000" }, { - "name" : "dnsaas.example", + "name" : "unit.tests", "id" : 2722347, "comment" : "Sample comment", "updated" : "2011-06-21T15:54:31.000+0000", From 21b3ffb509391d4aa37187b34254eb0f745097f8 Mon Sep 17 00:00:00 2001 From: Vietor Davis Date: Fri, 7 Jul 2017 18:37:04 -0700 Subject: [PATCH 003/141] Minor test updates for rackspace --- tests/test_octodns_source_rackspace.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_octodns_source_rackspace.py b/tests/test_octodns_source_rackspace.py index 88ccb5f..44ec1eb 100644 --- a/tests/test_octodns_source_rackspace.py +++ b/tests/test_octodns_source_rackspace.py @@ -91,9 +91,9 @@ class TestRackspaceSource(TestCase): zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals(14, len(zone.records)) + self.assertEquals(5, len(zone.records)) changes = expected.changes(zone, provider) - self.assertEquals(0, len(changes)) + self.assertEquals(18, len(changes)) # Used in a minute def assert_rrsets_callback(request, context): @@ -103,7 +103,8 @@ class TestRackspaceSource(TestCase): # No existing records -> creates for every record in expected with requests_mock() as mock: - mock.get(ANY, status_code=200, text=EMPTY_TEXT) + mock.get(re.compile('domains$'), status_code=200, text=LIST_DOMAINS_RESPONSE) + mock.get(re.compile('records'), status_code=200, text=EMPTY_TEXT) # post 201, is reponse to the create with data mock.patch(ANY, status_code=201, text=assert_rrsets_callback) From 0579ff6f2dda0be69197ce3cf4a9bf899159ffb0 Mon Sep 17 00:00:00 2001 From: Terrence Cole Date: Wed, 12 Jul 2017 16:34:22 -0700 Subject: [PATCH 004/141] Working push for A records. --- octodns/provider/rackspace.py | 173 ++-- ...sample-recordset-existing-nameservers.json | 29 + tests/test_octodns_source_rackspace.py | 801 ++++++++++++++---- 3 files changed, 755 insertions(+), 248 deletions(-) create mode 100644 tests/fixtures/rackspace-sample-recordset-existing-nameservers.json diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py index f6817ae..eef20d9 100644 --- a/octodns/provider/rackspace.py +++ b/octodns/provider/rackspace.py @@ -1,13 +1,10 @@ # # # - - from __future__ import absolute_import, division, print_function, \ unicode_literals from requests import HTTPError, Session, post -import json from collections import defaultdict import logging @@ -40,6 +37,10 @@ class RackspaceProvider(BaseProvider): sess.headers.update({'X-Auth-Token': auth_token}) self._sess = sess + # Map record type, name, and data to an id when populating so that + # we can find the id for update and delete operations. + self._id_map = {} + def _get_auth_token(self, username, api_key): ret = post('https://identity.api.rackspacecloud.com/v2.0/tokens', json={"auth": {"RAX-KSKEY:apiKeyCredentials": {"username": username, "apiKey": api_key}}}, @@ -91,9 +92,15 @@ class RackspaceProvider(BaseProvider): def _post(self, path, data=None): return self._request('POST', path, data=data) + def _put(self, path, data=None): + return self._request('PUT', path, data=data) + def _patch(self, path, data=None): return self._request('PATCH', path, data=data) + def _delete(self, path, data=None): + return self._request('DELETE', path, data=data) + def _data_for_multiple(self, rrset): # TODO: geo not supported return { @@ -204,7 +211,7 @@ class RackspaceProvider(BaseProvider): resp_data = None try: domain_id = self._get_zone_id_for(zone) - resp_data = self._request('GET', '/domains/{}/records'.format(domain_id), pagination_key='records') + resp_data = self._request('GET', 'domains/{}/records'.format(domain_id), pagination_key='records') self.log.debug('populate: loaded') except HTTPError as e: if e.response.status_code == 401: @@ -224,9 +231,9 @@ class RackspaceProvider(BaseProvider): if resp_data: records = self._group_records(resp_data) for record_type, records_of_type in records.items(): + if record_type == 'SOA': + continue for raw_record_name, record_set in records_of_type.items(): - if record_type == 'SOA': - continue data_for = getattr(self, '_data_for_{}'.format(record_type)) record_name = zone.hostname_from_fqdn(raw_record_name) record = Record.new(zone, record_name, data_for(record_set), @@ -239,6 +246,7 @@ class RackspaceProvider(BaseProvider): def _group_records(self, all_records): records = defaultdict(lambda: defaultdict(list)) for record in all_records: + self._id_map[(record['type'], record['name'], record['data'])] = record['id'] records[record['type']][record['name']].append(record) return records @@ -294,61 +302,45 @@ class RackspaceProvider(BaseProvider): } for v in record.values] def _mod_Create(self, change): - new = change.new - records_for = getattr(self, '_records_for_{}'.format(new._type)) - return { - 'name': new.fqdn, - 'type': new._type, - 'ttl': new.ttl, - 'changetype': 'REPLACE', - 'records': records_for(new) - } - - _mod_Update = _mod_Create + out = [] + for value in change.new.values: + out.append({ + 'name': change.new.fqdn, + 'type': change.new._type, + 'data': value, + 'ttl': change.new.ttl, + }) + return out + + def _mod_Update(self, change): + # A reduction in number of values in an update record needs + # to get upgraded into a Delete change for the removed values. + deleted_values = set(change.existing.values) - set(change.new.values) + delete_out = self._delete_given_change_values(change, deleted_values) + + update_out = [] + for value in change.new.values: + key = (change.existing._type, change.existing.fqdn, value) + rsid = self._id_map[key] + update_out.append({ + 'id': rsid, + 'name': change.new.fqdn, + 'data': value, + 'ttl': change.new.ttl, + }) + return update_out, delete_out def _mod_Delete(self, change): - existing = change.existing - records_for = getattr(self, '_records_for_{}'.format(existing._type)) - return { - 'name': existing.fqdn, - 'type': existing._type, - 'ttl': existing.ttl, - 'changetype': 'DELETE', - 'records': records_for(existing) - } + return self._delete_given_change_values(change, change.existing.values) - def _get_nameserver_record(self, existing): - return None - - def _extra_changes(self, existing, _): - self.log.debug('_extra_changes: zone=%s', existing.name) - - ns = self._get_nameserver_record(existing) - if not ns: - return [] - - # sorting mostly to make things deterministic for testing, but in - # theory it let us find what we're after quickier (though sorting would - # ve more exepensive.) - for record in sorted(existing.records): - if record == ns: - # We've found the top-level NS record, return any changes - change = record.changes(ns, self) - self.log.debug('_extra_changes: change=%s', change) - if change: - # We need to modify an existing record - return [change] - # No change is necessary - return [] - # No existing top-level NS - self.log.debug('_extra_changes: create') - return [Create(ns)] - - def _get_error(self, http_error): - try: - return http_error.response.json()['error'] - except Exception: - return '' + def _delete_given_change_values(self, change, values): + out = [] + for value in values: + key = (change.existing._type, change.existing.fqdn, value) + rsid = self._id_map[key] + out.append('id=' + rsid) + del self._id_map[key] + return out def _apply(self, plan): desired = plan.desired @@ -356,46 +348,29 @@ class RackspaceProvider(BaseProvider): self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name, len(changes)) - mods = [] + domain_id = self._get_zone_id_for(desired) + creates = [] + updates = [] + deletes = [] for change in changes: - class_name = change.__class__.__name__ - mods.append(getattr(self, '_mod_{}'.format(class_name))(change)) - self.log.debug('_apply: sending change request') - - try: - self._patch('zones/{}'.format(desired.name), - data={'rrsets': mods}) - self.log.debug('_apply: patched') - except HTTPError as e: - error = self._get_error(e) - if e.response.status_code != 422 or \ - not error.startswith('Could not find domain '): - self.log.error('_apply: status=%d, text=%s', - e.response.status_code, - e.response.text) - raise - self.log.info('_apply: creating zone=%s', desired.name) - # 422 means powerdns doesn't know anything about the requsted - # domain. We'll try to create it with the correct records instead - # of update. Hopefully all the mods are creates :-) - data = { - 'name': desired.name, - 'kind': 'Master', - 'masters': [], - 'nameservers': [], - 'rrsets': mods, - 'soa_edit_api': 'INCEPTION-INCREMENT', - 'serial': 0, - } - try: - self._post('zones', data) - except HTTPError as e: - self.log.error('_apply: status=%d, text=%s', - e.response.status_code, - e.response.text) - raise - self.log.debug('_apply: created') - - self.log.debug('_apply: complete') - + if change.__class__.__name__ == 'Create': + creates += self._mod_Create(change) + elif change.__class__.__name__ == 'Update': + add_updates, add_deletes = self._mod_Update(change) + updates += add_updates + deletes += add_deletes + elif change.__class__.__name__ == 'Delete': + deletes += self._mod_Delete(change) + + if creates: + data = {"records": sorted(creates, key=lambda v: v['name'])} + self._post('domains/{}/records'.format(domain_id), data=data) + + if updates: + data = {"records": sorted(updates, key=lambda v: v['name'])} + self._put('domains/{}/records'.format(domain_id), data=data) + + if deletes: + params = "&".join(sorted(deletes)) + self._delete('domains/{}/records?{}'.format(domain_id, params)) diff --git a/tests/fixtures/rackspace-sample-recordset-existing-nameservers.json b/tests/fixtures/rackspace-sample-recordset-existing-nameservers.json new file mode 100644 index 0000000..034aa44 --- /dev/null +++ b/tests/fixtures/rackspace-sample-recordset-existing-nameservers.json @@ -0,0 +1,29 @@ +{ + "totalEntries" : 3, + "records" : [{ + "name" : "unit.tests.", + "id" : "A-6822995", + "type" : "A", + "data" : "1.2.3.4", + "updated" : "2011-06-24T01:12:53.000+0000", + "ttl" : 60, + "created" : "2011-06-24T01:12:53.000+0000" + }, { + "name" : "unit.tests.", + "id" : "NS-454454", + "type" : "NS", + "data" : "8.8.8.8.", + "updated" : "2011-06-24T01:12:51.000+0000", + "ttl" : 600, + "created" : "2011-06-24T01:12:51.000+0000" + }, { + "name" : "unit.tests.", + "id" : "NS-454455", + "type" : "NS", + "data" : "9.9.9.9.", + "updated" : "2011-06-24T01:12:52.000+0000", + "ttl" : 600, + "created" : "2011-06-24T01:12:52.000+0000" + }], + "links" : [] +} diff --git a/tests/test_octodns_source_rackspace.py b/tests/test_octodns_source_rackspace.py index 44ec1eb..4c53e95 100644 --- a/tests/test_octodns_source_rackspace.py +++ b/tests/test_octodns_source_rackspace.py @@ -5,10 +5,11 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals +import json import re -from json import loads, dumps from os.path import dirname, join from unittest import TestCase +from urlparse import urlparse from requests import HTTPError from requests_mock import ANY, mock as requests_mock @@ -18,9 +19,11 @@ from octodns.provider.yaml import YamlProvider from octodns.record import Record from octodns.zone import Zone +from pprint import pprint + EMPTY_TEXT = ''' { - "totalEntries" : 6, + "totalEntries" : 0, "records" : [] } ''' @@ -37,51 +40,66 @@ with open('./tests/fixtures/rackspace-sample-recordset-page1.json') as fh: with open('./tests/fixtures/rackspace-sample-recordset-page2.json') as fh: RECORDS_PAGE_2 = fh.read() -def load_provider(): - with requests_mock() as mock: - mock.post(ANY, status_code=200, text=AUTH_RESPONSE) - return RackspaceProvider('test', 'api-key') - - -class TestRackspaceSource(TestCase): +with open('./tests/fixtures/rackspace-sample-recordset-existing-nameservers.json') as fh: + RECORDS_EXISTING_NAMESERVERS = fh.read() - def test_provider(self): - provider = load_provider() +class TestRackspaceProvider(TestCase): + def setUp(self): + with requests_mock() as mock: + mock.post(ANY, status_code=200, text=AUTH_RESPONSE) + self.provider = RackspaceProvider('test', 'api-key') + self.assertTrue(mock.called_once) - # Bad auth + def test_bad_auth(self): with requests_mock() as mock: mock.get(ANY, status_code=401, text='Unauthorized') with self.assertRaises(Exception) as ctx: zone = Zone('unit.tests.', []) - provider.populate(zone) + self.provider.populate(zone) self.assertTrue('unauthorized' in ctx.exception.message) + self.assertTrue(mock.called_once) - # General error + def test_server_error(self): with requests_mock() as mock: mock.get(ANY, status_code=502, text='Things caught fire') with self.assertRaises(HTTPError) as ctx: zone = Zone('unit.tests.', []) - provider.populate(zone) + self.provider.populate(zone) self.assertEquals(502, ctx.exception.response.status_code) + self.assertTrue(mock.called_once) - # Non-existant zone doesn't populate anything + def test_nonexistent_zone(self): + # Non-existent zone doesn't populate anything with requests_mock() as mock: mock.get(ANY, status_code=422, json={'error': "Could not find domain 'unit.tests.'"}) zone = Zone('unit.tests.', []) - provider.populate(zone) + self.provider.populate(zone) self.assertEquals(set(), zone.records) + self.assertTrue(mock.called_once) - # The rest of this is messy/complicated b/c it's dealing with mocking + def test_multipage_populate(self): + with requests_mock() as mock: + mock.get(re.compile('domains$'), status_code=200, text=LIST_DOMAINS_RESPONSE) + mock.get(re.compile('records'), status_code=200, text=RECORDS_PAGE_1) + mock.get(re.compile('records.*offset=3'), status_code=200, text=RECORDS_PAGE_2) + zone = Zone('unit.tests.', []) + self.provider.populate(zone) + self.assertEquals(5, len(zone.records)) + + def _load_full_config(self): expected = Zone('unit.tests.', []) source = YamlProvider('test', join(dirname(__file__), 'config')) source.populate(expected) - expected_n = len(expected.records) - 1 - self.assertEquals(14, expected_n) + self.assertEquals(15, len(expected.records)) + return expected + + def test_changes_are_formatted_correctly(self): + expected = self._load_full_config() # No diffs == no changes with requests_mock() as mock: @@ -90,27 +108,551 @@ class TestRackspaceSource(TestCase): mock.get(re.compile('records.*offset=3'), status_code=200, text=RECORDS_PAGE_2) zone = Zone('unit.tests.', []) - provider.populate(zone) - self.assertEquals(5, len(zone.records)) - changes = expected.changes(zone, provider) + self.provider.populate(zone) + changes = expected.changes(zone, self.provider) self.assertEquals(18, len(changes)) + def test_plan_disappearing_ns_records(self): + expected = Zone('unit.tests.', []) + expected.add_record(Record.new(expected, '', { + 'type': 'NS', + 'ttl': 600, + 'values': ['8.8.8.8.', '9.9.9.9.'] + })) + expected.add_record(Record.new(expected, 'sub', { + 'type': 'NS', + 'ttl': 600, + 'values': ['8.8.8.8.', '9.9.9.9.'] + })) + with requests_mock() as mock: + mock.get(re.compile('domains$'), status_code=200, text=LIST_DOMAINS_RESPONSE) + mock.get(re.compile('records'), status_code=200, text=EMPTY_TEXT) + + plan = self.provider.plan(expected) + self.assertTrue(mock.called) + + # OctoDNS does not propagate top-level NS records. + self.assertEquals(1, len(plan.changes)) + + def _test_apply_with_data(self, data): + expected = Zone('unit.tests.', []) + for record in data.OtherRecords: + expected.add_record(Record.new(expected, record['subdomain'], record['data'])) + + with requests_mock() as list_mock: + list_mock.get(re.compile('domains$'), status_code=200, text=LIST_DOMAINS_RESPONSE) + list_mock.get(re.compile('records'), status_code=200, json=data.OwnRecords) + plan = self.provider.plan(expected) + self.assertTrue(list_mock.called) + if not data.ExpectChanges: + self.assertFalse(plan) + return + + with requests_mock() as mock: + called = set() + + def make_assert_sending_right_body(expected): + def _assert_sending_right_body(request, _context): + called.add(request.method) + if request.method != 'DELETE': + self.assertEqual(request.headers['content-type'], 'application/json') + self.assertDictEqual(expected, json.loads(request.body)) + else: + parts = urlparse(request.url) + self.assertEqual(expected, parts.query) + return '' + return _assert_sending_right_body + + mock.get(re.compile('domains$'), status_code=200, text=LIST_DOMAINS_RESPONSE) + mock.post(re.compile('domains/.*/records$'), status_code=202, + text=make_assert_sending_right_body(data.ExpectedAdditions)) + mock.delete(re.compile('domains/.*/records?.*'), status_code=202, + text=make_assert_sending_right_body(data.ExpectedDeletions)) + mock.put(re.compile('domains/.*/records$'), status_code=202, + text=make_assert_sending_right_body(data.ExpectedUpdates)) + + self.provider.apply(plan) + self.assertTrue(data.ExpectedAdditions is None or "POST" in called) + self.assertTrue(data.ExpectedDeletions is None or "DELETE" in called) + self.assertTrue(data.ExpectedUpdates is None or "PUT" in called) + + def test_apply_no_change_empty(self): + class TestData(object): + OtherRecords = [] + OwnRecords = { + "totalEntries": 0, + "records": [] + } + ExpectChanges = False + ExpectedAdditions = None + ExpectedDeletions = None + ExpectedUpdates = None + return self._test_apply_with_data(TestData) + + def test_apply_no_change_a_records(self): + class TestData(object): + OtherRecords = [ + { + "subdomain": '', + "data": { + 'type': 'A', + 'ttl': 60, + 'values': ['1.2.3.4', '1.2.3.5', '1.2.3.6'] + } + } + ] + OwnRecords = { + "totalEntries": 3, + "records": [{ + "name": "unit.tests.", + "id": "A-111111", + "type": "A", + "data": "1.2.3.4", + "ttl": 60 + }, { + "name": "unit.tests.", + "id": "A-222222", + "type": "A", + "data": "1.2.3.5", + "ttl": 60 + }, { + "name": "unit.tests.", + "id": "A-333333", + "type": "A", + "data": "1.2.3.6", + "ttl": 60 + }] + } + ExpectChanges = False + ExpectedAdditions = None + ExpectedDeletions = None + ExpectedUpdates = None + return self._test_apply_with_data(TestData) + + def test_apply_no_change_a_records_cross_zone(self): + class TestData(object): + OtherRecords = [ + { + "subdomain": 'foo', + "data": { + 'type': 'A', + 'ttl': 60, + 'value': '1.2.3.4' + } + }, + { + "subdomain": 'bar', + "data": { + 'type': 'A', + 'ttl': 60, + 'value': '1.2.3.4' + } + } + ] + OwnRecords = { + "totalEntries": 3, + "records": [{ + "name": "foo.unit.tests.", + "id": "A-111111", + "type": "A", + "data": "1.2.3.4", + "ttl": 60 + }, { + "name": "bar.unit.tests.", + "id": "A-222222", + "type": "A", + "data": "1.2.3.4", + "ttl": 60 + }] + } + ExpectChanges = False + ExpectedAdditions = None + ExpectedDeletions = None + ExpectedUpdates = None + return self._test_apply_with_data(TestData) + + def test_apply_one_addition(self): + class TestData(object): + OtherRecords = [ + { + "subdomain": '', + "data": { + 'type': 'A', + 'ttl': 60, + 'value': '1.2.3.4' + } + } + ] + OwnRecords = { + "totalEntries": 0, + "records": [] + } + ExpectChanges = True + ExpectedAdditions = { + "records": [{ + "name": "unit.tests.", + "type": "A", + "data": "1.2.3.4", + "ttl": 60 + }] + } + ExpectedDeletions = None + ExpectedUpdates = None + return self._test_apply_with_data(TestData) + + def test_apply_multiple_additions_exploding(self): + class TestData(object): + OtherRecords = [ + { + "subdomain": '', + "data": { + 'type': 'A', + 'ttl': 60, + 'values': ['1.2.3.4', '1.2.3.5', '1.2.3.6'] + } + } + ] + OwnRecords = { + "totalEntries": 0, + "records": [] + } + ExpectChanges = True + ExpectedAdditions = { + "records": [{ + "name": "unit.tests.", + "type": "A", + "data": "1.2.3.4", + "ttl": 60 + }, { + "name": "unit.tests.", + "type": "A", + "data": "1.2.3.5", + "ttl": 60 + }, { + "name": "unit.tests.", + "type": "A", + "data": "1.2.3.6", + "ttl": 60 + }] + } + ExpectedDeletions = None + ExpectedUpdates = None + return self._test_apply_with_data(TestData) + + def test_apply_multiple_additions_namespaced(self): + class TestData(object): + OtherRecords = [{ + "subdomain": 'foo', + "data": { + 'type': 'A', + 'ttl': 60, + 'value': '1.2.3.4' + } + }, { + "subdomain": 'bar', + "data": { + 'type': 'A', + 'ttl': 60, + 'value': '1.2.3.4' + } + }] + OwnRecords = { + "totalEntries": 0, + "records": [] + } + ExpectChanges = True + ExpectedAdditions = { + "records": [{ + "name": "bar.unit.tests.", + "type": "A", + "data": "1.2.3.4", + "ttl": 60 + }, { + "name": "foo.unit.tests.", + "type": "A", + "data": "1.2.3.4", + "ttl": 60 + }] + } + ExpectedDeletions = None + ExpectedUpdates = None + return self._test_apply_with_data(TestData) + + def test_apply_single_deletion(self): + class TestData(object): + OtherRecords = [] + OwnRecords = { + "totalEntries": 1, + "records": [{ + "name": "unit.tests.", + "id": "A-111111", + "type": "A", + "data": "1.2.3.4", + "ttl": 60 + }] + } + ExpectChanges = True + ExpectedAdditions = None + ExpectedDeletions = "id=A-111111" + ExpectedUpdates = None + return self._test_apply_with_data(TestData) + + def test_apply_multiple_deletions(self): + class TestData(object): + OtherRecords = [ + { + "subdomain": '', + "data": { + 'type': 'A', + 'ttl': 60, + 'value': '1.2.3.5' + } + } + ] + OwnRecords = { + "totalEntries": 3, + "records": [{ + "name": "unit.tests.", + "id": "A-111111", + "type": "A", + "data": "1.2.3.4", + "ttl": 60 + }, { + "name": "unit.tests.", + "id": "A-222222", + "type": "A", + "data": "1.2.3.5", + "ttl": 60 + }, { + "name": "unit.tests.", + "id": "A-333333", + "type": "A", + "data": "1.2.3.6", + "ttl": 60 + }] + } + ExpectChanges = True + ExpectedAdditions = None + ExpectedDeletions = "id=A-111111&id=A-333333" + ExpectedUpdates = { + "records": [{ + "name": "unit.tests.", + "id": "A-222222", + "data": "1.2.3.5", + "ttl": 60 + }] + } + return self._test_apply_with_data(TestData) + + def test_apply_multiple_deletions_cross_zone(self): + class TestData(object): + OtherRecords = [ + { + "subdomain": '', + "data": { + 'type': 'A', + 'ttl': 60, + 'value': '1.2.3.4' + } + } + ] + OwnRecords = { + "totalEntries": 3, + "records": [{ + "name": "unit.tests.", + "id": "A-111111", + "type": "A", + "data": "1.2.3.4", + "ttl": 60 + }, { + "name": "foo.unit.tests.", + "id": "A-222222", + "type": "A", + "data": "1.2.3.5", + "ttl": 60 + }, { + "name": "bar.unit.tests.", + "id": "A-333333", + "type": "A", + "data": "1.2.3.6", + "ttl": 60 + }] + } + ExpectChanges = True + ExpectedAdditions = None + ExpectedDeletions = "id=A-222222&id=A-333333" + ExpectedUpdates = None + return self._test_apply_with_data(TestData) + + def test_apply_single_update(self): + class TestData(object): + OtherRecords = [ + { + "subdomain": '', + "data": { + 'type': 'A', + 'ttl': 3600, + 'value': '1.2.3.4' + } + } + ] + OwnRecords = { + "totalEntries": 1, + "records": [{ + "name": "unit.tests.", + "id": "A-111111", + "type": "A", + "data": "1.2.3.4", + "ttl": 60 + }] + } + ExpectChanges = True + ExpectedAdditions = None + ExpectedDeletions = None + ExpectedUpdates = { + "records": [{ + "name": "unit.tests.", + "id": "A-111111", + "data": "1.2.3.4", + "ttl": 3600 + }] + } + return self._test_apply_with_data(TestData) + + def test_apply_multiple_updates(self): + class TestData(object): + OtherRecords = [ + { + "subdomain": '', + "data": { + 'type': 'A', + 'ttl': 3600, + 'values': ['1.2.3.4', '1.2.3.5', '1.2.3.6'] + } + } + ] + OwnRecords = { + "totalEntries": 3, + "records": [{ + "name": "unit.tests.", + "id": "A-111111", + "type": "A", + "data": "1.2.3.4", + "ttl": 60 + }, { + "name": "unit.tests.", + "id": "A-222222", + "type": "A", + "data": "1.2.3.5", + "ttl": 60 + }, { + "name": "unit.tests.", + "id": "A-333333", + "type": "A", + "data": "1.2.3.6", + "ttl": 60 + }] + } + ExpectChanges = True + ExpectedAdditions = None + ExpectedDeletions = None + ExpectedUpdates = { + "records": [{ + "name": "unit.tests.", + "id": "A-111111", + "data": "1.2.3.4", + "ttl": 3600 + }, { + "name": "unit.tests.", + "id": "A-222222", + "data": "1.2.3.5", + "ttl": 3600 + }, { + "name": "unit.tests.", + "id": "A-333333", + "data": "1.2.3.6", + "ttl": 3600 + }] + } + return self._test_apply_with_data(TestData) + + def test_apply_multiple_updates_cross_zone(self): + class TestData(object): + OtherRecords = [ + { + "subdomain": 'foo', + "data": { + 'type': 'A', + 'ttl': 3600, + 'value': '1.2.3.4' + } + }, + { + "subdomain": 'bar', + "data": { + 'type': 'A', + 'ttl': 3600, + 'value': '1.2.3.4' + } + } + ] + OwnRecords = { + "totalEntries": 2, + "records": [{ + "name": "foo.unit.tests.", + "id": "A-111111", + "type": "A", + "data": "1.2.3.4", + "ttl": 60 + }, { + "name": "bar.unit.tests.", + "id": "A-222222", + "type": "A", + "data": "1.2.3.4", + "ttl": 60 + }] + } + ExpectChanges = True + ExpectedAdditions = None + ExpectedDeletions = None + ExpectedUpdates = { + "records": [{ + "name": "bar.unit.tests.", + "id": "A-222222", + "data": "1.2.3.4", + "ttl": 3600 + }, { + "name": "foo.unit.tests.", + "id": "A-111111", + "data": "1.2.3.4", + "ttl": 3600 + }] + } + return self._test_apply_with_data(TestData) + + def test_provider(self): + expected = self._load_full_config() + + # No existing records -> creates for every record in expected + with requests_mock() as mock: + mock.get(re.compile('domains$'), status_code=200, text=LIST_DOMAINS_RESPONSE) + mock.get(re.compile('records'), status_code=200, text=EMPTY_TEXT) + + plan = self.provider.plan(expected) + self.assertTrue(mock.called) + self.assertEquals(len(expected.records), len(plan.changes)) + # Used in a minute def assert_rrsets_callback(request, context): data = loads(request.body) self.assertEquals(expected_n, len(data['rrsets'])) return '' - # No existing records -> creates for every record in expected with requests_mock() as mock: - mock.get(re.compile('domains$'), status_code=200, text=LIST_DOMAINS_RESPONSE) - mock.get(re.compile('records'), status_code=200, text=EMPTY_TEXT) - # post 201, is reponse to the create with data + # post 201, is response to the create with data mock.patch(ANY, status_code=201, text=assert_rrsets_callback) - plan = provider.plan(expected) - self.assertEquals(expected_n, len(plan.changes)) - self.assertEquals(expected_n, provider.apply(plan)) + self.assertEquals(expected_n, self.provider.apply(plan)) # Non-existent zone -> creates for every record in expected # OMG this is fucking ugly, probably better to ditch requests_mocks and @@ -123,12 +665,12 @@ class TestRackspaceSource(TestCase): mock.get(ANY, status_code=422, text='') # patch 422's, unknown zone mock.patch(ANY, status_code=422, text=dumps(not_found)) - # post 201, is reponse to the create with data + # post 201, is response to the create with data mock.post(ANY, status_code=201, text=assert_rrsets_callback) - plan = provider.plan(expected) + plan = self.provider.plan(expected) self.assertEquals(expected_n, len(plan.changes)) - self.assertEquals(expected_n, provider.apply(plan)) + self.assertEquals(expected_n, self.provider.apply(plan)) with requests_mock() as mock: # get 422's, unknown zone @@ -138,8 +680,8 @@ class TestRackspaceSource(TestCase): mock.patch(ANY, status_code=422, text=dumps(data)) with self.assertRaises(HTTPError) as ctx: - plan = provider.plan(expected) - provider.apply(plan) + plan = self.provider.plan(expected) + self.provider.apply(plan) response = ctx.exception.response self.assertEquals(422, response.status_code) self.assertTrue('error' in response.json()) @@ -151,8 +693,8 @@ class TestRackspaceSource(TestCase): mock.patch(ANY, status_code=500, text='') with self.assertRaises(HTTPError): - plan = provider.plan(expected) - provider.apply(plan) + plan = self.provider.plan(expected) + self.provider.apply(plan) with requests_mock() as mock: # get 422's, unknown zone @@ -163,133 +705,94 @@ class TestRackspaceSource(TestCase): mock.post(ANY, status_code=422, text='Hello Word!') with self.assertRaises(HTTPError): - plan = provider.plan(expected) - provider.apply(plan) - - def test_small_change(self): - provider = load_provider() + plan = self.provider.plan(expected) + self.provider.apply(plan) + def test_plan_no_changes(self): expected = Zone('unit.tests.', []) - source = YamlProvider('test', join(dirname(__file__), 'config')) - source.populate(expected) - self.assertEquals(15, len(expected.records)) + expected.add_record(Record.new(expected, '', { + 'type': 'NS', + 'ttl': 600, + 'values': ['8.8.8.8.', '9.9.9.9.'] + })) + expected.add_record(Record.new(expected, '', { + 'type': 'A', + 'ttl': 60, + 'value': '1.2.3.4' + })) - # A small change to a single record with requests_mock() as mock: - mock.get(ANY, status_code=200, text=FULL_TEXT) - - missing = Zone(expected.name, []) - # Find and delete the SPF record - for record in expected.records: - if record._type != 'SPF': - missing.add_record(record) - - def assert_delete_callback(request, context): - self.assertEquals({ - 'rrsets': [{ - 'records': [ - {'content': '"v=spf1 ip4:192.168.0.1/16-all"', - 'disabled': False} - ], - 'changetype': 'DELETE', - 'type': 'SPF', - 'name': 'spf.unit.tests.', - 'ttl': 600 - }] - }, loads(request.body)) - return '' - - mock.patch(ANY, status_code=201, text=assert_delete_callback) - - plan = provider.plan(missing) - self.assertEquals(1, len(plan.changes)) - self.assertEquals(1, provider.apply(plan)) + mock.get(re.compile('domains/.*/records'), status_code=200, text=RECORDS_EXISTING_NAMESERVERS) + mock.get(re.compile('domains$'), status_code=200, text=LIST_DOMAINS_RESPONSE) - def test_existing_nameservers(self): - ns_values = ['8.8.8.8.', '9.9.9.9.'] - provider = load_provider() + plan = self.provider.plan(expected) + self.assertTrue(mock.called) + self.assertFalse(plan) + + def test_plan_remove_a_record(self): expected = Zone('unit.tests.', []) - ns_record = Record.new(expected, '', { + expected.add_record(Record.new(expected, '', { 'type': 'NS', 'ttl': 600, - 'values': ns_values - }) - expected.add_record(ns_record) + 'values': ['8.8.8.8.', '9.9.9.9.'] + })) - # no changes with requests_mock() as mock: - data = { - 'rrsets': [{ - 'comments': [], - 'name': 'unit.tests.', - 'records': [ - { - 'content': '8.8.8.8.', - 'disabled': False - }, - { - 'content': '9.9.9.9.', - 'disabled': False - } - ], - 'ttl': 600, - 'type': 'NS' - }, { - 'comments': [], - 'name': 'unit.tests.', - 'records': [{ - 'content': '1.2.3.4', - 'disabled': False, - }], - 'ttl': 60, - 'type': 'A' - }] - } - mock.get(ANY, status_code=200, json=data) - - unrelated_record = Record.new(expected, '', { - 'type': 'A', - 'ttl': 60, - 'value': '1.2.3.4' - }) - expected.add_record(unrelated_record) - plan = provider.plan(expected) - self.assertFalse(plan) - # remove it now that we don't need the unrelated change any longer - expected.records.remove(unrelated_record) + mock.get(re.compile('domains/.*/records'), status_code=200, text=RECORDS_EXISTING_NAMESERVERS) + mock.get(re.compile('domains$'), status_code=200, text=LIST_DOMAINS_RESPONSE) + + plan = self.provider.plan(expected) + self.assertTrue(mock.called) + self.assertEquals(1, len(plan.changes)) + self.assertEqual(plan.changes[0].existing.ttl, 60) + self.assertEqual(plan.changes[0].existing.values[0], '1.2.3.4') + + def test_plan_create_a_record(self): + expected = Zone('unit.tests.', []) + expected.add_record(Record.new(expected, '', { + 'type': 'NS', + 'ttl': 600, + 'values': ['8.8.8.8.', '9.9.9.9.'] + })) + expected.add_record(Record.new(expected, '', { + 'type': 'A', + 'ttl': 60, + 'values': ['1.2.3.4', '1.2.3.5'] + })) - # ttl diff with requests_mock() as mock: - data = { - 'rrsets': [{ - 'comments': [], - 'name': 'unit.tests.', - 'records': [ - { - 'content': '8.8.8.8.', - 'disabled': False - }, - { - 'content': '9.9.9.9.', - 'disabled': False - }, - ], - 'ttl': 3600, - 'type': 'NS' - }] - } - mock.get(ANY, status_code=200, json=data) + mock.get(re.compile('domains/.*/records'), status_code=200, text=RECORDS_EXISTING_NAMESERVERS) + mock.get(re.compile('domains$'), status_code=200, text=LIST_DOMAINS_RESPONSE) - plan = provider.plan(expected) + plan = self.provider.plan(expected) + self.assertTrue(mock.called) self.assertEquals(1, len(plan.changes)) + self.assertEqual(plan.changes[0].new.ttl, 60) + self.assertEqual(plan.changes[0].new.values[0], '1.2.3.4') + self.assertEqual(plan.changes[0].new.values[1], '1.2.3.5') + + def test_plan_change_ttl(self): + expected = Zone('unit.tests.', []) + expected.add_record(Record.new(expected, '', { + 'type': 'NS', + 'ttl': 600, + 'values': ['8.8.8.8.', '9.9.9.9.'] + })) + expected.add_record(Record.new(expected, '', { + 'type': 'A', + 'ttl': 86400, + 'value': '1.2.3.4' + })) - # create with requests_mock() as mock: - data = { - 'rrsets': [] - } - mock.get(ANY, status_code=200, json=data) + mock.get(re.compile('domains/.*/records'), status_code=200, text=RECORDS_EXISTING_NAMESERVERS) + mock.get(re.compile('domains$'), status_code=200, text=LIST_DOMAINS_RESPONSE) - plan = provider.plan(expected) - self.assertEquals(1, len(plan.changes)) + plan = self.provider.plan(expected) + + self.assertTrue(mock.called) + self.assertEqual(1, len(plan.changes)) + self.assertEqual(plan.changes[0].existing.ttl, 60) + self.assertEqual(plan.changes[0].new.ttl, 86400) + self.assertEqual(plan.changes[0].new.values[0], '1.2.3.4') From 823423054fa21b8a3b7d968bfa2def206912137d Mon Sep 17 00:00:00 2001 From: Terrence Cole Date: Wed, 12 Jul 2017 16:35:39 -0700 Subject: [PATCH 005/141] Rename the test file to reflect the new functionality. --- ...dns_source_rackspace.py => test_octodns_provider_rackspace.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/{test_octodns_source_rackspace.py => test_octodns_provider_rackspace.py} (100%) diff --git a/tests/test_octodns_source_rackspace.py b/tests/test_octodns_provider_rackspace.py similarity index 100% rename from tests/test_octodns_source_rackspace.py rename to tests/test_octodns_provider_rackspace.py From 01f8431d7440a4fa45eb76e40a2201e4ec7b7f7f Mon Sep 17 00:00:00 2001 From: Terrence Cole Date: Thu, 13 Jul 2017 11:44:09 -0700 Subject: [PATCH 006/141] Make formatting consistent and improve record type support. --- octodns/provider/rackspace.py | 299 ++++++++++-------- ...sample-recordset-existing-nameservers.json | 6 +- tests/test_octodns_provider_rackspace.py | 152 ++++++--- 3 files changed, 281 insertions(+), 176 deletions(-) diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py index eef20d9..110412b 100644 --- a/octodns/provider/rackspace.py +++ b/octodns/provider/rackspace.py @@ -12,6 +12,33 @@ from ..record import Create, Record from .base import BaseProvider +def add_trailing_dot(s): + assert s + assert s[-1] != '.' + return s + '.' + + +def remove_trailing_dot(s): + assert s + assert s[-1] == '.' + return s[:-1] + + +def strip_quotes(s): + assert s + assert len(s) > 2 + assert s[0] == '"' + assert s[-1] == '"' + return s[1:-1] + + +def add_quotes(s): + assert s + assert s[0] != '"' + assert s[-1] != '"' + return '"{}"'.format(s) + + class RackspaceProvider(BaseProvider): SUPPORTS_GEO = False TIMEOUT = 5 @@ -101,6 +128,10 @@ class RackspaceProvider(BaseProvider): def _delete(self, path, data=None): return self._request('DELETE', path, data=data) + @staticmethod + def _key_for_record(rs_record): + return rs_record['type'], rs_record['name'], rs_record['data'] + def _data_for_multiple(self, rrset): # TODO: geo not supported return { @@ -116,14 +147,14 @@ class RackspaceProvider(BaseProvider): # TODO: geo not supported return { 'type': rrset[0]['type'], - 'values': ["{}.".format(r['data']) for r in rrset], + 'values': [add_trailing_dot(r['data']) for r in rrset], 'ttl': rrset[0]['ttl'] } def _data_for_single(self, record): return { 'type': record[0]['type'], - 'value': "{}.".format(record[0]['data']), + 'value': add_trailing_dot(record[0]['data']), 'ttl': record[0]['ttl'] } @@ -134,7 +165,7 @@ class RackspaceProvider(BaseProvider): def _data_for_quoted(self, rrset): return { 'type': rrset['type'], - 'values': [r['content'][1:-1] for r in rrset['records']], + 'values': [strip_quotes(r['content']) for r in rrset['records']], 'ttl': rrset['ttl'] } @@ -146,7 +177,7 @@ class RackspaceProvider(BaseProvider): for record in rrset: values.append({ 'priority': record['priority'], - 'value': record['data'], + 'value': add_trailing_dot(record['data']), }) return { 'type': rrset[0]['type'], @@ -155,56 +186,59 @@ class RackspaceProvider(BaseProvider): } def _data_for_NAPTR(self, rrset): - values = [] - for record in rrset['records']: - order, preference, flags, service, regexp, replacement = \ - record['content'].split(' ', 5) - values.append({ - 'order': order, - 'preference': preference, - 'flags': flags[1:-1], - 'service': service[1:-1], - 'regexp': regexp[1:-1], - 'replacement': replacement, - }) - return { - 'type': rrset['type'], - 'values': values, - 'ttl': rrset['ttl'] - } + raise NotImplementedError("Missing support for reading NAPTR records") + # values = [] + # for record in rrset['records']: + # order, preference, flags, service, regexp, replacement = \ + # record['content'].split(' ', 5) + # values.append({ + # 'order': order, + # 'preference': preference, + # 'flags': flags[1:-1], + # 'service': service[1:-1], + # 'regexp': regexp[1:-1], + # 'replacement': replacement, + # }) + # return { + # 'type': rrset['type'], + # 'values': values, + # 'ttl': rrset['ttl'] + # } def _data_for_SSHFP(self, rrset): - values = [] - for record in rrset['records']: - algorithm, fingerprint_type, fingerprint = \ - record['content'].split(' ', 2) - values.append({ - 'algorithm': algorithm, - 'fingerprint_type': fingerprint_type, - 'fingerprint': fingerprint, - }) - return { - 'type': rrset['type'], - 'values': values, - 'ttl': rrset['ttl'] - } + raise NotImplementedError("Missing support for reading SSHFP records") + # values = [] + # for record in rrset['records']: + # algorithm, fingerprint_type, fingerprint = \ + # record['content'].split(' ', 2) + # values.append({ + # 'algorithm': algorithm, + # 'fingerprint_type': fingerprint_type, + # 'fingerprint': fingerprint, + # }) + # return { + # 'type': rrset['type'], + # 'values': values, + # 'ttl': rrset['ttl'] + # } def _data_for_SRV(self, rrset): - values = [] - for record in rrset['records']: - priority, weight, port, target = \ - record['content'].split(' ', 3) - values.append({ - 'priority': priority, - 'weight': weight, - 'port': port, - 'target': target, - }) - return { - 'type': rrset['type'], - 'values': values, - 'ttl': rrset['ttl'] - } + raise NotImplementedError("Missing support for reading SRV records") + # values = [] + # for record in rrset['records']: + # priority, weight, port, target = \ + # record['content'].split(' ', 3) + # values.append({ + # 'priority': priority, + # 'weight': weight, + # 'port': port, + # 'target': target, + # }) + # return { + # 'type': rrset['type'], + # 'values': values, + # 'ttl': rrset['ttl'] + # } def populate(self, zone, target=False): self.log.debug('populate: name=%s', zone.name) @@ -217,14 +251,10 @@ class RackspaceProvider(BaseProvider): if e.response.status_code == 401: # Nicer error message for auth problems raise Exception('Rackspace request unauthorized') - elif e.response.status_code == 422: - # 422 means powerdns doesn't know anything about the requsted - # domain. We'll just ignore it here and leave the zone - # untouched. - pass - else: - # just re-throw - raise + elif e.response.status_code == 404: + # Zone not found leaves the zone empty instead of failing. + return + raise before = len(zone.records) @@ -246,70 +276,82 @@ class RackspaceProvider(BaseProvider): def _group_records(self, all_records): records = defaultdict(lambda: defaultdict(list)) for record in all_records: - self._id_map[(record['type'], record['name'], record['data'])] = record['id'] + self._id_map[self._key_for_record(record)] = record['id'] records[record['type']][record['name']].append(record) return records - def _records_for_multiple(self, record): - return [{'content': v, 'disabled': False} - for v in record.values] - - _records_for_A = _records_for_multiple - _records_for_AAAA = _records_for_multiple - _records_for_NS = _records_for_multiple - - def _records_for_single(self, record): - return [{'content': record.value, 'disabled': False}] - - _records_for_ALIAS = _records_for_single - _records_for_CNAME = _records_for_single - _records_for_PTR = _records_for_single - - def _records_for_quoted(self, record): - return [{'content': '"{}"'.format(v), 'disabled': False} - for v in record.values] - - _records_for_SPF = _records_for_quoted - _records_for_TXT = _records_for_quoted - - def _records_for_MX(self, record): - return [{ - 'content': '{} {}'.format(v.priority, v.value), - 'disabled': False - } for v in record.values] - - def _records_for_NAPTR(self, record): - return [{ - 'content': '{} {} "{}" "{}" "{}" {}'.format(v.order, v.preference, - v.flags, v.service, - v.regexp, - v.replacement), - 'disabled': False - } for v in record.values] - - def _records_for_SSHFP(self, record): - return [{ - 'content': '{} {} {}'.format(v.algorithm, v.fingerprint_type, - v.fingerprint), - 'disabled': False - } for v in record.values] - - def _records_for_SRV(self, record): - return [{ - 'content': '{} {} {} {}'.format(v.priority, v.weight, v.port, - v.target), - 'disabled': False - } for v in record.values] + @staticmethod + def _record_for_ip(record, value): + return { + 'name': record.fqdn, + 'type': record._type, + 'data': value, + 'ttl': max(record.ttl, 300), + } + _record_for_A = _record_for_ip + _record_for_AAAA = _record_for_ip + + @staticmethod + def _record_for_named(record, value): + return { + 'name': record.fqdn, + 'type': record._type, + 'data': remove_trailing_dot(value), + 'ttl': max(record.ttl, 300), + } + _record_for_NS = _record_for_named + _record_for_ALIAS = _record_for_named + _record_for_CNAME = _record_for_named + _record_for_PTR = _record_for_named + + @staticmethod + def _record_for_quoted(record, value): + return { + 'name': record.fqdn, + 'type': record._type, + 'data': add_quotes(value), + 'ttl': max(record.ttl, 300), + } + _record_for_SPF = _record_for_quoted + _record_for_TXT = _record_for_quoted + + @staticmethod + def _record_for_MX(record, value): + return { + 'name': record.fqdn, + 'type': record._type, + 'data': remove_trailing_dot(value), + 'ttl': max(record.ttl, 300), + 'priority': record.priority + } + + @staticmethod + def _record_for_SRV(record, value): + raise NotImplementedError("Missing support for writing SRV records") + + def _record_for_NAPTR(self, record): + raise NotImplementedError("Missing support for writing NAPTR records") + # return [{ + # 'content': '{} {} "{}" "{}" "{}" {}'.format(v.order, v.preference, + # v.flags, v.service, + # v.regexp, + # v.replacement), + # 'disabled': False + # } for v in record.values] + + def _record_for_SSHFP(self, record): + raise NotImplementedError("Missing support for writing SSHFP records") + # return [{ + # 'content': '{} {} {}'.format(v.algorithm, v.fingerprint_type, + # v.fingerprint), + # 'disabled': False + # } for v in record.values] def _mod_Create(self, change): out = [] for value in change.new.values: - out.append({ - 'name': change.new.fqdn, - 'type': change.new._type, - 'data': value, - 'ttl': change.new.ttl, - }) + transformer = getattr(self, "_record_for_{}".format(change.new._type)) + out.append(transformer(change.new, value)) return out def _mod_Update(self, change): @@ -320,14 +362,16 @@ class RackspaceProvider(BaseProvider): update_out = [] for value in change.new.values: - key = (change.existing._type, change.existing.fqdn, value) - rsid = self._id_map[key] - update_out.append({ - 'id': rsid, - 'name': change.new.fqdn, - 'data': value, - 'ttl': change.new.ttl, - }) + transformer = getattr(self, "_record_for_{}".format(change.new._type)) + prior_rs_record = transformer(change.existing, value) + prior_key = self._key_for_record(prior_rs_record) + next_rs_record = transformer(change.new, value) + next_key = self._key_for_record(prior_rs_record) + next_rs_record["id"] = self._id_map[prior_key] + del next_rs_record["type"] + update_out.append(next_rs_record) + self._id_map[next_key] = self._id_map[prior_key] + del self._id_map[prior_key] return update_out, delete_out def _mod_Delete(self, change): @@ -336,9 +380,10 @@ class RackspaceProvider(BaseProvider): def _delete_given_change_values(self, change, values): out = [] for value in values: - key = (change.existing._type, change.existing.fqdn, value) - rsid = self._id_map[key] - out.append('id=' + rsid) + transformer = getattr(self, "_record_for_{}".format(change.existing._type)) + rs_record = transformer(change.existing, value) + key = self._key_for_record(rs_record) + out.append('id=' + self._id_map[key]) del self._id_map[key] return out @@ -363,7 +408,7 @@ class RackspaceProvider(BaseProvider): deletes += self._mod_Delete(change) if creates: - data = {"records": sorted(creates, key=lambda v: v['name'])} + data = {"records": sorted(creates, key=lambda v: v['type'] + v['name'] + v.get('data', ''))} self._post('domains/{}/records'.format(domain_id), data=data) if updates: diff --git a/tests/fixtures/rackspace-sample-recordset-existing-nameservers.json b/tests/fixtures/rackspace-sample-recordset-existing-nameservers.json index 034aa44..3e0f9cd 100644 --- a/tests/fixtures/rackspace-sample-recordset-existing-nameservers.json +++ b/tests/fixtures/rackspace-sample-recordset-existing-nameservers.json @@ -6,13 +6,13 @@ "type" : "A", "data" : "1.2.3.4", "updated" : "2011-06-24T01:12:53.000+0000", - "ttl" : 60, + "ttl" : 600, "created" : "2011-06-24T01:12:53.000+0000" }, { "name" : "unit.tests.", "id" : "NS-454454", "type" : "NS", - "data" : "8.8.8.8.", + "data" : "ns1.example.com", "updated" : "2011-06-24T01:12:51.000+0000", "ttl" : 600, "created" : "2011-06-24T01:12:51.000+0000" @@ -20,7 +20,7 @@ "name" : "unit.tests.", "id" : "NS-454455", "type" : "NS", - "data" : "9.9.9.9.", + "data" : "ns2.example.com", "updated" : "2011-06-24T01:12:52.000+0000", "ttl" : 600, "created" : "2011-06-24T01:12:52.000+0000" diff --git a/tests/test_octodns_provider_rackspace.py b/tests/test_octodns_provider_rackspace.py index 4c53e95..864f940 100644 --- a/tests/test_octodns_provider_rackspace.py +++ b/tests/test_octodns_provider_rackspace.py @@ -43,8 +43,10 @@ with open('./tests/fixtures/rackspace-sample-recordset-page2.json') as fh: with open('./tests/fixtures/rackspace-sample-recordset-existing-nameservers.json') as fh: RECORDS_EXISTING_NAMESERVERS = fh.read() + class TestRackspaceProvider(TestCase): def setUp(self): + self.maxDiff = 1000 with requests_mock() as mock: mock.post(ANY, status_code=200, text=AUTH_RESPONSE) self.provider = RackspaceProvider('test', 'api-key') @@ -73,7 +75,7 @@ class TestRackspaceProvider(TestCase): def test_nonexistent_zone(self): # Non-existent zone doesn't populate anything with requests_mock() as mock: - mock.get(ANY, status_code=422, + mock.get(ANY, status_code=404, json={'error': "Could not find domain 'unit.tests.'"}) zone = Zone('unit.tests.', []) @@ -196,7 +198,7 @@ class TestRackspaceProvider(TestCase): "subdomain": '', "data": { 'type': 'A', - 'ttl': 60, + 'ttl': 300, 'values': ['1.2.3.4', '1.2.3.5', '1.2.3.6'] } } @@ -208,19 +210,19 @@ class TestRackspaceProvider(TestCase): "id": "A-111111", "type": "A", "data": "1.2.3.4", - "ttl": 60 + "ttl": 300 }, { "name": "unit.tests.", "id": "A-222222", "type": "A", "data": "1.2.3.5", - "ttl": 60 + "ttl": 300 }, { "name": "unit.tests.", "id": "A-333333", "type": "A", "data": "1.2.3.6", - "ttl": 60 + "ttl": 300 }] } ExpectChanges = False @@ -236,7 +238,7 @@ class TestRackspaceProvider(TestCase): "subdomain": 'foo', "data": { 'type': 'A', - 'ttl': 60, + 'ttl': 300, 'value': '1.2.3.4' } }, @@ -244,7 +246,7 @@ class TestRackspaceProvider(TestCase): "subdomain": 'bar', "data": { 'type': 'A', - 'ttl': 60, + 'ttl': 300, 'value': '1.2.3.4' } } @@ -256,13 +258,13 @@ class TestRackspaceProvider(TestCase): "id": "A-111111", "type": "A", "data": "1.2.3.4", - "ttl": 60 + "ttl": 300 }, { "name": "bar.unit.tests.", "id": "A-222222", "type": "A", "data": "1.2.3.4", - "ttl": 60 + "ttl": 300 }] } ExpectChanges = False @@ -278,9 +280,17 @@ class TestRackspaceProvider(TestCase): "subdomain": '', "data": { 'type': 'A', - 'ttl': 60, + 'ttl': 300, 'value': '1.2.3.4' } + }, + { + "subdomain": 'foo', + "data": { + 'type': 'NS', + 'ttl': 300, + 'value': 'ns.example.com.' + } } ] OwnRecords = { @@ -293,7 +303,12 @@ class TestRackspaceProvider(TestCase): "name": "unit.tests.", "type": "A", "data": "1.2.3.4", - "ttl": 60 + "ttl": 300 + }, { + "name": "foo.unit.tests.", + "type": "NS", + "data": "ns.example.com", + "ttl": 300 }] } ExpectedDeletions = None @@ -307,9 +322,17 @@ class TestRackspaceProvider(TestCase): "subdomain": '', "data": { 'type': 'A', - 'ttl': 60, + 'ttl': 300, 'values': ['1.2.3.4', '1.2.3.5', '1.2.3.6'] } + }, + { + "subdomain": 'foo', + "data": { + 'type': 'NS', + 'ttl': 300, + 'values': ['ns1.example.com.', 'ns2.example.com.'] + } } ] OwnRecords = { @@ -322,17 +345,27 @@ class TestRackspaceProvider(TestCase): "name": "unit.tests.", "type": "A", "data": "1.2.3.4", - "ttl": 60 + "ttl": 300 }, { "name": "unit.tests.", "type": "A", "data": "1.2.3.5", - "ttl": 60 + "ttl": 300 }, { "name": "unit.tests.", "type": "A", "data": "1.2.3.6", - "ttl": 60 + "ttl": 300 + }, { + "name": "foo.unit.tests.", + "type": "NS", + "data": "ns1.example.com", + "ttl": 300 + }, { + "name": "foo.unit.tests.", + "type": "NS", + "data": "ns2.example.com", + "ttl": 300 }] } ExpectedDeletions = None @@ -345,16 +378,24 @@ class TestRackspaceProvider(TestCase): "subdomain": 'foo', "data": { 'type': 'A', - 'ttl': 60, + 'ttl': 300, 'value': '1.2.3.4' } }, { "subdomain": 'bar', "data": { 'type': 'A', - 'ttl': 60, + 'ttl': 300, 'value': '1.2.3.4' } + }, + { + "subdomain": 'foo', + "data": { + 'type': 'NS', + 'ttl': 300, + 'value': 'ns.example.com.' + } }] OwnRecords = { "totalEntries": 0, @@ -366,12 +407,17 @@ class TestRackspaceProvider(TestCase): "name": "bar.unit.tests.", "type": "A", "data": "1.2.3.4", - "ttl": 60 + "ttl": 300 }, { "name": "foo.unit.tests.", "type": "A", "data": "1.2.3.4", - "ttl": 60 + "ttl": 300 + }, { + "name": "foo.unit.tests.", + "type": "NS", + "data": "ns.example.com", + "ttl": 300 }] } ExpectedDeletions = None @@ -388,12 +434,18 @@ class TestRackspaceProvider(TestCase): "id": "A-111111", "type": "A", "data": "1.2.3.4", - "ttl": 60 + "ttl": 300 + }, { + "name": "foo.unit.tests.", + "id": "NS-111111", + "type": "NS", + "data": "ns.example.com", + "ttl": 300 }] } ExpectChanges = True ExpectedAdditions = None - ExpectedDeletions = "id=A-111111" + ExpectedDeletions = "id=A-111111&id=NS-111111" ExpectedUpdates = None return self._test_apply_with_data(TestData) @@ -404,7 +456,7 @@ class TestRackspaceProvider(TestCase): "subdomain": '', "data": { 'type': 'A', - 'ttl': 60, + 'ttl': 300, 'value': '1.2.3.5' } } @@ -416,30 +468,36 @@ class TestRackspaceProvider(TestCase): "id": "A-111111", "type": "A", "data": "1.2.3.4", - "ttl": 60 + "ttl": 300 }, { "name": "unit.tests.", "id": "A-222222", "type": "A", "data": "1.2.3.5", - "ttl": 60 + "ttl": 300 }, { "name": "unit.tests.", "id": "A-333333", "type": "A", "data": "1.2.3.6", - "ttl": 60 + "ttl": 300 + }, { + "name": "foo.unit.tests.", + "id": "NS-111111", + "type": "NS", + "data": "ns.example.com", + "ttl": 300 }] } ExpectChanges = True ExpectedAdditions = None - ExpectedDeletions = "id=A-111111&id=A-333333" + ExpectedDeletions = "id=A-111111&id=A-333333&id=NS-111111" ExpectedUpdates = { "records": [{ "name": "unit.tests.", "id": "A-222222", "data": "1.2.3.5", - "ttl": 60 + "ttl": 300 }] } return self._test_apply_with_data(TestData) @@ -451,7 +509,7 @@ class TestRackspaceProvider(TestCase): "subdomain": '', "data": { 'type': 'A', - 'ttl': 60, + 'ttl': 300, 'value': '1.2.3.4' } } @@ -463,19 +521,19 @@ class TestRackspaceProvider(TestCase): "id": "A-111111", "type": "A", "data": "1.2.3.4", - "ttl": 60 + "ttl": 300 }, { "name": "foo.unit.tests.", "id": "A-222222", "type": "A", "data": "1.2.3.5", - "ttl": 60 + "ttl": 300 }, { "name": "bar.unit.tests.", "id": "A-333333", "type": "A", "data": "1.2.3.6", - "ttl": 60 + "ttl": 300 }] } ExpectChanges = True @@ -503,7 +561,7 @@ class TestRackspaceProvider(TestCase): "id": "A-111111", "type": "A", "data": "1.2.3.4", - "ttl": 60 + "ttl": 300 }] } ExpectChanges = True @@ -538,19 +596,19 @@ class TestRackspaceProvider(TestCase): "id": "A-111111", "type": "A", "data": "1.2.3.4", - "ttl": 60 + "ttl": 300 }, { "name": "unit.tests.", "id": "A-222222", "type": "A", "data": "1.2.3.5", - "ttl": 60 + "ttl": 300 }, { "name": "unit.tests.", "id": "A-333333", "type": "A", "data": "1.2.3.6", - "ttl": 60 + "ttl": 300 }] } ExpectChanges = True @@ -603,13 +661,13 @@ class TestRackspaceProvider(TestCase): "id": "A-111111", "type": "A", "data": "1.2.3.4", - "ttl": 60 + "ttl": 300 }, { "name": "bar.unit.tests.", "id": "A-222222", "type": "A", "data": "1.2.3.4", - "ttl": 60 + "ttl": 300 }] } ExpectChanges = True @@ -630,6 +688,7 @@ class TestRackspaceProvider(TestCase): } return self._test_apply_with_data(TestData) + """ def test_provider(self): expected = self._load_full_config() @@ -707,17 +766,18 @@ class TestRackspaceProvider(TestCase): with self.assertRaises(HTTPError): plan = self.provider.plan(expected) self.provider.apply(plan) + """ def test_plan_no_changes(self): expected = Zone('unit.tests.', []) expected.add_record(Record.new(expected, '', { 'type': 'NS', 'ttl': 600, - 'values': ['8.8.8.8.', '9.9.9.9.'] + 'values': ['ns1.example.com.', 'ns2.example.com.'] })) expected.add_record(Record.new(expected, '', { 'type': 'A', - 'ttl': 60, + 'ttl': 600, 'value': '1.2.3.4' })) @@ -735,7 +795,7 @@ class TestRackspaceProvider(TestCase): expected.add_record(Record.new(expected, '', { 'type': 'NS', 'ttl': 600, - 'values': ['8.8.8.8.', '9.9.9.9.'] + 'values': ['ns1.example.com.', 'ns2.example.com.'] })) with requests_mock() as mock: @@ -745,7 +805,7 @@ class TestRackspaceProvider(TestCase): plan = self.provider.plan(expected) self.assertTrue(mock.called) self.assertEquals(1, len(plan.changes)) - self.assertEqual(plan.changes[0].existing.ttl, 60) + self.assertEqual(plan.changes[0].existing.ttl, 600) self.assertEqual(plan.changes[0].existing.values[0], '1.2.3.4') def test_plan_create_a_record(self): @@ -753,11 +813,11 @@ class TestRackspaceProvider(TestCase): expected.add_record(Record.new(expected, '', { 'type': 'NS', 'ttl': 600, - 'values': ['8.8.8.8.', '9.9.9.9.'] + 'values': ['ns1.example.com.', 'ns2.example.com.'] })) expected.add_record(Record.new(expected, '', { 'type': 'A', - 'ttl': 60, + 'ttl': 600, 'values': ['1.2.3.4', '1.2.3.5'] })) @@ -768,7 +828,7 @@ class TestRackspaceProvider(TestCase): plan = self.provider.plan(expected) self.assertTrue(mock.called) self.assertEquals(1, len(plan.changes)) - self.assertEqual(plan.changes[0].new.ttl, 60) + self.assertEqual(plan.changes[0].new.ttl, 600) self.assertEqual(plan.changes[0].new.values[0], '1.2.3.4') self.assertEqual(plan.changes[0].new.values[1], '1.2.3.5') @@ -777,7 +837,7 @@ class TestRackspaceProvider(TestCase): expected.add_record(Record.new(expected, '', { 'type': 'NS', 'ttl': 600, - 'values': ['8.8.8.8.', '9.9.9.9.'] + 'values': ['ns1.example.com.', 'ns2.example.com.'] })) expected.add_record(Record.new(expected, '', { 'type': 'A', @@ -793,6 +853,6 @@ class TestRackspaceProvider(TestCase): self.assertTrue(mock.called) self.assertEqual(1, len(plan.changes)) - self.assertEqual(plan.changes[0].existing.ttl, 60) + self.assertEqual(plan.changes[0].existing.ttl, 600) self.assertEqual(plan.changes[0].new.ttl, 86400) self.assertEqual(plan.changes[0].new.values[0], '1.2.3.4') From 92fb24f3fa57b06fc0ff16891fd9247e382a1be2 Mon Sep 17 00:00:00 2001 From: Terrence Cole Date: Thu, 13 Jul 2017 14:47:29 -0700 Subject: [PATCH 007/141] The provider constructor requires a pass-through id parameter. --- octodns/provider/rackspace.py | 2 +- tests/test_octodns_provider_rackspace.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py index 110412b..848811c 100644 --- a/octodns/provider/rackspace.py +++ b/octodns/provider/rackspace.py @@ -43,7 +43,7 @@ class RackspaceProvider(BaseProvider): SUPPORTS_GEO = False TIMEOUT = 5 - def __init__(self, username, api_key, *args, **kwargs): + def __init__(self, id, username, api_key, *args, **kwargs): ''' Rackspace API v1 Provider diff --git a/tests/test_octodns_provider_rackspace.py b/tests/test_octodns_provider_rackspace.py index 864f940..63f3de9 100644 --- a/tests/test_octodns_provider_rackspace.py +++ b/tests/test_octodns_provider_rackspace.py @@ -49,7 +49,7 @@ class TestRackspaceProvider(TestCase): self.maxDiff = 1000 with requests_mock() as mock: mock.post(ANY, status_code=200, text=AUTH_RESPONSE) - self.provider = RackspaceProvider('test', 'api-key') + self.provider = RackspaceProvider(id, 'test', 'api-key') self.assertTrue(mock.called_once) def test_bad_auth(self): From 3459064d0fc9335c48795cfb8c1caa4bd1fd3023 Mon Sep 17 00:00:00 2001 From: Terrence Cole Date: Thu, 13 Jul 2017 14:53:30 -0700 Subject: [PATCH 008/141] Use the proper arity when accessing quoted data records. --- octodns/provider/rackspace.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py index 848811c..e1931db 100644 --- a/octodns/provider/rackspace.py +++ b/octodns/provider/rackspace.py @@ -164,9 +164,9 @@ class RackspaceProvider(BaseProvider): def _data_for_quoted(self, rrset): return { - 'type': rrset['type'], - 'values': [strip_quotes(r['content']) for r in rrset['records']], - 'ttl': rrset['ttl'] + 'type': rrset[0]['type'], + 'values': [strip_quotes(r['content']) for r in rrset[0]['records']], + 'ttl': rrset[0]['ttl'] } _data_for_SPF = _data_for_quoted From 95fb613966163be8edc1cf42275bc038555bb6da Mon Sep 17 00:00:00 2001 From: Terrence Cole Date: Thu, 13 Jul 2017 14:56:04 -0700 Subject: [PATCH 009/141] Pull quoted data out of the correct field. --- octodns/provider/rackspace.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py index e1931db..be7b619 100644 --- a/octodns/provider/rackspace.py +++ b/octodns/provider/rackspace.py @@ -165,7 +165,7 @@ class RackspaceProvider(BaseProvider): def _data_for_quoted(self, rrset): return { 'type': rrset[0]['type'], - 'values': [strip_quotes(r['content']) for r in rrset[0]['records']], + 'values': [strip_quotes(r['data']) for r in rrset], 'ttl': rrset[0]['ttl'] } From b911fac90e041dd3845e2843780ac1703c79b37c Mon Sep 17 00:00:00 2001 From: Terrence Cole Date: Thu, 13 Jul 2017 15:04:16 -0700 Subject: [PATCH 010/141] RackSpace does not send back TXT records quoted. --- octodns/provider/rackspace.py | 43 ++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py index be7b619..8f80abf 100644 --- a/octodns/provider/rackspace.py +++ b/octodns/provider/rackspace.py @@ -162,15 +162,15 @@ class RackspaceProvider(BaseProvider): _data_for_CNAME = _data_for_single _data_for_PTR = _data_for_single - def _data_for_quoted(self, rrset): - return { - 'type': rrset[0]['type'], - 'values': [strip_quotes(r['data']) for r in rrset], - 'ttl': rrset[0]['ttl'] - } + # def _data_for_quoted(self, rrset): + # return { + # 'type': rrset[0]['type'], + # 'values': [strip_quotes(r['data']) for r in rrset], + # 'ttl': rrset[0]['ttl'] + # } - _data_for_SPF = _data_for_quoted - _data_for_TXT = _data_for_quoted + _data_for_SPF = _data_for_multiple + _data_for_TXT = _data_for_multiple def _data_for_MX(self, rrset): values = [] @@ -281,15 +281,15 @@ class RackspaceProvider(BaseProvider): return records @staticmethod - def _record_for_ip(record, value): + def _record_for_single(record, value): return { 'name': record.fqdn, 'type': record._type, 'data': value, 'ttl': max(record.ttl, 300), } - _record_for_A = _record_for_ip - _record_for_AAAA = _record_for_ip + _record_for_A = _record_for_single + _record_for_AAAA = _record_for_single @staticmethod def _record_for_named(record, value): @@ -304,16 +304,17 @@ class RackspaceProvider(BaseProvider): _record_for_CNAME = _record_for_named _record_for_PTR = _record_for_named - @staticmethod - def _record_for_quoted(record, value): - return { - 'name': record.fqdn, - 'type': record._type, - 'data': add_quotes(value), - 'ttl': max(record.ttl, 300), - } - _record_for_SPF = _record_for_quoted - _record_for_TXT = _record_for_quoted + # @staticmethod + # def _record_for_quoted(record, value): + # return { + # 'name': record.fqdn, + # 'type': record._type, + # 'data': add_quotes(value), + # 'ttl': max(record.ttl, 300), + # } + + _record_for_SPF = _record_for_single + _record_for_TXT = _record_for_single @staticmethod def _record_for_MX(record, value): From 057f50621e030ce1f47c630b4356f5723fba94a5 Mon Sep 17 00:00:00 2001 From: Terrence Cole Date: Thu, 13 Jul 2017 15:10:33 -0700 Subject: [PATCH 011/141] RackSpace does not escape ;. --- octodns/provider/rackspace.py | 48 +++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 19 deletions(-) diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py index 8f80abf..be8bda5 100644 --- a/octodns/provider/rackspace.py +++ b/octodns/provider/rackspace.py @@ -7,6 +7,7 @@ from __future__ import absolute_import, division, print_function, \ from requests import HTTPError, Session, post from collections import defaultdict import logging +import string from ..record import Create, Record from .base import BaseProvider @@ -39,6 +40,16 @@ def add_quotes(s): return '"{}"'.format(s) +def escape_semicolon(s): + assert s + return string.replace(s, ';', '\;') + + +def unescape_semicolon(s): + assert s + return string.replace(s, '\;', ';') + + class RackspaceProvider(BaseProvider): SUPPORTS_GEO = False TIMEOUT = 5 @@ -162,15 +173,15 @@ class RackspaceProvider(BaseProvider): _data_for_CNAME = _data_for_single _data_for_PTR = _data_for_single - # def _data_for_quoted(self, rrset): - # return { - # 'type': rrset[0]['type'], - # 'values': [strip_quotes(r['data']) for r in rrset], - # 'ttl': rrset[0]['ttl'] - # } + def _data_for_textual(self, rrset): + return { + 'type': rrset[0]['type'], + 'values': [escape_semicolon(r['data']) for r in rrset], + 'ttl': rrset[0]['ttl'] + } - _data_for_SPF = _data_for_multiple - _data_for_TXT = _data_for_multiple + _data_for_SPF = _data_for_textual + _data_for_TXT = _data_for_textual def _data_for_MX(self, rrset): values = [] @@ -304,17 +315,16 @@ class RackspaceProvider(BaseProvider): _record_for_CNAME = _record_for_named _record_for_PTR = _record_for_named - # @staticmethod - # def _record_for_quoted(record, value): - # return { - # 'name': record.fqdn, - # 'type': record._type, - # 'data': add_quotes(value), - # 'ttl': max(record.ttl, 300), - # } - - _record_for_SPF = _record_for_single - _record_for_TXT = _record_for_single + @staticmethod + def _record_for_textual(record, value): + return { + 'name': record.fqdn, + 'type': record._type, + 'data': unescape_semicolon(value), + 'ttl': max(record.ttl, 300), + } + _record_for_SPF = _record_for_textual + _record_for_TXT = _record_for_textual @staticmethod def _record_for_MX(record, value): From 0c31257b0f456caca4f49bf982abd052f6ce15e3 Mon Sep 17 00:00:00 2001 From: Terrence Cole Date: Mon, 17 Jul 2017 16:47:09 -0700 Subject: [PATCH 012/141] Use the correct record when computing the key. --- octodns/provider/rackspace.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py index be8bda5..6ed1d78 100644 --- a/octodns/provider/rackspace.py +++ b/octodns/provider/rackspace.py @@ -377,7 +377,7 @@ class RackspaceProvider(BaseProvider): prior_rs_record = transformer(change.existing, value) prior_key = self._key_for_record(prior_rs_record) next_rs_record = transformer(change.new, value) - next_key = self._key_for_record(prior_rs_record) + next_key = self._key_for_record(next_rs_record) next_rs_record["id"] = self._id_map[prior_key] del next_rs_record["type"] update_out.append(next_rs_record) From 10ff8301a542bb427325efbe0a5a40022ab12bcf Mon Sep 17 00:00:00 2001 From: Terrence Cole Date: Tue, 18 Jul 2017 10:02:57 -0700 Subject: [PATCH 013/141] RackSpace's "name" field is a "fully-qualified" name, but without the dot. --- octodns/provider/rackspace.py | 2 +- tests/test_octodns_provider_rackspace.py | 27 ++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py index 6ed1d78..52e41d4 100644 --- a/octodns/provider/rackspace.py +++ b/octodns/provider/rackspace.py @@ -294,7 +294,7 @@ class RackspaceProvider(BaseProvider): @staticmethod def _record_for_single(record, value): return { - 'name': record.fqdn, + 'name': remove_trailing_dot(record.fqdn), 'type': record._type, 'data': value, 'ttl': max(record.ttl, 300), diff --git a/tests/test_octodns_provider_rackspace.py b/tests/test_octodns_provider_rackspace.py index 63f3de9..aa82245 100644 --- a/tests/test_octodns_provider_rackspace.py +++ b/tests/test_octodns_provider_rackspace.py @@ -136,6 +136,33 @@ class TestRackspaceProvider(TestCase): # OctoDNS does not propagate top-level NS records. self.assertEquals(1, len(plan.changes)) + def test_fqdn_a_record(self): + expected = Zone('example.com.', []) + # expected.add_record(Record.new(expected, 'foo', '1.2.3.4')) + + with requests_mock() as list_mock: + list_mock.get(re.compile('domains$'), status_code=200, text=LIST_DOMAINS_RESPONSE) + list_mock.get(re.compile('records'), status_code=200, json={'records': [ + {'type': 'A', + 'name': 'foo.example.com', + 'id': 'A-111111', + 'data': '1.2.3.4', + 'ttl': 300}]}) + plan = self.provider.plan(expected) + self.assertTrue(list_mock.called) + self.assertEqual(1, len(plan.changes)) + self.assertTrue(plan.changes[0].existing.fqdn == 'foo.example.com.') + + with requests_mock() as mock: + def _assert_deleting(request, context): + parts = urlparse(request.url) + self.assertEqual('id=A-111111', parts.query) + mock.get(re.compile('domains$'), status_code=200, text=LIST_DOMAINS_RESPONSE) + mock.delete(re.compile('domains/.*/records?.*'), status_code=202, + text=_assert_deleting) + self.provider.apply(plan) + self.assertTrue(mock.called) + def _test_apply_with_data(self, data): expected = Zone('unit.tests.', []) for record in data.OtherRecords: From bb4d1c82b1540fa45393332e9526052024e3ffde Mon Sep 17 00:00:00 2001 From: Terrence Cole Date: Thu, 20 Jul 2017 15:46:08 -0700 Subject: [PATCH 014/141] Add a --quiet flag to suppress non-warning or error messages. --- octodns/cmds/args.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/octodns/cmds/args.py b/octodns/cmds/args.py index c84dd92..9322461 100644 --- a/octodns/cmds/args.py +++ b/octodns/cmds/args.py @@ -37,6 +37,8 @@ class ArgumentParser(_Base): _help = 'Increase verbosity to get details and help track down issues' self.add_argument('--debug', action='store_true', default=False, help=_help) + self.add_argument('--quiet', action='store_true', default=False, + help='Only print warning and error messages') args = super(ArgumentParser, self).parse_args() self._setup_logging(args, default_log_level) @@ -59,7 +61,11 @@ class ArgumentParser(_Base): handler.setFormatter(Formatter(fmt=fmt)) logger.addHandler(handler) - logger.level = DEBUG if args.debug else default_log_level + logger.level = default_log_level + if args.debug: + logger.level = DEBUG + elif args.quiet: + logger.level = WARN # boto is noisy, set it to warn getLogger('botocore').level = WARN From 4707b4654e391d31ef3015e87261d142afc0adf8 Mon Sep 17 00:00:00 2001 From: Terrence Cole Date: Thu, 20 Jul 2017 15:50:57 -0700 Subject: [PATCH 015/141] Add a delay to work around rackspace rate limiting. --- octodns/provider/rackspace.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py index 52e41d4..bcfcd5d 100644 --- a/octodns/provider/rackspace.py +++ b/octodns/provider/rackspace.py @@ -8,6 +8,7 @@ from requests import HTTPError, Session, post from collections import defaultdict import logging import string +import time from ..record import Create, Record from .base import BaseProvider @@ -71,6 +72,8 @@ class RackspaceProvider(BaseProvider): auth_token, dns_endpoint = self._get_auth_token(username, api_key) self.dns_endpoint = dns_endpoint + self.ratelimit_delay = kwargs.get('ratelimit_delay', 0) + sess = Session() sess.headers.update({'X-Auth-Token': auth_token}) self._sess = sess @@ -88,6 +91,7 @@ class RackspaceProvider(BaseProvider): def _get_zone_id_for(self, zone): ret = self._request('GET', 'domains', pagination_key='domains') + time.sleep(self.ratelimit_delay) if ret: return [x for x in ret if x['name'] == zone.name[:-1]][0]['id'] else: @@ -104,6 +108,7 @@ class RackspaceProvider(BaseProvider): def _request_for_url(self, method, url, data): resp = self._sess.request(method, url, json=data, timeout=self.TIMEOUT) + time.sleep(self.ratelimit_delay) self.log.debug('_request: status=%d', resp.status_code) resp.raise_for_status() return resp @@ -112,6 +117,7 @@ class RackspaceProvider(BaseProvider): acc = [] resp = self._sess.request(method, url, json=data, timeout=self.TIMEOUT) + time.sleep(self.ratelimit_delay) self.log.debug('_request: status=%d', resp.status_code) resp.raise_for_status() acc.extend(resp.json()[pagination_key]) From 803b002cd00b3f8bcde0943e37e826bf21d8f6ba Mon Sep 17 00:00:00 2001 From: Terrence Cole Date: Thu, 20 Jul 2017 15:59:32 -0700 Subject: [PATCH 016/141] Rackspace backoff is actually just required. --- octodns/provider/rackspace.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py index bcfcd5d..7b47cd6 100644 --- a/octodns/provider/rackspace.py +++ b/octodns/provider/rackspace.py @@ -55,7 +55,7 @@ class RackspaceProvider(BaseProvider): SUPPORTS_GEO = False TIMEOUT = 5 - def __init__(self, id, username, api_key, *args, **kwargs): + def __init__(self, id, username, api_key, ratelimit_delay, *args, **kwargs): ''' Rackspace API v1 Provider @@ -72,7 +72,7 @@ class RackspaceProvider(BaseProvider): auth_token, dns_endpoint = self._get_auth_token(username, api_key) self.dns_endpoint = dns_endpoint - self.ratelimit_delay = kwargs.get('ratelimit_delay', 0) + self.ratelimit_delay = ratelimit_delay sess = Session() sess.headers.update({'X-Auth-Token': auth_token}) From 8d86002382d298e1b14de9001f6780509753cdfb Mon Sep 17 00:00:00 2001 From: Terrence Cole Date: Thu, 20 Jul 2017 16:09:52 -0700 Subject: [PATCH 017/141] Environment variables are strings, so convert to a float first. --- octodns/provider/rackspace.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py index 7b47cd6..ef22415 100644 --- a/octodns/provider/rackspace.py +++ b/octodns/provider/rackspace.py @@ -72,7 +72,7 @@ class RackspaceProvider(BaseProvider): auth_token, dns_endpoint = self._get_auth_token(username, api_key) self.dns_endpoint = dns_endpoint - self.ratelimit_delay = ratelimit_delay + self.ratelimit_delay = float(ratelimit_delay) sess = Session() sess.headers.update({'X-Auth-Token': auth_token}) From a9f3384d114ec7823c55e3c10bc09221909f58ff Mon Sep 17 00:00:00 2001 From: Terrence Cole Date: Thu, 20 Jul 2017 16:24:43 -0700 Subject: [PATCH 018/141] Remove trailing dot on all record types that take an fqdn. --- octodns/provider/rackspace.py | 6 +- tests/test_octodns_provider_rackspace.py | 76 ++++++++++++------------ 2 files changed, 41 insertions(+), 41 deletions(-) diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py index ef22415..bf0e248 100644 --- a/octodns/provider/rackspace.py +++ b/octodns/provider/rackspace.py @@ -311,7 +311,7 @@ class RackspaceProvider(BaseProvider): @staticmethod def _record_for_named(record, value): return { - 'name': record.fqdn, + 'name': remove_trailing_dot(record.fqdn), 'type': record._type, 'data': remove_trailing_dot(value), 'ttl': max(record.ttl, 300), @@ -324,7 +324,7 @@ class RackspaceProvider(BaseProvider): @staticmethod def _record_for_textual(record, value): return { - 'name': record.fqdn, + 'name': remove_trailing_dot(record.fqdn), 'type': record._type, 'data': unescape_semicolon(value), 'ttl': max(record.ttl, 300), @@ -335,7 +335,7 @@ class RackspaceProvider(BaseProvider): @staticmethod def _record_for_MX(record, value): return { - 'name': record.fqdn, + 'name': remove_trailing_dot(record.fqdn), 'type': record._type, 'data': remove_trailing_dot(value), 'ttl': max(record.ttl, 300), diff --git a/tests/test_octodns_provider_rackspace.py b/tests/test_octodns_provider_rackspace.py index aa82245..d579faa 100644 --- a/tests/test_octodns_provider_rackspace.py +++ b/tests/test_octodns_provider_rackspace.py @@ -49,7 +49,7 @@ class TestRackspaceProvider(TestCase): self.maxDiff = 1000 with requests_mock() as mock: mock.post(ANY, status_code=200, text=AUTH_RESPONSE) - self.provider = RackspaceProvider(id, 'test', 'api-key') + self.provider = RackspaceProvider(id, 'test', 'api-key', '0') self.assertTrue(mock.called_once) def test_bad_auth(self): @@ -233,19 +233,19 @@ class TestRackspaceProvider(TestCase): OwnRecords = { "totalEntries": 3, "records": [{ - "name": "unit.tests.", + "name": "unit.tests", "id": "A-111111", "type": "A", "data": "1.2.3.4", "ttl": 300 }, { - "name": "unit.tests.", + "name": "unit.tests", "id": "A-222222", "type": "A", "data": "1.2.3.5", "ttl": 300 }, { - "name": "unit.tests.", + "name": "unit.tests", "id": "A-333333", "type": "A", "data": "1.2.3.6", @@ -281,13 +281,13 @@ class TestRackspaceProvider(TestCase): OwnRecords = { "totalEntries": 3, "records": [{ - "name": "foo.unit.tests.", + "name": "foo.unit.tests", "id": "A-111111", "type": "A", "data": "1.2.3.4", "ttl": 300 }, { - "name": "bar.unit.tests.", + "name": "bar.unit.tests", "id": "A-222222", "type": "A", "data": "1.2.3.4", @@ -327,12 +327,12 @@ class TestRackspaceProvider(TestCase): ExpectChanges = True ExpectedAdditions = { "records": [{ - "name": "unit.tests.", + "name": "unit.tests", "type": "A", "data": "1.2.3.4", "ttl": 300 }, { - "name": "foo.unit.tests.", + "name": "foo.unit.tests", "type": "NS", "data": "ns.example.com", "ttl": 300 @@ -369,27 +369,27 @@ class TestRackspaceProvider(TestCase): ExpectChanges = True ExpectedAdditions = { "records": [{ - "name": "unit.tests.", + "name": "unit.tests", "type": "A", "data": "1.2.3.4", "ttl": 300 }, { - "name": "unit.tests.", + "name": "unit.tests", "type": "A", "data": "1.2.3.5", "ttl": 300 }, { - "name": "unit.tests.", + "name": "unit.tests", "type": "A", "data": "1.2.3.6", "ttl": 300 }, { - "name": "foo.unit.tests.", + "name": "foo.unit.tests", "type": "NS", "data": "ns1.example.com", "ttl": 300 }, { - "name": "foo.unit.tests.", + "name": "foo.unit.tests", "type": "NS", "data": "ns2.example.com", "ttl": 300 @@ -431,17 +431,17 @@ class TestRackspaceProvider(TestCase): ExpectChanges = True ExpectedAdditions = { "records": [{ - "name": "bar.unit.tests.", + "name": "bar.unit.tests", "type": "A", "data": "1.2.3.4", "ttl": 300 }, { - "name": "foo.unit.tests.", + "name": "foo.unit.tests", "type": "A", "data": "1.2.3.4", "ttl": 300 }, { - "name": "foo.unit.tests.", + "name": "foo.unit.tests", "type": "NS", "data": "ns.example.com", "ttl": 300 @@ -457,13 +457,13 @@ class TestRackspaceProvider(TestCase): OwnRecords = { "totalEntries": 1, "records": [{ - "name": "unit.tests.", + "name": "unit.tests", "id": "A-111111", "type": "A", "data": "1.2.3.4", "ttl": 300 }, { - "name": "foo.unit.tests.", + "name": "foo.unit.tests", "id": "NS-111111", "type": "NS", "data": "ns.example.com", @@ -491,25 +491,25 @@ class TestRackspaceProvider(TestCase): OwnRecords = { "totalEntries": 3, "records": [{ - "name": "unit.tests.", + "name": "unit.tests", "id": "A-111111", "type": "A", "data": "1.2.3.4", "ttl": 300 }, { - "name": "unit.tests.", + "name": "unit.tests", "id": "A-222222", "type": "A", "data": "1.2.3.5", "ttl": 300 }, { - "name": "unit.tests.", + "name": "unit.tests", "id": "A-333333", "type": "A", "data": "1.2.3.6", "ttl": 300 }, { - "name": "foo.unit.tests.", + "name": "foo.unit.tests", "id": "NS-111111", "type": "NS", "data": "ns.example.com", @@ -521,7 +521,7 @@ class TestRackspaceProvider(TestCase): ExpectedDeletions = "id=A-111111&id=A-333333&id=NS-111111" ExpectedUpdates = { "records": [{ - "name": "unit.tests.", + "name": "unit.tests", "id": "A-222222", "data": "1.2.3.5", "ttl": 300 @@ -544,19 +544,19 @@ class TestRackspaceProvider(TestCase): OwnRecords = { "totalEntries": 3, "records": [{ - "name": "unit.tests.", + "name": "unit.tests", "id": "A-111111", "type": "A", "data": "1.2.3.4", "ttl": 300 }, { - "name": "foo.unit.tests.", + "name": "foo.unit.tests", "id": "A-222222", "type": "A", "data": "1.2.3.5", "ttl": 300 }, { - "name": "bar.unit.tests.", + "name": "bar.unit.tests", "id": "A-333333", "type": "A", "data": "1.2.3.6", @@ -584,7 +584,7 @@ class TestRackspaceProvider(TestCase): OwnRecords = { "totalEntries": 1, "records": [{ - "name": "unit.tests.", + "name": "unit.tests", "id": "A-111111", "type": "A", "data": "1.2.3.4", @@ -596,7 +596,7 @@ class TestRackspaceProvider(TestCase): ExpectedDeletions = None ExpectedUpdates = { "records": [{ - "name": "unit.tests.", + "name": "unit.tests", "id": "A-111111", "data": "1.2.3.4", "ttl": 3600 @@ -619,19 +619,19 @@ class TestRackspaceProvider(TestCase): OwnRecords = { "totalEntries": 3, "records": [{ - "name": "unit.tests.", + "name": "unit.tests", "id": "A-111111", "type": "A", "data": "1.2.3.4", "ttl": 300 }, { - "name": "unit.tests.", + "name": "unit.tests", "id": "A-222222", "type": "A", "data": "1.2.3.5", "ttl": 300 }, { - "name": "unit.tests.", + "name": "unit.tests", "id": "A-333333", "type": "A", "data": "1.2.3.6", @@ -643,17 +643,17 @@ class TestRackspaceProvider(TestCase): ExpectedDeletions = None ExpectedUpdates = { "records": [{ - "name": "unit.tests.", + "name": "unit.tests", "id": "A-111111", "data": "1.2.3.4", "ttl": 3600 }, { - "name": "unit.tests.", + "name": "unit.tests", "id": "A-222222", "data": "1.2.3.5", "ttl": 3600 }, { - "name": "unit.tests.", + "name": "unit.tests", "id": "A-333333", "data": "1.2.3.6", "ttl": 3600 @@ -684,13 +684,13 @@ class TestRackspaceProvider(TestCase): OwnRecords = { "totalEntries": 2, "records": [{ - "name": "foo.unit.tests.", + "name": "foo.unit.tests", "id": "A-111111", "type": "A", "data": "1.2.3.4", "ttl": 300 }, { - "name": "bar.unit.tests.", + "name": "bar.unit.tests", "id": "A-222222", "type": "A", "data": "1.2.3.4", @@ -702,12 +702,12 @@ class TestRackspaceProvider(TestCase): ExpectedDeletions = None ExpectedUpdates = { "records": [{ - "name": "bar.unit.tests.", + "name": "bar.unit.tests", "id": "A-222222", "data": "1.2.3.4", "ttl": 3600 }, { - "name": "foo.unit.tests.", + "name": "foo.unit.tests", "id": "A-111111", "data": "1.2.3.4", "ttl": 3600 From c185d28f146d04fffcd360abd5110982901affb5 Mon Sep 17 00:00:00 2001 From: Terrence Cole Date: Wed, 26 Jul 2017 12:49:30 -0700 Subject: [PATCH 019/141] Handle _ValueMixin record types as well as we handle _ValuesMixin records. --- octodns/provider/rackspace.py | 17 +++++++++++++---- tests/test_octodns_provider_rackspace.py | 19 +++++++++++++++++++ 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py index bf0e248..6af51fa 100644 --- a/octodns/provider/rackspace.py +++ b/octodns/provider/rackspace.py @@ -364,21 +364,30 @@ class RackspaceProvider(BaseProvider): # 'disabled': False # } for v in record.values] + def _get_values(self, record): + try: + return record.values + except AttributeError: + return [record.value] + def _mod_Create(self, change): out = [] - for value in change.new.values: + for value in self._get_values(change.new): transformer = getattr(self, "_record_for_{}".format(change.new._type)) out.append(transformer(change.new, value)) return out def _mod_Update(self, change): + existing_values = self._get_values(change.existing) + new_values = self._get_values(change.new) + # A reduction in number of values in an update record needs # to get upgraded into a Delete change for the removed values. - deleted_values = set(change.existing.values) - set(change.new.values) + deleted_values = set(existing_values) - set(new_values) delete_out = self._delete_given_change_values(change, deleted_values) update_out = [] - for value in change.new.values: + for value in new_values: transformer = getattr(self, "_record_for_{}".format(change.new._type)) prior_rs_record = transformer(change.existing, value) prior_key = self._key_for_record(prior_rs_record) @@ -392,7 +401,7 @@ class RackspaceProvider(BaseProvider): return update_out, delete_out def _mod_Delete(self, change): - return self._delete_given_change_values(change, change.existing.values) + return self._delete_given_change_values(change, self._get_values(change.existing)) def _delete_given_change_values(self, change, values): out = [] diff --git a/tests/test_octodns_provider_rackspace.py b/tests/test_octodns_provider_rackspace.py index d579faa..2862871 100644 --- a/tests/test_octodns_provider_rackspace.py +++ b/tests/test_octodns_provider_rackspace.py @@ -569,6 +569,25 @@ class TestRackspaceProvider(TestCase): ExpectedUpdates = None return self._test_apply_with_data(TestData) + def test_apply_delete_cname(self): + class TestData(object): + OtherRecords = [] + OwnRecords = { + "totalEntries": 3, + "records": [{ + "name": "foo.unit.tests", + "id": "CNAME-111111", + "type": "CNAME", + "data": "a.example.com", + "ttl": 300 + }] + } + ExpectChanges = True + ExpectedAdditions = None + ExpectedDeletions = "id=CNAME-111111" + ExpectedUpdates = None + return self._test_apply_with_data(TestData) + def test_apply_single_update(self): class TestData(object): OtherRecords = [ From 41617e69a77f3905956da2ff4a403954d1d96bac Mon Sep 17 00:00:00 2001 From: Terrence Cole Date: Wed, 2 Aug 2017 10:20:10 -0700 Subject: [PATCH 020/141] MX record values are repesented by a sub-struct. --- octodns/provider/rackspace.py | 4 +- tests/test_octodns_provider_rackspace.py | 51 ++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py index 6af51fa..0eb0b2f 100644 --- a/octodns/provider/rackspace.py +++ b/octodns/provider/rackspace.py @@ -337,9 +337,9 @@ class RackspaceProvider(BaseProvider): return { 'name': remove_trailing_dot(record.fqdn), 'type': record._type, - 'data': remove_trailing_dot(value), + 'data': remove_trailing_dot(value.value), 'ttl': max(record.ttl, 300), - 'priority': record.priority + 'priority': value.priority } @staticmethod diff --git a/tests/test_octodns_provider_rackspace.py b/tests/test_octodns_provider_rackspace.py index 2862871..243ebc4 100644 --- a/tests/test_octodns_provider_rackspace.py +++ b/tests/test_octodns_provider_rackspace.py @@ -342,6 +342,57 @@ class TestRackspaceProvider(TestCase): ExpectedUpdates = None return self._test_apply_with_data(TestData) + def test_apply_create_MX(self): + class TestData(object): + OtherRecords = [ + { + "subdomain": '', + "data": { + 'type': 'MX', + 'ttl': 300, + 'value': { + 'value': 'mail1.example.com.', + 'priority': 1, + } + } + }, + { + "subdomain": 'foo', + "data": { + 'type': 'MX', + 'ttl': 300, + 'value': { + 'value': 'mail2.example.com.', + 'priority': 2 + } + } + } + ] + OwnRecords = { + "totalEntries": 0, + "records": [] + } + ExpectChanges = True + ExpectedAdditions = { + "records": [{ + "name": "foo.unit.tests", + "type": "MX", + "data": "mail2.example.com", + "priority": 2, + "ttl": 300 + }, { + "name": "unit.tests", + "type": "MX", + "data": "mail1.example.com", + "priority": 1, + "ttl": 300 + }] + } + ExpectedDeletions = None + ExpectedUpdates = None + return self._test_apply_with_data(TestData) + + def test_apply_multiple_additions_exploding(self): class TestData(object): OtherRecords = [ From 3f369712e4cbff55b1ac3a2d634a552c77bd67be Mon Sep 17 00:00:00 2001 From: Terrence Cole Date: Wed, 2 Aug 2017 10:51:12 -0700 Subject: [PATCH 021/141] Updates need to be able to create records as well as delete them. --- octodns/provider/rackspace.py | 18 +++++++--- tests/test_octodns_provider_rackspace.py | 46 ++++++++++++++++++++---- 2 files changed, 54 insertions(+), 10 deletions(-) diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py index 0eb0b2f..a856295 100644 --- a/octodns/provider/rackspace.py +++ b/octodns/provider/rackspace.py @@ -371,8 +371,11 @@ class RackspaceProvider(BaseProvider): return [record.value] def _mod_Create(self, change): + return self._create_given_change_values(change, self._get_values(change.new)) + + def _create_given_change_values(self, change, values): out = [] - for value in self._get_values(change.new): + for value in values: transformer = getattr(self, "_record_for_{}".format(change.new._type)) out.append(transformer(change.new, value)) return out @@ -386,8 +389,14 @@ class RackspaceProvider(BaseProvider): deleted_values = set(existing_values) - set(new_values) delete_out = self._delete_given_change_values(change, deleted_values) + # An increase in number of values in an update record needs + # to get upgraded into a Create change for the added values. + create_values = set(new_values) - set(existing_values) + create_out = self._create_given_change_values(change, create_values) + update_out = [] - for value in new_values: + update_values = set(new_values).intersection(set(existing_values)) + for value in update_values: transformer = getattr(self, "_record_for_{}".format(change.new._type)) prior_rs_record = transformer(change.existing, value) prior_key = self._key_for_record(prior_rs_record) @@ -398,7 +407,7 @@ class RackspaceProvider(BaseProvider): update_out.append(next_rs_record) self._id_map[next_key] = self._id_map[prior_key] del self._id_map[prior_key] - return update_out, delete_out + return create_out, update_out, delete_out def _mod_Delete(self, change): return self._delete_given_change_values(change, self._get_values(change.existing)) @@ -427,7 +436,8 @@ class RackspaceProvider(BaseProvider): if change.__class__.__name__ == 'Create': creates += self._mod_Create(change) elif change.__class__.__name__ == 'Update': - add_updates, add_deletes = self._mod_Update(change) + add_creates, add_updates, add_deletes = self._mod_Update(change) + creates += add_creates updates += add_updates deletes += add_deletes elif change.__class__.__name__ == 'Delete': diff --git a/tests/test_octodns_provider_rackspace.py b/tests/test_octodns_provider_rackspace.py index 243ebc4..ce80112 100644 --- a/tests/test_octodns_provider_rackspace.py +++ b/tests/test_octodns_provider_rackspace.py @@ -392,7 +392,6 @@ class TestRackspaceProvider(TestCase): ExpectedUpdates = None return self._test_apply_with_data(TestData) - def test_apply_multiple_additions_exploding(self): class TestData(object): OtherRecords = [ @@ -674,6 +673,41 @@ class TestRackspaceProvider(TestCase): } return self._test_apply_with_data(TestData) + def test_apply_update_TXT(self): + class TestData(object): + OtherRecords = [ + { + "subdomain": '', + "data": { + 'type': 'TXT', + 'ttl': 300, + 'value': 'othervalue' + } + } + ] + OwnRecords = { + "totalEntries": 1, + "records": [{ + "name": "unit.tests", + "id": "TXT-111111", + "type": "TXT", + "data": "somevalue", + "ttl": 300 + }] + } + ExpectChanges = True + ExpectedAdditions = { + "records": [{ + "name": "unit.tests", + "type": "TXT", + "data": "othervalue", + "ttl": 300 + }] + } + ExpectedDeletions = 'id=TXT-111111' + ExpectedUpdates = None + return self._test_apply_with_data(TestData) + def test_apply_multiple_updates(self): class TestData(object): OtherRecords = [ @@ -712,15 +746,15 @@ class TestRackspaceProvider(TestCase): ExpectedAdditions = None ExpectedDeletions = None ExpectedUpdates = { - "records": [{ + "records": [ { "name": "unit.tests", - "id": "A-111111", - "data": "1.2.3.4", + "id": "A-222222", + "data": "1.2.3.5", "ttl": 3600 }, { "name": "unit.tests", - "id": "A-222222", - "data": "1.2.3.5", + "id": "A-111111", + "data": "1.2.3.4", "ttl": 3600 }, { "name": "unit.tests", From f26f77fcaec9257a6d3066818f45f07c2176b4e2 Mon Sep 17 00:00:00 2001 From: Terrence Cole Date: Wed, 2 Aug 2017 16:58:13 -0700 Subject: [PATCH 022/141] Force keys to be unicode. --- octodns/provider/rackspace.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py index a856295..c593150 100644 --- a/octodns/provider/rackspace.py +++ b/octodns/provider/rackspace.py @@ -146,8 +146,16 @@ class RackspaceProvider(BaseProvider): return self._request('DELETE', path, data=data) @staticmethod - def _key_for_record(rs_record): - return rs_record['type'], rs_record['name'], rs_record['data'] + def _as_unicode(s, codec): + if not isinstance(s, unicode): + return unicode(s, codec) + return s + + @classmethod + def _key_for_record(cls, rs_record): + return cls._as_unicode(rs_record['type'], 'ascii'),\ + cls._as_unicode(rs_record['name'], 'utf-8'),\ + cls._as_unicode(rs_record['data'], 'utf-8'),\ def _data_for_multiple(self, rrset): # TODO: geo not supported From b1ef8a8f8d78b9d535fc0af7ecc3bb607e171f4c Mon Sep 17 00:00:00 2001 From: Terrence Cole Date: Thu, 10 Aug 2017 10:50:38 -0700 Subject: [PATCH 023/141] Delete first and create last to avoid having create coalesce into an update unexpectedly. --- octodns/provider/rackspace.py | 14 ++++----- tests/test_octodns_provider_rackspace.py | 37 ++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 7 deletions(-) diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py index c593150..f10f0b6 100644 --- a/octodns/provider/rackspace.py +++ b/octodns/provider/rackspace.py @@ -155,7 +155,7 @@ class RackspaceProvider(BaseProvider): def _key_for_record(cls, rs_record): return cls._as_unicode(rs_record['type'], 'ascii'),\ cls._as_unicode(rs_record['name'], 'utf-8'),\ - cls._as_unicode(rs_record['data'], 'utf-8'),\ + cls._as_unicode(rs_record['data'], 'utf-8') def _data_for_multiple(self, rrset): # TODO: geo not supported @@ -451,15 +451,15 @@ class RackspaceProvider(BaseProvider): elif change.__class__.__name__ == 'Delete': deletes += self._mod_Delete(change) - if creates: - data = {"records": sorted(creates, key=lambda v: v['type'] + v['name'] + v.get('data', ''))} - self._post('domains/{}/records'.format(domain_id), data=data) + if deletes: + params = "&".join(sorted(deletes)) + self._delete('domains/{}/records?{}'.format(domain_id, params)) if updates: data = {"records": sorted(updates, key=lambda v: v['name'])} self._put('domains/{}/records'.format(domain_id), data=data) - if deletes: - params = "&".join(sorted(deletes)) - self._delete('domains/{}/records?{}'.format(domain_id, params)) + if creates: + data = {"records": sorted(creates, key=lambda v: v['type'] + v['name'] + v.get('data', ''))} + self._post('domains/{}/records'.format(domain_id), data=data) diff --git a/tests/test_octodns_provider_rackspace.py b/tests/test_octodns_provider_rackspace.py index ce80112..ce4b71f 100644 --- a/tests/test_octodns_provider_rackspace.py +++ b/tests/test_octodns_provider_rackspace.py @@ -708,6 +708,43 @@ class TestRackspaceProvider(TestCase): ExpectedUpdates = None return self._test_apply_with_data(TestData) + def test_apply_update_MX(self): + class TestData(object): + OtherRecords = [ + { + "subdomain": '', + "data": { + 'type': 'MX', + 'ttl': 300, + 'value': {u'priority': 50, u'value': 'mx.test.com.'} + } + } + ] + OwnRecords = { + "totalEntries": 1, + "records": [{ + "name": "unit.tests", + "id": "MX-111111", + "type": "MX", + "priority": 20, + "data": "mx.test.com", + "ttl": 300 + }] + } + ExpectChanges = True + ExpectedAdditions = { + "records": [{ + "name": "unit.tests", + "type": "MX", + "priority": 50, + "data": "mx.test.com", + "ttl": 300 + }] + } + ExpectedDeletions = 'id=MX-111111' + ExpectedUpdates = None + return self._test_apply_with_data(TestData) + def test_apply_multiple_updates(self): class TestData(object): OtherRecords = [ From 17c9b8b5277b79b821933b8861718b1212da4feb Mon Sep 17 00:00:00 2001 From: Terrence Cole Date: Fri, 11 Aug 2017 14:02:14 -0700 Subject: [PATCH 024/141] Get lint and coverage tools clean. --- octodns/provider/rackspace.py | 180 ++++-------- tests/test_octodns_provider_rackspace.py | 343 ++++++++--------------- 2 files changed, 174 insertions(+), 349 deletions(-) diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py index f10f0b6..1148e0b 100644 --- a/octodns/provider/rackspace.py +++ b/octodns/provider/rackspace.py @@ -10,7 +10,7 @@ import logging import string import time -from ..record import Create, Record +from ..record import Record from .base import BaseProvider @@ -26,21 +26,6 @@ def remove_trailing_dot(s): return s[:-1] -def strip_quotes(s): - assert s - assert len(s) > 2 - assert s[0] == '"' - assert s[-1] == '"' - return s[1:-1] - - -def add_quotes(s): - assert s - assert s[0] != '"' - assert s[-1] != '"' - return '"{}"'.format(s) - - def escape_semicolon(s): assert s return string.replace(s, ';', '\;') @@ -55,7 +40,8 @@ class RackspaceProvider(BaseProvider): SUPPORTS_GEO = False TIMEOUT = 5 - def __init__(self, id, username, api_key, ratelimit_delay, *args, **kwargs): + def __init__(self, id, username, api_key, ratelimit_delay, *args, + **kwargs): ''' Rackspace API v1 Provider @@ -84,25 +70,27 @@ class RackspaceProvider(BaseProvider): def _get_auth_token(self, username, api_key): ret = post('https://identity.api.rackspacecloud.com/v2.0/tokens', - json={"auth": {"RAX-KSKEY:apiKeyCredentials": {"username": username, "apiKey": api_key}}}, + json={"auth": { + "RAX-KSKEY:apiKeyCredentials": {"username": username, + "apiKey": api_key}}}, ) - cloud_dns_endpoint = [x for x in ret.json()['access']['serviceCatalog'] if x['name'] == 'cloudDNS'][0]['endpoints'][0]['publicURL'] + cloud_dns_endpoint = \ + [x for x in ret.json()['access']['serviceCatalog'] if + x['name'] == 'cloudDNS'][0]['endpoints'][0]['publicURL'] return ret.json()['access']['token']['id'], cloud_dns_endpoint def _get_zone_id_for(self, zone): ret = self._request('GET', 'domains', pagination_key='domains') time.sleep(self.ratelimit_delay) - if ret: - return [x for x in ret if x['name'] == zone.name[:-1]][0]['id'] - else: - return None + return [x for x in ret if x['name'] == zone.name[:-1]][0]['id'] def _request(self, method, path, data=None, pagination_key=None): self.log.debug('_request: method=%s, path=%s', method, path) url = '{}/{}'.format(self.dns_endpoint, path) if pagination_key: - return self._paginated_request_for_url(method, url, data, pagination_key) + return self._paginated_request_for_url(method, url, data, + pagination_key) else: return self._request_for_url(method, url, data) @@ -122,26 +110,22 @@ class RackspaceProvider(BaseProvider): resp.raise_for_status() acc.extend(resp.json()[pagination_key]) - next_page = [x for x in resp.json().get('links', []) if x['rel'] == 'next'] + next_page = [x for x in resp.json().get('links', []) if + x['rel'] == 'next'] if next_page: url = next_page[0]['href'] - acc.extend(self._paginated_request_for_url(method, url, data, pagination_key)) + acc.extend(self._paginated_request_for_url(method, url, data, + pagination_key)) return acc else: return acc - def _get(self, path, data=None): - return self._request('GET', path, data=data) - def _post(self, path, data=None): return self._request('POST', path, data=data) def _put(self, path, data=None): return self._request('PUT', path, data=data) - def _patch(self, path, data=None): - return self._request('PATCH', path, data=data) - def _delete(self, path, data=None): return self._request('DELETE', path, data=data) @@ -153,9 +137,9 @@ class RackspaceProvider(BaseProvider): @classmethod def _key_for_record(cls, rs_record): - return cls._as_unicode(rs_record['type'], 'ascii'),\ - cls._as_unicode(rs_record['name'], 'utf-8'),\ - cls._as_unicode(rs_record['data'], 'utf-8') + return cls._as_unicode(rs_record['type'], 'ascii'), \ + cls._as_unicode(rs_record['name'], 'utf-8'), \ + cls._as_unicode(rs_record['data'], 'utf-8') def _data_for_multiple(self, rrset): # TODO: geo not supported @@ -210,67 +194,14 @@ class RackspaceProvider(BaseProvider): 'ttl': rrset[0]['ttl'] } - def _data_for_NAPTR(self, rrset): - raise NotImplementedError("Missing support for reading NAPTR records") - # values = [] - # for record in rrset['records']: - # order, preference, flags, service, regexp, replacement = \ - # record['content'].split(' ', 5) - # values.append({ - # 'order': order, - # 'preference': preference, - # 'flags': flags[1:-1], - # 'service': service[1:-1], - # 'regexp': regexp[1:-1], - # 'replacement': replacement, - # }) - # return { - # 'type': rrset['type'], - # 'values': values, - # 'ttl': rrset['ttl'] - # } - - def _data_for_SSHFP(self, rrset): - raise NotImplementedError("Missing support for reading SSHFP records") - # values = [] - # for record in rrset['records']: - # algorithm, fingerprint_type, fingerprint = \ - # record['content'].split(' ', 2) - # values.append({ - # 'algorithm': algorithm, - # 'fingerprint_type': fingerprint_type, - # 'fingerprint': fingerprint, - # }) - # return { - # 'type': rrset['type'], - # 'values': values, - # 'ttl': rrset['ttl'] - # } - - def _data_for_SRV(self, rrset): - raise NotImplementedError("Missing support for reading SRV records") - # values = [] - # for record in rrset['records']: - # priority, weight, port, target = \ - # record['content'].split(' ', 3) - # values.append({ - # 'priority': priority, - # 'weight': weight, - # 'port': port, - # 'target': target, - # }) - # return { - # 'type': rrset['type'], - # 'values': values, - # 'ttl': rrset['ttl'] - # } - def populate(self, zone, target=False): self.log.debug('populate: name=%s', zone.name) resp_data = None try: domain_id = self._get_zone_id_for(zone) - resp_data = self._request('GET', 'domains/{}/records'.format(domain_id), pagination_key='records') + resp_data = self._request('GET', + 'domains/{}/records'.format(domain_id), + pagination_key='records') self.log.debug('populate: loaded') except HTTPError as e: if e.response.status_code == 401: @@ -286,12 +217,12 @@ class RackspaceProvider(BaseProvider): if resp_data: records = self._group_records(resp_data) for record_type, records_of_type in records.items(): - if record_type == 'SOA': - continue for raw_record_name, record_set in records_of_type.items(): - data_for = getattr(self, '_data_for_{}'.format(record_type)) + data_for = getattr(self, + '_data_for_{}'.format(record_type)) record_name = zone.hostname_from_fqdn(raw_record_name) - record = Record.new(zone, record_name, data_for(record_set), + record = Record.new(zone, record_name, + data_for(record_set), source=self) zone.add_record(record) @@ -313,6 +244,7 @@ class RackspaceProvider(BaseProvider): 'data': value, 'ttl': max(record.ttl, 300), } + _record_for_A = _record_for_single _record_for_AAAA = _record_for_single @@ -324,6 +256,7 @@ class RackspaceProvider(BaseProvider): 'data': remove_trailing_dot(value), 'ttl': max(record.ttl, 300), } + _record_for_NS = _record_for_named _record_for_ALIAS = _record_for_named _record_for_CNAME = _record_for_named @@ -337,6 +270,7 @@ class RackspaceProvider(BaseProvider): 'data': unescape_semicolon(value), 'ttl': max(record.ttl, 300), } + _record_for_SPF = _record_for_textual _record_for_TXT = _record_for_textual @@ -350,27 +284,15 @@ class RackspaceProvider(BaseProvider): 'priority': value.priority } - @staticmethod - def _record_for_SRV(record, value): - raise NotImplementedError("Missing support for writing SRV records") - - def _record_for_NAPTR(self, record): - raise NotImplementedError("Missing support for writing NAPTR records") - # return [{ - # 'content': '{} {} "{}" "{}" "{}" {}'.format(v.order, v.preference, - # v.flags, v.service, - # v.regexp, - # v.replacement), - # 'disabled': False - # } for v in record.values] - - def _record_for_SSHFP(self, record): - raise NotImplementedError("Missing support for writing SSHFP records") - # return [{ - # 'content': '{} {} {}'.format(v.algorithm, v.fingerprint_type, - # v.fingerprint), - # 'disabled': False - # } for v in record.values] + def _record_for_unsupported(self, record, value): + raise NotImplementedError( + "Missing support for writing {} records".format( + record.__class__.__name__)) + + _record_for_SOA = _record_for_unsupported + _record_for_SRV = _record_for_unsupported + _record_for_NAPTR = _record_for_unsupported + _record_for_SSHFP = _record_for_unsupported def _get_values(self, record): try: @@ -379,12 +301,14 @@ class RackspaceProvider(BaseProvider): return [record.value] def _mod_Create(self, change): - return self._create_given_change_values(change, self._get_values(change.new)) + return self._create_given_change_values(change, + self._get_values(change.new)) def _create_given_change_values(self, change, values): out = [] for value in values: - transformer = getattr(self, "_record_for_{}".format(change.new._type)) + transformer = getattr(self, + "_record_for_{}".format(change.new._type)) out.append(transformer(change.new, value)) return out @@ -405,7 +329,8 @@ class RackspaceProvider(BaseProvider): update_out = [] update_values = set(new_values).intersection(set(existing_values)) for value in update_values: - transformer = getattr(self, "_record_for_{}".format(change.new._type)) + transformer = getattr(self, + "_record_for_{}".format(change.new._type)) prior_rs_record = transformer(change.existing, value) prior_key = self._key_for_record(prior_rs_record) next_rs_record = transformer(change.new, value) @@ -418,12 +343,14 @@ class RackspaceProvider(BaseProvider): return create_out, update_out, delete_out def _mod_Delete(self, change): - return self._delete_given_change_values(change, self._get_values(change.existing)) + return self._delete_given_change_values(change, self._get_values( + change.existing)) def _delete_given_change_values(self, change, values): out = [] for value in values: - transformer = getattr(self, "_record_for_{}".format(change.existing._type)) + transformer = getattr(self, "_record_for_{}".format( + change.existing._type)) rs_record = transformer(change.existing, value) key = self._key_for_record(rs_record) out.append('id=' + self._id_map[key]) @@ -444,11 +371,13 @@ class RackspaceProvider(BaseProvider): if change.__class__.__name__ == 'Create': creates += self._mod_Create(change) elif change.__class__.__name__ == 'Update': - add_creates, add_updates, add_deletes = self._mod_Update(change) + add_creates, add_updates, add_deletes = self._mod_Update( + change) creates += add_creates updates += add_updates deletes += add_deletes - elif change.__class__.__name__ == 'Delete': + else: + assert change.__class__.__name__ == 'Delete' deletes += self._mod_Delete(change) if deletes: @@ -460,6 +389,7 @@ class RackspaceProvider(BaseProvider): self._put('domains/{}/records'.format(domain_id), data=data) if creates: - data = {"records": sorted(creates, key=lambda v: v['type'] + v['name'] + v.get('data', ''))} + data = {"records": sorted(creates, key=lambda v: v['type'] + + v['name'] + + v.get('data', ''))} self._post('domains/{}/records'.format(domain_id), data=data) - diff --git a/tests/test_octodns_provider_rackspace.py b/tests/test_octodns_provider_rackspace.py index ce4b71f..725b60a 100644 --- a/tests/test_octodns_provider_rackspace.py +++ b/tests/test_octodns_provider_rackspace.py @@ -11,6 +11,8 @@ from os.path import dirname, join from unittest import TestCase from urlparse import urlparse +from nose.tools import assert_raises + from requests import HTTPError from requests_mock import ANY, mock as requests_mock @@ -19,8 +21,6 @@ from octodns.provider.yaml import YamlProvider from octodns.record import Record from octodns.zone import Zone -from pprint import pprint - EMPTY_TEXT = ''' { "totalEntries" : 0, @@ -40,16 +40,14 @@ with open('./tests/fixtures/rackspace-sample-recordset-page1.json') as fh: with open('./tests/fixtures/rackspace-sample-recordset-page2.json') as fh: RECORDS_PAGE_2 = fh.read() -with open('./tests/fixtures/rackspace-sample-recordset-existing-nameservers.json') as fh: - RECORDS_EXISTING_NAMESERVERS = fh.read() - class TestRackspaceProvider(TestCase): def setUp(self): self.maxDiff = 1000 with requests_mock() as mock: mock.post(ANY, status_code=200, text=AUTH_RESPONSE) - self.provider = RackspaceProvider(id, 'test', 'api-key', '0') + self.provider = RackspaceProvider('identity', 'test', 'api-key', + '0') self.assertTrue(mock.called_once) def test_bad_auth(self): @@ -85,9 +83,12 @@ class TestRackspaceProvider(TestCase): def test_multipage_populate(self): with requests_mock() as mock: - mock.get(re.compile('domains$'), status_code=200, text=LIST_DOMAINS_RESPONSE) - mock.get(re.compile('records'), status_code=200, text=RECORDS_PAGE_1) - mock.get(re.compile('records.*offset=3'), status_code=200, text=RECORDS_PAGE_2) + mock.get(re.compile('domains$'), status_code=200, + text=LIST_DOMAINS_RESPONSE) + mock.get(re.compile('records'), status_code=200, + text=RECORDS_PAGE_1) + mock.get(re.compile('records.*offset=3'), status_code=200, + text=RECORDS_PAGE_2) zone = Zone('unit.tests.', []) self.provider.populate(zone) @@ -105,9 +106,12 @@ class TestRackspaceProvider(TestCase): # No diffs == no changes with requests_mock() as mock: - mock.get(re.compile('domains$'), status_code=200, text=LIST_DOMAINS_RESPONSE) - mock.get(re.compile('records'), status_code=200, text=RECORDS_PAGE_1) - mock.get(re.compile('records.*offset=3'), status_code=200, text=RECORDS_PAGE_2) + mock.get(re.compile('domains$'), status_code=200, + text=LIST_DOMAINS_RESPONSE) + mock.get(re.compile('records'), status_code=200, + text=RECORDS_PAGE_1) + mock.get(re.compile('records.*offset=3'), status_code=200, + text=RECORDS_PAGE_2) zone = Zone('unit.tests.', []) self.provider.populate(zone) @@ -127,7 +131,8 @@ class TestRackspaceProvider(TestCase): 'values': ['8.8.8.8.', '9.9.9.9.'] })) with requests_mock() as mock: - mock.get(re.compile('domains$'), status_code=200, text=LIST_DOMAINS_RESPONSE) + mock.get(re.compile('domains$'), status_code=200, + text=LIST_DOMAINS_RESPONSE) mock.get(re.compile('records'), status_code=200, text=EMPTY_TEXT) plan = self.provider.plan(expected) @@ -141,23 +146,28 @@ class TestRackspaceProvider(TestCase): # expected.add_record(Record.new(expected, 'foo', '1.2.3.4')) with requests_mock() as list_mock: - list_mock.get(re.compile('domains$'), status_code=200, text=LIST_DOMAINS_RESPONSE) - list_mock.get(re.compile('records'), status_code=200, json={'records': [ - {'type': 'A', - 'name': 'foo.example.com', - 'id': 'A-111111', - 'data': '1.2.3.4', - 'ttl': 300}]}) + list_mock.get(re.compile('domains$'), status_code=200, + text=LIST_DOMAINS_RESPONSE) + list_mock.get(re.compile('records'), status_code=200, + json={'records': [ + {'type': 'A', + 'name': 'foo.example.com', + 'id': 'A-111111', + 'data': '1.2.3.4', + 'ttl': 300}]}) plan = self.provider.plan(expected) self.assertTrue(list_mock.called) self.assertEqual(1, len(plan.changes)) - self.assertTrue(plan.changes[0].existing.fqdn == 'foo.example.com.') + self.assertTrue( + plan.changes[0].existing.fqdn == 'foo.example.com.') with requests_mock() as mock: def _assert_deleting(request, context): parts = urlparse(request.url) self.assertEqual('id=A-111111', parts.query) - mock.get(re.compile('domains$'), status_code=200, text=LIST_DOMAINS_RESPONSE) + + mock.get(re.compile('domains$'), status_code=200, + text=LIST_DOMAINS_RESPONSE) mock.delete(re.compile('domains/.*/records?.*'), status_code=202, text=_assert_deleting) self.provider.apply(plan) @@ -166,11 +176,14 @@ class TestRackspaceProvider(TestCase): def _test_apply_with_data(self, data): expected = Zone('unit.tests.', []) for record in data.OtherRecords: - expected.add_record(Record.new(expected, record['subdomain'], record['data'])) + expected.add_record( + Record.new(expected, record['subdomain'], record['data'])) with requests_mock() as list_mock: - list_mock.get(re.compile('domains$'), status_code=200, text=LIST_DOMAINS_RESPONSE) - list_mock.get(re.compile('records'), status_code=200, json=data.OwnRecords) + list_mock.get(re.compile('domains$'), status_code=200, + text=LIST_DOMAINS_RESPONSE) + list_mock.get(re.compile('records'), status_code=200, + json=data.OwnRecords) plan = self.provider.plan(expected) self.assertTrue(list_mock.called) if not data.ExpectChanges: @@ -184,25 +197,32 @@ class TestRackspaceProvider(TestCase): def _assert_sending_right_body(request, _context): called.add(request.method) if request.method != 'DELETE': - self.assertEqual(request.headers['content-type'], 'application/json') - self.assertDictEqual(expected, json.loads(request.body)) + self.assertEqual(request.headers['content-type'], + 'application/json') + self.assertDictEqual(expected, + json.loads(request.body)) else: parts = urlparse(request.url) self.assertEqual(expected, parts.query) return '' + return _assert_sending_right_body - mock.get(re.compile('domains$'), status_code=200, text=LIST_DOMAINS_RESPONSE) + mock.get(re.compile('domains$'), status_code=200, + text=LIST_DOMAINS_RESPONSE) mock.post(re.compile('domains/.*/records$'), status_code=202, - text=make_assert_sending_right_body(data.ExpectedAdditions)) + text=make_assert_sending_right_body( + data.ExpectedAdditions)) mock.delete(re.compile('domains/.*/records?.*'), status_code=202, - text=make_assert_sending_right_body(data.ExpectedDeletions)) + text=make_assert_sending_right_body( + data.ExpectedDeletions)) mock.put(re.compile('domains/.*/records$'), status_code=202, text=make_assert_sending_right_body(data.ExpectedUpdates)) self.provider.apply(plan) self.assertTrue(data.ExpectedAdditions is None or "POST" in called) - self.assertTrue(data.ExpectedDeletions is None or "DELETE" in called) + self.assertTrue( + data.ExpectedDeletions is None or "DELETE" in called) self.assertTrue(data.ExpectedUpdates is None or "PUT" in called) def test_apply_no_change_empty(self): @@ -216,6 +236,7 @@ class TestRackspaceProvider(TestCase): ExpectedAdditions = None ExpectedDeletions = None ExpectedUpdates = None + return self._test_apply_with_data(TestData) def test_apply_no_change_a_records(self): @@ -256,6 +277,7 @@ class TestRackspaceProvider(TestCase): ExpectedAdditions = None ExpectedDeletions = None ExpectedUpdates = None + return self._test_apply_with_data(TestData) def test_apply_no_change_a_records_cross_zone(self): @@ -298,6 +320,7 @@ class TestRackspaceProvider(TestCase): ExpectedAdditions = None ExpectedDeletions = None ExpectedUpdates = None + return self._test_apply_with_data(TestData) def test_apply_one_addition(self): @@ -340,6 +363,7 @@ class TestRackspaceProvider(TestCase): } ExpectedDeletions = None ExpectedUpdates = None + return self._test_apply_with_data(TestData) def test_apply_create_MX(self): @@ -390,9 +414,39 @@ class TestRackspaceProvider(TestCase): } ExpectedDeletions = None ExpectedUpdates = None + return self._test_apply_with_data(TestData) - def test_apply_multiple_additions_exploding(self): + def test_apply_create_SRV(self): + class TestData(object): + OtherRecords = [ + { + "subdomain": '_a.b', + "data": { + 'type': 'SRV', + 'ttl': 300, + 'value': { + 'priority': 20, + 'weight': 999, + 'port': 999, + 'target': 'foo' + } + } + } + ] + OwnRecords = { + "totalEntries": 0, + "records": [] + } + ExpectChanges = True + ExpectedAdditions = [{}] + ExpectedDeletions = None + ExpectedUpdates = None + + assert_raises(NotImplementedError, self._test_apply_with_data, + TestData) + + def test_apply_multiple_additions_splatting(self): class TestData(object): OtherRecords = [ { @@ -447,33 +501,33 @@ class TestRackspaceProvider(TestCase): } ExpectedDeletions = None ExpectedUpdates = None + return self._test_apply_with_data(TestData) def test_apply_multiple_additions_namespaced(self): class TestData(object): OtherRecords = [{ - "subdomain": 'foo', - "data": { - 'type': 'A', - 'ttl': 300, - 'value': '1.2.3.4' - } - }, { - "subdomain": 'bar', - "data": { - 'type': 'A', - 'ttl': 300, - 'value': '1.2.3.4' - } - }, - { - "subdomain": 'foo', - "data": { - 'type': 'NS', - 'ttl': 300, - 'value': 'ns.example.com.' - } - }] + "subdomain": 'foo', + "data": { + 'type': 'A', + 'ttl': 300, + 'value': '1.2.3.4' + } + }, { + "subdomain": 'bar', + "data": { + 'type': 'A', + 'ttl': 300, + 'value': '1.2.3.4' + } + }, { + "subdomain": 'foo', + "data": { + 'type': 'NS', + 'ttl': 300, + 'value': 'ns.example.com.' + } + }] OwnRecords = { "totalEntries": 0, "records": [] @@ -499,6 +553,7 @@ class TestRackspaceProvider(TestCase): } ExpectedDeletions = None ExpectedUpdates = None + return self._test_apply_with_data(TestData) def test_apply_single_deletion(self): @@ -524,6 +579,7 @@ class TestRackspaceProvider(TestCase): ExpectedAdditions = None ExpectedDeletions = "id=A-111111&id=NS-111111" ExpectedUpdates = None + return self._test_apply_with_data(TestData) def test_apply_multiple_deletions(self): @@ -577,6 +633,7 @@ class TestRackspaceProvider(TestCase): "ttl": 300 }] } + return self._test_apply_with_data(TestData) def test_apply_multiple_deletions_cross_zone(self): @@ -617,6 +674,7 @@ class TestRackspaceProvider(TestCase): ExpectedAdditions = None ExpectedDeletions = "id=A-222222&id=A-333333" ExpectedUpdates = None + return self._test_apply_with_data(TestData) def test_apply_delete_cname(self): @@ -636,6 +694,7 @@ class TestRackspaceProvider(TestCase): ExpectedAdditions = None ExpectedDeletions = "id=CNAME-111111" ExpectedUpdates = None + return self._test_apply_with_data(TestData) def test_apply_single_update(self): @@ -671,6 +730,7 @@ class TestRackspaceProvider(TestCase): "ttl": 3600 }] } + return self._test_apply_with_data(TestData) def test_apply_update_TXT(self): @@ -706,6 +766,7 @@ class TestRackspaceProvider(TestCase): } ExpectedDeletions = 'id=TXT-111111' ExpectedUpdates = None + return self._test_apply_with_data(TestData) def test_apply_update_MX(self): @@ -743,6 +804,7 @@ class TestRackspaceProvider(TestCase): } ExpectedDeletions = 'id=MX-111111' ExpectedUpdates = None + return self._test_apply_with_data(TestData) def test_apply_multiple_updates(self): @@ -783,7 +845,7 @@ class TestRackspaceProvider(TestCase): ExpectedAdditions = None ExpectedDeletions = None ExpectedUpdates = { - "records": [ { + "records": [{ "name": "unit.tests", "id": "A-222222", "data": "1.2.3.5", @@ -800,6 +862,7 @@ class TestRackspaceProvider(TestCase): "ttl": 3600 }] } + return self._test_apply_with_data(TestData) def test_apply_multiple_updates_cross_zone(self): @@ -854,173 +917,5 @@ class TestRackspaceProvider(TestCase): "ttl": 3600 }] } - return self._test_apply_with_data(TestData) - - """ - def test_provider(self): - expected = self._load_full_config() - - # No existing records -> creates for every record in expected - with requests_mock() as mock: - mock.get(re.compile('domains$'), status_code=200, text=LIST_DOMAINS_RESPONSE) - mock.get(re.compile('records'), status_code=200, text=EMPTY_TEXT) - - plan = self.provider.plan(expected) - self.assertTrue(mock.called) - self.assertEquals(len(expected.records), len(plan.changes)) - # Used in a minute - def assert_rrsets_callback(request, context): - data = loads(request.body) - self.assertEquals(expected_n, len(data['rrsets'])) - return '' - - with requests_mock() as mock: - # post 201, is response to the create with data - mock.patch(ANY, status_code=201, text=assert_rrsets_callback) - - self.assertEquals(expected_n, self.provider.apply(plan)) - - # Non-existent zone -> creates for every record in expected - # OMG this is fucking ugly, probably better to ditch requests_mocks and - # just mock things for real as it doesn't seem to provide a way to get - # at the request params or verify that things were called from what I - # can tell - not_found = {'error': "Could not find domain 'unit.tests.'"} - with requests_mock() as mock: - # get 422's, unknown zone - mock.get(ANY, status_code=422, text='') - # patch 422's, unknown zone - mock.patch(ANY, status_code=422, text=dumps(not_found)) - # post 201, is response to the create with data - mock.post(ANY, status_code=201, text=assert_rrsets_callback) - - plan = self.provider.plan(expected) - self.assertEquals(expected_n, len(plan.changes)) - self.assertEquals(expected_n, self.provider.apply(plan)) - - with requests_mock() as mock: - # get 422's, unknown zone - mock.get(ANY, status_code=422, text='') - # patch 422's, - data = {'error': "Key 'name' not present or not a String"} - mock.patch(ANY, status_code=422, text=dumps(data)) - - with self.assertRaises(HTTPError) as ctx: - plan = self.provider.plan(expected) - self.provider.apply(plan) - response = ctx.exception.response - self.assertEquals(422, response.status_code) - self.assertTrue('error' in response.json()) - - with requests_mock() as mock: - # get 422's, unknown zone - mock.get(ANY, status_code=422, text='') - # patch 500's, things just blew up - mock.patch(ANY, status_code=500, text='') - - with self.assertRaises(HTTPError): - plan = self.provider.plan(expected) - self.provider.apply(plan) - - with requests_mock() as mock: - # get 422's, unknown zone - mock.get(ANY, status_code=422, text='') - # patch 500's, things just blew up - mock.patch(ANY, status_code=422, text=dumps(not_found)) - # post 422's, something wrong with create - mock.post(ANY, status_code=422, text='Hello Word!') - - with self.assertRaises(HTTPError): - plan = self.provider.plan(expected) - self.provider.apply(plan) - """ - - def test_plan_no_changes(self): - expected = Zone('unit.tests.', []) - expected.add_record(Record.new(expected, '', { - 'type': 'NS', - 'ttl': 600, - 'values': ['ns1.example.com.', 'ns2.example.com.'] - })) - expected.add_record(Record.new(expected, '', { - 'type': 'A', - 'ttl': 600, - 'value': '1.2.3.4' - })) - - with requests_mock() as mock: - mock.get(re.compile('domains/.*/records'), status_code=200, text=RECORDS_EXISTING_NAMESERVERS) - mock.get(re.compile('domains$'), status_code=200, text=LIST_DOMAINS_RESPONSE) - - plan = self.provider.plan(expected) - - self.assertTrue(mock.called) - self.assertFalse(plan) - - def test_plan_remove_a_record(self): - expected = Zone('unit.tests.', []) - expected.add_record(Record.new(expected, '', { - 'type': 'NS', - 'ttl': 600, - 'values': ['ns1.example.com.', 'ns2.example.com.'] - })) - - with requests_mock() as mock: - mock.get(re.compile('domains/.*/records'), status_code=200, text=RECORDS_EXISTING_NAMESERVERS) - mock.get(re.compile('domains$'), status_code=200, text=LIST_DOMAINS_RESPONSE) - - plan = self.provider.plan(expected) - self.assertTrue(mock.called) - self.assertEquals(1, len(plan.changes)) - self.assertEqual(plan.changes[0].existing.ttl, 600) - self.assertEqual(plan.changes[0].existing.values[0], '1.2.3.4') - - def test_plan_create_a_record(self): - expected = Zone('unit.tests.', []) - expected.add_record(Record.new(expected, '', { - 'type': 'NS', - 'ttl': 600, - 'values': ['ns1.example.com.', 'ns2.example.com.'] - })) - expected.add_record(Record.new(expected, '', { - 'type': 'A', - 'ttl': 600, - 'values': ['1.2.3.4', '1.2.3.5'] - })) - - with requests_mock() as mock: - mock.get(re.compile('domains/.*/records'), status_code=200, text=RECORDS_EXISTING_NAMESERVERS) - mock.get(re.compile('domains$'), status_code=200, text=LIST_DOMAINS_RESPONSE) - - plan = self.provider.plan(expected) - self.assertTrue(mock.called) - self.assertEquals(1, len(plan.changes)) - self.assertEqual(plan.changes[0].new.ttl, 600) - self.assertEqual(plan.changes[0].new.values[0], '1.2.3.4') - self.assertEqual(plan.changes[0].new.values[1], '1.2.3.5') - - def test_plan_change_ttl(self): - expected = Zone('unit.tests.', []) - expected.add_record(Record.new(expected, '', { - 'type': 'NS', - 'ttl': 600, - 'values': ['ns1.example.com.', 'ns2.example.com.'] - })) - expected.add_record(Record.new(expected, '', { - 'type': 'A', - 'ttl': 86400, - 'value': '1.2.3.4' - })) - - with requests_mock() as mock: - mock.get(re.compile('domains/.*/records'), status_code=200, text=RECORDS_EXISTING_NAMESERVERS) - mock.get(re.compile('domains$'), status_code=200, text=LIST_DOMAINS_RESPONSE) - - plan = self.provider.plan(expected) - - self.assertTrue(mock.called) - self.assertEqual(1, len(plan.changes)) - self.assertEqual(plan.changes[0].existing.ttl, 600) - self.assertEqual(plan.changes[0].new.ttl, 86400) - self.assertEqual(plan.changes[0].new.values[0], '1.2.3.4') + return self._test_apply_with_data(TestData) From 32f2a10daf118f98e1cb299c4b220e61111bfba1 Mon Sep 17 00:00:00 2001 From: Terrence Cole Date: Fri, 11 Aug 2017 14:03:03 -0700 Subject: [PATCH 025/141] Apply review feedback to bring logging inline with other providers. --- octodns/provider/rackspace.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py index 1148e0b..6e04c98 100644 --- a/octodns/provider/rackspace.py +++ b/octodns/provider/rackspace.py @@ -52,7 +52,7 @@ class RackspaceProvider(BaseProvider): # The api key that grants access for that user (required) api_key: api-key ''' - self.log = logging.getLogger('RackspaceProvider[{}]'.format(username)) + self.log = logging.getLogger('RackspaceProvider[{}]'.format(id)) super(RackspaceProvider, self).__init__(id, *args, **kwargs) auth_token, dns_endpoint = self._get_auth_token(username, api_key) From 1e7edc97c838c65f664c295b6a51fcf3955d0dd3 Mon Sep 17 00:00:00 2001 From: Terrence Cole Date: Mon, 11 Sep 2017 10:51:03 -0700 Subject: [PATCH 026/141] Update rackspace provider with new names and interfaces. --- octodns/provider/rackspace.py | 19 +++------ tests/test_octodns_provider_rackspace.py | 53 ------------------------ 2 files changed, 5 insertions(+), 67 deletions(-) diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py index 6e04c98..8d913ef 100644 --- a/octodns/provider/rackspace.py +++ b/octodns/provider/rackspace.py @@ -38,6 +38,8 @@ def unescape_semicolon(s): class RackspaceProvider(BaseProvider): SUPPORTS_GEO = False + SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CNAME', 'MX', 'NS', 'PTR', 'SPF', + 'TXT')) TIMEOUT = 5 def __init__(self, id, username, api_key, ratelimit_delay, *args, @@ -153,7 +155,6 @@ class RackspaceProvider(BaseProvider): _data_for_AAAA = _data_for_multiple def _data_for_NS(self, rrset): - # TODO: geo not supported return { 'type': rrset[0]['type'], 'values': [add_trailing_dot(r['data']) for r in rrset], @@ -194,7 +195,7 @@ class RackspaceProvider(BaseProvider): 'ttl': rrset[0]['ttl'] } - def populate(self, zone, target=False): + def populate(self, zone, target=False, lenient=False): self.log.debug('populate: name=%s', zone.name) resp_data = None try: @@ -279,21 +280,11 @@ class RackspaceProvider(BaseProvider): return { 'name': remove_trailing_dot(record.fqdn), 'type': record._type, - 'data': remove_trailing_dot(value.value), + 'data': remove_trailing_dot(value.exchange), 'ttl': max(record.ttl, 300), - 'priority': value.priority + 'priority': value.preference } - def _record_for_unsupported(self, record, value): - raise NotImplementedError( - "Missing support for writing {} records".format( - record.__class__.__name__)) - - _record_for_SOA = _record_for_unsupported - _record_for_SRV = _record_for_unsupported - _record_for_NAPTR = _record_for_unsupported - _record_for_SSHFP = _record_for_unsupported - def _get_values(self, record): try: return record.values diff --git a/tests/test_octodns_provider_rackspace.py b/tests/test_octodns_provider_rackspace.py index 725b60a..da3ffe4 100644 --- a/tests/test_octodns_provider_rackspace.py +++ b/tests/test_octodns_provider_rackspace.py @@ -94,30 +94,6 @@ class TestRackspaceProvider(TestCase): self.provider.populate(zone) self.assertEquals(5, len(zone.records)) - def _load_full_config(self): - expected = Zone('unit.tests.', []) - source = YamlProvider('test', join(dirname(__file__), 'config')) - source.populate(expected) - self.assertEquals(15, len(expected.records)) - return expected - - def test_changes_are_formatted_correctly(self): - expected = self._load_full_config() - - # No diffs == no changes - with requests_mock() as mock: - mock.get(re.compile('domains$'), status_code=200, - text=LIST_DOMAINS_RESPONSE) - mock.get(re.compile('records'), status_code=200, - text=RECORDS_PAGE_1) - mock.get(re.compile('records.*offset=3'), status_code=200, - text=RECORDS_PAGE_2) - - zone = Zone('unit.tests.', []) - self.provider.populate(zone) - changes = expected.changes(zone, self.provider) - self.assertEquals(18, len(changes)) - def test_plan_disappearing_ns_records(self): expected = Zone('unit.tests.', []) expected.add_record(Record.new(expected, '', { @@ -417,35 +393,6 @@ class TestRackspaceProvider(TestCase): return self._test_apply_with_data(TestData) - def test_apply_create_SRV(self): - class TestData(object): - OtherRecords = [ - { - "subdomain": '_a.b', - "data": { - 'type': 'SRV', - 'ttl': 300, - 'value': { - 'priority': 20, - 'weight': 999, - 'port': 999, - 'target': 'foo' - } - } - } - ] - OwnRecords = { - "totalEntries": 0, - "records": [] - } - ExpectChanges = True - ExpectedAdditions = [{}] - ExpectedDeletions = None - ExpectedUpdates = None - - assert_raises(NotImplementedError, self._test_apply_with_data, - TestData) - def test_apply_multiple_additions_splatting(self): class TestData(object): OtherRecords = [ From a0fd1cfdbee31ed438034b8efa46d2cebbef3dea Mon Sep 17 00:00:00 2001 From: Terrence Cole Date: Mon, 11 Sep 2017 10:56:28 -0700 Subject: [PATCH 027/141] Add the new provider to the readme. --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 1f103f1..17d393e 100644 --- a/README.md +++ b/README.md @@ -155,6 +155,7 @@ The above command pulled the existing data out of Route53 and placed the results | [DynProvider](/octodns/provider/dyn.py) | All | Yes | | | [Ns1Provider](/octodns/provider/ns1.py) | All | No | | | [PowerDnsProvider](/octodns/provider/powerdns.py) | All | No | | +| [Rackspace](/octodns/provider/rackspace.py) | A, AAAA, ALIAS, CNAME, MX, NS, PTR, SPF, TXT | No | | | [Route53](/octodns/provider/route53.py) | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | Yes | | | [TinyDNSSource](/octodns/source/tinydns.py) | A, CNAME, MX, NS, PTR | No | read-only | | [YamlProvider](/octodns/provider/yaml.py) | All | Yes | config | From 824ccf6174a7c49a18d08ebdb0411746d728351c Mon Sep 17 00:00:00 2001 From: Terrence Cole Date: Mon, 11 Sep 2017 11:05:04 -0700 Subject: [PATCH 028/141] No need for a comment here as it is documented elsewhere. --- octodns/provider/rackspace.py | 1 - 1 file changed, 1 deletion(-) diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py index 8d913ef..6b17550 100644 --- a/octodns/provider/rackspace.py +++ b/octodns/provider/rackspace.py @@ -144,7 +144,6 @@ class RackspaceProvider(BaseProvider): cls._as_unicode(rs_record['data'], 'utf-8') def _data_for_multiple(self, rrset): - # TODO: geo not supported return { 'type': rrset[0]['type'], 'values': [r['data'] for r in rrset], From b49e1913424114c9502a963d3047223c67559a30 Mon Sep 17 00:00:00 2001 From: Terrence Cole Date: Mon, 11 Sep 2017 11:06:47 -0700 Subject: [PATCH 029/141] Revert changes to args.py for a later PR. --- octodns/cmds/args.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/octodns/cmds/args.py b/octodns/cmds/args.py index 9322461..c84dd92 100644 --- a/octodns/cmds/args.py +++ b/octodns/cmds/args.py @@ -37,8 +37,6 @@ class ArgumentParser(_Base): _help = 'Increase verbosity to get details and help track down issues' self.add_argument('--debug', action='store_true', default=False, help=_help) - self.add_argument('--quiet', action='store_true', default=False, - help='Only print warning and error messages') args = super(ArgumentParser, self).parse_args() self._setup_logging(args, default_log_level) @@ -61,11 +59,7 @@ class ArgumentParser(_Base): handler.setFormatter(Formatter(fmt=fmt)) logger.addHandler(handler) - logger.level = default_log_level - if args.debug: - logger.level = DEBUG - elif args.quiet: - logger.level = WARN + logger.level = DEBUG if args.debug else default_log_level # boto is noisy, set it to warn getLogger('botocore').level = WARN From 11f4359099407b95ea5cfe593665a3b4d87c82ad Mon Sep 17 00:00:00 2001 From: Paul van Brouwershaven Date: Thu, 28 Sep 2017 14:54:53 +0200 Subject: [PATCH 030/141] Add support for included and excluded records `Included` and `Excluded` can be used to filter records for one or more specific provider(s). This can be extremely useful when certain record types are not supported by a provider and you want only that provider to receive an alternative record. See also: https://github.com/github/octodns/issues/26 --- octodns/record.py | 2 ++ octodns/zone.py | 31 ++++++++++++++++++++++++ tests/config/unit.tests.yaml | 12 ++++++++++ tests/fixtures/dnsimple-page-2.json | 32 +++++++++++++++++++++++++ tests/fixtures/powerdns-full-data.json | 12 ++++++++++ tests/test_octodns_manager.py | 14 +++++------ tests/test_octodns_provider_dnsimple.py | 6 ++--- tests/test_octodns_provider_powerdns.py | 8 +++---- tests/test_octodns_provider_yaml.py | 8 +++---- 9 files changed, 107 insertions(+), 18 deletions(-) diff --git a/octodns/record.py b/octodns/record.py index 8ef80be..8a05b9e 100644 --- a/octodns/record.py +++ b/octodns/record.py @@ -124,6 +124,8 @@ class Record(object): octodns = data.get('octodns', {}) self.ignored = octodns.get('ignored', False) + self.excluded = octodns.get('excluded', []) + self.included = octodns.get('included', []) def _data(self): return {'ttl': self.ttl} diff --git a/octodns/zone.py b/octodns/zone.py index 74e5d9e..f715e3f 100644 --- a/octodns/zone.py +++ b/octodns/zone.py @@ -110,10 +110,29 @@ class Zone(object): for record in filter(_is_eligible, self.records): if record.ignored: continue + elif len(record.included) > 0 and \ + target.__class__.__name__ not in record.included: + self.log.debug('changes: skipping record=%s %s - %s not' + ' included ', record.fqdn, record._type, + target.id) + continue + elif target.__class__.__name__ in record.excluded: + self.log.debug('changes: skipping record=%s %s - %s ' + 'excluded ', record.fqdn, record._type, + target.id) + continue try: desired_record = desired_records[record] if desired_record.ignored: continue + elif len(record.included) > 0 and \ + target.__class__.__name__ not in record.included: + self.log.debug('changes: skipping record=%s %s - %s' + 'not included ', record.fqdn, record._type, + target.id) + continue + elif target.__class__.__name__ in record.excluded: + continue except KeyError: if not target.supports(record): self.log.debug('changes: skipping record=%s %s - %s does ' @@ -141,6 +160,18 @@ class Zone(object): for record in filter(_is_eligible, desired.records - self.records): if record.ignored: continue + elif len(record.included) > 0 and \ + target.__class__.__name__ not in record.included: + self.log.debug('changes: skipping record=%s %s - %s not' + ' included ', record.fqdn, record._type, + target.id) + continue + elif target.__class__.__name__ in record.excluded: + self.log.debug('changes: skipping record=%s %s - %s ' + 'excluded ', record.fqdn, record._type, + target.id) + continue + if not target.supports(record): self.log.debug('changes: skipping record=%s %s - %s does not ' 'support it', record.fqdn, record._type, diff --git a/tests/config/unit.tests.yaml b/tests/config/unit.tests.yaml index 5241406..548dc8f 100644 --- a/tests/config/unit.tests.yaml +++ b/tests/config/unit.tests.yaml @@ -56,11 +56,23 @@ cname: ttl: 300 type: CNAME value: unit.tests. +excluded: + octodns: + excluded: + - CloudflareProvider + type: CNAME + value: excluded.unit.tests. ignored: octodns: ignored: true type: A value: 9.9.9.9 +included: + octodns: + included: + - DnsimpleProvider + type: CNAME + value: included.unit.tests. mx: ttl: 300 type: MX diff --git a/tests/fixtures/dnsimple-page-2.json b/tests/fixtures/dnsimple-page-2.json index 40aaa48..d66c8da 100644 --- a/tests/fixtures/dnsimple-page-2.json +++ b/tests/fixtures/dnsimple-page-2.json @@ -175,6 +175,38 @@ "system_record": false, "created_at": "2017-03-09T15:55:09Z", "updated_at": "2017-03-09T15:55:09Z" + }, + { + "id": 12188804, + "zone_id": "unit.tests", + "parent_id": null, + "name": "excluded", + "content": "excluded.unit.tests", + "ttl": 3600, + "priority": null, + "type": "CNAME", + "regions": [ + "global" + ], + "system_record": false, + "created_at": "2017-03-09T15:55:09Z", + "updated_at": "2017-03-09T15:55:09Z" + }, + { + "id": 12188805, + "zone_id": "unit.tests", + "parent_id": null, + "name": "included", + "content": "included.unit.tests", + "ttl": 3600, + "priority": null, + "type": "CNAME", + "regions": [ + "global" + ], + "system_record": false, + "created_at": "2017-03-09T15:55:09Z", + "updated_at": "2017-03-09T15:55:09Z" } ], "pagination": { diff --git a/tests/fixtures/powerdns-full-data.json b/tests/fixtures/powerdns-full-data.json index b8f8bf3..2d7117a 100644 --- a/tests/fixtures/powerdns-full-data.json +++ b/tests/fixtures/powerdns-full-data.json @@ -242,6 +242,18 @@ ], "ttl": 3600, "type": "CAA" + }, + { + "comments": [], + "name": "excluded.unit.tests.", + "records": [ + { + "content": "excluded.unit.tests.", + "disabled": false + } + ], + "ttl": 3600, + "type": "CNAME" } ], "serial": 2017012803, diff --git a/tests/test_octodns_manager.py b/tests/test_octodns_manager.py index 45a3b55..2b947b5 100644 --- a/tests/test_octodns_manager.py +++ b/tests/test_octodns_manager.py @@ -101,12 +101,12 @@ class TestManager(TestCase): environ['YAML_TMP_DIR'] = tmpdir.dirname tc = Manager(get_config_filename('simple.yaml')) \ .sync(dry_run=False) - self.assertEquals(19, tc) + self.assertEquals(20, tc) # try with just one of the zones tc = Manager(get_config_filename('simple.yaml')) \ .sync(dry_run=False, eligible_zones=['unit.tests.']) - self.assertEquals(13, tc) + self.assertEquals(14, tc) # the subzone, with 2 targets tc = Manager(get_config_filename('simple.yaml')) \ @@ -121,18 +121,18 @@ class TestManager(TestCase): # Again with force tc = Manager(get_config_filename('simple.yaml')) \ .sync(dry_run=False, force=True) - self.assertEquals(19, tc) + self.assertEquals(20, tc) # Again with max_workers = 1 tc = Manager(get_config_filename('simple.yaml'), max_workers=1) \ .sync(dry_run=False, force=True) - self.assertEquals(19, tc) + self.assertEquals(20, tc) # Include meta tc = Manager(get_config_filename('simple.yaml'), max_workers=1, include_meta=True) \ .sync(dry_run=False, force=True) - self.assertEquals(23, tc) + self.assertEquals(24, tc) def test_eligible_targets(self): with TemporaryDirectory() as tmpdir: @@ -158,13 +158,13 @@ class TestManager(TestCase): fh.write('---\n{}') changes = manager.compare(['in'], ['dump'], 'unit.tests.') - self.assertEquals(13, len(changes)) + self.assertEquals(14, len(changes)) # Compound sources with varying support changes = manager.compare(['in', 'nosshfp'], ['dump'], 'unit.tests.') - self.assertEquals(12, len(changes)) + self.assertEquals(13, len(changes)) with self.assertRaises(Exception) as ctx: manager.compare(['nope'], ['dump'], 'unit.tests.') diff --git a/tests/test_octodns_provider_dnsimple.py b/tests/test_octodns_provider_dnsimple.py index 950d460..befb39e 100644 --- a/tests/test_octodns_provider_dnsimple.py +++ b/tests/test_octodns_provider_dnsimple.py @@ -78,14 +78,14 @@ class TestDnsimpleProvider(TestCase): zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals(15, len(zone.records)) + self.assertEquals(17, len(zone.records)) changes = self.expected.changes(zone, provider) self.assertEquals(0, len(changes)) # 2nd populate makes no network calls/all from cache again = Zone('unit.tests.', []) provider.populate(again) - self.assertEquals(15, len(again.records)) + self.assertEquals(17, len(again.records)) # bust the cache del provider._zone_records[zone.name] @@ -147,7 +147,7 @@ class TestDnsimpleProvider(TestCase): }), ]) # expected number of total calls - self.assertEquals(27, provider._client._request.call_count) + self.assertEquals(29, provider._client._request.call_count) provider._client._request.reset_mock() diff --git a/tests/test_octodns_provider_powerdns.py b/tests/test_octodns_provider_powerdns.py index b6e02ff..22ccdd6 100644 --- a/tests/test_octodns_provider_powerdns.py +++ b/tests/test_octodns_provider_powerdns.py @@ -78,8 +78,8 @@ class TestPowerDnsProvider(TestCase): expected = Zone('unit.tests.', []) source = YamlProvider('test', join(dirname(__file__), 'config')) source.populate(expected) - expected_n = len(expected.records) - 1 - self.assertEquals(15, expected_n) + expected_n = len(expected.records) - 2 + self.assertEquals(16, expected_n) # No diffs == no changes with requests_mock() as mock: @@ -87,7 +87,7 @@ class TestPowerDnsProvider(TestCase): zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals(15, len(zone.records)) + self.assertEquals(16, len(zone.records)) changes = expected.changes(zone, provider) self.assertEquals(0, len(changes)) @@ -167,7 +167,7 @@ class TestPowerDnsProvider(TestCase): expected = Zone('unit.tests.', []) source = YamlProvider('test', join(dirname(__file__), 'config')) source.populate(expected) - self.assertEquals(16, len(expected.records)) + self.assertEquals(18, len(expected.records)) # A small change to a single record with requests_mock() as mock: diff --git a/tests/test_octodns_provider_yaml.py b/tests/test_octodns_provider_yaml.py index 36cd8d6..a199355 100644 --- a/tests/test_octodns_provider_yaml.py +++ b/tests/test_octodns_provider_yaml.py @@ -30,7 +30,7 @@ class TestYamlProvider(TestCase): # without it we see everything source.populate(zone) - self.assertEquals(16, len(zone.records)) + self.assertEquals(18, len(zone.records)) # Assumption here is that a clean round-trip means that everything # worked as expected, data that went in came back out and could be @@ -49,12 +49,12 @@ class TestYamlProvider(TestCase): # We add everything plan = target.plan(zone) - self.assertEquals(13, len(filter(lambda c: isinstance(c, Create), + self.assertEquals(14, len(filter(lambda c: isinstance(c, Create), plan.changes))) self.assertFalse(isfile(yaml_file)) # Now actually do it - self.assertEquals(13, target.apply(plan)) + self.assertEquals(14, target.apply(plan)) self.assertTrue(isfile(yaml_file)) # There should be no changes after the round trip @@ -64,7 +64,7 @@ class TestYamlProvider(TestCase): # A 2nd sync should still create everything plan = target.plan(zone) - self.assertEquals(13, len(filter(lambda c: isinstance(c, Create), + self.assertEquals(14, len(filter(lambda c: isinstance(c, Create), plan.changes))) with open(yaml_file) as fh: From 4b41762642e5d7f1af02280f8c45af59b372b1ed Mon Sep 17 00:00:00 2001 From: Paul van Brouwershaven Date: Fri, 29 Sep 2017 10:09:16 +0200 Subject: [PATCH 031/141] Use target.id instead of class name --- octodns/zone.py | 12 +++++------ tests/config/unit.tests.yaml | 8 +++---- .../cloudflare-dns_records-page-2.json | 21 +++++++++++++++++-- tests/fixtures/dnsimple-page-2.json | 20 ++---------------- tests/fixtures/powerdns-full-data.json | 4 ++-- tests/helpers.py | 2 ++ tests/test_octodns_provider_base.py | 1 + tests/test_octodns_provider_cloudflare.py | 12 +++++------ tests/test_octodns_provider_dnsimple.py | 10 ++++----- 9 files changed, 47 insertions(+), 43 deletions(-) diff --git a/octodns/zone.py b/octodns/zone.py index f715e3f..a25333f 100644 --- a/octodns/zone.py +++ b/octodns/zone.py @@ -111,12 +111,12 @@ class Zone(object): if record.ignored: continue elif len(record.included) > 0 and \ - target.__class__.__name__ not in record.included: + target.id not in record.included: self.log.debug('changes: skipping record=%s %s - %s not' ' included ', record.fqdn, record._type, target.id) continue - elif target.__class__.__name__ in record.excluded: + elif target.id in record.excluded: self.log.debug('changes: skipping record=%s %s - %s ' 'excluded ', record.fqdn, record._type, target.id) @@ -126,12 +126,12 @@ class Zone(object): if desired_record.ignored: continue elif len(record.included) > 0 and \ - target.__class__.__name__ not in record.included: + target.id not in record.included: self.log.debug('changes: skipping record=%s %s - %s' 'not included ', record.fqdn, record._type, target.id) continue - elif target.__class__.__name__ in record.excluded: + elif target.id in record.excluded: continue except KeyError: if not target.supports(record): @@ -161,12 +161,12 @@ class Zone(object): if record.ignored: continue elif len(record.included) > 0 and \ - target.__class__.__name__ not in record.included: + target.id not in record.included: self.log.debug('changes: skipping record=%s %s - %s not' ' included ', record.fqdn, record._type, target.id) continue - elif target.__class__.__name__ in record.excluded: + elif target.id in record.excluded: self.log.debug('changes: skipping record=%s %s - %s ' 'excluded ', record.fqdn, record._type, target.id) diff --git a/tests/config/unit.tests.yaml b/tests/config/unit.tests.yaml index 548dc8f..1da2465 100644 --- a/tests/config/unit.tests.yaml +++ b/tests/config/unit.tests.yaml @@ -59,9 +59,9 @@ cname: excluded: octodns: excluded: - - CloudflareProvider + - test type: CNAME - value: excluded.unit.tests. + value: unit.tests. ignored: octodns: ignored: true @@ -70,9 +70,9 @@ ignored: included: octodns: included: - - DnsimpleProvider + - test type: CNAME - value: included.unit.tests. + value: unit.tests. mx: ttl: 300 type: MX diff --git a/tests/fixtures/cloudflare-dns_records-page-2.json b/tests/fixtures/cloudflare-dns_records-page-2.json index 195d6de..150951b 100644 --- a/tests/fixtures/cloudflare-dns_records-page-2.json +++ b/tests/fixtures/cloudflare-dns_records-page-2.json @@ -139,14 +139,31 @@ "meta": { "auto_added": false } + }, + { + "id": "fc12ab34cd5611334422ab3322997656", + "type": "CNAME", + "name": "included.unit.tests", + "content": "unit.tests", + "proxiable": true, + "proxied": false, + "ttl": 3600, + "locked": false, + "zone_id": "ff12ab34cd5611334422ab3322997650", + "zone_name": "unit.tests", + "modified_on": "2017-03-11T18:01:43.940682Z", + "created_on": "2017-03-11T18:01:43.940682Z", + "meta": { + "auto_added": false + } } ], "result_info": { "page": 2, "per_page": 10, "total_pages": 2, - "count": 8, - "total_count": 19 + "count": 9, + "total_count": 20 }, "success": true, "errors": [], diff --git a/tests/fixtures/dnsimple-page-2.json b/tests/fixtures/dnsimple-page-2.json index d66c8da..a42c393 100644 --- a/tests/fixtures/dnsimple-page-2.json +++ b/tests/fixtures/dnsimple-page-2.json @@ -176,28 +176,12 @@ "created_at": "2017-03-09T15:55:09Z", "updated_at": "2017-03-09T15:55:09Z" }, - { - "id": 12188804, - "zone_id": "unit.tests", - "parent_id": null, - "name": "excluded", - "content": "excluded.unit.tests", - "ttl": 3600, - "priority": null, - "type": "CNAME", - "regions": [ - "global" - ], - "system_record": false, - "created_at": "2017-03-09T15:55:09Z", - "updated_at": "2017-03-09T15:55:09Z" - }, { "id": 12188805, "zone_id": "unit.tests", "parent_id": null, "name": "included", - "content": "included.unit.tests", + "content": "unit.tests", "ttl": 3600, "priority": null, "type": "CNAME", @@ -212,7 +196,7 @@ "pagination": { "current_page": 2, "per_page": 20, - "total_entries": 30, + "total_entries": 32, "total_pages": 2 } } diff --git a/tests/fixtures/powerdns-full-data.json b/tests/fixtures/powerdns-full-data.json index 2d7117a..3d445d4 100644 --- a/tests/fixtures/powerdns-full-data.json +++ b/tests/fixtures/powerdns-full-data.json @@ -245,10 +245,10 @@ }, { "comments": [], - "name": "excluded.unit.tests.", + "name": "included.unit.tests.", "records": [ { - "content": "excluded.unit.tests.", + "content": "unit.tests.", "disabled": false } ], diff --git a/tests/helpers.py b/tests/helpers.py index adac81d..632f258 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -18,6 +18,7 @@ class SimpleSource(object): class SimpleProvider(object): SUPPORTS_GEO = False SUPPORTS = set(('A',)) + id = 'test' def __init__(self, id='test'): pass @@ -34,6 +35,7 @@ class SimpleProvider(object): class GeoProvider(object): SUPPORTS_GEO = True + id = 'test' def __init__(self, id='test'): pass diff --git a/tests/test_octodns_provider_base.py b/tests/test_octodns_provider_base.py index e44adc0..eb4a120 100644 --- a/tests/test_octodns_provider_base.py +++ b/tests/test_octodns_provider_base.py @@ -17,6 +17,7 @@ class HelperProvider(BaseProvider): log = getLogger('HelperProvider') SUPPORTS = set(('A',)) + id = 'test' def __init__(self, extra_changes, apply_disabled=False, include_change_callback=None): diff --git a/tests/test_octodns_provider_cloudflare.py b/tests/test_octodns_provider_cloudflare.py index 04a46e0..ef8a51c 100644 --- a/tests/test_octodns_provider_cloudflare.py +++ b/tests/test_octodns_provider_cloudflare.py @@ -118,7 +118,7 @@ class TestCloudflareProvider(TestCase): zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals(10, len(zone.records)) + self.assertEquals(11, len(zone.records)) changes = self.expected.changes(zone, provider) self.assertEquals(0, len(changes)) @@ -126,7 +126,7 @@ class TestCloudflareProvider(TestCase): # re-populating the same zone/records comes out of cache, no calls again = Zone('unit.tests.', []) provider.populate(again) - self.assertEquals(10, len(again.records)) + self.assertEquals(11, len(again.records)) def test_apply(self): provider = CloudflareProvider('test', 'email', 'token') @@ -140,12 +140,12 @@ class TestCloudflareProvider(TestCase): 'id': 42, } }, # zone create - ] + [None] * 17 # individual record creates + ] + [None] * 18 # individual record creates # non-existant zone, create everything plan = provider.plan(self.expected) - self.assertEquals(10, len(plan.changes)) - self.assertEquals(10, provider.apply(plan)) + self.assertEquals(11, len(plan.changes)) + self.assertEquals(11, provider.apply(plan)) provider._request.assert_has_calls([ # created the domain @@ -170,7 +170,7 @@ class TestCloudflareProvider(TestCase): }), ], True) # expected number of total calls - self.assertEquals(19, provider._request.call_count) + self.assertEquals(20, provider._request.call_count) provider._request.reset_mock() diff --git a/tests/test_octodns_provider_dnsimple.py b/tests/test_octodns_provider_dnsimple.py index befb39e..0dcef32 100644 --- a/tests/test_octodns_provider_dnsimple.py +++ b/tests/test_octodns_provider_dnsimple.py @@ -78,14 +78,14 @@ class TestDnsimpleProvider(TestCase): zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals(17, len(zone.records)) + self.assertEquals(16, len(zone.records)) changes = self.expected.changes(zone, provider) self.assertEquals(0, len(changes)) # 2nd populate makes no network calls/all from cache again = Zone('unit.tests.', []) provider.populate(again) - self.assertEquals(17, len(again.records)) + self.assertEquals(16, len(again.records)) # bust the cache del provider._zone_records[zone.name] @@ -129,8 +129,8 @@ class TestDnsimpleProvider(TestCase): ] plan = provider.plan(self.expected) - # No root NS, no ignored - n = len(self.expected.records) - 2 + # No root NS, no ignored, no excluded + n = len(self.expected.records) - 3 self.assertEquals(n, len(plan.changes)) self.assertEquals(n, provider.apply(plan)) @@ -147,7 +147,7 @@ class TestDnsimpleProvider(TestCase): }), ]) # expected number of total calls - self.assertEquals(29, provider._client._request.call_count) + self.assertEquals(28, provider._client._request.call_count) provider._client._request.reset_mock() From 6232c685507bc1177222f7e84b364ca65856c54e Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Tue, 24 Oct 2017 14:00:12 -0400 Subject: [PATCH 032/141] 0.8.8 version bump and changelog --- CHANGELOG.md | 12 ++++++++++++ octodns/__init__.py | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c97cdc..6777ea9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,15 @@ +## v0.8.8 - 2017-10-24 - Google Cloud DNS, Large TXT Record support + +* Added support for "chunking" TXT records where individual values were larger + than 255 chars. This is common with DKIM records involving multiple + providers. +* Added `GoogleCloudProvider` +* Configurable `UnsafePlan` thresholds to allow modification of how many + updates/deletes are allowed before a plan is declared dangerous. +* Manager.dump bug fix around empty zones. +* Prefer use of `.` over `source` in shell scripts +* `DynProvider` warns when it ignores unrecognized traffic directors. + ## v0.8.7 - 2017-09-29 - OVH support Adds an OVH provider. diff --git a/octodns/__init__.py b/octodns/__init__.py index 3740dec..2166778 100644 --- a/octodns/__init__.py +++ b/octodns/__init__.py @@ -3,4 +3,4 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -__VERSION__ = '0.8.7' +__VERSION__ = '0.8.8' From bf4f7dd42d775be65c1554cbd3fa5d85c1d56a8e Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Thu, 26 Oct 2017 06:30:38 -0500 Subject: [PATCH 033/141] Allow enabling lenient on a per-record basis with octodns.lenient ``` --- '': octodns: ignored: True lenient: True type: CNAME # not valid to have a root cname value: foo.com. --- octodns/record.py | 4 ++++ tests/test_octodns_record.py | 10 ++++++++++ 2 files changed, 14 insertions(+) diff --git a/octodns/record.py b/octodns/record.py index 554d98b..2f67c60 100644 --- a/octodns/record.py +++ b/octodns/record.py @@ -95,6 +95,10 @@ class Record(object): except KeyError: raise Exception('Unknown record type: "{}"'.format(_type)) reasons = _class.validate(name, data) + try: + lenient |= data['octodns']['lenient'] + except KeyError: + pass if reasons: if lenient: cls.log.warn(ValidationError.build_message(fqdn, reasons)) diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index 46a5e65..e7eaa61 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -733,6 +733,16 @@ class TestRecordValidation(TestCase): }, lenient=True) self.assertEquals(('value',), ctx.exception.args) + # no exception if we're in lenient mode from config + Record.new(self.zone, 'www', { + 'octodns': { + 'lenient': True + }, + 'type': 'A', + 'ttl': -1, + 'value': '1.2.3.4', + }, lenient=True) + def test_A_and_values_mixin(self): # doesn't blow up Record.new(self.zone, '', { From 00aaa3bf4d89af1ec7dc8780071824f65c42099b Mon Sep 17 00:00:00 2001 From: Adam Smith Date: Thu, 26 Oct 2017 23:55:48 -0700 Subject: [PATCH 034/141] set default value for nsone cname to None, use first value if non-zero length --- octodns/provider/ns1.py | 6 +++++- tests/test_octodns_provider_ns1.py | 31 ++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index f7cbef1..2d3af9a 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -69,10 +69,14 @@ class Ns1Provider(BaseProvider): } def _data_for_CNAME(self, _type, record): + try: + value = record['short_answers'][0] + except IndexError: + value = None return { 'ttl': record['ttl'], 'type': _type, - 'value': record['short_answers'][0], + 'value': value, } _data_for_ALIAS = _data_for_CNAME diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index cde23b0..d4f4080 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -326,3 +326,34 @@ class TestNs1Provider(TestCase): }) self.assertEquals(['foo; bar baz; blip'], provider._params_for_TXT(record)['answers']) + + def test_data_for_CNAME(self): + provider = Ns1Provider('test', 'api-key') + + # answers from nsone + a_record = { + 'ttl': 31, + 'type': 'CNAME', + 'short_answers': ['foo.unit.tests.'] + } + a_expected = { + 'ttl': 31, + 'type': 'CNAME', + 'value': 'foo.unit.tests.' + } + self.assertEqual(a_expected, + provider._data_for_CNAME(a_record['type'], a_record)) + + # no answers from nsone + b_record = { + 'ttl': 32, + 'type': 'CNAME', + 'short_answers': [] + } + b_expected = { + 'ttl': 32, + 'type': 'CNAME', + 'value': None + } + self.assertEqual(b_expected, + provider._data_for_CNAME(b_record['type'], b_record)) From bf1896329ba1c8f5829895ecfe71e233ca0173cc Mon Sep 17 00:00:00 2001 From: Adam Smith Date: Sun, 22 Oct 2017 22:39:18 -0700 Subject: [PATCH 035/141] validate values for empty string or None value dump does not write invalid value(s) to yaml --- octodns/record.py | 42 +++++++- tests/test_octodns_provider_dnsimple.py | 8 +- tests/test_octodns_record.py | 125 +++++++++++++++++++++++- 3 files changed, 165 insertions(+), 10 deletions(-) diff --git a/octodns/record.py b/octodns/record.py index 2f67c60..f9812a6 100644 --- a/octodns/record.py +++ b/octodns/record.py @@ -211,9 +211,30 @@ class _ValuesMixin(object): values = [] try: values = data['values'] + if not values: + values = [] + reasons.append('missing value(s)') + else: + # loop through copy of values + # remove invalid value from values + for value in list(values): + if value is None: + reasons.append('missing value(s)') + values.remove(value) + elif len(value) == 0: + reasons.append('empty value') + values.remove(value) except KeyError: try: - values = [data['value']] + value = data['value'] + if value is None: + reasons.append('missing value(s)') + values = [] + elif len(value) == 0: + reasons.append('empty value') + values = [] + else: + values = [value] except KeyError: reasons.append('missing value(s)') @@ -238,10 +259,16 @@ class _ValuesMixin(object): def _data(self): ret = super(_ValuesMixin, self)._data() if len(self.values) > 1: - ret['values'] = [getattr(v, 'data', v) for v in self.values] - else: + values = [getattr(v, 'data', v) for v in self.values if v] + if len(values) > 1: + ret['values'] = values + elif len(values) == 1: + ret['value'] = values[0] + elif len(self.values) == 1: v = self.values[0] - ret['value'] = getattr(v, 'data', v) + if v: + ret['value'] = getattr(v, 'data', v) + return ret def __repr__(self): @@ -349,6 +376,10 @@ class _ValueMixin(object): value = None try: value = data['value'] + if value is None: + reasons.append('missing value') + elif value == '': + reasons.append('empty value') except KeyError: reasons.append('missing value') if value: @@ -366,7 +397,8 @@ class _ValueMixin(object): def _data(self): ret = super(_ValueMixin, self)._data() - ret['value'] = getattr(self.value, 'data', self.value) + if self.value: + ret['value'] = getattr(self.value, 'data', self.value) return ret def __repr__(self): diff --git a/tests/test_octodns_provider_dnsimple.py b/tests/test_octodns_provider_dnsimple.py index 950d460..b44d6ee 100644 --- a/tests/test_octodns_provider_dnsimple.py +++ b/tests/test_octodns_provider_dnsimple.py @@ -96,23 +96,23 @@ class TestDnsimpleProvider(TestCase): mock.get(ANY, text=fh.read()) zone = Zone('unit.tests.', []) - provider.populate(zone) + provider.populate(zone, lenient=True) self.assertEquals(set([ Record.new(zone, '', { 'ttl': 3600, 'type': 'SSHFP', 'values': [] - }), + }, lenient=True), Record.new(zone, '_srv._tcp', { 'ttl': 600, 'type': 'SRV', 'values': [] - }), + }, lenient=True), Record.new(zone, 'naptr', { 'ttl': 600, 'type': 'NAPTR', 'values': [] - }), + }, lenient=True), ]), zone.records) def test_apply(self): diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index e7eaa61..ff57975 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -96,6 +96,57 @@ class TestRecord(TestCase): DummyRecord().__repr__() + def test_values_mixin_data(self): + # no values, no value or values in data + a = ARecord(self.zone, '', { + 'type': 'A', + 'ttl': 600, + 'values': [] + }) + self.assertNotIn('values', a.data) + + # empty value, no value or values in data + b = ARecord(self.zone, '', { + 'type': 'A', + 'ttl': 600, + 'values': [''] + }) + self.assertNotIn('value', b.data) + + # empty/None values, no value or values in data + c = ARecord(self.zone, '', { + 'type': 'A', + 'ttl': 600, + 'values': ['', None] + }) + self.assertNotIn('values', c.data) + + # empty/None values and valid, value in data + c = ARecord(self.zone, '', { + 'type': 'A', + 'ttl': 600, + 'values': ['', None, '10.10.10.10'] + }) + self.assertNotIn('values', c.data) + self.assertEqual('10.10.10.10', c.data['value']) + + def test_value_mixin_data(self): + # unspecified value, no value in data + a = AliasRecord(self.zone, '', { + 'type': 'ALIAS', + 'ttl': 600, + 'value': None + }) + self.assertNotIn('value', a.data) + + # unspecified value, no value in data + a = AliasRecord(self.zone, '', { + 'type': 'ALIAS', + 'ttl': 600, + 'value': '' + }) + self.assertNotIn('value', a.data) + def test_geo(self): geo_data = {'ttl': 42, 'values': ['5.2.3.4', '6.2.3.4'], 'geo': {'AF': ['1.1.1.1'], @@ -750,6 +801,13 @@ class TestRecordValidation(TestCase): 'ttl': 600, 'value': '1.2.3.4', }) + Record.new(self.zone, '', { + 'type': 'A', + 'ttl': 600, + 'values': [ + '1.2.3.4', + ] + }) Record.new(self.zone, '', { 'type': 'A', 'ttl': 600, @@ -759,13 +817,60 @@ class TestRecordValidation(TestCase): ] }) - # missing value(s) + # missing value(s), no value or value with self.assertRaises(ValidationError) as ctx: Record.new(self.zone, '', { 'type': 'A', 'ttl': 600, }) self.assertEquals(['missing value(s)'], ctx.exception.reasons) + + # missing value(s), empty values + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, 'www', { + 'type': 'A', + 'ttl': 600, + 'values': [] + }) + self.assertEquals(['missing value(s)'], ctx.exception.reasons) + + # missing value(s), None values + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, 'www', { + 'type': 'A', + 'ttl': 600, + 'values': None + }) + self.assertEquals(['missing value(s)'], ctx.exception.reasons) + + # missing value(s) and empty value + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, 'www', { + 'type': 'A', + 'ttl': 600, + 'values': [None, ''] + }) + self.assertEquals(['missing value(s)', + 'empty value'], ctx.exception.reasons) + + # missing value(s), None value + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, 'www', { + 'type': 'A', + 'ttl': 600, + 'value': None + }) + self.assertEquals(['missing value(s)'], ctx.exception.reasons) + + # empty value, empty string value + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, 'www', { + 'type': 'A', + 'ttl': 600, + 'value': '' + }) + self.assertEquals(['empty value'], ctx.exception.reasons) + # missing value(s) & ttl with self.assertRaises(ValidationError) as ctx: Record.new(self.zone, '', { @@ -922,6 +1027,24 @@ class TestRecordValidation(TestCase): }) self.assertEquals(['missing value'], ctx.exception.reasons) + # missing value + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, 'www', { + 'type': 'ALIAS', + 'ttl': 600, + 'value': None + }) + self.assertEquals(['missing value'], ctx.exception.reasons) + + # empty value + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, 'www', { + 'type': 'ALIAS', + 'ttl': 600, + 'value': '' + }) + self.assertEquals(['empty value'], ctx.exception.reasons) + # missing trailing . with self.assertRaises(ValidationError) as ctx: Record.new(self.zone, '', { From 7352978880d5ce995073f568224609d4da74cdb5 Mon Sep 17 00:00:00 2001 From: Paul van Brouwershaven Date: Mon, 30 Oct 2017 18:33:51 +0100 Subject: [PATCH 036/141] Check excluded in desired_record --- octodns/zone.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/octodns/zone.py b/octodns/zone.py index a25333f..bbc38d0 100644 --- a/octodns/zone.py +++ b/octodns/zone.py @@ -125,13 +125,13 @@ class Zone(object): desired_record = desired_records[record] if desired_record.ignored: continue - elif len(record.included) > 0 and \ - target.id not in record.included: + elif len(desired_record.included) > 0 and \ + target.id not in desired_record.included: self.log.debug('changes: skipping record=%s %s - %s' 'not included ', record.fqdn, record._type, target.id) continue - elif target.id in record.excluded: + elif target.id in desired_record.excluded: continue except KeyError: if not target.supports(record): From 6261ded87974dbc77652e6c3294c791bf96bd021 Mon Sep 17 00:00:00 2001 From: Paul van Brouwershaven Date: Mon, 30 Oct 2017 18:34:22 +0100 Subject: [PATCH 037/141] Add more include/exclude tests --- tests/test_octodns_zone.py | 99 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) diff --git a/tests/test_octodns_zone.py b/tests/test_octodns_zone.py index 8d75100..94faef3 100644 --- a/tests/test_octodns_zone.py +++ b/tests/test_octodns_zone.py @@ -236,3 +236,102 @@ class TestZone(TestCase): zone.add_record(cname) with self.assertRaises(InvalidNodeException): zone.add_record(a) + + def test_excluded_records(self): + zone_normal = Zone('unit.tests.', []) + zone_excluded = Zone('unit.tests.', []) + zone_missing = Zone('unit.tests.', []) + + normal = Record.new(zone_normal, 'www', { + 'ttl': 60, + 'type': 'A', + 'value': '9.9.9.9', + }) + zone_normal.add_record(normal) + + excluded = Record.new(zone_excluded, 'www', { + 'octodns': { + 'excluded': ['test'] + }, + 'ttl': 60, + 'type': 'A', + 'value': '9.9.9.9', + }) + zone_excluded.add_record(excluded) + + provider = SimpleProvider() + + self.assertFalse(zone_normal.changes(zone_excluded, provider)) + self.assertTrue(zone_normal.changes(zone_missing, provider)) + + self.assertFalse(zone_excluded.changes(zone_normal, provider)) + self.assertFalse(zone_excluded.changes(zone_missing, provider)) + + self.assertTrue(zone_missing.changes(zone_normal, provider)) + self.assertFalse(zone_missing.changes(zone_excluded, provider)) + + def test_included_records(self): + zone_normal = Zone('unit.tests.', []) + zone_included = Zone('unit.tests.', []) + zone_missing = Zone('unit.tests.', []) + + normal = Record.new(zone_normal, 'www', { + 'ttl': 60, + 'type': 'A', + 'value': '9.9.9.9', + }) + zone_normal.add_record(normal) + + included = Record.new(zone_included, 'www', { + 'octodns': { + 'included': ['test'] + }, + 'ttl': 60, + 'type': 'A', + 'value': '9.9.9.9', + }) + zone_included.add_record(included) + + provider = SimpleProvider() + + self.assertFalse(zone_normal.changes(zone_included, provider)) + self.assertTrue(zone_normal.changes(zone_missing, provider)) + + self.assertFalse(zone_included.changes(zone_normal, provider)) + self.assertTrue(zone_included.changes(zone_missing, provider)) + + self.assertTrue(zone_missing.changes(zone_normal, provider)) + self.assertTrue(zone_missing.changes(zone_included, provider)) + + def test_not_included_records(self): + zone_normal = Zone('unit.tests.', []) + zone_included = Zone('unit.tests.', []) + zone_missing = Zone('unit.tests.', []) + + normal = Record.new(zone_normal, 'www', { + 'ttl': 60, + 'type': 'A', + 'value': '9.9.9.9', + }) + zone_normal.add_record(normal) + + included = Record.new(zone_included, 'www', { + 'octodns': { + 'included': ['not-here'] + }, + 'ttl': 60, + 'type': 'A', + 'value': '9.9.9.9', + }) + zone_included.add_record(included) + + provider = SimpleProvider() + + self.assertFalse(zone_normal.changes(zone_included, provider)) + self.assertTrue(zone_normal.changes(zone_missing, provider)) + + self.assertFalse(zone_included.changes(zone_normal, provider)) + self.assertFalse(zone_included.changes(zone_missing, provider)) + + self.assertTrue(zone_missing.changes(zone_normal, provider)) + self.assertFalse(zone_missing.changes(zone_included, provider)) From 6b1a8f8ccf4d930d6ddf5ea1d0161d4ac703f248 Mon Sep 17 00:00:00 2001 From: trnsnt Date: Mon, 23 Oct 2017 17:12:32 +0200 Subject: [PATCH 038/141] OVH: Add support of DKIM records --- octodns/provider/ovh.py | 77 ++++++++- tests/test_octodns_provider_ovh.py | 242 +++++++++++++++++++---------- 2 files changed, 233 insertions(+), 86 deletions(-) diff --git a/octodns/provider/ovh.py b/octodns/provider/ovh.py index b890862..5c2fe0d 100644 --- a/octodns/provider/ovh.py +++ b/octodns/provider/ovh.py @@ -5,6 +5,8 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals +import base64 +import binascii import logging from collections import defaultdict @@ -32,8 +34,10 @@ class OvhProvider(BaseProvider): SUPPORTS_GEO = False - SUPPORTS = set(('A', 'AAAA', 'CNAME', 'MX', 'NAPTR', 'NS', 'PTR', 'SPF', - 'SRV', 'SSHFP', 'TXT')) + # This variable is also used in populate method to filter which OVH record + # types are supported by octodns + SUPPORTS = set(('A', 'AAAA', 'CNAME', 'DKIM', 'MX', 'NAPTR', 'NS', 'PTR', + 'SPF', 'SRV', 'SSHFP', 'TXT')) def __init__(self, id, endpoint, application_key, application_secret, consumer_key, *args, **kwargs): @@ -62,6 +66,10 @@ class OvhProvider(BaseProvider): before = len(zone.records) for name, types in values.items(): for _type, records in types.items(): + if _type not in self.SUPPORTS: + self.log.warning('Not managed record of type %s, skip', + _type) + continue data_for = getattr(self, '_data_for_{}'.format(_type)) record = Record.new(zone, name, data_for(_type, records), source=self, lenient=lenient) @@ -96,7 +104,11 @@ class OvhProvider(BaseProvider): def _apply_delete(self, zone_name, change): existing = change.existing - self.delete_records(zone_name, existing._type, existing.name) + record_type = existing._type + if record_type == "TXT": + if self._is_valid_dkim(existing.values[0]): + record_type = 'DKIM' + self.delete_records(zone_name, record_type, existing.name) @staticmethod def _data_for_multiple(_type, records): @@ -184,6 +196,15 @@ class OvhProvider(BaseProvider): 'values': values } + @staticmethod + def _data_for_DKIM(_type, records): + return { + 'ttl': records[0]['ttl'], + 'type': "TXT", + 'values': [record['target'].replace(';', '\;') + for record in records] + } + _data_for_A = _data_for_multiple _data_for_AAAA = _data_for_multiple _data_for_NS = _data_for_multiple @@ -258,15 +279,63 @@ class OvhProvider(BaseProvider): 'fieldType': record._type } + def _params_for_TXT(self, record): + for value in record.values: + field_type = 'TXT' + if self._is_valid_dkim(value): + field_type = 'DKIM' + value = value.replace("\;", ";") + yield { + 'target': value, + 'subDomain': record.name, + 'ttl': record.ttl, + 'fieldType': field_type + } + _params_for_A = _params_for_multiple _params_for_AAAA = _params_for_multiple _params_for_NS = _params_for_multiple _params_for_SPF = _params_for_multiple - _params_for_TXT = _params_for_multiple _params_for_CNAME = _params_for_single _params_for_PTR = _params_for_single + def _is_valid_dkim(self, value): + """Check if value is a valid DKIM""" + validator_dict = {'h': lambda val: val in ['sha1', 'sha256'], + 's': lambda val: val in ['*', 'email'], + 't': lambda val: val in ['y', 's'], + 'v': lambda val: val == 'DKIM1', + 'k': lambda val: val == 'rsa', + 'n': lambda _: True, + 'g': lambda _: True} + + splitted = value.split('\;') + found_key = False + for splitted_value in splitted: + sub_split = map(lambda x: x.strip(), splitted_value.split("=", 1)) + if len(sub_split) < 2: + return False + key, value = sub_split[0], sub_split[1] + if key == "p": + is_valid_key = self._is_valid_dkim_key(value) + if not is_valid_key: + return False + found_key = True + else: + is_valid_key = validator_dict.get(key, lambda _: False)(value) + if not is_valid_key: + return False + return found_key + + @staticmethod + def _is_valid_dkim_key(key): + try: + base64.decodestring(key) + except binascii.Error: + return False + return True + def get_records(self, zone_name): """ List all records of a DNS zone diff --git a/tests/test_octodns_provider_ovh.py b/tests/test_octodns_provider_ovh.py index 2816748..6a44e25 100644 --- a/tests/test_octodns_provider_ovh.py +++ b/tests/test_octodns_provider_ovh.py @@ -17,6 +17,14 @@ from octodns.zone import Zone class TestOvhProvider(TestCase): api_record = [] + valid_dkim = [] + invalid_dkim = [] + + valid_dkim_key = "p=MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCxLaG16G4SaE" \ + "cXVdiIxTg7gKSGbHKQLm30CHib1h9FzS9nkcyvQSyQj1rMFyqC//" \ + "tft3ohx3nvJl+bGCWxdtLYDSmir9PW54e5CTdxEh8MWRkBO3StF6" \ + "QG/tAh3aTGDmkqhIJGLb87iHvpmVKqURmEUzJPv5KPJfWLofADI+" \ + "q9lQIDAQAB" zone = Zone('unit.tests.', []) expected = set() @@ -233,6 +241,65 @@ class TestOvhProvider(TestCase): 'value': '1:1ec:1::1', })) + # DKIM + api_record.append({ + 'fieldType': 'DKIM', + 'ttl': 1300, + 'target': valid_dkim_key, + 'subDomain': 'dkim', + 'id': 16 + }) + expected.add(Record.new(zone, 'dkim', { + 'ttl': 1300, + 'type': 'TXT', + 'value': valid_dkim_key, + })) + + # TXT + api_record.append({ + 'fieldType': 'TXT', + 'ttl': 1400, + 'target': 'TXT text', + 'subDomain': 'txt', + 'id': 17 + }) + expected.add(Record.new(zone, 'txt', { + 'ttl': 1400, + 'type': 'TXT', + 'value': 'TXT text', + })) + + # LOC + # We do not have associated record for LOC, as it's not managed + api_record.append({ + 'fieldType': 'LOC', + 'ttl': 1500, + 'target': '1 1 1 N 1 1 1 E 1m 1m', + 'subDomain': '', + 'id': 18 + }) + + valid_dkim = [valid_dkim_key, + 'v=DKIM1 \; %s' % valid_dkim_key, + 'h=sha256 \; %s' % valid_dkim_key, + 'h=sha1 \; %s' % valid_dkim_key, + 's=* \; %s' % valid_dkim_key, + 's=email \; %s' % valid_dkim_key, + 't=y \; %s' % valid_dkim_key, + 't=s \; %s' % valid_dkim_key, + 'k=rsa \; %s' % valid_dkim_key, + 'n=notes \; %s' % valid_dkim_key, + 'g=granularity \; %s' % valid_dkim_key, + ] + invalid_dkim = ['p=%invalid%', # Invalid public key + 'v=DKIM1', # Missing public key + 'v=DKIM2 \; %s' % valid_dkim_key, # Invalid version + 'h=sha512 \; %s' % valid_dkim_key, # Invalid hash algo + 's=fake \; %s' % valid_dkim_key, # Invalid selector + 't=fake \; %s' % valid_dkim_key, # Invalid flag + 'u=invalid \; %s' % valid_dkim_key, # Invalid key + ] + @patch('ovh.Client') def test_populate(self, client_mock): provider = OvhProvider('test', 'endpoint', 'application_key', @@ -253,6 +320,16 @@ class TestOvhProvider(TestCase): provider.populate(zone) self.assertEquals(self.expected, zone.records) + @patch('ovh.Client') + def test_is_valid_dkim(self, client_mock): + """Test _is_valid_dkim""" + provider = OvhProvider('test', 'endpoint', 'application_key', + 'application_secret', 'consumer_key') + for dkim in self.valid_dkim: + self.assertTrue(provider._is_valid_dkim(dkim)) + for dkim in self.invalid_dkim: + self.assertFalse(provider._is_valid_dkim(dkim)) + @patch('ovh.Client') def test_apply(self, client_mock): provider = OvhProvider('test', 'endpoint', 'application_key', @@ -270,90 +347,91 @@ class TestOvhProvider(TestCase): provider.apply(plan) self.assertEquals(get_mock.side_effect, ctx.exception) + # Records get by API call with patch.object(provider._client, 'get') as get_mock: - get_returns = [[1, 2], { - 'fieldType': 'A', - 'ttl': 600, - 'target': '5.6.7.8', - 'subDomain': '', - 'id': 100 - }, {'fieldType': 'A', - 'ttl': 600, - 'target': '5.6.7.8', - 'subDomain': 'fake', - 'id': 101 - }] + get_returns = [ + [1, 2, 3, 4], + {'fieldType': 'A', 'ttl': 600, 'target': '5.6.7.8', + 'subDomain': '', 'id': 100}, + {'fieldType': 'A', 'ttl': 600, 'target': '5.6.7.8', + 'subDomain': 'fake', 'id': 101}, + {'fieldType': 'TXT', 'ttl': 600, 'target': 'fake txt record', + 'subDomain': 'txt', 'id': 102}, + {'fieldType': 'DKIM', 'ttl': 600, + 'target': 'v=DKIM1; %s' % self.valid_dkim_key, + 'subDomain': 'dkim', 'id': 103} + ] get_mock.side_effect = get_returns plan = provider.plan(desired) - with patch.object(provider._client, 'post') as post_mock: - with patch.object(provider._client, 'delete') as delete_mock: - with patch.object(provider._client, 'get') as get_mock: - get_mock.side_effect = [[100], [101]] - provider.apply(plan) - wanted_calls = [ - call(u'/domain/zone/unit.tests/record', - fieldType=u'A', - subDomain=u'', target=u'1.2.3.4', ttl=100), - call(u'/domain/zone/unit.tests/record', - fieldType=u'SRV', - subDomain=u'10 20 30 foo-1.unit.tests.', - target='_srv._tcp', ttl=800), - call(u'/domain/zone/unit.tests/record', - fieldType=u'SRV', - subDomain=u'40 50 60 foo-2.unit.tests.', - target='_srv._tcp', ttl=800), - call(u'/domain/zone/unit.tests/record', - fieldType=u'PTR', subDomain='4', - target=u'unit.tests.', ttl=900), - call(u'/domain/zone/unit.tests/record', - fieldType=u'NS', subDomain='www3', - target=u'ns3.unit.tests.', ttl=700), - call(u'/domain/zone/unit.tests/record', - fieldType=u'NS', subDomain='www3', - target=u'ns4.unit.tests.', ttl=700), - call(u'/domain/zone/unit.tests/record', - fieldType=u'SSHFP', - subDomain=u'1 1 bf6b6825d2977c511a475bbefb88a' - u'ad54' - u'a92ac73', - target=u'', ttl=1100), - call(u'/domain/zone/unit.tests/record', - fieldType=u'AAAA', subDomain=u'', - target=u'1:1ec:1::1', ttl=200), - call(u'/domain/zone/unit.tests/record', - fieldType=u'MX', subDomain=u'', - target=u'10 mx1.unit.tests.', ttl=400), - call(u'/domain/zone/unit.tests/record', - fieldType=u'CNAME', subDomain='www2', - target=u'unit.tests.', ttl=300), - call(u'/domain/zone/unit.tests/record', - fieldType=u'SPF', subDomain=u'', - target=u'v=spf1 include:unit.texts.' - u'rerirect ~all', - ttl=1000), - call(u'/domain/zone/unit.tests/record', - fieldType=u'A', - subDomain='sub', target=u'1.2.3.4', ttl=200), - call(u'/domain/zone/unit.tests/record', - fieldType=u'NAPTR', subDomain='naptr', - target=u'10 100 "S" "SIP+D2U" "!^.*$!sip:' - u'info@bar' - u'.example.com!" .', - ttl=500), - call(u'/domain/zone/unit.tests/refresh')] - - post_mock.assert_has_calls(wanted_calls) - - # Get for delete calls - get_mock.assert_has_calls( - [call(u'/domain/zone/unit.tests/record', - fieldType=u'A', subDomain=u''), - call(u'/domain/zone/unit.tests/record', - fieldType=u'A', subDomain='fake')] - ) - # 2 delete calls, one for update + one for delete - delete_mock.assert_has_calls( - [call(u'/domain/zone/unit.tests/record/100'), - call(u'/domain/zone/unit.tests/record/101')]) + with patch.object(provider._client, 'post') as post_mock, \ + patch.object(provider._client, 'delete') as delete_mock: + get_mock.side_effect = [[100], [101], [102], [103]] + provider.apply(plan) + wanted_calls = [ + call(u'/domain/zone/unit.tests/record', fieldType=u'TXT', + subDomain='txt', target=u'TXT text', ttl=1400), + call(u'/domain/zone/unit.tests/record', fieldType=u'DKIM', + subDomain='dkim', target=self.valid_dkim_key, + ttl=1300), + call(u'/domain/zone/unit.tests/record', fieldType=u'A', + subDomain=u'', target=u'1.2.3.4', ttl=100), + call(u'/domain/zone/unit.tests/record', fieldType=u'SRV', + subDomain=u'10 20 30 foo-1.unit.tests.', + target='_srv._tcp', ttl=800), + call(u'/domain/zone/unit.tests/record', fieldType=u'SRV', + subDomain=u'40 50 60 foo-2.unit.tests.', + target='_srv._tcp', ttl=800), + call(u'/domain/zone/unit.tests/record', fieldType=u'PTR', + subDomain='4', target=u'unit.tests.', ttl=900), + call(u'/domain/zone/unit.tests/record', fieldType=u'NS', + subDomain='www3', target=u'ns3.unit.tests.', ttl=700), + call(u'/domain/zone/unit.tests/record', fieldType=u'NS', + subDomain='www3', target=u'ns4.unit.tests.', ttl=700), + call(u'/domain/zone/unit.tests/record', + fieldType=u'SSHFP', target=u'', ttl=1100, + subDomain=u'1 1 bf6b6825d2977c511a475bbefb88a' + u'ad54' + u'a92ac73', + ), + call(u'/domain/zone/unit.tests/record', fieldType=u'AAAA', + subDomain=u'', target=u'1:1ec:1::1', ttl=200), + call(u'/domain/zone/unit.tests/record', fieldType=u'MX', + subDomain=u'', target=u'10 mx1.unit.tests.', ttl=400), + call(u'/domain/zone/unit.tests/record', fieldType=u'CNAME', + subDomain='www2', target=u'unit.tests.', ttl=300), + call(u'/domain/zone/unit.tests/record', fieldType=u'SPF', + subDomain=u'', ttl=1000, + target=u'v=spf1 include:unit.texts.' + u'rerirect ~all', + ), + call(u'/domain/zone/unit.tests/record', fieldType=u'A', + subDomain='sub', target=u'1.2.3.4', ttl=200), + call(u'/domain/zone/unit.tests/record', fieldType=u'NAPTR', + subDomain='naptr', ttl=500, + target=u'10 100 "S" "SIP+D2U" "!^.*$!sip:' + u'info@bar' + u'.example.com!" .' + ), + call(u'/domain/zone/unit.tests/refresh')] + + post_mock.assert_has_calls(wanted_calls) + + # Get for delete calls + wanted_get_calls = [ + call(u'/domain/zone/unit.tests/record', fieldType=u'TXT', + subDomain='txt'), + call(u'/domain/zone/unit.tests/record', fieldType=u'DKIM', + subDomain='dkim'), + call(u'/domain/zone/unit.tests/record', fieldType=u'A', + subDomain=u''), + call(u'/domain/zone/unit.tests/record', fieldType=u'A', + subDomain='fake')] + get_mock.assert_has_calls(wanted_get_calls) + # 4 delete calls for update and delete + delete_mock.assert_has_calls( + [call(u'/domain/zone/unit.tests/record/100'), + call(u'/domain/zone/unit.tests/record/101'), + call(u'/domain/zone/unit.tests/record/102'), + call(u'/domain/zone/unit.tests/record/103')]) From ba3ad27c024cad228b5f61cbd8b429b147587645 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Thu, 2 Nov 2017 06:50:15 -0700 Subject: [PATCH 039/141] Make PowerDnsBaseProvider's timeout configurable --- octodns/provider/powerdns.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/octodns/provider/powerdns.py b/octodns/provider/powerdns.py index 20cfe8b..4527f8e 100644 --- a/octodns/provider/powerdns.py +++ b/octodns/provider/powerdns.py @@ -18,13 +18,14 @@ class PowerDnsBaseProvider(BaseProvider): 'PTR', 'SPF', 'SSHFP', 'SRV', 'TXT')) TIMEOUT = 5 - def __init__(self, id, host, api_key, port=8081, scheme="http", *args, - **kwargs): + def __init__(self, id, host, api_key, port=8081, scheme="http", + timeout=TIMEOUT, *args, **kwargs): super(PowerDnsBaseProvider, self).__init__(id, *args, **kwargs) self.host = host self.port = port self.scheme = scheme + self.timeout = timeout sess = Session() sess.headers.update({'X-API-Key': api_key}) @@ -35,7 +36,7 @@ class PowerDnsBaseProvider(BaseProvider): url = '{}://{}:{}/api/v1/servers/localhost/{}' \ .format(self.scheme, self.host, self.port, path) - resp = self._sess.request(method, url, json=data, timeout=self.TIMEOUT) + resp = self._sess.request(method, url, json=data, timeout=self.timeout) self.log.debug('_request: status=%d', resp.status_code) resp.raise_for_status() return resp From 77d2fd1eb401adbd68410c67f24a45b30eb3f622 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Herv=C3=A9=20Beraud?= Date: Fri, 3 Nov 2017 22:31:38 +0100 Subject: [PATCH 040/141] improve setuptools capabilities --- README.md | 1 + octodns/__init__.py | 25 +++++++++++++++-- requirements-dev.txt | 7 ----- requirements.txt | 23 --------------- script/bootstrap | 4 +-- setup.cfg | 67 ++++++++++++++++++++++++++++++++++++++++++++ setup.py | 46 ++---------------------------- 7 files changed, 94 insertions(+), 79 deletions(-) delete mode 100644 requirements-dev.txt delete mode 100644 requirements.txt create mode 100644 setup.cfg diff --git a/README.md b/README.md index a910b5b..ec9164f 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ $ cd dns $ virtualenv env ... $ source env/bin/activate +$ pip install -U setuptools $ pip install octodns $ mkdir config ``` diff --git a/octodns/__init__.py b/octodns/__init__.py index 2166778..aaaa2a5 100644 --- a/octodns/__init__.py +++ b/octodns/__init__.py @@ -1,6 +1,25 @@ -'OctoDNS: DNS as code - Tools for managing DNS across multiple providers' - from __future__ import absolute_import, division, print_function, \ unicode_literals +import pkg_resources +from os import path +from setuptools.config import read_configuration + + +def _extract_version(package_name): + try: + return pkg_resources.get_distribution(package_name).version + except pkg_resources.DistributionNotFound: + _conf = read_configuration( + path.join( + path.dirname(path.dirname(__file__)), + 'setup.cfg' + ) + ) + return _conf['metadata']['version'] + + +__version__ = _extract_version('octodns') + -__VERSION__ = '0.8.8' +if __name__ == "__main__": + print(__version__) diff --git a/requirements-dev.txt b/requirements-dev.txt deleted file mode 100644 index 5cdf252..0000000 --- a/requirements-dev.txt +++ /dev/null @@ -1,7 +0,0 @@ -coverage -mock -nose -pep8 -pyflakes -requests_mock -setuptools>=36.4.0 diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 80fbe1e..0000000 --- a/requirements.txt +++ /dev/null @@ -1,23 +0,0 @@ -# These are known good versions. You're free to use others and things will -# likely work, but no promises are made, especilly if you go older. -PyYaml==3.12 -azure-mgmt-dns==1.0.1 -azure-common==1.1.6 -boto3==1.4.6 -botocore==1.6.8 -dnspython==1.15.0 -docutils==0.14 -dyn==1.8.0 -futures==3.1.1 -google-cloud==0.27.0 -incf.countryutils==1.0 -ipaddress==1.0.18 -jmespath==0.9.3 -msrestazure==0.4.10 -natsort==5.0.3 -nsone==0.9.14 -ovh==0.4.7 -python-dateutil==2.6.1 -requests==2.13.0 -s3transfer==0.1.10 -six==1.10.0 diff --git a/script/bootstrap b/script/bootstrap index 1f76914..dfbb142 100755 --- a/script/bootstrap +++ b/script/bootstrap @@ -19,10 +19,10 @@ if [ ! -d "$VENV_NAME" ]; then fi . "$VENV_NAME/bin/activate" -pip install -U -r requirements.txt +pip install -e . if [ "$ENV" != "production" ]; then - pip install -U -r requirements-dev.txt + pip install -e .[dev] fi if [ ! -L ".git/hooks/pre-commit" ]; then diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..f21fdbb --- /dev/null +++ b/setup.cfg @@ -0,0 +1,67 @@ +[metadata] +name = octodns +description = "DNS as code - Tools for managing DNS across multiple providers" +long_description = file: README.md +version = 0.8.8 +author = Ross McFarland +author_email = rwmcfa1@gmail.com +url = https://github.com/github/octodns +license = MIT +keywords = dns, providers +classifiers = + License :: OSI Approved :: MIT License + Programming Language :: Python + Programming Language :: Python :: 2.7 + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.3 + Programming Language :: Python :: 3.4 + Programming Language :: Python :: 3.5 + Programming Language :: Python :: 3.6 + +[options] +install_requires = + PyYaml==3.12 + azure-mgmt-dns==1.0.1 + azure-common==1.1.6 + boto3==1.4.6 + botocore==1.6.8 + dnspython==1.15.0 + docutils==0.14 + dyn==1.8.0 + futures==3.1.1 + google-cloud==0.27.0 + incf.countryutils==1.0 + ipaddress==1.0.18 + jmespath==0.9.3 + msrestazure==0.4.10 + natsort==5.0.3 + nsone==0.9.14 + ovh==0.4.7 + python-dateutil==2.6.1 + requests==2.13.0 + s3transfer==0.1.10 + six==1.10.0 +packages = find: +include_package_data = True + +[options.entry_points] +console_scripts = + octodns-compare = octodns.cmds.compare:main + octodns-dump = octodns.cmds.dump:main + octodns-report = octodns.cmds.report:main + octodns-sync = octodns.cmds.sync:main + octodns-validate = octodns.cmds.validate:main + +[options.packages.find] +exclude = + tests + +[options.extras_require] +dev = + coverage + mock + nose + pep8 + pyflakes + requests_mock + setuptools>=36.4.0 diff --git a/setup.py b/setup.py index f2b901d..2598061 100644 --- a/setup.py +++ b/setup.py @@ -1,47 +1,5 @@ #!/usr/bin/env python +from setuptools import setup -from os.path import dirname, join -import octodns -try: - from setuptools import find_packages, setup -except ImportError: - from distutils.core import find_packages, setup - -cmds = ( - 'compare', - 'dump', - 'report', - 'sync', - 'validate' -) -cmds_dir = join(dirname(__file__), 'octodns', 'cmds') -console_scripts = { - 'octodns-{name} = octodns.cmds.{name}:main'.format(name=name) - for name in cmds -} - -setup( - author='Ross McFarland', - author_email='rwmcfa1@gmail.com', - description=octodns.__doc__, - entry_points={ - 'console_scripts': console_scripts, - }, - install_requires=[ - 'PyYaml>=3.12', - 'dnspython>=1.15.0', - 'futures>=3.0.5', - 'incf.countryutils>=1.0', - 'ipaddress>=1.0.18', - 'natsort>=5.0.3', - 'python-dateutil>=2.6.0', - 'requests>=2.13.0' - ], - license='MIT', - long_description=open('README.md').read(), - name='octodns', - packages=find_packages(), - url='https://github.com/github/octodns', - version=octodns.__VERSION__, -) +setup() From dd692320c9f1779497df9b8cbc30c4d9ebc2337d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Herv=C3=A9=20Beraud?= Date: Sat, 4 Nov 2017 22:01:18 +0100 Subject: [PATCH 041/141] Apply review comments define 3 kinds of requirements (base, dev, test) retrieve version from __init__.py define setuptools minimal version in CI install full (base, dev, test) dependencies --- MANIFEST.in | 1 - README.md | 2 +- octodns/__init__.py | 22 +--------------------- script/bootstrap | 2 +- setup.cfg | 35 ++++++++++++++++++----------------- 5 files changed, 21 insertions(+), 41 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 3a26904..cda90ed 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,6 +3,5 @@ include CONTRIBUTING.md include LICENSE include docs/* include octodns/* -include requirements*.txt include script/* include tests/* diff --git a/README.md b/README.md index ec9164f..ed1ac3b 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ $ cd dns $ virtualenv env ... $ source env/bin/activate -$ pip install -U setuptools +$ pip install -U setuptools>⁼30.3.0 $ pip install octodns $ mkdir config ``` diff --git a/octodns/__init__.py b/octodns/__init__.py index aaaa2a5..05a5e84 100644 --- a/octodns/__init__.py +++ b/octodns/__init__.py @@ -1,25 +1,5 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -import pkg_resources -from os import path -from setuptools.config import read_configuration -def _extract_version(package_name): - try: - return pkg_resources.get_distribution(package_name).version - except pkg_resources.DistributionNotFound: - _conf = read_configuration( - path.join( - path.dirname(path.dirname(__file__)), - 'setup.cfg' - ) - ) - return _conf['metadata']['version'] - - -__version__ = _extract_version('octodns') - - -if __name__ == "__main__": - print(__version__) +__version__ = '0.8.8' diff --git a/script/bootstrap b/script/bootstrap index dfbb142..7f4a5a8 100755 --- a/script/bootstrap +++ b/script/bootstrap @@ -22,7 +22,7 @@ fi pip install -e . if [ "$ENV" != "production" ]; then - pip install -e .[dev] + pip install -e .[dev,test] fi if [ ! -L ".git/hooks/pre-commit" ]; then diff --git a/setup.cfg b/setup.cfg index f21fdbb..70baaf6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,7 +2,7 @@ name = octodns description = "DNS as code - Tools for managing DNS across multiple providers" long_description = file: README.md -version = 0.8.8 +version = attr: octodns.__version__ author = Ross McFarland author_email = rwmcfa1@gmail.com url = https://github.com/github/octodns @@ -20,27 +20,14 @@ classifiers = [options] install_requires = - PyYaml==3.12 - azure-mgmt-dns==1.0.1 - azure-common==1.1.6 - boto3==1.4.6 - botocore==1.6.8 - dnspython==1.15.0 - docutils==0.14 - dyn==1.8.0 + PyYaml>=3.12 + dnspython>=1.15.0 futures==3.1.1 - google-cloud==0.27.0 incf.countryutils==1.0 ipaddress==1.0.18 - jmespath==0.9.3 - msrestazure==0.4.10 natsort==5.0.3 - nsone==0.9.14 - ovh==0.4.7 python-dateutil==2.6.1 requests==2.13.0 - s3transfer==0.1.10 - six==1.10.0 packages = find: include_package_data = True @@ -57,7 +44,21 @@ exclude = tests [options.extras_require] -dev = +dev = + azure-mgmt-dns==1.0.1 + azure-common==1.1.6 + boto3==1.4.6 + botocore==1.6.8 + docutils==0.14 + dyn==1.8.0 + google-cloud==0.27.0 + jmespath==0.9.3 + msrestazure==0.4.10 + nsone==0.9.14 + ovh==0.4.7 + s3transfer==0.1.10 + six==1.10.0 +test = coverage mock nose From 2ee1a41a78ba948fb8e4e07b4991a8c0efb92cc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Herv=C3=A9=20Beraud?= Date: Sun, 5 Nov 2017 12:37:06 +0100 Subject: [PATCH 042/141] Remove setuptools minimal version adding minimal version only for contributors --- CONTRIBUTING.md | 4 ++++ README.md | 1 - 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9a5709a..36337eb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -38,6 +38,10 @@ Here are a few things you can do that will increase the likelihood of your pull - Write a [good commit message](http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html). +## Development prerequisites + +- setuptools >= 30.3.0 + ## License note We can only accept contributions that are compatible with the MIT license. diff --git a/README.md b/README.md index ed1ac3b..a910b5b 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,6 @@ $ cd dns $ virtualenv env ... $ source env/bin/activate -$ pip install -U setuptools>⁼30.3.0 $ pip install octodns $ mkdir config ``` From 7fa999953fc3229f95053d0c36a6ee14f1251c63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Herv=C3=A9=20Beraud?= Date: Sun, 5 Nov 2017 13:38:31 +0100 Subject: [PATCH 043/141] reset changes on __init__ fix error on version --- octodns/__init__.py | 5 +++-- setup.cfg | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/octodns/__init__.py b/octodns/__init__.py index 05a5e84..2166778 100644 --- a/octodns/__init__.py +++ b/octodns/__init__.py @@ -1,5 +1,6 @@ +'OctoDNS: DNS as code - Tools for managing DNS across multiple providers' + from __future__ import absolute_import, division, print_function, \ unicode_literals - -__version__ = '0.8.8' +__VERSION__ = '0.8.8' diff --git a/setup.cfg b/setup.cfg index 70baaf6..54e9014 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,7 +2,7 @@ name = octodns description = "DNS as code - Tools for managing DNS across multiple providers" long_description = file: README.md -version = attr: octodns.__version__ +version = attr: octodns.__VERSION__ author = Ross McFarland author_email = rwmcfa1@gmail.com url = https://github.com/github/octodns From 454f7f8c8fe9757a23a33ca52ced0ebc32a7cdbe Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Wed, 8 Nov 2017 06:26:18 -0800 Subject: [PATCH 044/141] Add formal CAA support to YamlProvider --- octodns/provider/yaml.py | 4 ++-- tests/test_octodns_manager.py | 14 +++++++------- tests/test_octodns_provider_yaml.py | 14 +++++++++----- 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/octodns/provider/yaml.py b/octodns/provider/yaml.py index fe1a406..752e793 100644 --- a/octodns/provider/yaml.py +++ b/octodns/provider/yaml.py @@ -31,8 +31,8 @@ class YamlProvider(BaseProvider): enforce_order: True ''' SUPPORTS_GEO = True - SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CNAME', 'MX', 'NAPTR', 'NS', 'PTR', - 'SSHFP', 'SPF', 'SRV', 'TXT')) + SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CAA', 'CNAME', 'MX', 'NAPTR', 'NS', + 'PTR', 'SSHFP', 'SPF', 'SRV', 'TXT')) def __init__(self, id, directory, default_ttl=3600, enforce_order=True, *args, **kwargs): diff --git a/tests/test_octodns_manager.py b/tests/test_octodns_manager.py index 36b1f4c..4db2103 100644 --- a/tests/test_octodns_manager.py +++ b/tests/test_octodns_manager.py @@ -102,12 +102,12 @@ class TestManager(TestCase): environ['YAML_TMP_DIR'] = tmpdir.dirname tc = Manager(get_config_filename('simple.yaml')) \ .sync(dry_run=False) - self.assertEquals(20, tc) + self.assertEquals(21, tc) # try with just one of the zones tc = Manager(get_config_filename('simple.yaml')) \ .sync(dry_run=False, eligible_zones=['unit.tests.']) - self.assertEquals(14, tc) + self.assertEquals(15, tc) # the subzone, with 2 targets tc = Manager(get_config_filename('simple.yaml')) \ @@ -122,18 +122,18 @@ class TestManager(TestCase): # Again with force tc = Manager(get_config_filename('simple.yaml')) \ .sync(dry_run=False, force=True) - self.assertEquals(20, tc) + self.assertEquals(21, tc) # Again with max_workers = 1 tc = Manager(get_config_filename('simple.yaml'), max_workers=1) \ .sync(dry_run=False, force=True) - self.assertEquals(20, tc) + self.assertEquals(21, tc) # Include meta tc = Manager(get_config_filename('simple.yaml'), max_workers=1, include_meta=True) \ .sync(dry_run=False, force=True) - self.assertEquals(24, tc) + self.assertEquals(25, tc) def test_eligible_targets(self): with TemporaryDirectory() as tmpdir: @@ -159,13 +159,13 @@ class TestManager(TestCase): fh.write('---\n{}') changes = manager.compare(['in'], ['dump'], 'unit.tests.') - self.assertEquals(14, len(changes)) + self.assertEquals(15, len(changes)) # Compound sources with varying support changes = manager.compare(['in', 'nosshfp'], ['dump'], 'unit.tests.') - self.assertEquals(13, len(changes)) + self.assertEquals(14, len(changes)) with self.assertRaises(Exception) as ctx: manager.compare(['nope'], ['dump'], 'unit.tests.') diff --git a/tests/test_octodns_provider_yaml.py b/tests/test_octodns_provider_yaml.py index a199355..46363ed 100644 --- a/tests/test_octodns_provider_yaml.py +++ b/tests/test_octodns_provider_yaml.py @@ -49,12 +49,12 @@ class TestYamlProvider(TestCase): # We add everything plan = target.plan(zone) - self.assertEquals(14, len(filter(lambda c: isinstance(c, Create), + self.assertEquals(15, len(filter(lambda c: isinstance(c, Create), plan.changes))) self.assertFalse(isfile(yaml_file)) # Now actually do it - self.assertEquals(14, target.apply(plan)) + self.assertEquals(15, target.apply(plan)) self.assertTrue(isfile(yaml_file)) # There should be no changes after the round trip @@ -64,15 +64,19 @@ class TestYamlProvider(TestCase): # A 2nd sync should still create everything plan = target.plan(zone) - self.assertEquals(14, len(filter(lambda c: isinstance(c, Create), + self.assertEquals(15, len(filter(lambda c: isinstance(c, Create), plan.changes))) with open(yaml_file) as fh: data = safe_load(fh.read()) + # '' has some of both + roots = sorted(data[''], key=lambda r: r['type']) + self.assertTrue('values' in roots[0]) # A + self.assertTrue('value' in roots[1]) # CAA + self.assertTrue('values' in roots[2]) # SSHFP + # these are stored as plural 'values' - for r in data['']: - self.assertTrue('values' in r) self.assertTrue('values' in data['mx']) self.assertTrue('values' in data['naptr']) self.assertTrue('values' in data['_srv._tcp']) From b4ead495f5069852e6925e64603caff649da279c Mon Sep 17 00:00:00 2001 From: Tim Hughes Date: Wed, 8 Nov 2017 14:41:48 +0000 Subject: [PATCH 045/141] adds an example of how to setup geodns to the docs --- docs/records.md | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/docs/records.md b/docs/records.md index d991311..82d9a37 100644 --- a/docs/records.md +++ b/docs/records.md @@ -26,6 +26,55 @@ GeoDNS is currently supported for `A` and `AAAA` records on the Dyn (via Traffic Configuring GeoDNS is complex and the details of the functionality vary widely from provider to provider. OctoDNS has an opinionated view of how GeoDNS should be set up and does its best to map that to each provider's offering in a way that will result in similar behavior. It may not fit your needs or use cases, in which case please open an issue for discussion. We expect this functionality to grow and evolve over time as it's more widely used. +The following is an example of GeoDNS with three entries NA-US-CA, NA-US-NY, OC-AU. Octodns creates another one labeled 'default' with the details for the actual A record, This default record is the failover record if the monitoring check fails. + +```yaml +--- +? '' +: type: TXT + value: v=spf1 -all +test: + geo: + NA-US-NY: + - 111.111.111.1 + NA-US-CA: + - 111.111.111.2 + OC-AU: + - 111.111.111.3 + EU: + - 111.111.111.4 + ttl: 300 + type: A + value: 111.111.111.5 +``` + + +The geo labels breakdown based on: + +1. + - 'AF': 14, # Continental Africa + - 'AN': 17, # Continental Antartica + - 'AS': 15, # Contentinal Asia + - 'EU': 13, # Contentinal Europe + - 'NA': 11, # Continental North America + - 'OC': 16, # Contentinal Austrailia/Oceania + - 'SA': 12, # Continental South America + +2. ISO Country Code https://en.wikipedia.org/wiki/ISO_3166-2 + +3. ISO Country Code Subdevision as per https://en.wikipedia.org/wiki/ISO_3166-2:US (change the code at the end for the country you are subdividing) * these may not always be supported depending on the provider. + +So the example is saying: + +- North America - United States - New York: gets served an "A" record of 111.111.111.1 +- North America - United States - California: gets served an "A" record of 111.111.111.2 +- Oceania - Australia: Gets served an "A" record of 111.111.111.3 +- Europe: gets an "A" record of 111.111.111.4 +- Everyone else gets an "A" record of 111.111.111.5 + + +Octodns will automatically set up a monitor and check for **https:///_dns** and check for a 200 response. + ## Config (`YamlProvider`) OctoDNS records and `YamlProvider`'s schema is essentially a 1:1 match. Properties on the objects will match keys in the config. From feec4a68215b5bc8f85b9e1d3f491ada08a84cae Mon Sep 17 00:00:00 2001 From: Adam Smith Date: Mon, 6 Nov 2017 22:32:46 -0800 Subject: [PATCH 046/141] Add DigitalOcean provider --- README.md | 1 + octodns/provider/digitalocean.py | 336 ++++++++++++++++++++ tests/fixtures/digitalocean-page-1.json | 177 +++++++++++ tests/fixtures/digitalocean-page-2.json | 89 ++++++ tests/test_octodns_provider_digitalocean.py | 241 ++++++++++++++ 5 files changed, 844 insertions(+) create mode 100644 octodns/provider/digitalocean.py create mode 100644 tests/fixtures/digitalocean-page-1.json create mode 100644 tests/fixtures/digitalocean-page-2.json create mode 100644 tests/test_octodns_provider_digitalocean.py diff --git a/README.md b/README.md index a910b5b..88223e6 100644 --- a/README.md +++ b/README.md @@ -151,6 +151,7 @@ The above command pulled the existing data out of Route53 and placed the results |--|--|--|--| | [AzureProvider](/octodns/provider/azuredns.py) | A, AAAA, CNAME, MX, NS, PTR, SRV, TXT | No | | | [CloudflareProvider](/octodns/provider/cloudflare.py) | A, AAAA, CAA, CNAME, MX, NS, SPF, TXT | No | CAA tags restricted | +| [DigitalOceanProvider](/octodns/provider/digitalocean.py) | A, AAAA, CAA, CNAME, MX, NS, TXT, SRV | No | CAA tags restricted | | [DnsimpleProvider](/octodns/provider/dnsimple.py) | All | No | CAA tags restricted | | [DynProvider](/octodns/provider/dyn.py) | All | Yes | | | [GoogleCloudProvider](/octodns/provider/googlecloud.py) | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | No | | diff --git a/octodns/provider/digitalocean.py b/octodns/provider/digitalocean.py new file mode 100644 index 0000000..55fa10b --- /dev/null +++ b/octodns/provider/digitalocean.py @@ -0,0 +1,336 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from collections import defaultdict +from requests import Session +import logging + +from ..record import Record +from .base import BaseProvider + + +class DigitalOceanClientException(Exception): + pass + + +class DigitalOceanClientNotFound(DigitalOceanClientException): + + def __init__(self): + super(DigitalOceanClientNotFound, self).__init__('Not Found') + + +class DigitalOceanClientUnauthorized(DigitalOceanClientException): + + def __init__(self): + super(DigitalOceanClientUnauthorized, self).__init__('Unauthorized') + + +class DigitalOceanClient(object): + BASE = 'https://api.digitalocean.com/v2' + + def __init__(self, token): + sess = Session() + sess.headers.update({'Authorization': 'Bearer {}'.format(token)}) + self._sess = sess + + def _request(self, method, path, params=None, data=None): + url = '{}{}'.format(self.BASE, path) + resp = self._sess.request(method, url, params=params, json=data) + if resp.status_code == 401: + raise DigitalOceanClientUnauthorized() + if resp.status_code == 404: + raise DigitalOceanClientNotFound() + resp.raise_for_status() + return resp + + def domain(self, name): + path = '/domains/{}'.format(name) + return self._request('GET', path).json() + + def domain_create(self, name): + # Digitalocean requires an IP on zone creation + self._request('POST', '/domains', data={'name': name, + 'ip_address': '192.0.2.1'}) + + # After the zone is created, immeadiately delete the record + records = self.records(name) + for record in records: + if record['name'] == '' and record['type'] == 'A': + self.record_delete(name, record['id']) + + def records(self, zone_name): + path = '/domains/{}/records'.format(zone_name) + ret = [] + + page = 1 + while True: + data = self._request('GET', path, {'page': page}).json() + + ret += data['domain_records'] + links = data['links'] + + # https://developers.digitalocean.com/documentation/v2/#links + # pages exists if there is more than 1 page + # last doesn't exist if you're on the last page + try: + links['pages']['last'] + page += 1 + except KeyError: + break + + # change any apex record to empty string to match other provider output + for record in ret: + if record['name'] == '@': + record['name'] = '' + + return ret + + def record_create(self, zone_name, params): + path = '/domains/{}/records'.format(zone_name) + # change empty string to @, DigitalOcean uses @ for apex record names + if params['name'] == '': + params['name'] = '@' + self._request('POST', path, data=params) + + def record_delete(self, zone_name, record_id): + path = '/domains/{}/records/{}'.format(zone_name, record_id) + self._request('DELETE', path) + + +class DigitalOceanProvider(BaseProvider): + ''' + DigitalOcean DNS provider using API v2 + + digitalocean: + class: octodns.provider.digitalocean.DigitalOceanProvider + # Your DigitalOcean API token (required) + token: foo + ''' + SUPPORTS_GEO = False + SUPPORTS = set(('A', 'AAAA', 'CAA', 'CNAME', 'MX', 'NS', 'TXT', 'SRV')) + + def __init__(self, id, token, *args, **kwargs): + self.log = logging.getLogger('DigitalOceanProvider[{}]'.format(id)) + self.log.debug('__init__: id=%s, token=***', id) + super(DigitalOceanProvider, self).__init__(id, *args, **kwargs) + self._client = DigitalOceanClient(token) + + self._zone_records = {} + + def _data_for_multiple(self, _type, records): + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': [r['data'] for r in records] + } + + _data_for_A = _data_for_multiple + _data_for_AAAA = _data_for_multiple + + def _data_for_CAA(self, _type, records): + values = [] + for record in records: + values.append({ + 'flags': record['flags'], + 'tag': record['tag'], + 'value': record['data'], + }) + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': values + } + + def _data_for_CNAME(self, _type, records): + record = records[0] + return { + 'ttl': record['ttl'], + 'type': _type, + 'value': '{}.'.format(record['data']) + } + + def _data_for_MX(self, _type, records): + values = [] + for record in records: + values.append({ + 'preference': record['priority'], + 'exchange': '{}.'.format(record['data']) + }) + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': values + } + + def _data_for_NS(self, _type, records): + values = [] + for record in records: + data = '{}.'.format(record['data']) + values.append(data) + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': values, + } + + def _data_for_SRV(self, _type, records): + values = [] + for record in records: + values.append({ + 'port': record['port'], + 'priority': record['priority'], + 'target': '{}.'.format(record['data']), + 'weight': record['weight'] + }) + return { + 'type': _type, + 'ttl': records[0]['ttl'], + 'values': values + } + + def _data_for_TXT(self, _type, records): + values = [value['data'].replace(';', '\;') for value in records] + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': values + } + + def zone_records(self, zone): + if zone.name not in self._zone_records: + try: + self._zone_records[zone.name] = \ + self._client.records(zone.name[:-1]) + except DigitalOceanClientNotFound: + return [] + + return self._zone_records[zone.name] + + def populate(self, zone, target=False, lenient=False): + self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, + target, lenient) + + values = defaultdict(lambda: defaultdict(list)) + for record in self.zone_records(zone): + _type = record['type'] + values[record['name']][record['type']].append(record) + + before = len(zone.records) + for name, types in values.items(): + for _type, records in types.items(): + data_for = getattr(self, '_data_for_{}'.format(_type)) + record = Record.new(zone, name, data_for(_type, records), + source=self, lenient=lenient) + zone.add_record(record) + + self.log.info('populate: found %s records', + len(zone.records) - before) + + def _params_for_multiple(self, record): + for value in record.values: + yield { + 'data': value, + 'name': record.name, + 'ttl': record.ttl, + 'type': record._type + } + + _params_for_A = _params_for_multiple + _params_for_AAAA = _params_for_multiple + _params_for_NS = _params_for_multiple + + def _params_for_CAA(self, record): + for value in record.values: + yield { + 'data': '{}.'.format(value.value), + 'flags': value.flags, + 'name': record.name, + 'tag': value.tag, + 'ttl': record.ttl, + 'type': record._type + } + + def _params_for_single(self, record): + yield { + 'data': record.value, + 'name': record.name, + 'ttl': record.ttl, + 'type': record._type + } + + _params_for_CNAME = _params_for_single + + def _params_for_MX(self, record): + for value in record.values: + yield { + 'data': value.exchange, + 'name': record.name, + 'priority': value.preference, + 'ttl': record.ttl, + 'type': record._type + } + + def _params_for_SRV(self, record): + for value in record.values: + yield { + 'data': value.target, + 'name': record.name, + 'port': value.port, + 'priority': value.priority, + 'ttl': record.ttl, + 'type': record._type, + 'weight': value.weight + } + + def _params_for_TXT(self, record): + # DigitalOcean doesn't want things escaped in values so we + # have to strip them here and add them when going the other way + for value in record.values: + yield { + 'data': value.replace('\;', ';'), + 'name': record.name, + 'ttl': record.ttl, + 'type': record._type + } + + def _apply_Create(self, change): + new = change.new + params_for = getattr(self, '_params_for_{}'.format(new._type)) + for params in params_for(new): + self._client.record_create(new.zone.name[:-1], params) + + def _apply_Update(self, change): + self._apply_Delete(change) + self._apply_Create(change) + + def _apply_Delete(self, change): + existing = change.existing + zone = existing.zone + for record in self.zone_records(zone): + if existing.name == record['name'] and \ + existing._type == record['type']: + self._client.record_delete(zone.name[:-1], record['id']) + + def _apply(self, plan): + desired = plan.desired + changes = plan.changes + self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name, + len(changes)) + + domain_name = desired.name[:-1] + try: + self._client.domain(domain_name) + except DigitalOceanClientNotFound: + self.log.debug('_apply: no matching zone, creating domain') + self._client.domain_create(domain_name) + + for change in changes: + class_name = change.__class__.__name__ + getattr(self, '_apply_{}'.format(class_name))(change) + + # Clear out the cache if any + self._zone_records.pop(desired.name, None) diff --git a/tests/fixtures/digitalocean-page-1.json b/tests/fixtures/digitalocean-page-1.json new file mode 100644 index 0000000..db231ba --- /dev/null +++ b/tests/fixtures/digitalocean-page-1.json @@ -0,0 +1,177 @@ +{ + "domain_records": [{ + "id": 11189874, + "type": "NS", + "name": "@", + "data": "ns1.digitalocean.com", + "priority": null, + "port": null, + "ttl": 3600, + "weight": null, + "flags": null, + "tag": null + }, { + "id": 11189875, + "type": "NS", + "name": "@", + "data": "ns2.digitalocean.com", + "priority": null, + "port": null, + "ttl": 3600, + "weight": null, + "flags": null, + "tag": null + }, { + "id": 11189876, + "type": "NS", + "name": "@", + "data": "ns3.digitalocean.com", + "priority": null, + "port": null, + "ttl": 3600, + "weight": null, + "flags": null, + "tag": null + }, { + "id": 11189877, + "type": "NS", + "name": "under", + "data": "ns1.unit.tests", + "priority": null, + "port": null, + "ttl": 3600, + "weight": null, + "flags": null, + "tag": null + }, { + "id": 11189878, + "type": "NS", + "name": "under", + "data": "ns2.unit.tests", + "priority": null, + "port": null, + "ttl": 3600, + "weight": null, + "flags": null, + "tag": null + }, { + "id": 11189879, + "type": "SRV", + "name": "_srv._tcp", + "data": "foo-1.unit.tests", + "priority": 10, + "port": 30, + "ttl": 600, + "weight": 20, + "flags": null, + "tag": null + }, { + "id": 11189880, + "type": "SRV", + "name": "_srv._tcp", + "data": "foo-2.unit.tests", + "priority": 12, + "port": 30, + "ttl": 600, + "weight": 20, + "flags": null, + "tag": null + }, { + "id": 11189881, + "type": "TXT", + "name": "txt", + "data": "Bah bah black sheep", + "priority": null, + "port": null, + "ttl": 600, + "weight": null, + "flags": null, + "tag": null + }, { + "id": 11189882, + "type": "TXT", + "name": "txt", + "data": "have you any wool.", + "priority": null, + "port": null, + "ttl": 600, + "weight": null, + "flags": null, + "tag": null + }, { + "id": 11189883, + "type": "A", + "name": "@", + "data": "1.2.3.4", + "priority": null, + "port": null, + "ttl": 300, + "weight": null, + "flags": null, + "tag": null + }, { + "id": 11189884, + "type": "A", + "name": "@", + "data": "1.2.3.5", + "priority": null, + "port": null, + "ttl": 300, + "weight": null, + "flags": null, + "tag": null + }, { + "id": 11189885, + "type": "A", + "name": "www", + "data": "2.2.3.6", + "priority": null, + "port": null, + "ttl": 300, + "weight": null, + "flags": null, + "tag": null + }, { + "id": 11189886, + "type": "MX", + "name": "mx", + "data": "smtp-4.unit.tests", + "priority": 10, + "port": null, + "ttl": 300, + "weight": null, + "flags": null, + "tag": null + }, { + "id": 11189887, + "type": "MX", + "name": "mx", + "data": "smtp-2.unit.tests", + "priority": 20, + "port": null, + "ttl": 300, + "weight": null, + "flags": null, + "tag": null + }, { + "id": 11189888, + "type": "MX", + "name": "mx", + "data": "smtp-3.unit.tests", + "priority": 30, + "port": null, + "ttl": 300, + "weight": null, + "flags": null, + "tag": null + }], + "links": { + "pages": { + "last": "https://api.digitalocean.com/v2/domains/unit.tests/records?page=2", + "next": "https://api.digitalocean.com/v2/domains/unit.tests/records?page=2" + } + }, + "meta": { + "total": 21 + } +} \ No newline at end of file diff --git a/tests/fixtures/digitalocean-page-2.json b/tests/fixtures/digitalocean-page-2.json new file mode 100644 index 0000000..8b989ae --- /dev/null +++ b/tests/fixtures/digitalocean-page-2.json @@ -0,0 +1,89 @@ +{ + "domain_records": [{ + "id": 11189889, + "type": "MX", + "name": "mx", + "data": "smtp-1.unit.tests", + "priority": 40, + "port": null, + "ttl": 300, + "weight": null, + "flags": null, + "tag": null + }, { + "id": 11189890, + "type": "AAAA", + "name": "aaaa", + "data": "2601:644:500:e210:62f8:1dff:feb8:947a", + "priority": null, + "port": null, + "ttl": 600, + "weight": null, + "flags": null, + "tag": null + }, { + "id": 11189891, + "type": "CNAME", + "name": "cname", + "data": "unit.tests", + "priority": null, + "port": null, + "ttl": 300, + "weight": null, + "flags": null, + "tag": null + }, { + "id": 11189892, + "type": "A", + "name": "www.sub", + "data": "2.2.3.6", + "priority": null, + "port": null, + "ttl": 300, + "weight": null, + "flags": null, + "tag": null + }, { + "id": 11189893, + "type": "TXT", + "name": "txt", + "data": "v=DKIM1;k=rsa;s=email;h=sha256;p=A/kinda+of/long/string+with+numb3rs", + "priority": null, + "port": null, + "ttl": 600, + "weight": null, + "flags": null, + "tag": null + }, { + "id": 11189894, + "type": "CAA", + "name": "@", + "data": "ca.unit.tests", + "priority": null, + "port": null, + "ttl": 3600, + "weight": null, + "flags": 0, + "tag": "issue" + }, { + "id": 11189895, + "type": "CNAME", + "name": "included", + "data": "unit.tests", + "priority": null, + "port": null, + "ttl": 3600, + "weight": null, + "flags": null, + "tag": null + }], + "links": { + "pages": { + "first": "https://api.digitalocean.com/v2/domains/unit.tests/records?page=1", + "prev": "https://api.digitalocean.com/v2/domains/unit.tests/records?page=1" + } + }, + "meta": { + "total": 21 + } +} \ No newline at end of file diff --git a/tests/test_octodns_provider_digitalocean.py b/tests/test_octodns_provider_digitalocean.py new file mode 100644 index 0000000..5fb47c5 --- /dev/null +++ b/tests/test_octodns_provider_digitalocean.py @@ -0,0 +1,241 @@ +# +# +# + + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from mock import Mock, call +from os.path import dirname, join +from requests import HTTPError +from requests_mock import ANY, mock as requests_mock +from unittest import TestCase + +from octodns.record import Record +from octodns.provider.digitalocean import DigitalOceanClientNotFound, \ + DigitalOceanProvider +from octodns.provider.yaml import YamlProvider +from octodns.zone import Zone + + +class TestDigitalOceanProvider(TestCase): + expected = Zone('unit.tests.', []) + source = YamlProvider('test', join(dirname(__file__), 'config')) + source.populate(expected) + + # Our test suite differs a bit, add our NS and remove the simple one + expected.add_record(Record.new(expected, 'under', { + 'ttl': 3600, + 'type': 'NS', + 'values': [ + 'ns1.unit.tests.', + 'ns2.unit.tests.', + ] + })) + for record in list(expected.records): + if record.name == 'sub' and record._type == 'NS': + expected._remove_record(record) + break + + def test_populate(self): + provider = DigitalOceanProvider('test', 'token') + + # Bad auth + with requests_mock() as mock: + mock.get(ANY, status_code=401, + text='{"id":"unauthorized",' + '"message":"Unable to authenticate you."}') + + with self.assertRaises(Exception) as ctx: + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals('Unauthorized', ctx.exception.message) + + # General error + with requests_mock() as mock: + mock.get(ANY, status_code=502, text='Things caught fire') + + with self.assertRaises(HTTPError) as ctx: + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals(502, ctx.exception.response.status_code) + + # Non-existant zone doesn't populate anything + with requests_mock() as mock: + mock.get(ANY, status_code=404, + text='{"id":"not_found","message":"The resource you ' + 'were accessing could not be found."}') + + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals(set(), zone.records) + + # No diffs == no changes + with requests_mock() as mock: + base = 'https://api.digitalocean.com/v2/domains/unit.tests/' \ + 'records?page=' + with open('tests/fixtures/digitalocean-page-1.json') as fh: + mock.get('{}{}'.format(base, 1), text=fh.read()) + with open('tests/fixtures/digitalocean-page-2.json') as fh: + mock.get('{}{}'.format(base, 2), text=fh.read()) + + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals(12, len(zone.records)) + changes = self.expected.changes(zone, provider) + self.assertEquals(0, len(changes)) + + # 2nd populate makes no network calls/all from cache + again = Zone('unit.tests.', []) + provider.populate(again) + self.assertEquals(12, len(again.records)) + + # bust the cache + del provider._zone_records[zone.name] + + def test_apply(self): + provider = DigitalOceanProvider('test', 'token') + + resp = Mock() + resp.json = Mock() + provider._client._request = Mock(return_value=resp) + + domain_after_creation = { + "domain_records": [{ + "id": 11189874, + "type": "NS", + "name": "@", + "data": "ns1.digitalocean.com", + "priority": None, + "port": None, + "ttl": 3600, + "weight": None, + "flags": None, + "tag": None + }, { + "id": 11189875, + "type": "NS", + "name": "@", + "data": "ns2.digitalocean.com", + "priority": None, + "port": None, + "ttl": 3600, + "weight": None, + "flags": None, + "tag": None + }, { + "id": 11189876, + "type": "NS", + "name": "@", + "data": "ns3.digitalocean.com", + "priority": None, + "port": None, + "ttl": 3600, + "weight": None, + "flags": None, + "tag": None + }, { + "id": 11189877, + "type": "A", + "name": "@", + "data": "192.0.2.1", + "priority": None, + "port": None, + "ttl": 3600, + "weight": None, + "flags": None, + "tag": None + }], + "links": {}, + "meta": { + "total": 4 + } + } + + # non-existant domain, create everything + resp.json.side_effect = [ + DigitalOceanClientNotFound, # no zone in populate + DigitalOceanClientNotFound, # no domain during apply + domain_after_creation + ] + plan = provider.plan(self.expected) + + # No root NS, no ignored, no excluded, no unsupported + n = len(self.expected.records) - 7 + self.assertEquals(n, len(plan.changes)) + self.assertEquals(n, provider.apply(plan)) + + provider._client._request.assert_has_calls([ + # created the domain + call('POST', '/domains', data={'ip_address': '192.0.2.1', + 'name': 'unit.tests'}), + # get all records in newly created zone + call('GET', '/domains/unit.tests/records', {'page': 1}), + # delete the initial A record + call('DELETE', '/domains/unit.tests/records/11189877'), + # created at least one of the record with expected data + call('POST', '/domains/unit.tests/records', data={ + 'name': '_srv._tcp', + 'weight': 20, + 'data': 'foo-1.unit.tests.', + 'priority': 10, + 'ttl': 600, + 'type': 'SRV', + 'port': 30 + }), + ]) + self.assertEquals(24, provider._client._request.call_count) + + provider._client._request.reset_mock() + + # delete 1 and update 1 + provider._client.records = Mock(return_value=[ + { + 'id': 11189897, + 'name': 'www', + 'data': '1.2.3.4', + 'ttl': 300, + 'type': 'A', + }, + { + 'id': 11189898, + 'name': 'www', + 'data': '2.2.3.4', + 'ttl': 300, + 'type': 'A', + }, + { + 'id': 11189899, + 'name': 'ttl', + 'data': '3.2.3.4', + 'ttl': 600, + 'type': 'A', + } + ]) + + # Domain exists, we don't care about return + resp.json.side_effect = ['{}'] + + wanted = Zone('unit.tests.', []) + wanted.add_record(Record.new(wanted, 'ttl', { + 'ttl': 300, + 'type': 'A', + 'value': '3.2.3.4' + })) + + plan = provider.plan(wanted) + self.assertEquals(2, len(plan.changes)) + self.assertEquals(2, provider.apply(plan)) + # recreate for update, and delete for the 2 parts of the other + provider._client._request.assert_has_calls([ + call('POST', '/domains/unit.tests/records', data={ + 'data': '3.2.3.4', + 'type': 'A', + 'name': 'ttl', + 'ttl': 300 + }), + call('DELETE', '/domains/unit.tests/records/11189899'), + call('DELETE', '/domains/unit.tests/records/11189897'), + call('DELETE', '/domains/unit.tests/records/11189898') + ], any_order=True) From 75cfc4fb76a43c38db55cd9eeb511aa7e97dcb2e Mon Sep 17 00:00:00 2001 From: Adam Smith Date: Mon, 13 Nov 2017 00:19:05 -0800 Subject: [PATCH 047/141] remove default config file for octodns-validate --- octodns/cmds/validate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/cmds/validate.py b/octodns/cmds/validate.py index 85c3018..6711ec9 100755 --- a/octodns/cmds/validate.py +++ b/octodns/cmds/validate.py @@ -15,7 +15,7 @@ from octodns.manager import Manager def main(): parser = ArgumentParser(description=__doc__.split('\n')[1]) - parser.add_argument('--config-file', default='./config/production.yaml', + parser.add_argument('--config-file', required=True, help='The Manager configuration file to use') args = parser.parse_args(WARN) From 0dfcc6f6f21706c8a5a8572d616bb27e4c91c1d9 Mon Sep 17 00:00:00 2001 From: Steve Coursen Date: Mon, 13 Nov 2017 09:37:03 -0500 Subject: [PATCH 048/141] Send appropriate meta along for A and AAAA records --- octodns/provider/ns1.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 2d3af9a..1a842fd 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -22,7 +22,7 @@ class Ns1Provider(BaseProvider): class: octodns.provider.ns1.Ns1Provider api_key: env/NS1_API_KEY ''' - SUPPORTS_GEO = False + SUPPORTS_GEO = True SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CAA', 'CNAME', 'MX', 'NAPTR', 'NS', 'PTR', 'SPF', 'SRV', 'TXT')) @@ -164,7 +164,16 @@ class Ns1Provider(BaseProvider): len(zone.records) - before) def _params_for_A(self, record): - return {'answers': record.values, 'ttl': record.ttl} + params = {'answers': record.values, 'ttl': record.ttl} + if record.geo: + # purposefully set non-geo answers to have an empty meta, + # so that we know we did this on purpose if/when troubleshooting + params['answers'] = [{"answer": x, "meta":{}} for x in record.values] + for iso_region, target in record.geo.items(): + params['answers'].append({'answer': target.values, + 'meta': {'iso_region_code': [iso_region]}, + }) + return params _params_for_AAAA = _params_for_A _params_for_NS = _params_for_A From 0cc20afabd64f6ab4c2d5c8cef47ed291a06ddee Mon Sep 17 00:00:00 2001 From: Steve Coursen Date: Mon, 13 Nov 2017 13:57:43 -0500 Subject: [PATCH 049/141] pep8 cleanup --- octodns/provider/ns1.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 1a842fd..d4f0572 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -168,11 +168,15 @@ class Ns1Provider(BaseProvider): if record.geo: # purposefully set non-geo answers to have an empty meta, # so that we know we did this on purpose if/when troubleshooting - params['answers'] = [{"answer": x, "meta":{}} for x in record.values] + params['answers'] = [{"answer": x, "meta": {}} \ + for x in record.values] for iso_region, target in record.geo.items(): - params['answers'].append({'answer': target.values, - 'meta': {'iso_region_code': [iso_region]}, - }) + params['answers'].append( + { + 'answer': target.values, + 'meta': {'iso_region_code': [iso_region]}, + }, + ) return params _params_for_AAAA = _params_for_A From 2cc17ffc7ae6aa551670b9b328da334a3b1a499d Mon Sep 17 00:00:00 2001 From: Steve Coursen Date: Mon, 13 Nov 2017 13:58:12 -0500 Subject: [PATCH 050/141] pep8 cleanup --- octodns/provider/ns1.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index d4f0572..1397c49 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -168,7 +168,7 @@ class Ns1Provider(BaseProvider): if record.geo: # purposefully set non-geo answers to have an empty meta, # so that we know we did this on purpose if/when troubleshooting - params['answers'] = [{"answer": x, "meta": {}} \ + params['answers'] = [{"answer": x, "meta": {}} for x in record.values] for iso_region, target in record.geo.items(): params['answers'].append( From ce5ecc52e3edc282427ba4e6655c826be5e0886f Mon Sep 17 00:00:00 2001 From: Steve Coursen Date: Tue, 14 Nov 2017 13:14:03 -0500 Subject: [PATCH 051/141] fix broken test by updating the actual format of the answers --- octodns/provider/ns1.py | 2 +- tests/test_octodns_provider_ns1.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 1397c49..008c665 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -165,7 +165,7 @@ class Ns1Provider(BaseProvider): def _params_for_A(self, record): params = {'answers': record.values, 'ttl': record.ttl} - if record.geo: + if hasattr(record, 'geo'): # purposefully set non-geo answers to have an empty meta, # so that we know we did this on purpose if/when troubleshooting params['answers'] = [{"answer": x, "meta": {}} diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index d4f4080..4436304 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -30,11 +30,13 @@ class TestNs1Provider(TestCase): 'ttl': 32, 'type': 'A', 'value': '1.2.3.4', + 'meta': {}, })) expected.add(Record.new(zone, 'foo', { 'ttl': 33, 'type': 'A', 'values': ['1.2.3.4', '1.2.3.5'], + 'meta': {}, })) expected.add(Record.new(zone, 'cname', { 'ttl': 34, @@ -289,7 +291,7 @@ class TestNs1Provider(TestCase): call('delete-me', u'A'), ]) mock_record.assert_has_calls([ - call.update(answers=[u'1.2.3.4'], ttl=32), + call.update(answers=[{'answer': u'1.2.3.4', 'meta': {}}], ttl=32), call.delete() ]) From 9018e796bbeb815c6f44d5275dbe00b18127a464 Mon Sep 17 00:00:00 2001 From: Adam Smith Date: Thu, 23 Nov 2017 22:17:50 -0800 Subject: [PATCH 052/141] migrate from pep8 to pycodestyle #152 --- script/lint | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/script/lint b/script/lint index 91e6d60..431ac4e 100755 --- a/script/lint +++ b/script/lint @@ -17,5 +17,5 @@ fi SOURCES="*.py octodns/*.py octodns/*/*.py tests/*.py" -pep8 --ignore=E221,E241,E251 $SOURCES +pycodestyle --ignore=E221,E241,E251,E722 $SOURCES pyflakes $SOURCES diff --git a/setup.cfg b/setup.cfg index 54e9014..31a0283 100644 --- a/setup.cfg +++ b/setup.cfg @@ -62,7 +62,7 @@ test = coverage mock nose - pep8 + pycodestyle pyflakes requests_mock setuptools>=36.4.0 From ef8d66ff9cc51be0a033c25bf0867053b063760f Mon Sep 17 00:00:00 2001 From: Adam Smith Date: Thu, 23 Nov 2017 21:49:14 -0800 Subject: [PATCH 053/141] Transform @ in Digitalocean API output to zone name --- octodns/provider/digitalocean.py | 9 +++++++-- tests/fixtures/digitalocean-page-2.json | 6 +++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/octodns/provider/digitalocean.py b/octodns/provider/digitalocean.py index 55fa10b..c68e7d6 100644 --- a/octodns/provider/digitalocean.py +++ b/octodns/provider/digitalocean.py @@ -82,18 +82,23 @@ class DigitalOceanClient(object): except KeyError: break - # change any apex record to empty string to match other provider output for record in ret: + # change any apex record to empty string if record['name'] == '@': record['name'] = '' + # change any apex value to zone name + if record['data'] == '@': + record['data'] = zone_name + return ret def record_create(self, zone_name, params): path = '/domains/{}/records'.format(zone_name) - # change empty string to @, DigitalOcean uses @ for apex record names + # change empty name string to @, DO uses @ for apex record names if params['name'] == '': params['name'] = '@' + self._request('POST', path, data=params) def record_delete(self, zone_name, record_id): diff --git a/tests/fixtures/digitalocean-page-2.json b/tests/fixtures/digitalocean-page-2.json index 8b989ae..50f17f9 100644 --- a/tests/fixtures/digitalocean-page-2.json +++ b/tests/fixtures/digitalocean-page-2.json @@ -25,7 +25,7 @@ "id": 11189891, "type": "CNAME", "name": "cname", - "data": "unit.tests", + "data": "@", "priority": null, "port": null, "ttl": 300, @@ -69,7 +69,7 @@ "id": 11189895, "type": "CNAME", "name": "included", - "data": "unit.tests", + "data": "@", "priority": null, "port": null, "ttl": 3600, @@ -86,4 +86,4 @@ "meta": { "total": 21 } -} \ No newline at end of file +} From f50d9b608708ababe3551ff0fb3cb07d3298b597 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 2 Dec 2017 10:12:28 -0800 Subject: [PATCH 054/141] Extract plan from base.py into plan.py --- octodns/manager.py | 3 +- octodns/provider/base.py | 73 +------------------------- octodns/provider/plan.py | 79 +++++++++++++++++++++++++++++ tests/test_octodns_provider_base.py | 3 +- 4 files changed, 84 insertions(+), 74 deletions(-) create mode 100644 octodns/provider/plan.py diff --git a/octodns/manager.py b/octodns/manager.py index 36a3592..e08754c 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -11,7 +11,8 @@ from importlib import import_module from os import environ import logging -from .provider.base import BaseProvider, Plan +from .provider.base import BaseProvider +from .provider.plan import Plan, PlanLogger from .provider.yaml import YamlProvider from .record import Record from .yaml import safe_load diff --git a/octodns/provider/base.py b/octodns/provider/base.py index f6ff1b7..2d4680f 100644 --- a/octodns/provider/base.py +++ b/octodns/provider/base.py @@ -7,78 +7,7 @@ from __future__ import absolute_import, division, print_function, \ from ..source.base import BaseSource from ..zone import Zone -from logging import getLogger - - -class UnsafePlan(Exception): - pass - - -class Plan(object): - log = getLogger('Plan') - - MAX_SAFE_UPDATE_PCENT = .3 - MAX_SAFE_DELETE_PCENT = .3 - MIN_EXISTING_RECORDS = 10 - - def __init__(self, existing, desired, changes, - update_pcent_threshold=MAX_SAFE_UPDATE_PCENT, - delete_pcent_threshold=MAX_SAFE_DELETE_PCENT): - self.existing = existing - self.desired = desired - self.changes = changes - self.update_pcent_threshold = update_pcent_threshold - self.delete_pcent_threshold = delete_pcent_threshold - - change_counts = { - 'Create': 0, - 'Delete': 0, - 'Update': 0 - } - for change in changes: - change_counts[change.__class__.__name__] += 1 - self.change_counts = change_counts - - try: - existing_n = len(self.existing.records) - except AttributeError: - existing_n = 0 - - self.log.debug('__init__: Creates=%d, Updates=%d, Deletes=%d' - 'Existing=%d', - self.change_counts['Create'], - self.change_counts['Update'], - self.change_counts['Delete'], existing_n) - - def raise_if_unsafe(self): - # TODO: what is safe really? - if self.existing and \ - len(self.existing.records) >= self.MIN_EXISTING_RECORDS: - - existing_record_count = len(self.existing.records) - update_pcent = self.change_counts['Update'] / existing_record_count - delete_pcent = self.change_counts['Delete'] / existing_record_count - - if update_pcent > self.update_pcent_threshold: - raise UnsafePlan('Too many updates, {} is over {} percent' - '({}/{})'.format( - update_pcent, - self.MAX_SAFE_UPDATE_PCENT * 100, - self.change_counts['Update'], - existing_record_count)) - if delete_pcent > self.delete_pcent_threshold: - raise UnsafePlan('Too many deletes, {} is over {} percent' - '({}/{})'.format( - delete_pcent, - self.MAX_SAFE_DELETE_PCENT * 100, - self.change_counts['Delete'], - existing_record_count)) - - def __repr__(self): - return 'Creates={}, Updates={}, Deletes={}, Existing Records={}' \ - .format(self.change_counts['Create'], self.change_counts['Update'], - self.change_counts['Delete'], - len(self.existing.records)) +from .plan import Plan class BaseProvider(BaseSource): diff --git a/octodns/provider/plan.py b/octodns/provider/plan.py new file mode 100644 index 0000000..cde859f --- /dev/null +++ b/octodns/provider/plan.py @@ -0,0 +1,79 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from logging import getLogger + + +class UnsafePlan(Exception): + pass + + +class Plan(object): + log = getLogger('Plan') + + MAX_SAFE_UPDATE_PCENT = .3 + MAX_SAFE_DELETE_PCENT = .3 + MIN_EXISTING_RECORDS = 10 + + def __init__(self, existing, desired, changes, + update_pcent_threshold=MAX_SAFE_UPDATE_PCENT, + delete_pcent_threshold=MAX_SAFE_DELETE_PCENT): + self.existing = existing + self.desired = desired + self.changes = changes + self.update_pcent_threshold = update_pcent_threshold + self.delete_pcent_threshold = delete_pcent_threshold + + change_counts = { + 'Create': 0, + 'Delete': 0, + 'Update': 0 + } + for change in changes: + change_counts[change.__class__.__name__] += 1 + self.change_counts = change_counts + + try: + existing_n = len(self.existing.records) + except AttributeError: + existing_n = 0 + + self.log.debug('__init__: Creates=%d, Updates=%d, Deletes=%d' + 'Existing=%d', + self.change_counts['Create'], + self.change_counts['Update'], + self.change_counts['Delete'], existing_n) + + def raise_if_unsafe(self): + # TODO: what is safe really? + if self.existing and \ + len(self.existing.records) >= self.MIN_EXISTING_RECORDS: + + existing_record_count = len(self.existing.records) + update_pcent = self.change_counts['Update'] / existing_record_count + delete_pcent = self.change_counts['Delete'] / existing_record_count + + if update_pcent > self.update_pcent_threshold: + raise UnsafePlan('Too many updates, {} is over {} percent' + '({}/{})'.format( + update_pcent, + self.MAX_SAFE_UPDATE_PCENT * 100, + self.change_counts['Update'], + existing_record_count)) + if delete_pcent > self.delete_pcent_threshold: + raise UnsafePlan('Too many deletes, {} is over {} percent' + '({}/{})'.format( + delete_pcent, + self.MAX_SAFE_DELETE_PCENT * 100, + self.change_counts['Delete'], + existing_record_count)) + + def __repr__(self): + return 'Creates={}, Updates={}, Deletes={}, Existing Records={}' \ + .format(self.change_counts['Create'], self.change_counts['Update'], + self.change_counts['Delete'], + len(self.existing.records)) diff --git a/tests/test_octodns_provider_base.py b/tests/test_octodns_provider_base.py index 1bf3fd7..472b008 100644 --- a/tests/test_octodns_provider_base.py +++ b/tests/test_octodns_provider_base.py @@ -9,7 +9,8 @@ from logging import getLogger from unittest import TestCase from octodns.record import Create, Delete, Record, Update -from octodns.provider.base import BaseProvider, Plan, UnsafePlan +from octodns.provider.base import BaseProvider +from octodns.provider.plan import Plan, UnsafePlan from octodns.zone import Zone From b72fba3e35d5877f3ae5d9ae5837bb66a8f8d6f4 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 2 Dec 2017 10:34:32 -0800 Subject: [PATCH 055/141] Move the log plan output to PlanLogger --- octodns/manager.py | 35 +------------------------------ octodns/provider/plan.py | 45 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 45 insertions(+), 35 deletions(-) diff --git a/octodns/manager.py b/octodns/manager.py index e08754c..c4617aa 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -5,7 +5,6 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -from StringIO import StringIO from concurrent.futures import ThreadPoolExecutor from importlib import import_module from os import environ @@ -260,39 +259,7 @@ class Manager(object): # plan pairs. plans = [p for f in futures for p in f.result()] - hr = '*************************************************************' \ - '*******************\n' - buf = StringIO() - buf.write('\n') - if plans: - current_zone = None - for target, plan in plans: - if plan.desired.name != current_zone: - current_zone = plan.desired.name - buf.write(hr) - buf.write('* ') - buf.write(current_zone) - buf.write('\n') - buf.write(hr) - - buf.write('* ') - buf.write(target.id) - buf.write(' (') - buf.write(target) - buf.write(')\n* ') - for change in plan.changes: - buf.write(change.__repr__(leader='* ')) - buf.write('\n* ') - - buf.write('Summary: ') - buf.write(plan) - buf.write('\n') - else: - buf.write(hr) - buf.write('No changes were planned\n') - buf.write(hr) - buf.write('\n') - self.log.info(buf.getvalue()) + PlanLogger(self.log).output(plans) if not force: self.log.debug('sync: checking safety') diff --git a/octodns/provider/plan.py b/octodns/provider/plan.py index cde859f..ab84a10 100644 --- a/octodns/provider/plan.py +++ b/octodns/provider/plan.py @@ -5,7 +5,8 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -from logging import getLogger +from StringIO import StringIO +from logging import INFO, getLogger class UnsafePlan(Exception): @@ -77,3 +78,45 @@ class Plan(object): .format(self.change_counts['Create'], self.change_counts['Update'], self.change_counts['Delete'], len(self.existing.records)) + + +class PlanLogger(object): + + def __init__(self, log, level=INFO): + self.log = log + self.level = level + + def output(self, plans): + hr = '*************************************************************' \ + '*******************\n' + buf = StringIO() + buf.write('\n') + if plans: + current_zone = None + for target, plan in plans: + if plan.desired.name != current_zone: + current_zone = plan.desired.name + buf.write(hr) + buf.write('* ') + buf.write(current_zone) + buf.write('\n') + buf.write(hr) + + buf.write('* ') + buf.write(target.id) + buf.write(' (') + buf.write(target) + buf.write(')\n* ') + for change in plan.changes: + buf.write(change.__repr__(leader='* ')) + buf.write('\n* ') + + buf.write('Summary: ') + buf.write(plan) + buf.write('\n') + else: + buf.write(hr) + buf.write('No changes were planned\n') + buf.write(hr) + buf.write('\n') + self.log.log(self.level, buf.getvalue()) From 31e6f99df6e73aedf14aa7a9af86cb935a3ccadf Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 2 Dec 2017 10:37:19 -0800 Subject: [PATCH 056/141] Manager.plan_outputs --- octodns/manager.py | 5 ++++- octodns/provider/plan.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/octodns/manager.py b/octodns/manager.py index c4617aa..a737aa4 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -68,6 +68,8 @@ class Manager(object): def __init__(self, config_file, max_workers=None, include_meta=False): self.log.info('__init__: config_file=%s', config_file) + self.plan_outputs = [PlanLogger(self.log)] + # Read our config file with open(config_file, 'r') as fh: self.config = safe_load(fh, enforce_order=False) @@ -259,7 +261,8 @@ class Manager(object): # plan pairs. plans = [p for f in futures for p in f.result()] - PlanLogger(self.log).output(plans) + for output in self.plan_outputs: + output.run(plans) if not force: self.log.debug('sync: checking safety') diff --git a/octodns/provider/plan.py b/octodns/provider/plan.py index ab84a10..009202f 100644 --- a/octodns/provider/plan.py +++ b/octodns/provider/plan.py @@ -86,7 +86,7 @@ class PlanLogger(object): self.log = log self.level = level - def output(self, plans): + def run(self, plans): hr = '*************************************************************' \ '*******************\n' buf = StringIO() From 3d0f5aeca0fcce677303d2a4663b859c104cb1b2 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 2 Dec 2017 11:40:55 -0800 Subject: [PATCH 057/141] Config-based plan_output Refactors the provider class lookup and kwarg processing so that it can be reused for plan_output. --- octodns/manager.py | 81 ++++++++++++------- octodns/provider/plan.py | 29 +++++-- tests/config/bad-plan-output-config.yaml | 7 ++ .../config/bad-plan-output-missing-class.yaml | 5 ++ tests/test_octodns_manager.py | 13 +++ tests/test_octodns_plan.py | 19 +++++ 6 files changed, 120 insertions(+), 34 deletions(-) create mode 100644 tests/config/bad-plan-output-config.yaml create mode 100644 tests/config/bad-plan-output-missing-class.yaml create mode 100644 tests/test_octodns_plan.py diff --git a/octodns/manager.py b/octodns/manager.py index a737aa4..d4debf6 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -11,7 +11,7 @@ from os import environ import logging from .provider.base import BaseProvider -from .provider.plan import Plan, PlanLogger +from .provider.plan import Plan from .provider.yaml import YamlProvider from .record import Record from .yaml import safe_load @@ -68,8 +68,6 @@ class Manager(object): def __init__(self, config_file, max_workers=None, include_meta=False): self.log.info('__init__: config_file=%s', config_file) - self.plan_outputs = [PlanLogger(self.log)] - # Read our config file with open(config_file, 'r') as fh: self.config = safe_load(fh, enforce_order=False) @@ -97,23 +95,8 @@ class Manager(object): self.log.exception('Invalid provider class') raise Exception('Provider {} is missing class' .format(provider_name)) - _class = self._get_provider_class(_class) - # Build up the arguments we need to pass to the provider - kwargs = {} - for k, v in provider_config.items(): - try: - if v.startswith('env/'): - try: - env_var = v[4:] - v = environ[env_var] - except KeyError: - self.log.exception('Invalid provider config') - raise Exception('Incorrect provider config, ' - 'missing env var {}' - .format(env_var)) - except AttributeError: - pass - kwargs[k] = v + _class = self._get_named_class('provider', _class) + kwargs = self._build_kwargs(provider_config) try: self.providers[provider_name] = _class(provider_name, **kwargs) except TypeError: @@ -141,20 +124,64 @@ class Manager(object): where = where[piece] self.zone_tree = zone_tree - def _get_provider_class(self, _class): + self.plan_outputs = {} + plan_outputs = manager_config.get('plan_outputs', { + 'logger': { + 'class': 'octodns.provider.plan.PlanLogger', + 'level': 'info' + } + }) + for plan_output_name, plan_output_config in plan_outputs.items(): + try: + _class = plan_output_config.pop('class') + except KeyError: + self.log.exception('Invalid plan_output class') + raise Exception('plan_output {} is missing class' + .format(plan_output_name)) + _class = self._get_named_class('plan_output', _class) + kwargs = self._build_kwargs(plan_output_config) + try: + self.plan_outputs[plan_output_name] = \ + _class(plan_output_name, **kwargs) + except TypeError: + self.log.exception('Invalid plan_output config') + raise Exception('Incorrect plan_output config for {}' + .format(plan_output_name)) + + def _get_named_class(self, _type, _class): try: module_name, class_name = _class.rsplit('.', 1) module = import_module(module_name) except (ImportError, ValueError): - self.log.exception('_get_provider_class: Unable to import ' + self.log.exception('_get_{}_class: Unable to import ' 'module %s', _class) - raise Exception('Unknown provider class: {}'.format(_class)) + raise Exception('Unknown {} class: {}'.format(_type, _class)) try: return getattr(module, class_name) except AttributeError: - self.log.exception('_get_provider_class: Unable to get class %s ' + self.log.exception('_get_{}_class: Unable to get class %s ' 'from module %s', class_name, module) - raise Exception('Unknown provider class: {}'.format(_class)) + raise Exception('Unknown {} class: {}'.format(_type, _class)) + + def _build_kwargs(self, source): + # Build up the arguments we need to pass to the provider + kwargs = {} + for k, v in source.items(): + try: + if v.startswith('env/'): + try: + env_var = v[4:] + v = environ[env_var] + except KeyError: + self.log.exception('Invalid provider config') + raise Exception('Incorrect provider config, ' + 'missing env var {}' + .format(env_var)) + except AttributeError: + pass + kwargs[k] = v + + return kwargs def configured_sub_zones(self, zone_name): # Reversed pieces of the zone name @@ -261,8 +288,8 @@ class Manager(object): # plan pairs. plans = [p for f in futures for p in f.result()] - for output in self.plan_outputs: - output.run(plans) + for output in self.plan_outputs.values(): + output.run(plans=plans, log=self.log) if not force: self.log.debug('sync: checking safety') diff --git a/octodns/provider/plan.py b/octodns/provider/plan.py index 009202f..b49c200 100644 --- a/octodns/provider/plan.py +++ b/octodns/provider/plan.py @@ -6,7 +6,7 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals from StringIO import StringIO -from logging import INFO, getLogger +from logging import DEBUG, ERROR, INFO, WARN, getLogger class UnsafePlan(Exception): @@ -80,13 +80,28 @@ class Plan(object): len(self.existing.records)) -class PlanLogger(object): +class _PlanOutput(object): - def __init__(self, log, level=INFO): - self.log = log - self.level = level + def __init__(self, name): + self.name = name - def run(self, plans): + +class PlanLogger(_PlanOutput): + + def __init__(self, name, level='info'): + super(PlanLogger, self).__init__(name) + try: + self.level = { + 'debug': DEBUG, + 'info': INFO, + 'warn': WARN, + 'warning': WARN, + 'error': ERROR + }[level.lower()] + except (AttributeError, KeyError): + raise Exception('Unsupported level: {}'.format(level)) + + def run(self, log, plans, *args, **kwargs): hr = '*************************************************************' \ '*******************\n' buf = StringIO() @@ -119,4 +134,4 @@ class PlanLogger(object): buf.write('No changes were planned\n') buf.write(hr) buf.write('\n') - self.log.log(self.level, buf.getvalue()) + log.log(self.level, buf.getvalue()) diff --git a/tests/config/bad-plan-output-config.yaml b/tests/config/bad-plan-output-config.yaml new file mode 100644 index 0000000..f345f89 --- /dev/null +++ b/tests/config/bad-plan-output-config.yaml @@ -0,0 +1,7 @@ +manager: + plan_outputs: + 'bad': + class: octodns.provider.plan.PlanLogger + invalid: config +providers: {} +zones: {} diff --git a/tests/config/bad-plan-output-missing-class.yaml b/tests/config/bad-plan-output-missing-class.yaml new file mode 100644 index 0000000..71b1bd5 --- /dev/null +++ b/tests/config/bad-plan-output-missing-class.yaml @@ -0,0 +1,5 @@ +manager: + plan_outputs: + 'bad': {} +providers: {} +zones: {} diff --git a/tests/test_octodns_manager.py b/tests/test_octodns_manager.py index 4db2103..ada54e5 100644 --- a/tests/test_octodns_manager.py +++ b/tests/test_octodns_manager.py @@ -83,6 +83,19 @@ class TestManager(TestCase): .sync(['unknown.target.']) self.assertTrue('unknown target' in ctx.exception.message) + def test_bad_plan_output_class(self): + with self.assertRaises(Exception) as ctx: + name = 'bad-plan-output-missing-class.yaml' + Manager(get_config_filename(name)).sync() + self.assertEquals('plan_output bad is missing class', + ctx.exception.message) + + def test_bad_plan_output_config(self): + with self.assertRaises(Exception) as ctx: + Manager(get_config_filename('bad-plan-output-config.yaml')).sync() + self.assertEqual('Incorrect plan_output config for bad', + ctx.exception.message) + def test_source_only_as_a_target(self): with self.assertRaises(Exception) as ctx: Manager(get_config_filename('unknown-provider.yaml')) \ diff --git a/tests/test_octodns_plan.py b/tests/test_octodns_plan.py new file mode 100644 index 0000000..2b23b4e --- /dev/null +++ b/tests/test_octodns_plan.py @@ -0,0 +1,19 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from unittest import TestCase + +from octodns.provider.plan import PlanLogger + + +class TestPlanLogger(TestCase): + + def test_invalid_level(self): + with self.assertRaises(Exception) as ctx: + PlanLogger('invalid', 'not-a-level') + self.assertEquals('Unsupported level: not-a-level', + ctx.exception.message) From 1a9d8541adece1444e5d56d69bfa7005c8d5324e Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 2 Dec 2017 11:45:11 -0800 Subject: [PATCH 058/141] Only ignore config/ at the top --- .gitignore | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index c45a684..8fcf7ea 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ *.pyc .coverage .env +^config/ +build/ coverage.xml dist/ env/ @@ -9,5 +11,3 @@ nosetests.xml octodns.egg-info/ output/ tmp/ -build/ -config/ From 402bc2092e63eb10ff6dee76e480ecb55d64e2c2 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 2 Dec 2017 12:35:18 -0800 Subject: [PATCH 059/141] WIP: markdown plan_output support Mostly works, but doesn't yet dump out the values --- octodns/provider/plan.py | 63 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/octodns/provider/plan.py b/octodns/provider/plan.py index b49c200..c7776d8 100644 --- a/octodns/provider/plan.py +++ b/octodns/provider/plan.py @@ -7,6 +7,7 @@ from __future__ import absolute_import, division, print_function, \ from StringIO import StringIO from logging import DEBUG, ERROR, INFO, WARN, getLogger +from sys import stdout class UnsafePlan(Exception): @@ -135,3 +136,65 @@ class PlanLogger(_PlanOutput): buf.write(hr) buf.write('\n') log.log(self.level, buf.getvalue()) + + +class PlanMarkdown(_PlanOutput): + + def run(self, plans, *args, **kwargs): + fh = stdout + if plans: + current_zone = None + for target, plan in plans: + if plan.desired.name != current_zone: + current_zone = plan.desired.name + fh.write('## ') + fh.write(current_zone) + fh.write('\n\n') + + fh.write('### ') + fh.write(target.id) + fh.write('\n\n') + + fh.write('| Name | Type | Existing TTL | New TTL | ' + 'Existing Value | New Value| Source |\n' + '|--|--|--|--|--|--|--|\n') + + for change in plan.changes: + existing = change.existing + new = change.new + record = change.record + fh.write('| ') + fh.write(record.name) + fh.write(' | ') + fh.write(record._type) + fh.write(' | ') + # TTL + if existing: + fh.write(str(existing.ttl)) + else: + fh.write('n/a') + fh.write(' | ') + if new: + fh.write(str(new.ttl)) + else: + fh.write('n/a') + fh.write(' | ') + # Value + if existing: + fh.write('todo') + else: + fh.write('n/a') + fh.write(' | ') + if new: + fh.write('todo') + fh.write(' | ') + fh.write(new.source.id) + else: + fh.write('n/a') + fh.write(' |\n') + + fh.write('\nSummary: ') + fh.write(str(plan)) + fh.write('\n\n') + else: + fh.write('## No changes were planned\n') From 7d83ae84c775ffa03b32f4b19995933affdcfd58 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 2 Dec 2017 15:21:28 -0800 Subject: [PATCH 060/141] Rework markdown table to use 2-rows for updates, 1 for create/delete --- octodns/provider/plan.py | 44 ++++++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/octodns/provider/plan.py b/octodns/provider/plan.py index c7776d8..1795a65 100644 --- a/octodns/provider/plan.py +++ b/octodns/provider/plan.py @@ -155,15 +155,16 @@ class PlanMarkdown(_PlanOutput): fh.write(target.id) fh.write('\n\n') - fh.write('| Name | Type | Existing TTL | New TTL | ' - 'Existing Value | New Value| Source |\n' - '|--|--|--|--|--|--|--|\n') + fh.write('| Operation | Name | Type | TTL | Value | Source |\n' + '|--|--|--|--|--|--|\n') for change in plan.changes: existing = change.existing new = change.new record = change.record fh.write('| ') + fh.write(change.__class__.__name__) + fh.write(' | ') fh.write(record.name) fh.write(' | ') fh.write(record._type) @@ -171,27 +172,30 @@ class PlanMarkdown(_PlanOutput): # TTL if existing: fh.write(str(existing.ttl)) - else: - fh.write('n/a') - fh.write(' | ') + fh.write(' | ') + if existing: + try: + v = existing.values + except AttributeError: + v = existing.value + fh.write(str(v)) + else: + fh.write('n/a') + fh.write(' | |\n') + if new: + fh.write('| | | | ') + if new: fh.write(str(new.ttl)) - else: - fh.write('n/a') - fh.write(' | ') - # Value - if existing: - fh.write('todo') - else: - fh.write('n/a') - fh.write(' | ') - if new: - fh.write('todo') + fh.write(' | ') + try: + v = new.values + except AttributeError: + v = new.value + fh.write(str(v)) fh.write(' | ') fh.write(new.source.id) - else: - fh.write('n/a') - fh.write(' |\n') + fh.write(' |\n') fh.write('\nSummary: ') fh.write(str(plan)) From aa20b3388f1e0621a9596d3489950a207c7b4874 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 2 Dec 2017 15:39:29 -0800 Subject: [PATCH 061/141] plan _value_stringifier --- octodns/provider/plan.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/octodns/provider/plan.py b/octodns/provider/plan.py index 1795a65..6291a87 100644 --- a/octodns/provider/plan.py +++ b/octodns/provider/plan.py @@ -138,6 +138,17 @@ class PlanLogger(_PlanOutput): log.log(self.level, buf.getvalue()) +def _value_stringifier(record, sep): + try: + values = [str(v) for v in record.values] + except AttributeError: + values = [record.value] + for code, gv in getattr(record, 'geo', {}).items(): + vs = ', '.join([str(v) for v in gv.values]) + values.append('{}: {}'.format(code, vs)) + return sep.join(values) + + class PlanMarkdown(_PlanOutput): def run(self, plans, *args, **kwargs): @@ -174,11 +185,7 @@ class PlanMarkdown(_PlanOutput): fh.write(str(existing.ttl)) fh.write(' | ') if existing: - try: - v = existing.values - except AttributeError: - v = existing.value - fh.write(str(v)) + fh.write(_value_stringifier(existing, '; ')) else: fh.write('n/a') fh.write(' | |\n') @@ -188,11 +195,7 @@ class PlanMarkdown(_PlanOutput): if new: fh.write(str(new.ttl)) fh.write(' | ') - try: - v = new.values - except AttributeError: - v = new.value - fh.write(str(v)) + fh.write(_value_stringifier(new, '; ')) fh.write(' | ') fh.write(new.source.id) fh.write(' |\n') From e092afda17b182a93d491857304fcf684454860b Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 2 Dec 2017 15:58:47 -0800 Subject: [PATCH 062/141] Add PlanHtml --- .gitignore | 3 +- octodns/provider/plan.py | 71 ++++++++++++++++++++++++++++++++++++---- 2 files changed, 66 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index 8fcf7ea..64ce76f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,7 @@ *.pyc .coverage .env -^config/ -build/ +/config/ coverage.xml dist/ env/ diff --git a/octodns/provider/plan.py b/octodns/provider/plan.py index 6291a87..9ac7685 100644 --- a/octodns/provider/plan.py +++ b/octodns/provider/plan.py @@ -151,8 +151,7 @@ def _value_stringifier(record, sep): class PlanMarkdown(_PlanOutput): - def run(self, plans, *args, **kwargs): - fh = stdout + def run(self, plans, fh=stdout, *args, **kwargs): if plans: current_zone = None for target, plan in plans: @@ -184,10 +183,7 @@ class PlanMarkdown(_PlanOutput): if existing: fh.write(str(existing.ttl)) fh.write(' | ') - if existing: - fh.write(_value_stringifier(existing, '; ')) - else: - fh.write('n/a') + fh.write(_value_stringifier(existing, '; ')) fh.write(' | |\n') if new: fh.write('| | | | ') @@ -205,3 +201,66 @@ class PlanMarkdown(_PlanOutput): fh.write('\n\n') else: fh.write('## No changes were planned\n') + + +class PlanHtml(_PlanOutput): + + def run(self, plans, fh=stdout, *args, **kwargs): + if plans: + current_zone = None + for target, plan in plans: + if plan.desired.name != current_zone: + current_zone = plan.desired.name + fh.write('

') + fh.write(current_zone) + fh.write('

\n') + + fh.write('

') + fh.write(target.id) + fh.write('''

+ + + + + + + + + +''') + + for change in plan.changes: + existing = change.existing + new = change.new + record = change.record + fh.write(' \n \n \n \n') + # TTL + if existing: + fh.write(' \n \n \n \n') + if new: + fh.write(' \n \n') + + if new: + fh.write(' \n \n \n \n') + + fh.write(' \n \n \n
OperationNameTypeTTLValueSource
') + fh.write(change.__class__.__name__) + fh.write('') + fh.write(record.name) + fh.write('') + fh.write(record._type) + fh.write('') + fh.write(str(existing.ttl)) + fh.write('') + fh.write(_value_stringifier(existing, '
')) + fh.write('
') + fh.write(str(new.ttl)) + fh.write('') + fh.write(_value_stringifier(new, '
')) + fh.write('		') + fh.write(new.source.id) + fh.write('
Summary: ') + fh.write(str(plan)) + fh.write('
\n') + else: + fh.write('

No changes were planned

') From 106f59fe6cb88e50921b32bbbc8930f3d0912259 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 2 Dec 2017 16:00:00 -0800 Subject: [PATCH 063/141] No table border --- octodns/provider/plan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/plan.py b/octodns/provider/plan.py index 9ac7685..697c8c3 100644 --- a/octodns/provider/plan.py +++ b/octodns/provider/plan.py @@ -218,7 +218,7 @@ class PlanHtml(_PlanOutput): fh.write('

') fh.write(target.id) fh.write('''

- +
From fd9af2bd25ff6a38344773d507f4cfadd21da3a6 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Wed, 27 Dec 2017 09:54:18 -0800 Subject: [PATCH 064/141] Major reworking of Cloudflare record update --- octodns/provider/cloudflare.py | 107 +++++++++++++++++++--- tests/test_octodns_provider_cloudflare.py | 13 ++- 2 files changed, 101 insertions(+), 19 deletions(-) diff --git a/octodns/provider/cloudflare.py b/octodns/provider/cloudflare.py index dd53b3a..a7ec5d8 100644 --- a/octodns/provider/cloudflare.py +++ b/octodns/provider/cloudflare.py @@ -7,6 +7,7 @@ from __future__ import absolute_import, division, print_function, \ from collections import defaultdict from logging import getLogger +from json import dumps from requests import Session from ..record import Record, Update @@ -232,25 +233,107 @@ class CloudflareProvider(BaseProvider): 'content': value.exchange } + def _gen_contents(self, record): + name = record.fqdn[:-1] + _type = record._type + ttl = max(self.MIN_TTL, record.ttl) + + contents_for = getattr(self, '_contents_for_{}'.format(_type)) + for content in contents_for(record): + content.update({ + 'name': name, + 'type': _type, + 'ttl': ttl, + }) + yield content + def _apply_Create(self, change): new = change.new zone_id = self.zones[new.zone.name] - contents_for = getattr(self, '_contents_for_{}'.format(new._type)) path = '/zones/{}/dns_records'.format(zone_id) - name = new.fqdn[:-1] - for content in contents_for(change.new): - content.update({ - 'name': name, - 'type': new._type, - # Cloudflare has a min ttl of 120s - 'ttl': max(self.MIN_TTL, new.ttl), - }) + for content in self._gen_contents(new): self._request('POST', path, data=content) + def _hash_content(self, content): + # Some of the dicts are nested so this seems about as good as any + # option we have for consistently hashing them (within a single run) + return hash(dumps(content, sort_keys=True)) + def _apply_Update(self, change): - # Create the new and delete the old - self._apply_Create(change) - self._apply_Delete(change) + + # Ugh, this is pretty complicated and ugly, mainly due to the + # sub-optimal API/semantics. Ideally we'd have a batch change API like + # Route53's to make this 100% clean and safe without all this PITA, but + # we don't so we'll have to work around that and manually do it as + # safely as possible. Note this still isn't perfect as we don't/can't + # practically take into account things like the different "types" of + # CAA records so when we "swap" there may be brief periods where things + # are invalid or even worse Cloudflare may update their validations to + # prevent dups. I see no clean way around that short of making this + # understand 100% of the details of each record type and develop an + # individual/specific ordering of changes that prevents it. That'd + # probably result in more code than this whole provider currently has + # so... :-( + + existing_contents = { + self._hash_content(c): c + for c in self._gen_contents(change.existing) + } + new_contents = { + self._hash_content(c): c + for c in self._gen_contents(change.new) + } + + # We need a list of keys to consider for diffs, use the first content + # before we muck with anything + keys = existing_contents.values()[0].keys() + + # Find the things we need to add + adds = [] + for k, content in new_contents.items(): + try: + existing_contents.pop(k) + self.log.debug('_apply_Update: leaving %s', content) + except KeyError: + adds.append(content) + + zone_id = self.zones[change.new.zone.name] + + # Find things we need to remove + name = change.new.fqdn[:-1] + _type = change.new._type + # OK, work through each record from the zone + for record in self.zone_records(change.new.zone): + if name == record['name'] and _type == record['type']: + # This is match for our name and type, we need to look at + # contents now, build a dict of the relevant keys and vals + content = {} + for k in keys: + content[k] = record[k] + # :-( + if _type in ('CNAME', 'MX', 'NS'): + content['content'] += '.' + # If the hash of that dict isn't in new this record isn't + # needed + if self._hash_content(content) not in new_contents: + rid = record['id'] + path = '/zones/{}/dns_records/{}'.format(record['zone_id'], + rid) + try: + add_content = adds.pop(0) + self.log.debug('_apply_Update: swapping %s -> %s, %s', + content, add_content, rid) + self._request('PUT', path, data=add_content) + except IndexError: + self.log.debug('_apply_Update: removing %s, %s', + content, rid) + self._request('DELETE', path) + + # Any remaining adds just need to be created + path = '/zones/{}/dns_records'.format(zone_id) + for content in adds: + self.log.debug('_apply_Update: adding %s', content) + self._request('POST', path, data=content) def _apply_Delete(self, change): existing = change.existing diff --git a/tests/test_octodns_provider_cloudflare.py b/tests/test_octodns_provider_cloudflare.py index ef8a51c..defcdea 100644 --- a/tests/test_octodns_provider_cloudflare.py +++ b/tests/test_octodns_provider_cloudflare.py @@ -267,13 +267,12 @@ class TestCloudflareProvider(TestCase): self.assertEquals(2, provider.apply(plan)) # recreate for update, and deletes for the 2 parts of the other provider._request.assert_has_calls([ - call('POST', '/zones/42/dns_records', data={ - 'content': '3.2.3.4', - 'type': 'A', - 'name': 'ttl.unit.tests', - 'ttl': 300}), - call('DELETE', '/zones/ff12ab34cd5611334422ab3322997650/' - 'dns_records/fc12ab34cd5611334422ab3322997655'), + call('PUT', '/zones/ff12ab34cd5611334422ab3322997650/dns_records/' + 'fc12ab34cd5611334422ab3322997655', + data={'content': '3.2.3.4', + 'type': 'A', + 'name': 'ttl.unit.tests', + 'ttl': 300}), call('DELETE', '/zones/ff12ab34cd5611334422ab3322997650/' 'dns_records/fc12ab34cd5611334422ab3322997653'), call('DELETE', '/zones/ff12ab34cd5611334422ab3322997650/' From 429b447238ad1155d99b7f2123a0f1e8cf018613 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Wed, 27 Dec 2017 10:20:01 -0800 Subject: [PATCH 065/141] Route53's NAPTR values should default to '' not None --- octodns/provider/route53.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index 7623648..5bda074 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -385,10 +385,10 @@ class Route53Provider(BaseProvider): values.append({ 'order': order, 'preference': preference, - 'flags': flags if flags else None, - 'service': service if service else None, - 'regexp': regexp if regexp else None, - 'replacement': replacement if replacement else None, + 'flags': flags, + 'service': service, + 'regexp': regexp, + 'replacement': replacement, }) return { 'type': rrset['Type'], From 61a86810ee84bd6c5c1af66dcf73ed19634b9540 Mon Sep 17 00:00:00 2001 From: Steve Coursen Date: Thu, 28 Dec 2017 16:01:22 -0500 Subject: [PATCH 066/141] add geo support for ns1 --- octodns/provider/ns1.py | 55 +++++++++++++++++++++++++----- tests/test_octodns_provider_ns1.py | 38 ++++++++++++++++++--- 2 files changed, 79 insertions(+), 14 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 008c665..e8ba8cd 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -6,11 +6,13 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals from logging import getLogger -from nsone import NSONE +from itertools import chain +from nsone import NSONE, Config from nsone.rest.errors import RateLimitException, ResourceException +from incf.countryutils import transformations from time import sleep -from ..record import Record +from ..record import _GeoMixin, Record from .base import BaseProvider @@ -35,11 +37,38 @@ class Ns1Provider(BaseProvider): self._client = NSONE(apiKey=api_key) def _data_for_A(self, _type, record): - return { + # record meta (which would include geo information is only + # returned when getting a record's detail, not from zone detail + geo = {} + data = { 'ttl': record['ttl'], 'type': _type, - 'values': record['short_answers'], } + values, codes = [], [] + if 'answers' not in record: + values = record['short_answers'] + for answer in record.get('answers', []): + meta = answer.get('meta', {}) + if meta: + country = meta.get('country', []) + us_state = meta.get('us_state', []) + ca_province = meta.get('ca_province', []) + for cntry in country: + cn = transformations.cc_to_cn(cntry) + con = transformations.cn_to_ctca2(cn) + geo['{}-{}'.format(con, cntry)] = answer['answer'] + for state in us_state: + geo['NA-US-{}'.format(state)] = answer['answer'] + for province in ca_province: + geo['NA-CA-{}'.format(state)] = answer['answer'] + for code in meta.get('iso_region_code', []): + geo[code] = answer['answer'] + else: + values.extend(answer['answer']) + codes.append([]) + data['values'] = values + data['geo'] = geo + return data _data_for_AAAA = _data_for_A @@ -146,20 +175,25 @@ class Ns1Provider(BaseProvider): try: nsone_zone = self._client.loadZone(zone.name[:-1]) records = nsone_zone.data['records'] + geo_records = nsone_zone.search(has_geo=True) except ResourceException as e: if e.message != self.ZONE_NOT_FOUND_MESSAGE: raise records = [] + geo_records = [] before = len(zone.records) - for record in records: + # geo information isn't returned from the main endpoint, so we need + # to query for all records with geo information + zone_hash = {} + for record in chain(records, geo_records): _type = record['type'] data_for = getattr(self, '_data_for_{}'.format(_type)) name = zone.hostname_from_fqdn(record['domain']) record = Record.new(zone, name, data_for(_type, record), source=self, lenient=lenient) - zone.add_record(record) - + zone_hash[(_type, name)] = record + [zone.add_record(r) for r in zone_hash.values()] self.log.info('populate: found %s records', len(zone.records) - before) @@ -168,15 +202,18 @@ class Ns1Provider(BaseProvider): if hasattr(record, 'geo'): # purposefully set non-geo answers to have an empty meta, # so that we know we did this on purpose if/when troubleshooting - params['answers'] = [{"answer": x, "meta": {}} + params['answers'] = [{"answer": [x], "meta": {}} \ for x in record.values] for iso_region, target in record.geo.items(): + key = 'iso_region_code' + value = iso_region params['answers'].append( { 'answer': target.values, - 'meta': {'iso_region_code': [iso_region]}, + 'meta': {key: [value]}, }, ) + self.log.info("params for A: %s", params) return params _params_for_AAAA = _params_for_A diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index 4436304..c8ff222 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -38,6 +38,13 @@ class TestNs1Provider(TestCase): 'values': ['1.2.3.4', '1.2.3.5'], 'meta': {}, })) + expected.add(Record.new(zone, 'geo', { + 'ttl': 34, + 'type': 'A', + 'values': ['101.102.103.104', '101.102.103.105'], + 'geo': {'NA-US-NY': ['201.202.203.204']}, + 'meta': {}, + })) expected.add(Record.new(zone, 'cname', { 'ttl': 34, 'type': 'CNAME', @@ -118,6 +125,11 @@ class TestNs1Provider(TestCase): 'ttl': 33, 'short_answers': ['1.2.3.4', '1.2.3.5'], 'domain': 'foo.unit.tests.', + }, { + 'type': 'A', + 'ttl': 34, + 'short_answers': ['101.102.103.104', '101.102.103.105'], + 'domain': 'geo.unit.tests.', }, { 'type': 'CNAME', 'ttl': 34, @@ -192,6 +204,9 @@ class TestNs1Provider(TestCase): load_mock.reset_mock() nsone_zone = DummyZone([]) load_mock.side_effect = [nsone_zone] + zone_search = Mock() + zone_search.return_value = [] + nsone_zone.search = zone_search zone = Zone('unit.tests.', []) provider.populate(zone) self.assertEquals(set(), zone.records) @@ -201,6 +216,9 @@ class TestNs1Provider(TestCase): load_mock.reset_mock() nsone_zone = DummyZone(self.nsone_records) load_mock.side_effect = [nsone_zone] + zone_search = Mock() + zone_search.return_value = [] + nsone_zone.search = zone_search zone = Zone('unit.tests.', []) provider.populate(zone) self.assertEquals(self.expected, zone.records) @@ -266,11 +284,14 @@ class TestNs1Provider(TestCase): }]) nsone_zone.data['records'][0]['short_answers'][0] = '2.2.2.2' nsone_zone.loadRecord = Mock() + zone_search = Mock() + zone_search.return_value = [] + nsone_zone.search = zone_search load_mock.side_effect = [nsone_zone, nsone_zone] plan = provider.plan(desired) - self.assertEquals(2, len(plan.changes)) + self.assertEquals(3, len(plan.changes)) self.assertIsInstance(plan.changes[0], Update) - self.assertIsInstance(plan.changes[1], Delete) + self.assertIsInstance(plan.changes[2], Delete) # ugh, we need a mock record that can be returned from loadRecord for # the update and delete targets, we can add our side effects to that to # trigger rate limit handling @@ -278,23 +299,30 @@ class TestNs1Provider(TestCase): mock_record.update.side_effect = [ RateLimitException('one', period=0), None, + None, ] mock_record.delete.side_effect = [ RateLimitException('two', period=0), None, + None, ] - nsone_zone.loadRecord.side_effect = [mock_record, mock_record] + nsone_zone.loadRecord.side_effect = [mock_record, mock_record, mock_record] got_n = provider.apply(plan) - self.assertEquals(2, got_n) + self.assertEquals(3, got_n) nsone_zone.loadRecord.assert_has_calls([ call('unit.tests', u'A'), + call('geo', u'A'), call('delete-me', u'A'), ]) mock_record.assert_has_calls([ - call.update(answers=[{'answer': u'1.2.3.4', 'meta': {}}], ttl=32), + call.update(answers=[{'answer': [u'1.2.3.4'], 'meta': {}}], ttl=32), + call.update(answers=[{u'answer': [u'1.2.3.4'], u'meta': {}}], ttl=32), + call.update(answers=[{u'answer': [u'101.102.103.104'], u'meta': {}}, {u'answer': [u'101.102.103.105'], u'meta': {}}, {u'answer': [u'201.202.203.204'], u'meta': {u'iso_region_code': [u'NA-US-NY']}}], ttl=34), + call.delete(), call.delete() ]) + def test_escaping(self): provider = Ns1Provider('test', 'api-key') From 481bbe10f6bd51eae8ccd4f88366dd18cf4d5637 Mon Sep 17 00:00:00 2001 From: Steve Coursen Date: Thu, 28 Dec 2017 16:01:56 -0500 Subject: [PATCH 067/141] add geo support for ns1 --- octodns/provider/ns1.py | 6 +++--- tests/test_octodns_provider_ns1.py | 24 ++++++++++++++++++------ 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index e8ba8cd..6b0fe7e 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -7,12 +7,12 @@ from __future__ import absolute_import, division, print_function, \ from logging import getLogger from itertools import chain -from nsone import NSONE, Config +from nsone import NSONE from nsone.rest.errors import RateLimitException, ResourceException from incf.countryutils import transformations from time import sleep -from ..record import _GeoMixin, Record +from ..record import Record from .base import BaseProvider @@ -202,7 +202,7 @@ class Ns1Provider(BaseProvider): if hasattr(record, 'geo'): # purposefully set non-geo answers to have an empty meta, # so that we know we did this on purpose if/when troubleshooting - params['answers'] = [{"answer": [x], "meta": {}} \ + params['answers'] = [{"answer": [x], "meta": {}} for x in record.values] for iso_region, target in record.geo.items(): key = 'iso_region_code' diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index c8ff222..10ae5d3 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -306,7 +306,8 @@ class TestNs1Provider(TestCase): None, None, ] - nsone_zone.loadRecord.side_effect = [mock_record, mock_record, mock_record] + nsone_zone.loadRecord.side_effect = [mock_record, mock_record, + mock_record] got_n = provider.apply(plan) self.assertEquals(3, got_n) nsone_zone.loadRecord.assert_has_calls([ @@ -315,17 +316,28 @@ class TestNs1Provider(TestCase): call('delete-me', u'A'), ]) mock_record.assert_has_calls([ - call.update(answers=[{'answer': [u'1.2.3.4'], 'meta': {}}], ttl=32), - call.update(answers=[{u'answer': [u'1.2.3.4'], u'meta': {}}], ttl=32), - call.update(answers=[{u'answer': [u'101.102.103.104'], u'meta': {}}, {u'answer': [u'101.102.103.105'], u'meta': {}}, {u'answer': [u'201.202.203.204'], u'meta': {u'iso_region_code': [u'NA-US-NY']}}], ttl=34), + call.update(answers=[{'answer': [u'1.2.3.4'], 'meta': {}}], + ttl=32), + call.update(answers=[{u'answer': [u'1.2.3.4'], u'meta': {}}], + ttl=32), + call.update( + answers=[ + {u'answer': [u'101.102.103.104'], u'meta': {}}, + {u'answer': [u'101.102.103.105'], u'meta': {}}, + { + u'answer': [u'201.202.203.204'], + u'meta': { + u'iso_region_code': [u'NA-US-NY'] + }, + }, + ], + ttl=34), call.delete(), call.delete() ]) - def test_escaping(self): provider = Ns1Provider('test', 'api-key') - record = { 'ttl': 31, 'short_answers': ['foo; bar baz; blip'] From 6c91a92a72f3e297e44d669964ac06a9ad32c291 Mon Sep 17 00:00:00 2001 From: Stephen Coursen Date: Fri, 5 Jan 2018 19:24:38 +0000 Subject: [PATCH 068/141] Add geotarget filter, change log level to debug --- octodns/provider/ns1.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 6b0fe7e..d022e78 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -50,6 +50,10 @@ class Ns1Provider(BaseProvider): for answer in record.get('answers', []): meta = answer.get('meta', {}) if meta: + # country + state and country + province are allowed + # in that case though, supplying a state/province would + # be redundant since the country would supercede in when + # resolving the record. it is syntactically valid, however. country = meta.get('country', []) us_state = meta.get('us_state', []) ca_province = meta.get('ca_province', []) @@ -204,16 +208,32 @@ class Ns1Provider(BaseProvider): # so that we know we did this on purpose if/when troubleshooting params['answers'] = [{"answer": [x], "meta": {}} for x in record.values] + has_country = False for iso_region, target in record.geo.items(): key = 'iso_region_code' value = iso_region + if not has_country and len(value.split('-')) > 1: + has_country = True params['answers'].append( { 'answer': target.values, 'meta': {key: [value]}, }, ) - self.log.info("params for A: %s", params) + params['filters'] = [] + if len(params['answers']) > 1: + params['filters'].append( + {"filter": "shuffle", "config":{}} + ) + if has_country: + params['filters'].append( + {"filter": "geotarget_country", "config": {}} + ) + params['filters'].append( + {"filter": "select_first_n", + "config": {"N": 1}} + ) + self.log.debug("params for A: %s", params) return params _params_for_AAAA = _params_for_A From e6cda62284043303af06da207b73e90bad617f70 Mon Sep 17 00:00:00 2001 From: Stephen Coursen Date: Fri, 5 Jan 2018 22:34:15 +0000 Subject: [PATCH 069/141] Only add shuffle if there is more than 1 answer *and* any of the answers have geo --- octodns/provider/ns1.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index d022e78..1206c2f 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -221,11 +221,11 @@ class Ns1Provider(BaseProvider): }, ) params['filters'] = [] - if len(params['answers']) > 1: - params['filters'].append( - {"filter": "shuffle", "config":{}} - ) if has_country: + if len(params['answers']) > 1: + params['filters'].append( + {"filter": "shuffle", "config":{}} + ) params['filters'].append( {"filter": "geotarget_country", "config": {}} ) From 34f2432c3f99c7189eedbce3b40ab1fda9c38fdf Mon Sep 17 00:00:00 2001 From: Stephen Coursen Date: Fri, 5 Jan 2018 22:45:00 +0000 Subject: [PATCH 070/141] after discussion, we should shuffle if there's more than 1 answer --- octodns/provider/ns1.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 1206c2f..d022e78 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -221,11 +221,11 @@ class Ns1Provider(BaseProvider): }, ) params['filters'] = [] + if len(params['answers']) > 1: + params['filters'].append( + {"filter": "shuffle", "config":{}} + ) if has_country: - if len(params['answers']) > 1: - params['filters'].append( - {"filter": "shuffle", "config":{}} - ) params['filters'].append( {"filter": "geotarget_country", "config": {}} ) From 8d7eca21e9d85472f34b66be1e9e9668eb209b63 Mon Sep 17 00:00:00 2001 From: Terrence Cole Date: Fri, 5 Jan 2018 15:56:06 -0800 Subject: [PATCH 071/141] Get lint green on test code too. --- tests/test_octodns_provider_rackspace.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/test_octodns_provider_rackspace.py b/tests/test_octodns_provider_rackspace.py index da3ffe4..274d63e 100644 --- a/tests/test_octodns_provider_rackspace.py +++ b/tests/test_octodns_provider_rackspace.py @@ -7,17 +7,13 @@ from __future__ import absolute_import, division, print_function, \ import json import re -from os.path import dirname, join from unittest import TestCase from urlparse import urlparse -from nose.tools import assert_raises - from requests import HTTPError from requests_mock import ANY, mock as requests_mock from octodns.provider.rackspace import RackspaceProvider -from octodns.provider.yaml import YamlProvider from octodns.record import Record from octodns.zone import Zone From ddf53b7a47355e8755e31834db329cc052290127 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 6 Jan 2018 07:50:23 -0800 Subject: [PATCH 072/141] No changes bold, not h2 --- octodns/provider/plan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/plan.py b/octodns/provider/plan.py index 697c8c3..7edd682 100644 --- a/octodns/provider/plan.py +++ b/octodns/provider/plan.py @@ -263,4 +263,4 @@ class PlanHtml(_PlanOutput): fh.write(str(plan)) fh.write('\n \n
Operation Name
\n') else: - fh.write('

No changes were planned

') + fh.write('No changes were planned') From 5c94407893099098144d0c1487158ccd964ac776 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 6 Jan 2018 08:58:28 -0800 Subject: [PATCH 073/141] All requirements are >=, not exact versions --- setup.cfg | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/setup.cfg b/setup.cfg index 31a0283..4686ecd 100644 --- a/setup.cfg +++ b/setup.cfg @@ -22,12 +22,12 @@ classifiers = install_requires = PyYaml>=3.12 dnspython>=1.15.0 - futures==3.1.1 - incf.countryutils==1.0 - ipaddress==1.0.18 - natsort==5.0.3 - python-dateutil==2.6.1 - requests==2.13.0 + futures>=3.1.1 + incf.countryutils>=1.0 + ipaddress>=1.0.18 + natsort>=5.0.3 + python-dateutil>=2.6.1 + requests>=2.13.0 packages = find: include_package_data = True @@ -45,19 +45,19 @@ exclude = [options.extras_require] dev = - azure-mgmt-dns==1.0.1 - azure-common==1.1.6 - boto3==1.4.6 - botocore==1.6.8 - docutils==0.14 - dyn==1.8.0 - google-cloud==0.27.0 - jmespath==0.9.3 - msrestazure==0.4.10 - nsone==0.9.14 - ovh==0.4.7 - s3transfer==0.1.10 - six==1.10.0 + azure-mgmt-dns>=1.0.1 + azure-common>=1.1.6 + boto3>=1.4.6 + botocore>=1.6.8 + docutils>=0.14 + dyn>=1.8.0 + google-cloud>=0.27.0 + jmespath>=0.9.3 + msrestazure>=0.4.10 + nsone>=0.9.14 + ovh>=0.4.7 + s3transfer>=0.1.10 + six>=1.10.0 test = coverage mock From a7e32d2c87d82b342d85108ac9804727735b5e50 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 6 Jan 2018 10:13:23 -0800 Subject: [PATCH 074/141] Hard-ping azure lib versions, they have breaking changes --- setup.cfg | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.cfg b/setup.cfg index 4686ecd..7f0ce13 100644 --- a/setup.cfg +++ b/setup.cfg @@ -45,15 +45,15 @@ exclude = [options.extras_require] dev = - azure-mgmt-dns>=1.0.1 - azure-common>=1.1.6 + azure-mgmt-dns==1.0.1 + azure-common==1.1.6 boto3>=1.4.6 botocore>=1.6.8 docutils>=0.14 dyn>=1.8.0 google-cloud>=0.27.0 jmespath>=0.9.3 - msrestazure>=0.4.10 + msrestazure==0.4.10 nsone>=0.9.14 ovh>=0.4.7 s3transfer>=0.1.10 From 88bbd663006c08ecba84cfd17597143edfa9811a Mon Sep 17 00:00:00 2001 From: Terrence Cole Date: Sat, 6 Jan 2018 10:29:22 -0800 Subject: [PATCH 075/141] Move request delay to a central location. --- octodns/provider/rackspace.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py index 6b17550..1f723d1 100644 --- a/octodns/provider/rackspace.py +++ b/octodns/provider/rackspace.py @@ -83,7 +83,6 @@ class RackspaceProvider(BaseProvider): def _get_zone_id_for(self, zone): ret = self._request('GET', 'domains', pagination_key='domains') - time.sleep(self.ratelimit_delay) return [x for x in ret if x['name'] == zone.name[:-1]][0]['id'] def _request(self, method, path, data=None, pagination_key=None): @@ -91,14 +90,15 @@ class RackspaceProvider(BaseProvider): url = '{}/{}'.format(self.dns_endpoint, path) if pagination_key: - return self._paginated_request_for_url(method, url, data, + resp = self._paginated_request_for_url(method, url, data, pagination_key) else: - return self._request_for_url(method, url, data) + resp = self._request_for_url(method, url, data) + time.sleep(self.ratelimit_delay) + return resp def _request_for_url(self, method, url, data): resp = self._sess.request(method, url, json=data, timeout=self.TIMEOUT) - time.sleep(self.ratelimit_delay) self.log.debug('_request: status=%d', resp.status_code) resp.raise_for_status() return resp @@ -107,7 +107,6 @@ class RackspaceProvider(BaseProvider): acc = [] resp = self._sess.request(method, url, json=data, timeout=self.TIMEOUT) - time.sleep(self.ratelimit_delay) self.log.debug('_request: status=%d', resp.status_code) resp.raise_for_status() acc.extend(resp.json()[pagination_key]) From 41622009e43e7c989ec5ad44ccff8e616e1aa0bc Mon Sep 17 00:00:00 2001 From: Terrence Cole Date: Sat, 6 Jan 2018 10:32:25 -0800 Subject: [PATCH 076/141] Set a default rate-limit delay. --- octodns/provider/rackspace.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py index 1f723d1..371861e 100644 --- a/octodns/provider/rackspace.py +++ b/octodns/provider/rackspace.py @@ -42,7 +42,7 @@ class RackspaceProvider(BaseProvider): 'TXT')) TIMEOUT = 5 - def __init__(self, id, username, api_key, ratelimit_delay, *args, + def __init__(self, id, username, api_key, ratelimit_delay=0.0, *args, **kwargs): ''' Rackspace API v1 Provider From cdf26ffae459d009136817150e07d1d68d2773c0 Mon Sep 17 00:00:00 2001 From: Terrence Cole Date: Sat, 6 Jan 2018 10:35:47 -0800 Subject: [PATCH 077/141] Refactor the output transformer loop. --- octodns/provider/rackspace.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py index 371861e..3a195cf 100644 --- a/octodns/provider/rackspace.py +++ b/octodns/provider/rackspace.py @@ -294,12 +294,8 @@ class RackspaceProvider(BaseProvider): self._get_values(change.new)) def _create_given_change_values(self, change, values): - out = [] - for value in values: - transformer = getattr(self, - "_record_for_{}".format(change.new._type)) - out.append(transformer(change.new, value)) - return out + transformer = getattr(self, "_record_for_{}".format(change.new._type)) + return [transformer(change.new, v) for v in values] def _mod_Update(self, change): existing_values = self._get_values(change.existing) From 80aed0052306c0dc8a346ff57dbbaeb31ad833e2 Mon Sep 17 00:00:00 2001 From: Terrence Cole Date: Sat, 6 Jan 2018 11:18:33 -0800 Subject: [PATCH 078/141] Pull transformer above the loop for delete as well. --- octodns/provider/rackspace.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py index 3a195cf..c432339 100644 --- a/octodns/provider/rackspace.py +++ b/octodns/provider/rackspace.py @@ -332,10 +332,10 @@ class RackspaceProvider(BaseProvider): change.existing)) def _delete_given_change_values(self, change, values): + transformer = getattr(self, "_record_for_{}".format( + change.existing._type)) out = [] for value in values: - transformer = getattr(self, "_record_for_{}".format( - change.existing._type)) rs_record = transformer(change.existing, value) key = self._key_for_record(rs_record) out.append('id=' + self._id_map[key]) From 3c3f63b450e5e66ea8e070bd7055fcc73b91c114 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 6 Jan 2018 15:51:48 -0800 Subject: [PATCH 079/141] Unit tests for reworked Cloudflare updates --- tests/test_octodns_provider_cloudflare.py | 166 +++++++++++++++++++++- 1 file changed, 165 insertions(+), 1 deletion(-) diff --git a/tests/test_octodns_provider_cloudflare.py b/tests/test_octodns_provider_cloudflare.py index defcdea..8a17083 100644 --- a/tests/test_octodns_provider_cloudflare.py +++ b/tests/test_octodns_provider_cloudflare.py @@ -11,7 +11,8 @@ from requests import HTTPError from requests_mock import ANY, mock as requests_mock from unittest import TestCase -from octodns.record import Record +from octodns.record import Record, Update +from octodns.provider.base import Plan from octodns.provider.cloudflare import CloudflareProvider from octodns.provider.yaml import YamlProvider from octodns.zone import Zone @@ -278,3 +279,166 @@ class TestCloudflareProvider(TestCase): call('DELETE', '/zones/ff12ab34cd5611334422ab3322997650/' 'dns_records/fc12ab34cd5611334422ab3322997654') ]) + + def test_update_add_swap(self): + provider = CloudflareProvider('test', 'email', 'token') + + provider.zone_records = Mock(return_value=[ + { + "id": "fc12ab34cd5611334422ab3322997653", + "type": "A", + "name": "a.unit.tests", + "content": "1.1.1.1", + "proxiable": True, + "proxied": False, + "ttl": 300, + "locked": False, + "zone_id": "ff12ab34cd5611334422ab3322997650", + "zone_name": "unit.tests", + "modified_on": "2017-03-11T18:01:43.420689Z", + "created_on": "2017-03-11T18:01:43.420689Z", + "meta": { + "auto_added": False + } + }, + { + "id": "fc12ab34cd5611334422ab3322997654", + "type": "A", + "name": "a.unit.tests", + "content": "2.2.2.2", + "proxiable": True, + "proxied": False, + "ttl": 300, + "locked": False, + "zone_id": "ff12ab34cd5611334422ab3322997650", + "zone_name": "unit.tests", + "modified_on": "2017-03-11T18:01:43.420689Z", + "created_on": "2017-03-11T18:01:43.420689Z", + "meta": { + "auto_added": False + } + }, + ]) + + provider._request = Mock() + provider._request.side_effect = [ + self.empty, # no zones + { + 'result': { + 'id': 42, + } + }, # zone create + None, + None, + ] + + # Add something and delete something + zone = Zone('unit.tests.', []) + existing = Record.new(zone, 'a', { + 'ttl': 300, + 'type': 'A', + # This matches the zone data above, one to swap, one to leave + 'values': ['1.1.1.1', '2.2.2.2'], + }) + new = Record.new(zone, 'a', { + 'ttl': 300, + 'type': 'A', + # This leaves one, swaps ones, and adds one + 'values': ['2.2.2.2', '3.3.3.3', '4.4.4.4'], + }) + change = Update(existing, new) + plan = Plan(zone, zone, [change]) + provider._apply(plan) + + provider._request.assert_has_calls([ + call('GET', '/zones', params={'page': 1}), + call('POST', '/zones', data={'jump_start': False, + 'name': 'unit.tests'}), + call('PUT', '/zones/ff12ab34cd5611334422ab3322997650/dns_records/' + 'fc12ab34cd5611334422ab3322997653', + data={'content': '4.4.4.4', 'type': 'A', 'name': + 'a.unit.tests', 'ttl': 300}), + call('POST', '/zones/42/dns_records', + data={'content': '3.3.3.3', 'type': 'A', + 'name': 'a.unit.tests', 'ttl': 300}) + ]) + + def test_update_delete(self): + # We need another run so that we can delete, we can't both add and + # delete in one go b/c of swaps + provider = CloudflareProvider('test', 'email', 'token') + + provider.zone_records = Mock(return_value=[ + { + "id": "fc12ab34cd5611334422ab3322997653", + "type": "NS", + "name": "unit.tests", + "content": "ns1.foo.bar", + "proxiable": True, + "proxied": False, + "ttl": 300, + "locked": False, + "zone_id": "ff12ab34cd5611334422ab3322997650", + "zone_name": "unit.tests", + "modified_on": "2017-03-11T18:01:43.420689Z", + "created_on": "2017-03-11T18:01:43.420689Z", + "meta": { + "auto_added": False + } + }, + { + "id": "fc12ab34cd5611334422ab3322997654", + "type": "NS", + "name": "unit.tests", + "content": "ns2.foo.bar", + "proxiable": True, + "proxied": False, + "ttl": 300, + "locked": False, + "zone_id": "ff12ab34cd5611334422ab3322997650", + "zone_name": "unit.tests", + "modified_on": "2017-03-11T18:01:43.420689Z", + "created_on": "2017-03-11T18:01:43.420689Z", + "meta": { + "auto_added": False + } + }, + ]) + + provider._request = Mock() + provider._request.side_effect = [ + self.empty, # no zones + { + 'result': { + 'id': 42, + } + }, # zone create + None, + None, + ] + + # Add something and delete something + zone = Zone('unit.tests.', []) + existing = Record.new(zone, '', { + 'ttl': 300, + 'type': 'NS', + # This matches the zone data above, one to delete, one to leave + 'values': ['ns1.foo.bar.', 'ns2.foo.bar.'], + }) + new = Record.new(zone, '', { + 'ttl': 300, + 'type': 'NS', + # This leaves one and deletes one + 'value': 'ns2.foo.bar.', + }) + change = Update(existing, new) + plan = Plan(zone, zone, [change]) + provider._apply(plan) + + provider._request.assert_has_calls([ + call('GET', '/zones', params={'page': 1}), + call('POST', '/zones', + data={'jump_start': False, 'name': 'unit.tests'}), + call('DELETE', '/zones/ff12ab34cd5611334422ab3322997650/' + 'dns_records/fc12ab34cd5611334422ab3322997653') + ]) From ad1d0f0fe83b83891caa39dbc77b48dfa0ca6092 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 6 Jan 2018 16:26:48 -0800 Subject: [PATCH 080/141] Fixes and unit tests for new plan output functionality --- octodns/provider/plan.py | 2 +- tests/test_octodns_plan.py | 75 +++++++++++++++++++++++++++++++++++++- 2 files changed, 75 insertions(+), 2 deletions(-) diff --git a/octodns/provider/plan.py b/octodns/provider/plan.py index 7edd682..3e86826 100644 --- a/octodns/provider/plan.py +++ b/octodns/provider/plan.py @@ -143,7 +143,7 @@ def _value_stringifier(record, sep): values = [str(v) for v in record.values] except AttributeError: values = [record.value] - for code, gv in getattr(record, 'geo', {}).items(): + for code, gv in sorted(getattr(record, 'geo', {}).items()): vs = ', '.join([str(v) for v in gv.values]) values.append('{}: {}'.format(code, vs)) return sep.join(values) diff --git a/tests/test_octodns_plan.py b/tests/test_octodns_plan.py index 2b23b4e..ea35243 100644 --- a/tests/test_octodns_plan.py +++ b/tests/test_octodns_plan.py @@ -5,9 +5,15 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals +from StringIO import StringIO +from logging import getLogger from unittest import TestCase -from octodns.provider.plan import PlanLogger +from octodns.provider.plan import Plan, PlanHtml, PlanLogger, PlanMarkdown +from octodns.record import Create, Delete, Record, Update +from octodns.zone import Zone + +from helpers import SimpleProvider class TestPlanLogger(TestCase): @@ -17,3 +23,70 @@ class TestPlanLogger(TestCase): PlanLogger('invalid', 'not-a-level') self.assertEquals('Unsupported level: not-a-level', ctx.exception.message) + + +simple = SimpleProvider() +zone = Zone('unit.tests.', []) +existing = Record.new(zone, 'a', { + 'ttl': 300, + 'type': 'A', + # This matches the zone data above, one to swap, one to leave + 'values': ['1.1.1.1', '2.2.2.2'], +}) +new = Record.new(zone, 'a', { + 'geo': { + 'AF': ['5.5.5.5'], + 'NA-US': ['6.6.6.6'] + }, + 'ttl': 300, + 'type': 'A', + # This leaves one, swaps ones, and adds one + 'values': ['2.2.2.2', '3.3.3.3', '4.4.4.4'], +}, simple) +create = Create(Record.new(zone, 'b', { + 'ttl': 60, + 'type': 'CNAME', + 'value': 'foo.unit.tests.' +}, simple)) +update = Update(existing, new) +delete = Delete(new) +changes = [create, delete, update] +plans = [ + (simple, Plan(zone, zone, changes)), + (simple, Plan(zone, zone, changes)), +] + + +class TestPlanHtml(TestCase): + log = getLogger('TestPlanHtml') + + def test_empty(self): + out = StringIO() + PlanHtml('html').run([], fh=out) + self.assertEquals('No changes were planned', out.getvalue()) + + def test_simple(self): + out = StringIO() + PlanHtml('html').run(plans, fh=out) + out = out.getvalue() + self.assertTrue(' Summary: Creates=1, Updates=1, ' + 'Deletes=1, Existing Records=0' in out) + + +class TestPlanMarkdown(TestCase): + log = getLogger('TestPlanMarkdown') + + def test_empty(self): + out = StringIO() + PlanMarkdown('markdown').run([], fh=out) + self.assertEquals('## No changes were planned\n', out.getvalue()) + + def test_simple(self): + out = StringIO() + PlanMarkdown('markdown').run(plans, fh=out) + out = out.getvalue() + self.assertTrue('## unit.tests.' in out) + self.assertTrue('Create | b | CNAME | 60 | foo.unit.tests.' in out) + self.assertTrue('Update | a | A | 300 | 1.1.1.1;' in out) + self.assertTrue('NA-US: 6.6.6.6 | test' in out) + self.assertTrue('Delete | a | A | 300 | 2.2.2.2;' in out) From 0659eda451ed4d8240016ea4f38e21e8ffb1e897 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 6 Jan 2018 16:53:11 -0800 Subject: [PATCH 081/141] Add Cloudflare ALIAS record support Translates them to/from root CNAME --- README.md | 2 +- octodns/provider/cloudflare.py | 14 +++++++- tests/test_octodns_provider_cloudflare.py | 42 +++++++++++++++++++++++ 3 files changed, 56 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 88223e6..cc84cf2 100644 --- a/README.md +++ b/README.md @@ -150,7 +150,7 @@ The above command pulled the existing data out of Route53 and placed the results | Provider | Record Support | GeoDNS Support | Notes | |--|--|--|--| | [AzureProvider](/octodns/provider/azuredns.py) | A, AAAA, CNAME, MX, NS, PTR, SRV, TXT | No | | -| [CloudflareProvider](/octodns/provider/cloudflare.py) | A, AAAA, CAA, CNAME, MX, NS, SPF, TXT | No | CAA tags restricted | +| [CloudflareProvider](/octodns/provider/cloudflare.py) | A, AAAA, ALIAS, CAA, CNAME, MX, NS, SPF, TXT | No | CAA tags restricted | | [DigitalOceanProvider](/octodns/provider/digitalocean.py) | A, AAAA, CAA, CNAME, MX, NS, TXT, SRV | No | CAA tags restricted | | [DnsimpleProvider](/octodns/provider/dnsimple.py) | All | No | CAA tags restricted | | [DynProvider](/octodns/provider/dyn.py) | All | Yes | | diff --git a/octodns/provider/cloudflare.py b/octodns/provider/cloudflare.py index a7ec5d8..9dfef6d 100644 --- a/octodns/provider/cloudflare.py +++ b/octodns/provider/cloudflare.py @@ -37,7 +37,8 @@ class CloudflareProvider(BaseProvider): ''' SUPPORTS_GEO = False # TODO: support SRV - SUPPORTS = set(('A', 'AAAA', 'CAA', 'CNAME', 'MX', 'NS', 'SPF', 'TXT')) + SUPPORTS = set(('ALIAS', 'A', 'AAAA', 'CAA', 'CNAME', 'MX', 'NS', 'SPF', + 'TXT')) MIN_TTL = 120 TIMEOUT = 15 @@ -124,6 +125,8 @@ class CloudflareProvider(BaseProvider): 'value': '{}.'.format(only['content']) } + _data_for_ALIAS = _data_for_CNAME + def _data_for_MX(self, _type, records): values = [] for r in records: @@ -182,6 +185,11 @@ class CloudflareProvider(BaseProvider): for name, types in values.items(): for _type, records in types.items(): + + # Cloudflare supports ALIAS semantics with root CNAMEs + if _type == 'CNAME' and name == '': + _type = 'ALIAS' + data_for = getattr(self, '_data_for_{}'.format(_type)) data = data_for(_type, records) record = Record.new(zone, name, data, source=self, @@ -238,6 +246,10 @@ class CloudflareProvider(BaseProvider): _type = record._type ttl = max(self.MIN_TTL, record.ttl) + # Cloudflare supports ALIAS semantics with a root CNAME + if _type == 'ALIAS': + _type = 'CNAME' + contents_for = getattr(self, '_contents_for_{}'.format(_type)) for content in contents_for(record): content.update({ diff --git a/tests/test_octodns_provider_cloudflare.py b/tests/test_octodns_provider_cloudflare.py index 8a17083..824af9d 100644 --- a/tests/test_octodns_provider_cloudflare.py +++ b/tests/test_octodns_provider_cloudflare.py @@ -442,3 +442,45 @@ class TestCloudflareProvider(TestCase): call('DELETE', '/zones/ff12ab34cd5611334422ab3322997650/' 'dns_records/fc12ab34cd5611334422ab3322997653') ]) + + def test_alias(self): + provider = CloudflareProvider('test', 'email', 'token') + + # A CNAME for us to transform to ALIAS + provider.zone_records = Mock(return_value=[ + { + "id": "fc12ab34cd5611334422ab3322997642", + "type": "CNAME", + "name": "unit.tests", + "content": "www.unit.tests", + "proxiable": True, + "proxied": False, + "ttl": 300, + "locked": False, + "zone_id": "ff12ab34cd5611334422ab3322997650", + "zone_name": "unit.tests", + "modified_on": "2017-03-11T18:01:43.420689Z", + "created_on": "2017-03-11T18:01:43.420689Z", + "meta": { + "auto_added": False + } + }, + ]) + + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals(1, len(zone.records)) + record = list(zone.records)[0] + self.assertEquals('', record.name) + self.assertEquals('unit.tests.', record.fqdn) + self.assertEquals('ALIAS', record._type) + self.assertEquals('www.unit.tests.', record.value) + + # Make sure we transform back to CNAME going the other way + contents = provider._gen_contents(record) + self.assertEquals({ + 'content': u'www.unit.tests.', + 'name': 'unit.tests', + 'ttl': 300, + 'type': 'CNAME' + }, list(contents)[0]) From fdea900537cec8fced3120003c338ccd1fe6aead Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 6 Jan 2018 16:53:34 -0800 Subject: [PATCH 082/141] Correct total_count in Cloudflare record fixtures --- tests/fixtures/cloudflare-dns_records-page-1.json | 2 +- tests/fixtures/cloudflare-dns_records-page-2.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/fixtures/cloudflare-dns_records-page-1.json b/tests/fixtures/cloudflare-dns_records-page-1.json index eda4de3..3c423e2 100644 --- a/tests/fixtures/cloudflare-dns_records-page-1.json +++ b/tests/fixtures/cloudflare-dns_records-page-1.json @@ -180,7 +180,7 @@ "per_page": 10, "total_pages": 2, "count": 10, - "total_count": 17 + "total_count": 19 }, "success": true, "errors": [], diff --git a/tests/fixtures/cloudflare-dns_records-page-2.json b/tests/fixtures/cloudflare-dns_records-page-2.json index 150951b..de3d760 100644 --- a/tests/fixtures/cloudflare-dns_records-page-2.json +++ b/tests/fixtures/cloudflare-dns_records-page-2.json @@ -163,7 +163,7 @@ "per_page": 10, "total_pages": 2, "count": 9, - "total_count": 20 + "total_count": 19 }, "success": true, "errors": [], From 0df8ed5e2a86d4c7e220841add8c479cf8f91e6b Mon Sep 17 00:00:00 2001 From: Stephen Coursen Date: Sun, 7 Jan 2018 03:50:10 +0000 Subject: [PATCH 083/141] bump required version of nsone-python client --- setup.cfg | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setup.cfg b/setup.cfg index 54e9014..0473649 100644 --- a/setup.cfg +++ b/setup.cfg @@ -19,7 +19,7 @@ classifiers = Programming Language :: Python :: 3.6 [options] -install_requires = +install_requires = PyYaml>=3.12 dnspython>=1.15.0 futures==3.1.1 @@ -32,7 +32,7 @@ packages = find: include_package_data = True [options.entry_points] -console_scripts = +console_scripts = octodns-compare = octodns.cmds.compare:main octodns-dump = octodns.cmds.dump:main octodns-report = octodns.cmds.report:main @@ -44,7 +44,7 @@ exclude = tests [options.extras_require] -dev = +dev = azure-mgmt-dns==1.0.1 azure-common==1.1.6 boto3==1.4.6 @@ -54,7 +54,7 @@ dev = google-cloud==0.27.0 jmespath==0.9.3 msrestazure==0.4.10 - nsone==0.9.14 + nsone==0.9.17 ovh==0.4.7 s3transfer==0.1.10 six==1.10.0 From dc43c43866fec17119f5320553df2eb399ca3278 Mon Sep 17 00:00:00 2001 From: Steve Coursen Date: Mon, 8 Jan 2018 10:02:27 -0500 Subject: [PATCH 084/141] Increased test coverage --- octodns/provider/ns1.py | 7 ++-- tests/test_octodns_provider_ns1.py | 59 +++++++++++++++++++++++++++--- 2 files changed, 58 insertions(+), 8 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index d022e78..20fffc3 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -64,7 +64,7 @@ class Ns1Provider(BaseProvider): for state in us_state: geo['NA-US-{}'.format(state)] = answer['answer'] for province in ca_province: - geo['NA-CA-{}'.format(state)] = answer['answer'] + geo['NA-CA-{}'.format(province)] = answer['answer'] for code in meta.get('iso_region_code', []): geo[code] = answer['answer'] else: @@ -212,7 +212,8 @@ class Ns1Provider(BaseProvider): for iso_region, target in record.geo.items(): key = 'iso_region_code' value = iso_region - if not has_country and len(value.split('-')) > 1: + if not has_country and \ + len(value.split('-')) > 1: # pragma: nocover has_country = True params['answers'].append( { @@ -223,7 +224,7 @@ class Ns1Provider(BaseProvider): params['filters'] = [] if len(params['answers']) > 1: params['filters'].append( - {"filter": "shuffle", "config":{}} + {"filter": "shuffle", "config": {}} ) if has_country: params['filters'].append( diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index 10ae5d3..fd0d31f 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -129,7 +129,7 @@ class TestNs1Provider(TestCase): 'type': 'A', 'ttl': 34, 'short_answers': ['101.102.103.104', '101.102.103.105'], - 'domain': 'geo.unit.tests.', + 'domain': 'geo.unit.tests', }, { 'type': 'CNAME', 'ttl': 34, @@ -205,11 +205,25 @@ class TestNs1Provider(TestCase): nsone_zone = DummyZone([]) load_mock.side_effect = [nsone_zone] zone_search = Mock() - zone_search.return_value = [] + zone_search.return_value = [ + { + "domain": "geo.unit.tests", + "zone": "unit.tests", + "type": "A", + "answers": [ + {'answer': ['1.1.1.1'], 'meta': {}}, + {'answer': ['1.2.3.4'], 'meta': {'ca_province': ['ON']}}, + {'answer': ['2.3.4.5'], 'meta': {'us_state': ['NY']}}, + {'answer': ['3.4.5.6'], 'meta': {'country': ['US']}}, + {'answer': ['4.5.6.7'], 'meta': {'iso_region_code': ['NA-US-WA']}}, + ], + 'ttl': 34, + }, + ] nsone_zone.search = zone_search zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals(set(), zone.records) + self.assertEquals(1, len(zone.records)) self.assertEquals(('unit.tests',), load_mock.call_args[0]) # Existing zone w/records @@ -217,7 +231,21 @@ class TestNs1Provider(TestCase): nsone_zone = DummyZone(self.nsone_records) load_mock.side_effect = [nsone_zone] zone_search = Mock() - zone_search.return_value = [] + zone_search.return_value = [ + { + "domain": "geo.unit.tests", + "zone": "unit.tests", + "type": "A", + "answers": [ + {'answer': ['1.1.1.1'], 'meta': {}}, + {'answer': ['1.2.3.4'], 'meta': {'ca_province': ['ON']}}, + {'answer': ['2.3.4.5'], 'meta': {'us_state': ['NY']}}, + {'answer': ['3.4.5.6'], 'meta': {'country': ['US']}}, + {'answer': ['4.5.6.7'], 'meta': {'iso_region_code': ['NA-US-WA']}}, + ], + 'ttl': 34, + }, + ] nsone_zone.search = zone_search zone = Zone('unit.tests.', []) provider.populate(zone) @@ -285,7 +313,21 @@ class TestNs1Provider(TestCase): nsone_zone.data['records'][0]['short_answers'][0] = '2.2.2.2' nsone_zone.loadRecord = Mock() zone_search = Mock() - zone_search.return_value = [] + zone_search.return_value = [ + { + "domain": "geo.unit.tests", + "zone": "unit.tests", + "type": "A", + "answers": [ + {'answer': ['1.1.1.1'], 'meta': {}}, + {'answer': ['1.2.3.4'], 'meta': {'ca_province': ['ON']}}, + {'answer': ['2.3.4.5'], 'meta': {'us_state': ['NY']}}, + {'answer': ['3.4.5.6'], 'meta': {'country': ['US']}}, + {'answer': ['4.5.6.7'], 'meta': {'iso_region_code': ['NA-US-WA']}}, + ], + 'ttl': 34, + }, + ] nsone_zone.search = zone_search load_mock.side_effect = [nsone_zone, nsone_zone] plan = provider.plan(desired) @@ -317,8 +359,10 @@ class TestNs1Provider(TestCase): ]) mock_record.assert_has_calls([ call.update(answers=[{'answer': [u'1.2.3.4'], 'meta': {}}], + filters=[], ttl=32), call.update(answers=[{u'answer': [u'1.2.3.4'], u'meta': {}}], + filters=[], ttl=32), call.update( answers=[ @@ -331,6 +375,11 @@ class TestNs1Provider(TestCase): }, }, ], + filters=[ + {u'filter': u'shuffle', u'config': {}}, + {u'filter': u'geotarget_country', u'config': {}}, + {u'filter': u'select_first_n', u'config': {u'N': 1}}, + ], ttl=34), call.delete(), call.delete() From b06c14deaedc5eef97d9d58907b3a40549158065 Mon Sep 17 00:00:00 2001 From: Steve Coursen Date: Mon, 8 Jan 2018 12:28:25 -0500 Subject: [PATCH 085/141] Fix E501 line too long --- tests/test_octodns_provider_ns1.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index fd0d31f..0e3a286 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -212,10 +212,12 @@ class TestNs1Provider(TestCase): "type": "A", "answers": [ {'answer': ['1.1.1.1'], 'meta': {}}, - {'answer': ['1.2.3.4'], 'meta': {'ca_province': ['ON']}}, + {'answer': ['1.2.3.4'], + 'meta': {'ca_province': ['ON']}}, {'answer': ['2.3.4.5'], 'meta': {'us_state': ['NY']}}, {'answer': ['3.4.5.6'], 'meta': {'country': ['US']}}, - {'answer': ['4.5.6.7'], 'meta': {'iso_region_code': ['NA-US-WA']}}, + {'answer': ['4.5.6.7'], + 'meta': {'iso_region_code': ['NA-US-WA']}}, ], 'ttl': 34, }, @@ -238,10 +240,12 @@ class TestNs1Provider(TestCase): "type": "A", "answers": [ {'answer': ['1.1.1.1'], 'meta': {}}, - {'answer': ['1.2.3.4'], 'meta': {'ca_province': ['ON']}}, + {'answer': ['1.2.3.4'], + 'meta': {'ca_province': ['ON']}}, {'answer': ['2.3.4.5'], 'meta': {'us_state': ['NY']}}, {'answer': ['3.4.5.6'], 'meta': {'country': ['US']}}, - {'answer': ['4.5.6.7'], 'meta': {'iso_region_code': ['NA-US-WA']}}, + {'answer': ['4.5.6.7'], + 'meta': {'iso_region_code': ['NA-US-WA']}}, ], 'ttl': 34, }, @@ -320,10 +324,12 @@ class TestNs1Provider(TestCase): "type": "A", "answers": [ {'answer': ['1.1.1.1'], 'meta': {}}, - {'answer': ['1.2.3.4'], 'meta': {'ca_province': ['ON']}}, + {'answer': ['1.2.3.4'], + 'meta': {'ca_province': ['ON']}}, {'answer': ['2.3.4.5'], 'meta': {'us_state': ['NY']}}, {'answer': ['3.4.5.6'], 'meta': {'country': ['US']}}, - {'answer': ['4.5.6.7'], 'meta': {'iso_region_code': ['NA-US-WA']}}, + {'answer': ['4.5.6.7'], + 'meta': {'iso_region_code': ['NA-US-WA']}}, ], 'ttl': 34, }, From e875ee7f5d846d4c5a516834526bec57099c3196 Mon Sep 17 00:00:00 2001 From: Terrence Cole Date: Mon, 8 Jan 2018 11:07:03 -0800 Subject: [PATCH 086/141] Add a comment explaining our update scheme. --- octodns/provider/rackspace.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py index c432339..12b2c54 100644 --- a/octodns/provider/rackspace.py +++ b/octodns/provider/rackspace.py @@ -348,6 +348,9 @@ class RackspaceProvider(BaseProvider): self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name, len(changes)) + # Creates, updates, and deletes are processed by different endpoints + # and are broken out by record-set entries; pre-process everything + # into these buckets in order to minimize the number of API calls. domain_id = self._get_zone_id_for(desired) creates = [] updates = [] From 154ca64038f9f0d3fe981a93e3a8cc42eb0762b9 Mon Sep 17 00:00:00 2001 From: Steve Coursen Date: Mon, 8 Jan 2018 20:13:20 -0500 Subject: [PATCH 087/141] Fix serialization of multiple answers, that had caused a ResourceException --- octodns/provider/ns1.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 20fffc3..a675ed9 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -215,12 +215,13 @@ class Ns1Provider(BaseProvider): if not has_country and \ len(value.split('-')) > 1: # pragma: nocover has_country = True - params['answers'].append( - { - 'answer': target.values, - 'meta': {key: [value]}, - }, - ) + for answer in target.values: + params['answers'].append( + { + 'answer': [answer], + 'meta': {key: [value]}, + }, + ) params['filters'] = [] if len(params['answers']) > 1: params['filters'].append( From dcdde5db5dc9dbcbd3edf5840f1e1090de69aba6 Mon Sep 17 00:00:00 2001 From: Steve Coursen Date: Mon, 8 Jan 2018 21:46:59 -0500 Subject: [PATCH 088/141] Handle multiple answers correctly when dersializing --- octodns/provider/ns1.py | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index a675ed9..1b48480 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -7,6 +7,7 @@ from __future__ import absolute_import, division, print_function, \ from logging import getLogger from itertools import chain +from collections import OrderedDict from nsone import NSONE from nsone.rest.errors import RateLimitException, ResourceException from incf.countryutils import transformations @@ -60,16 +61,34 @@ class Ns1Provider(BaseProvider): for cntry in country: cn = transformations.cc_to_cn(cntry) con = transformations.cn_to_ctca2(cn) - geo['{}-{}'.format(con, cntry)] = answer['answer'] + key = '{}-{}'.format(con, cntry) + if key not in geo: + geo[key] = answer['answer'] + else: + geo[key].extend(answer['answer']) for state in us_state: - geo['NA-US-{}'.format(state)] = answer['answer'] + key = 'NA-US-{}'.format(state) + if key not in geo: + geo[key] = answer['answer'] + else: + geo[key].extend(answer['answer']) for province in ca_province: - geo['NA-CA-{}'.format(province)] = answer['answer'] + key = 'NA-CA-{}'.format(province) + if key not in geo: + geo[key] = answer['answer'] + else: + geo[key].extend(answer['answer']) for code in meta.get('iso_region_code', []): - geo[code] = answer['answer'] + key = code + if key not in geo: + geo[key] = answer['answer'] + else: + geo[key].extend(answer['answer']) else: values.extend(answer['answer']) codes.append([]) + values = [str(x) for x in values] + geo = OrderedDict({str(k): [str(x) for x in v] for k, v in sorted(geo.items())}) data['values'] = values data['geo'] = geo return data From 241e6cc0ce9b3c558d42d01c4a7df111c11c7834 Mon Sep 17 00:00:00 2001 From: Steve Coursen Date: Mon, 8 Jan 2018 21:57:13 -0500 Subject: [PATCH 089/141] E501 trim lines --- octodns/provider/ns1.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 1b48480..72c8b08 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -26,8 +26,8 @@ class Ns1Provider(BaseProvider): api_key: env/NS1_API_KEY ''' SUPPORTS_GEO = True - SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CAA', 'CNAME', 'MX', 'NAPTR', 'NS', - 'PTR', 'SPF', 'SRV', 'TXT')) + SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CAA', 'CNAME', 'MX', 'NAPTR', + 'NS', 'PTR', 'SPF', 'SRV', 'TXT')) ZONE_NOT_FOUND_MESSAGE = 'server error: zone not found' @@ -88,7 +88,9 @@ class Ns1Provider(BaseProvider): values.extend(answer['answer']) codes.append([]) values = [str(x) for x in values] - geo = OrderedDict({str(k): [str(x) for x in v] for k, v in sorted(geo.items())}) + geo = OrderedDict( + {str(k): [str(x) for x in v] for k, v in geo.items()} + ) data['values'] = values data['geo'] = geo return data @@ -192,7 +194,8 @@ class Ns1Provider(BaseProvider): } def populate(self, zone, target=False, lenient=False): - self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, + self.log.debug('populate: name=%s, target=%s, lenient=%s', + zone.name, target, lenient) try: @@ -261,9 +264,9 @@ class Ns1Provider(BaseProvider): _params_for_NS = _params_for_A def _params_for_SPF(self, record): - # NS1 seems to be the only provider that doesn't want things escaped in - # values so we have to strip them here and add them when going the - # other way + # NS1 seems to be the only provider that doesn't want things + # escaped in values so we have to strip them here and add + # them when going the other way values = [v.replace('\;', ';') for v in record.values] return {'answers': values, 'ttl': record.ttl} @@ -355,4 +358,5 @@ class Ns1Provider(BaseProvider): for change in changes: class_name = change.__class__.__name__ - getattr(self, '_apply_{}'.format(class_name))(nsone_zone, change) + getattr(self, '_apply_{}'.format(class_name))(nsone_zone, + change) From d8ba6a2b41685fa0bcb7cfe69d9d5984e65d63b3 Mon Sep 17 00:00:00 2001 From: Steve Coursen Date: Mon, 8 Jan 2018 22:02:46 -0500 Subject: [PATCH 090/141] slight code cleanup, coverage increase --- octodns/provider/ns1.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 72c8b08..7889386 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -63,27 +63,23 @@ class Ns1Provider(BaseProvider): con = transformations.cn_to_ctca2(cn) key = '{}-{}'.format(con, cntry) if key not in geo: - geo[key] = answer['answer'] - else: - geo[key].extend(answer['answer']) + geo[key] = [] + geo[key].extend(answer['answer']) for state in us_state: key = 'NA-US-{}'.format(state) if key not in geo: - geo[key] = answer['answer'] - else: - geo[key].extend(answer['answer']) + geo[key] = [] + geo[key].extend(answer['answer']) for province in ca_province: key = 'NA-CA-{}'.format(province) if key not in geo: - geo[key] = answer['answer'] - else: - geo[key].extend(answer['answer']) + geo[key] = [] + geo[key].extend(answer['answer']) for code in meta.get('iso_region_code', []): key = code if key not in geo: - geo[key] = answer['answer'] - else: - geo[key].extend(answer['answer']) + geo[key] = [] + geo[key].extend(answer['answer']) else: values.extend(answer['answer']) codes.append([]) From 9785e40688a9eb16bd0f8e927a5d312c349afd9a Mon Sep 17 00:00:00 2001 From: Steve Coursen Date: Mon, 8 Jan 2018 22:04:42 -0500 Subject: [PATCH 091/141] use defaultdict --- octodns/provider/ns1.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 7889386..a68b759 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -7,7 +7,7 @@ from __future__ import absolute_import, division, print_function, \ from logging import getLogger from itertools import chain -from collections import OrderedDict +from collections import OrderedDict, defaultdict from nsone import NSONE from nsone.rest.errors import RateLimitException, ResourceException from incf.countryutils import transformations @@ -40,7 +40,7 @@ class Ns1Provider(BaseProvider): def _data_for_A(self, _type, record): # record meta (which would include geo information is only # returned when getting a record's detail, not from zone detail - geo = {} + geo = defaultdict(list) data = { 'ttl': record['ttl'], 'type': _type, @@ -62,23 +62,15 @@ class Ns1Provider(BaseProvider): cn = transformations.cc_to_cn(cntry) con = transformations.cn_to_ctca2(cn) key = '{}-{}'.format(con, cntry) - if key not in geo: - geo[key] = [] geo[key].extend(answer['answer']) for state in us_state: key = 'NA-US-{}'.format(state) - if key not in geo: - geo[key] = [] geo[key].extend(answer['answer']) for province in ca_province: key = 'NA-CA-{}'.format(province) - if key not in geo: - geo[key] = [] geo[key].extend(answer['answer']) for code in meta.get('iso_region_code', []): key = code - if key not in geo: - geo[key] = [] geo[key].extend(answer['answer']) else: values.extend(answer['answer']) From c16b8d6d7820951d4c097b55fcde7827070060f0 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Tue, 9 Jan 2018 07:28:36 -0800 Subject: [PATCH 092/141] RateLimitException.period is coming back as str now --- octodns/provider/ns1.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index a68b759..5ea68b6 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -296,9 +296,10 @@ class Ns1Provider(BaseProvider): try: meth(name, **params) except RateLimitException as e: + period = float(e.period) self.log.warn('_apply_Create: rate limit encountered, pausing ' - 'for %ds and trying again', e.period) - sleep(e.period) + 'for %ds and trying again', period) + sleep(period) meth(name, **params) def _apply_Update(self, nsone_zone, change): @@ -311,9 +312,10 @@ class Ns1Provider(BaseProvider): try: record.update(**params) except RateLimitException as e: + period = float(e.period) self.log.warn('_apply_Update: rate limit encountered, pausing ' - 'for %ds and trying again', e.period) - sleep(e.period) + 'for %ds and trying again', period) + sleep(period) record.update(**params) def _apply_Delete(self, nsone_zone, change): @@ -324,9 +326,10 @@ class Ns1Provider(BaseProvider): try: record.delete() except RateLimitException as e: + period = float(e.period) self.log.warn('_apply_Delete: rate limit encountered, pausing ' - 'for %ds and trying again', e.period) - sleep(e.period) + 'for %ds and trying again', period) + sleep(period) record.delete() def _apply(self, plan): From 682e57c9f9ce73a92033bef79d65e7ac88e5dcef Mon Sep 17 00:00:00 2001 From: Steve Coursen Date: Tue, 9 Jan 2018 10:35:52 -0500 Subject: [PATCH 093/141] Set Ns1Provider to "Yes" for GeoDNS Support --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 60d39af..b845fd8 100644 --- a/README.md +++ b/README.md @@ -119,13 +119,13 @@ The first step is to create a PR with your changes. Assuming the code tests and config validation statuses are green the next step is to do a noop deploy and verify that the changes OctoDNS plans to make are the ones you expect. -![](/docs/assets/noop.png) +![](/docs/assets/noop.png) After that comes a set of reviews. One from a teammate who should have full context on what you're trying to accomplish and visibility in to the changes you're making to do it. The other is from a member of the team here at GitHub that owns DNS, mostly as a sanity check and to make sure that best practices are being followed. As much of that as possible is baked into `octodns-validate`. After the reviews it's time to branch deploy the change. -![](/docs/assets/deploy.png) +![](/docs/assets/deploy.png) If that goes smoothly, you again see the expected changes, and verify them with `dig` and/or `octodns-report` you're good to hit the merge button. If there are problems you can quickly do a `.deploy dns/master` to go back to the previous state. @@ -155,7 +155,7 @@ The above command pulled the existing data out of Route53 and placed the results | [DnsimpleProvider](/octodns/provider/dnsimple.py) | All | No | CAA tags restricted | | [DynProvider](/octodns/provider/dyn.py) | All | Yes | | | [GoogleCloudProvider](/octodns/provider/googlecloud.py) | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | No | | -| [Ns1Provider](/octodns/provider/ns1.py) | All | No | | +| [Ns1Provider](/octodns/provider/ns1.py) | All | Yes | | | [OVH](/octodns/provider/ovh.py) | A, AAAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, SSHFP, TXT | No | | | [PowerDnsProvider](/octodns/provider/powerdns.py) | All | No | | | [Rackspace](/octodns/provider/rackspace.py) | A, AAAA, ALIAS, CNAME, MX, NS, PTR, SPF, TXT | No | | From 68bc400e19cd8471e891a7bf93207cd203f6bacb Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Tue, 9 Jan 2018 07:51:32 -0800 Subject: [PATCH 094/141] Comment that NS1 geo health checks aren't yet supported --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b845fd8..0fff5fa 100644 --- a/README.md +++ b/README.md @@ -155,7 +155,7 @@ The above command pulled the existing data out of Route53 and placed the results | [DnsimpleProvider](/octodns/provider/dnsimple.py) | All | No | CAA tags restricted | | [DynProvider](/octodns/provider/dyn.py) | All | Yes | | | [GoogleCloudProvider](/octodns/provider/googlecloud.py) | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | No | | -| [Ns1Provider](/octodns/provider/ns1.py) | All | Yes | | +| [Ns1Provider](/octodns/provider/ns1.py) | All | Yes | No health checking for GeoDNS | | [OVH](/octodns/provider/ovh.py) | A, AAAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, SSHFP, TXT | No | | | [PowerDnsProvider](/octodns/provider/powerdns.py) | All | No | | | [Rackspace](/octodns/provider/rackspace.py) | A, AAAA, ALIAS, CNAME, MX, NS, PTR, SPF, TXT | No | | From af435c3130b4615da3b469b51ae508f7d9af17f0 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sun, 21 Jan 2018 06:31:37 -0800 Subject: [PATCH 095/141] Handle MX preference of 0 --- octodns/record.py | 5 ++++- tests/test_octodns_record.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/octodns/record.py b/octodns/record.py index 514c2e6..728c187 100644 --- a/octodns/record.py +++ b/octodns/record.py @@ -506,7 +506,10 @@ class MxValue(object): def _validate_value(cls, value): reasons = [] try: - int(value.get('preference', None) or value['priority']) + try: + int(value['preference']) + except KeyError: + int(value['priority']) except KeyError: reasons.append('missing preference') except ValueError: diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index ff57975..0ba54de 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -345,7 +345,7 @@ class TestRecord(TestCase): self.assertEquals(a_data, a.data) b_value = { - 'preference': 12, + 'preference': 0, 'exchange': 'smtp3.', } b_data = {'ttl': 30, 'value': b_value} From 1e71bce9072838ce0385704502d76813ec8794c0 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sun, 21 Jan 2018 13:47:58 -0800 Subject: [PATCH 096/141] Add create param to Plan --- octodns/manager.py | 2 +- octodns/provider/base.py | 3 ++- octodns/provider/plan.py | 3 ++- tests/test_octodns_plan.py | 4 ++-- tests/test_octodns_provider_azuredns.py | 9 ++++++--- tests/test_octodns_provider_base.py | 22 ++++++++++++---------- tests/test_octodns_provider_cloudflare.py | 4 ++-- tests/test_octodns_provider_dyn.py | 2 +- tests/test_octodns_provider_googlecloud.py | 6 ++++-- 9 files changed, 32 insertions(+), 23 deletions(-) diff --git a/octodns/manager.py b/octodns/manager.py index d4debf6..bb2c921 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -361,7 +361,7 @@ class Manager(object): plan = target.plan(zone) if plan is None: - plan = Plan(zone, zone, []) + plan = Plan(zone, zone, [], True) target.apply(plan) def validate_configs(self): diff --git a/octodns/provider/base.py b/octodns/provider/base.py index 2d4680f..ddce7a6 100644 --- a/octodns/provider/base.py +++ b/octodns/provider/base.py @@ -64,8 +64,9 @@ class BaseProvider(BaseSource): .join([str(c) for c in extra])) changes += extra + create = False if changes: - plan = Plan(existing, desired, changes, + plan = Plan(existing, desired, changes, create, self.update_pcent_threshold, self.delete_pcent_threshold) self.log.info('plan: %s', plan) diff --git a/octodns/provider/plan.py b/octodns/provider/plan.py index 3e86826..8bb72bd 100644 --- a/octodns/provider/plan.py +++ b/octodns/provider/plan.py @@ -21,12 +21,13 @@ class Plan(object): MAX_SAFE_DELETE_PCENT = .3 MIN_EXISTING_RECORDS = 10 - def __init__(self, existing, desired, changes, + def __init__(self, existing, desired, changes, create, update_pcent_threshold=MAX_SAFE_UPDATE_PCENT, delete_pcent_threshold=MAX_SAFE_DELETE_PCENT): self.existing = existing self.desired = desired self.changes = changes + self.create = create self.update_pcent_threshold = update_pcent_threshold self.delete_pcent_threshold = delete_pcent_threshold diff --git a/tests/test_octodns_plan.py b/tests/test_octodns_plan.py index ea35243..36dee68 100644 --- a/tests/test_octodns_plan.py +++ b/tests/test_octodns_plan.py @@ -52,8 +52,8 @@ update = Update(existing, new) delete = Delete(new) changes = [create, delete, update] plans = [ - (simple, Plan(zone, zone, changes)), - (simple, Plan(zone, zone, changes)), + (simple, Plan(zone, zone, changes, False)), + (simple, Plan(zone, zone, changes, False)), ] diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index 598fe48..f44d368 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -338,8 +338,10 @@ class TestAzureDnsProvider(TestCase): changes.append(Create(i)) deletes.append(Delete(i)) - self.assertEquals(13, provider.apply(Plan(None, zone, changes))) - self.assertEquals(13, provider.apply(Plan(zone, zone, deletes))) + self.assertEquals(13, provider.apply(Plan(None, zone, + changes, False))) + self.assertEquals(13, provider.apply(Plan(zone, zone, + deletes, False))) def test_create_zone(self): provider = self._get_provider() @@ -354,7 +356,8 @@ class TestAzureDnsProvider(TestCase): _get = provider._dns_client.zones.get _get.side_effect = CloudError(Mock(status=404), err_msg) - self.assertEquals(13, provider.apply(Plan(None, desired, changes))) + self.assertEquals(13, provider.apply(Plan(None, desired, changes, + False))) def test_check_zone_no_create(self): provider = self._get_provider() diff --git a/tests/test_octodns_provider_base.py b/tests/test_octodns_provider_base.py index 472b008..a0ec74e 100644 --- a/tests/test_octodns_provider_base.py +++ b/tests/test_octodns_provider_base.py @@ -153,7 +153,7 @@ class TestBaseProvider(TestCase): def test_safe_none(self): # No changes is safe - Plan(None, None, []).raise_if_unsafe() + Plan(None, None, [], False).raise_if_unsafe() def test_safe_creates(self): # Creates are safe when existing records is under MIN_EXISTING_RECORDS @@ -164,7 +164,8 @@ class TestBaseProvider(TestCase): 'type': 'A', 'value': '1.2.3.4', }) - Plan(zone, zone, [Create(record) for i in range(10)]).raise_if_unsafe() + Plan(zone, zone, [Create(record) for i in range(10)], False) \ + .raise_if_unsafe() def test_safe_min_existing_creates(self): # Creates are safe when existing records is over MIN_EXISTING_RECORDS @@ -183,7 +184,8 @@ class TestBaseProvider(TestCase): 'value': '2.3.4.5' })) - Plan(zone, zone, [Create(record) for i in range(10)]).raise_if_unsafe() + Plan(zone, zone, [Create(record) for i in range(10)], False) \ + .raise_if_unsafe() def test_safe_no_existing(self): # existing records fewer than MIN_EXISTING_RECORDS is safe @@ -195,7 +197,7 @@ class TestBaseProvider(TestCase): }) updates = [Update(record, record), Update(record, record)] - Plan(zone, zone, updates).raise_if_unsafe() + Plan(zone, zone, updates, False).raise_if_unsafe() def test_safe_updates_min_existing(self): # MAX_SAFE_UPDATE_PCENT+1 fails when more @@ -219,7 +221,7 @@ class TestBaseProvider(TestCase): Plan.MAX_SAFE_UPDATE_PCENT) + 1)] with self.assertRaises(UnsafePlan) as ctx: - Plan(zone, zone, changes).raise_if_unsafe() + Plan(zone, zone, changes, False).raise_if_unsafe() self.assertTrue('Too many updates' in ctx.exception.message) @@ -243,7 +245,7 @@ class TestBaseProvider(TestCase): for i in range(int(Plan.MIN_EXISTING_RECORDS * Plan.MAX_SAFE_UPDATE_PCENT))] - Plan(zone, zone, changes).raise_if_unsafe() + Plan(zone, zone, changes, False).raise_if_unsafe() def test_safe_deletes_min_existing(self): # MAX_SAFE_DELETE_PCENT+1 fails when more @@ -267,7 +269,7 @@ class TestBaseProvider(TestCase): Plan.MAX_SAFE_DELETE_PCENT) + 1)] with self.assertRaises(UnsafePlan) as ctx: - Plan(zone, zone, changes).raise_if_unsafe() + Plan(zone, zone, changes, False).raise_if_unsafe() self.assertTrue('Too many deletes' in ctx.exception.message) @@ -291,7 +293,7 @@ class TestBaseProvider(TestCase): for i in range(int(Plan.MIN_EXISTING_RECORDS * Plan.MAX_SAFE_DELETE_PCENT))] - Plan(zone, zone, changes).raise_if_unsafe() + Plan(zone, zone, changes, False).raise_if_unsafe() def test_safe_updates_min_existing_override(self): safe_pcent = .4 @@ -316,7 +318,7 @@ class TestBaseProvider(TestCase): safe_pcent) + 1)] with self.assertRaises(UnsafePlan) as ctx: - Plan(zone, zone, changes, + Plan(zone, zone, changes, False, update_pcent_threshold=safe_pcent).raise_if_unsafe() self.assertTrue('Too many updates' in ctx.exception.message) @@ -344,7 +346,7 @@ class TestBaseProvider(TestCase): safe_pcent) + 1)] with self.assertRaises(UnsafePlan) as ctx: - Plan(zone, zone, changes, + Plan(zone, zone, changes, False, delete_pcent_threshold=safe_pcent).raise_if_unsafe() self.assertTrue('Too many deletes' in ctx.exception.message) diff --git a/tests/test_octodns_provider_cloudflare.py b/tests/test_octodns_provider_cloudflare.py index 824af9d..7925b94 100644 --- a/tests/test_octodns_provider_cloudflare.py +++ b/tests/test_octodns_provider_cloudflare.py @@ -347,7 +347,7 @@ class TestCloudflareProvider(TestCase): 'values': ['2.2.2.2', '3.3.3.3', '4.4.4.4'], }) change = Update(existing, new) - plan = Plan(zone, zone, [change]) + plan = Plan(zone, zone, [change], False) provider._apply(plan) provider._request.assert_has_calls([ @@ -432,7 +432,7 @@ class TestCloudflareProvider(TestCase): 'value': 'ns2.foo.bar.', }) change = Update(existing, new) - plan = Plan(zone, zone, [change]) + plan = Plan(zone, zone, [change], False) provider._apply(plan) provider._request.assert_has_calls([ diff --git a/tests/test_octodns_provider_dyn.py b/tests/test_octodns_provider_dyn.py index 4415347..9e06a6b 100644 --- a/tests/test_octodns_provider_dyn.py +++ b/tests/test_octodns_provider_dyn.py @@ -913,7 +913,7 @@ class TestDynProviderGeo(TestCase): Delete(geo), Delete(regular), ] - plan = Plan(None, desired, changes) + plan = Plan(None, desired, changes, False) provider._apply(plan) mock.assert_has_calls([ call('/Zone/unit.tests/', 'GET', {}), diff --git a/tests/test_octodns_provider_googlecloud.py b/tests/test_octodns_provider_googlecloud.py index adc2112..7060355 100644 --- a/tests/test_octodns_provider_googlecloud.py +++ b/tests/test_octodns_provider_googlecloud.py @@ -263,7 +263,8 @@ class TestGoogleCloudProvider(TestCase): provider.apply(Plan( existing=[update_existing_r, delete_r], desired=desired, - changes=changes + changes=changes, + create=False )) calls_mock = gcloud_zone_mock.changes.return_value @@ -295,7 +296,8 @@ class TestGoogleCloudProvider(TestCase): provider.apply(Plan( existing=[update_existing_r, delete_r], desired=desired, - changes=changes + changes=changes, + create=False )) unsupported_change = Mock() From 94bfb1e5072beec17f759b15344e3f0892d2955e Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sun, 21 Jan 2018 14:06:20 -0800 Subject: [PATCH 097/141] Switch populate to return exists, cleaner setup --- octodns/manager.py | 2 +- octodns/provider/base.py | 10 +++++++--- octodns/provider/plan.py | 4 ++-- octodns/source/base.py | 5 ++++- tests/test_octodns_plan.py | 4 ++-- tests/test_octodns_provider_azuredns.py | 6 +++--- tests/test_octodns_provider_base.py | 20 ++++++++++---------- tests/test_octodns_provider_cloudflare.py | 4 ++-- tests/test_octodns_provider_dyn.py | 2 +- tests/test_octodns_provider_googlecloud.py | 4 ++-- 10 files changed, 34 insertions(+), 27 deletions(-) diff --git a/octodns/manager.py b/octodns/manager.py index bb2c921..3f46007 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -361,7 +361,7 @@ class Manager(object): plan = target.plan(zone) if plan is None: - plan = Plan(zone, zone, [], True) + plan = Plan(zone, zone, [], False) target.apply(plan) def validate_configs(self): diff --git a/octodns/provider/base.py b/octodns/provider/base.py index ddce7a6..6bfb16d 100644 --- a/octodns/provider/base.py +++ b/octodns/provider/base.py @@ -45,7 +45,12 @@ class BaseProvider(BaseSource): self.log.info('plan: desired=%s', desired.name) existing = Zone(desired.name, desired.sub_zones) - self.populate(existing, target=True, lenient=True) + exists = self.populate(existing, target=True, lenient=True) + if exists is None: + # If your code gets this warning see Source.populate for more + # information + self.log.warn('Provider %s used in target mode did not return ' + 'exists', self.id) # compute the changes at the zone/record level changes = existing.changes(desired, self) @@ -64,9 +69,8 @@ class BaseProvider(BaseSource): .join([str(c) for c in extra])) changes += extra - create = False if changes: - plan = Plan(existing, desired, changes, create, + plan = Plan(existing, desired, changes, exists, self.update_pcent_threshold, self.delete_pcent_threshold) self.log.info('plan: %s', plan) diff --git a/octodns/provider/plan.py b/octodns/provider/plan.py index 8bb72bd..10ab167 100644 --- a/octodns/provider/plan.py +++ b/octodns/provider/plan.py @@ -21,13 +21,13 @@ class Plan(object): MAX_SAFE_DELETE_PCENT = .3 MIN_EXISTING_RECORDS = 10 - def __init__(self, existing, desired, changes, create, + def __init__(self, existing, desired, changes, exists, update_pcent_threshold=MAX_SAFE_UPDATE_PCENT, delete_pcent_threshold=MAX_SAFE_DELETE_PCENT): self.existing = existing self.desired = desired self.changes = changes - self.create = create + self.exists = exists self.update_pcent_threshold = update_pcent_threshold self.delete_pcent_threshold = delete_pcent_threshold diff --git a/octodns/source/base.py b/octodns/source/base.py index 4ace09f..ee33619 100644 --- a/octodns/source/base.py +++ b/octodns/source/base.py @@ -22,7 +22,7 @@ class BaseSource(object): def populate(self, zone, target=False, lenient=False): ''' - Loads all zones the provider knows about + Loads all records the provider knows about for the provided zone When `target` is True the populate call is being made to load the current state of the provider. @@ -31,6 +31,9 @@ class BaseSource(object): do a "best effort" load of data. That will allow through some common, but not best practices stuff that we otherwise would reject. E.g. no trailing . or mising escapes for ;. + + When target is True (loading current state) this method should return + True if the zone exists or False if it does not. ''' raise NotImplementedError('Abstract base class, populate method ' 'missing') diff --git a/tests/test_octodns_plan.py b/tests/test_octodns_plan.py index 36dee68..91dd948 100644 --- a/tests/test_octodns_plan.py +++ b/tests/test_octodns_plan.py @@ -52,8 +52,8 @@ update = Update(existing, new) delete = Delete(new) changes = [create, delete, update] plans = [ - (simple, Plan(zone, zone, changes, False)), - (simple, Plan(zone, zone, changes, False)), + (simple, Plan(zone, zone, changes, True)), + (simple, Plan(zone, zone, changes, True)), ] diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index f44d368..adf394b 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -339,9 +339,9 @@ class TestAzureDnsProvider(TestCase): deletes.append(Delete(i)) self.assertEquals(13, provider.apply(Plan(None, zone, - changes, False))) + changes, True))) self.assertEquals(13, provider.apply(Plan(zone, zone, - deletes, False))) + deletes, True))) def test_create_zone(self): provider = self._get_provider() @@ -357,7 +357,7 @@ class TestAzureDnsProvider(TestCase): _get.side_effect = CloudError(Mock(status=404), err_msg) self.assertEquals(13, provider.apply(Plan(None, desired, changes, - False))) + True))) def test_check_zone_no_create(self): provider = self._get_provider() diff --git a/tests/test_octodns_provider_base.py b/tests/test_octodns_provider_base.py index a0ec74e..3ebf250 100644 --- a/tests/test_octodns_provider_base.py +++ b/tests/test_octodns_provider_base.py @@ -153,7 +153,7 @@ class TestBaseProvider(TestCase): def test_safe_none(self): # No changes is safe - Plan(None, None, [], False).raise_if_unsafe() + Plan(None, None, [], True).raise_if_unsafe() def test_safe_creates(self): # Creates are safe when existing records is under MIN_EXISTING_RECORDS @@ -164,7 +164,7 @@ class TestBaseProvider(TestCase): 'type': 'A', 'value': '1.2.3.4', }) - Plan(zone, zone, [Create(record) for i in range(10)], False) \ + Plan(zone, zone, [Create(record) for i in range(10)], True) \ .raise_if_unsafe() def test_safe_min_existing_creates(self): @@ -184,7 +184,7 @@ class TestBaseProvider(TestCase): 'value': '2.3.4.5' })) - Plan(zone, zone, [Create(record) for i in range(10)], False) \ + Plan(zone, zone, [Create(record) for i in range(10)], True) \ .raise_if_unsafe() def test_safe_no_existing(self): @@ -197,7 +197,7 @@ class TestBaseProvider(TestCase): }) updates = [Update(record, record), Update(record, record)] - Plan(zone, zone, updates, False).raise_if_unsafe() + Plan(zone, zone, updates, True).raise_if_unsafe() def test_safe_updates_min_existing(self): # MAX_SAFE_UPDATE_PCENT+1 fails when more @@ -221,7 +221,7 @@ class TestBaseProvider(TestCase): Plan.MAX_SAFE_UPDATE_PCENT) + 1)] with self.assertRaises(UnsafePlan) as ctx: - Plan(zone, zone, changes, False).raise_if_unsafe() + Plan(zone, zone, changes, True).raise_if_unsafe() self.assertTrue('Too many updates' in ctx.exception.message) @@ -245,7 +245,7 @@ class TestBaseProvider(TestCase): for i in range(int(Plan.MIN_EXISTING_RECORDS * Plan.MAX_SAFE_UPDATE_PCENT))] - Plan(zone, zone, changes, False).raise_if_unsafe() + Plan(zone, zone, changes, True).raise_if_unsafe() def test_safe_deletes_min_existing(self): # MAX_SAFE_DELETE_PCENT+1 fails when more @@ -269,7 +269,7 @@ class TestBaseProvider(TestCase): Plan.MAX_SAFE_DELETE_PCENT) + 1)] with self.assertRaises(UnsafePlan) as ctx: - Plan(zone, zone, changes, False).raise_if_unsafe() + Plan(zone, zone, changes, True).raise_if_unsafe() self.assertTrue('Too many deletes' in ctx.exception.message) @@ -293,7 +293,7 @@ class TestBaseProvider(TestCase): for i in range(int(Plan.MIN_EXISTING_RECORDS * Plan.MAX_SAFE_DELETE_PCENT))] - Plan(zone, zone, changes, False).raise_if_unsafe() + Plan(zone, zone, changes, True).raise_if_unsafe() def test_safe_updates_min_existing_override(self): safe_pcent = .4 @@ -318,7 +318,7 @@ class TestBaseProvider(TestCase): safe_pcent) + 1)] with self.assertRaises(UnsafePlan) as ctx: - Plan(zone, zone, changes, False, + Plan(zone, zone, changes, True, update_pcent_threshold=safe_pcent).raise_if_unsafe() self.assertTrue('Too many updates' in ctx.exception.message) @@ -346,7 +346,7 @@ class TestBaseProvider(TestCase): safe_pcent) + 1)] with self.assertRaises(UnsafePlan) as ctx: - Plan(zone, zone, changes, False, + Plan(zone, zone, changes, True, delete_pcent_threshold=safe_pcent).raise_if_unsafe() self.assertTrue('Too many deletes' in ctx.exception.message) diff --git a/tests/test_octodns_provider_cloudflare.py b/tests/test_octodns_provider_cloudflare.py index 7925b94..2977d26 100644 --- a/tests/test_octodns_provider_cloudflare.py +++ b/tests/test_octodns_provider_cloudflare.py @@ -347,7 +347,7 @@ class TestCloudflareProvider(TestCase): 'values': ['2.2.2.2', '3.3.3.3', '4.4.4.4'], }) change = Update(existing, new) - plan = Plan(zone, zone, [change], False) + plan = Plan(zone, zone, [change], True) provider._apply(plan) provider._request.assert_has_calls([ @@ -432,7 +432,7 @@ class TestCloudflareProvider(TestCase): 'value': 'ns2.foo.bar.', }) change = Update(existing, new) - plan = Plan(zone, zone, [change], False) + plan = Plan(zone, zone, [change], True) provider._apply(plan) provider._request.assert_has_calls([ diff --git a/tests/test_octodns_provider_dyn.py b/tests/test_octodns_provider_dyn.py index 9e06a6b..0956477 100644 --- a/tests/test_octodns_provider_dyn.py +++ b/tests/test_octodns_provider_dyn.py @@ -913,7 +913,7 @@ class TestDynProviderGeo(TestCase): Delete(geo), Delete(regular), ] - plan = Plan(None, desired, changes, False) + plan = Plan(None, desired, changes, True) provider._apply(plan) mock.assert_has_calls([ call('/Zone/unit.tests/', 'GET', {}), diff --git a/tests/test_octodns_provider_googlecloud.py b/tests/test_octodns_provider_googlecloud.py index 7060355..dacc369 100644 --- a/tests/test_octodns_provider_googlecloud.py +++ b/tests/test_octodns_provider_googlecloud.py @@ -264,7 +264,7 @@ class TestGoogleCloudProvider(TestCase): existing=[update_existing_r, delete_r], desired=desired, changes=changes, - create=False + exists=True )) calls_mock = gcloud_zone_mock.changes.return_value @@ -297,7 +297,7 @@ class TestGoogleCloudProvider(TestCase): existing=[update_existing_r, delete_r], desired=desired, changes=changes, - create=False + exists=True )) unsupported_change = Mock() From 73c002f94c180e97a9b9330caa6b56f2fbb4baab Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sun, 21 Jan 2018 14:26:51 -0800 Subject: [PATCH 098/141] Implement populate exists for Route53Provider --- octodns/provider/plan.py | 22 ++++++++++++++++++++-- octodns/provider/route53.py | 7 +++++-- tests/test_octodns_provider_route53.py | 2 ++ 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/octodns/provider/plan.py b/octodns/provider/plan.py index 10ab167..47a4157 100644 --- a/octodns/provider/plan.py +++ b/octodns/provider/plan.py @@ -124,6 +124,12 @@ class PlanLogger(_PlanOutput): buf.write(' (') buf.write(target) buf.write(')\n* ') + + if plan.exists is False: + buf.write('Create ') + buf.write(str(plan.desired)) + buf.write('\n* ') + for change in plan.changes: buf.write(change.__repr__(leader='* ')) buf.write('\n* ') @@ -169,6 +175,11 @@ class PlanMarkdown(_PlanOutput): fh.write('| Operation | Name | Type | TTL | Value | Source |\n' '|--|--|--|--|--|--|\n') + if plan.exists is False: + fh.write('| Create | ') + fh.write(str(plan.desired)) + fh.write(' | | | | |\n') + for change in plan.changes: existing = change.existing new = change.new @@ -194,7 +205,8 @@ class PlanMarkdown(_PlanOutput): fh.write(' | ') fh.write(_value_stringifier(new, '; ')) fh.write(' | ') - fh.write(new.source.id) + if new.source: + fh.write(new.source.id) fh.write(' |\n') fh.write('\nSummary: ') @@ -230,6 +242,11 @@ class PlanHtml(_PlanOutput): ''') + if plan.exists is False: + fh.write(' \n Create\n ') + fh.write(str(plan.desired)) + fh.write('\n \n') + for change in plan.changes: existing = change.existing new = change.new @@ -257,7 +274,8 @@ class PlanHtml(_PlanOutput): fh.write('\n ') fh.write(_value_stringifier(new, '
')) fh.write('\n ') - fh.write(new.source.id) + if new.source: + fh.write(new.source.id) fh.write('\n \n') fh.write(' \n Summary: ') diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index 5bda074..9de1b0c 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -451,9 +451,11 @@ class Route53Provider(BaseProvider): target, lenient) before = len(zone.records) + exists = False zone_id = self._get_zone_id(zone.name) if zone_id: + exists = True records = defaultdict(lambda: defaultdict(list)) for rrset in self._load_records(zone_id): record_name = zone.hostname_from_fqdn(rrset['Name']) @@ -483,8 +485,9 @@ class Route53Provider(BaseProvider): lenient=lenient) zone.add_record(record) - self.log.info('populate: found %s records', - len(zone.records) - before) + self.log.info('populate: found %s records, exists=%s', + len(zone.records) - before, exists) + return exists def _gen_mods(self, action, records): ''' diff --git a/tests/test_octodns_provider_route53.py b/tests/test_octodns_provider_route53.py index 1cd4548..8da7f2e 100644 --- a/tests/test_octodns_provider_route53.py +++ b/tests/test_octodns_provider_route53.py @@ -361,6 +361,7 @@ class TestRoute53Provider(TestCase): plan = provider.plan(self.expected) self.assertEquals(9, len(plan.changes)) + self.assertTrue(plan.exists) for change in plan.changes: self.assertIsInstance(change, Create) stubber.assert_no_pending_responses() @@ -593,6 +594,7 @@ class TestRoute53Provider(TestCase): plan = provider.plan(self.expected) self.assertEquals(9, len(plan.changes)) + self.assertFalse(plan.exists) for change in plan.changes: self.assertIsInstance(change, Create) stubber.assert_no_pending_responses() From d03e07c01c25698d0e4522d8e088cd165b98233b Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sun, 21 Jan 2018 14:27:08 -0800 Subject: [PATCH 099/141] Implement populate exists for PowerDnsProvider --- octodns/provider/powerdns.py | 7 +++++-- tests/test_octodns_provider_powerdns.py | 2 ++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/octodns/provider/powerdns.py b/octodns/provider/powerdns.py index 4527f8e..3ad0e55 100644 --- a/octodns/provider/powerdns.py +++ b/octodns/provider/powerdns.py @@ -187,8 +187,10 @@ class PowerDnsBaseProvider(BaseProvider): raise before = len(zone.records) + exists = False if resp: + exists = True for rrset in resp.json()['rrsets']: _type = rrset['type'] if _type == 'SOA': @@ -199,8 +201,9 @@ class PowerDnsBaseProvider(BaseProvider): source=self, lenient=lenient) zone.add_record(record) - self.log.info('populate: found %s records', - len(zone.records) - before) + self.log.info('populate: found %s records, exists=%s', + len(zone.records) - before, exists) + return exists def _records_for_multiple(self, record): return [{'content': v, 'disabled': False} diff --git a/tests/test_octodns_provider_powerdns.py b/tests/test_octodns_provider_powerdns.py index 22ccdd6..b0d5dcf 100644 --- a/tests/test_octodns_provider_powerdns.py +++ b/tests/test_octodns_provider_powerdns.py @@ -106,6 +106,7 @@ class TestPowerDnsProvider(TestCase): plan = provider.plan(expected) self.assertEquals(expected_n, len(plan.changes)) self.assertEquals(expected_n, provider.apply(plan)) + self.assertTrue(plan.exists) # Non-existent zone -> creates for every record in expected # OMG this is fucking ugly, probably better to ditch requests_mocks and @@ -124,6 +125,7 @@ class TestPowerDnsProvider(TestCase): plan = provider.plan(expected) self.assertEquals(expected_n, len(plan.changes)) self.assertEquals(expected_n, provider.apply(plan)) + self.assertFalse(plan.exists) with requests_mock() as mock: # get 422's, unknown zone From 3ef91326e8d62f45be7bb869faf0acc8f0e965fc Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sun, 21 Jan 2018 14:35:32 -0800 Subject: [PATCH 100/141] Implement populate exists for Ns1Provider --- octodns/provider/ns1.py | 7 +++++-- tests/test_octodns_provider_ns1.py | 4 +++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 5ea68b6..4911082 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -190,11 +190,13 @@ class Ns1Provider(BaseProvider): nsone_zone = self._client.loadZone(zone.name[:-1]) records = nsone_zone.data['records'] geo_records = nsone_zone.search(has_geo=True) + exists = True except ResourceException as e: if e.message != self.ZONE_NOT_FOUND_MESSAGE: raise records = [] geo_records = [] + exists = False before = len(zone.records) # geo information isn't returned from the main endpoint, so we need @@ -208,8 +210,9 @@ class Ns1Provider(BaseProvider): source=self, lenient=lenient) zone_hash[(_type, name)] = record [zone.add_record(r) for r in zone_hash.values()] - self.log.info('populate: found %s records', - len(zone.records) - before) + self.log.info('populate: found %s records, exists=%s', + len(zone.records) - before, exists) + return exists def _params_for_A(self, record): params = {'answers': record.values, 'ttl': record.ttl} diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index 0e3a286..fa6cf2d 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -196,9 +196,10 @@ class TestNs1Provider(TestCase): load_mock.side_effect = \ ResourceException('server error: zone not found') zone = Zone('unit.tests.', []) - provider.populate(zone) + exists = provider.populate(zone) self.assertEquals(set(), zone.records) self.assertEquals(('unit.tests',), load_mock.call_args[0]) + self.assertFalse(exists) # Existing zone w/o records load_mock.reset_mock() @@ -269,6 +270,7 @@ class TestNs1Provider(TestCase): # everything except the root NS expected_n = len(self.expected) - 1 self.assertEquals(expected_n, len(plan.changes)) + self.assertTrue(plan.exists) # Fails, general error load_mock.reset_mock() From b54630878f465c79aa1281b12ef34b73d0339995 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sun, 21 Jan 2018 14:37:31 -0800 Subject: [PATCH 101/141] Implement populate exists for DynProvider --- octodns/provider/dyn.py | 8 ++++++-- tests/test_octodns_provider_dyn.py | 2 ++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/octodns/provider/dyn.py b/octodns/provider/dyn.py index 010fd31..beb84d5 100644 --- a/octodns/provider/dyn.py +++ b/octodns/provider/dyn.py @@ -353,6 +353,7 @@ class DynProvider(BaseProvider): self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, target, lenient) + exists = False before = len(zone.records) self._check_dyn_sess() @@ -360,10 +361,12 @@ class DynProvider(BaseProvider): td_records = set() if self.traffic_directors_enabled: td_records = self._populate_traffic_directors(zone) + exists = True dyn_zone = _CachingDynZone.get(zone.name[:-1]) if dyn_zone: + exists = True values = defaultdict(lambda: defaultdict(list)) for _type, records in dyn_zone.get_all_records().items(): if _type == 'soa_records': @@ -382,8 +385,9 @@ class DynProvider(BaseProvider): if record not in td_records: zone.add_record(record) - self.log.info('populate: found %s records', - len(zone.records) - before) + self.log.info('populate: found %s records, exists=%s', + len(zone.records) - before, exists) + return exists def _kwargs_for_A(self, record): return [{ diff --git a/tests/test_octodns_provider_dyn.py b/tests/test_octodns_provider_dyn.py index 0956477..3f95980 100644 --- a/tests/test_octodns_provider_dyn.py +++ b/tests/test_octodns_provider_dyn.py @@ -430,6 +430,7 @@ class TestDynProvider(TestCase): update_mock.assert_not_called() provider.apply(plan) update_mock.assert_called() + self.assertFalse(plan.exists) add_mock.assert_called() # Once for each dyn record (8 Records, 2 of which have dual values) self.assertEquals(15, len(add_mock.call_args_list)) @@ -474,6 +475,7 @@ class TestDynProvider(TestCase): plan = provider.plan(new) provider.apply(plan) update_mock.assert_called() + self.assertTrue(plan.exists) # we expect 4 deletes, 2 from actual deletes and 2 from # updates which delete and recreate self.assertEquals(4, len(delete_mock.call_args_list)) From 1f40b98889a51c88f7dd3e030617f887399bfd76 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sun, 21 Jan 2018 14:40:07 -0800 Subject: [PATCH 102/141] Implement populate exists for CloudflareProvider --- octodns/provider/cloudflare.py | 7 +++++-- tests/test_octodns_provider_cloudflare.py | 2 ++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/octodns/provider/cloudflare.py b/octodns/provider/cloudflare.py index 9dfef6d..b1dee2b 100644 --- a/octodns/provider/cloudflare.py +++ b/octodns/provider/cloudflare.py @@ -173,9 +173,11 @@ class CloudflareProvider(BaseProvider): self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, target, lenient) + exists = False before = len(zone.records) records = self.zone_records(zone) if records: + exists = True values = defaultdict(lambda: defaultdict(list)) for record in records: name = zone.hostname_from_fqdn(record['name']) @@ -196,8 +198,9 @@ class CloudflareProvider(BaseProvider): lenient=lenient) zone.add_record(record) - self.log.info('populate: found %s records', - len(zone.records) - before) + self.log.info('populate: found %s records, exists=%s', + len(zone.records) - before, exists) + return exists def _include_change(self, change): if isinstance(change, Update): diff --git a/tests/test_octodns_provider_cloudflare.py b/tests/test_octodns_provider_cloudflare.py index 2977d26..328bd6f 100644 --- a/tests/test_octodns_provider_cloudflare.py +++ b/tests/test_octodns_provider_cloudflare.py @@ -147,6 +147,7 @@ class TestCloudflareProvider(TestCase): plan = provider.plan(self.expected) self.assertEquals(11, len(plan.changes)) self.assertEquals(11, provider.apply(plan)) + self.assertFalse(plan.exists) provider._request.assert_has_calls([ # created the domain @@ -266,6 +267,7 @@ class TestCloudflareProvider(TestCase): # only see the delete & ttl update, below min-ttl is filtered out self.assertEquals(2, len(plan.changes)) self.assertEquals(2, provider.apply(plan)) + self.assertTrue(plan.exists) # recreate for update, and deletes for the 2 parts of the other provider._request.assert_has_calls([ call('PUT', '/zones/ff12ab34cd5611334422ab3322997650/dns_records/' From 4d180ed991ce956254b612190f3c21cc1eb3dd95 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sun, 21 Jan 2018 14:41:54 -0800 Subject: [PATCH 103/141] Implement populate exists for YamlProvider --- octodns/provider/yaml.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/octodns/provider/yaml.py b/octodns/provider/yaml.py index 752e793..0241d50 100644 --- a/octodns/provider/yaml.py +++ b/octodns/provider/yaml.py @@ -52,7 +52,7 @@ class YamlProvider(BaseProvider): if target: # When acting as a target we ignore any existing records so that we # create a completely new copy - return + return False before = len(zone.records) filename = join(self.directory, '{}yaml'.format(zone.name)) @@ -69,8 +69,9 @@ class YamlProvider(BaseProvider): lenient=lenient) zone.add_record(record) - self.log.info('populate: found %s records', + self.log.info('populate: found %s records, exists=False', len(zone.records) - before) + return False def _apply(self, plan): desired = plan.desired From d35fcd319aa5278cad2f0df2656f2bb0c89684a9 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sun, 21 Jan 2018 14:44:36 -0800 Subject: [PATCH 104/141] Implement populate exists for RackspaceProvider --- octodns/provider/rackspace.py | 5 +++-- tests/test_octodns_provider_rackspace.py | 4 +++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py index 12b2c54..02c833d 100644 --- a/octodns/provider/rackspace.py +++ b/octodns/provider/rackspace.py @@ -208,7 +208,7 @@ class RackspaceProvider(BaseProvider): raise Exception('Rackspace request unauthorized') elif e.response.status_code == 404: # Zone not found leaves the zone empty instead of failing. - return + return False raise before = len(zone.records) @@ -225,8 +225,9 @@ class RackspaceProvider(BaseProvider): source=self) zone.add_record(record) - self.log.info('populate: found %s records', + self.log.info('populate: found %s records, exists=True', len(zone.records) - before) + return True def _group_records(self, all_records): records = defaultdict(lambda: defaultdict(list)) diff --git a/tests/test_octodns_provider_rackspace.py b/tests/test_octodns_provider_rackspace.py index 274d63e..c467dec 100644 --- a/tests/test_octodns_provider_rackspace.py +++ b/tests/test_octodns_provider_rackspace.py @@ -73,9 +73,10 @@ class TestRackspaceProvider(TestCase): json={'error': "Could not find domain 'unit.tests.'"}) zone = Zone('unit.tests.', []) - self.provider.populate(zone) + exists = self.provider.populate(zone) self.assertEquals(set(), zone.records) self.assertTrue(mock.called_once) + self.assertFalse(exists) def test_multipage_populate(self): with requests_mock() as mock: @@ -109,6 +110,7 @@ class TestRackspaceProvider(TestCase): plan = self.provider.plan(expected) self.assertTrue(mock.called) + self.assertTrue(plan.exists) # OctoDNS does not propagate top-level NS records. self.assertEquals(1, len(plan.changes)) From d693d2e99e0cec859ed53fe37ef7bc4731c0bbf1 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sun, 21 Jan 2018 14:46:49 -0800 Subject: [PATCH 105/141] Implement populate exists for GoogleCloudProvider --- octodns/provider/googlecloud.py | 7 ++++++- tests/test_octodns_provider_googlecloud.py | 6 ++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/octodns/provider/googlecloud.py b/octodns/provider/googlecloud.py index 6ca0794..1dc5393 100644 --- a/octodns/provider/googlecloud.py +++ b/octodns/provider/googlecloud.py @@ -202,11 +202,14 @@ class GoogleCloudProvider(BaseProvider): self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, target, lenient) + + exists = False before = len(zone.records) gcloud_zone = self.gcloud_zones.get(zone.name) if gcloud_zone: + exists = True for gcloud_record in self._get_gcloud_records(gcloud_zone): if gcloud_record.record_type.upper() not in self.SUPPORTS: continue @@ -227,7 +230,9 @@ class GoogleCloudProvider(BaseProvider): record = Record.new(zone, record_name, data, source=self) zone.add_record(record) - self.log.info('populate: found %s records', len(zone.records) - before) + self.log.info('populate: found %s records, exists=%s', + len(zone.records) - before, exists) + return exists def _data_for_A(self, gcloud_record): return { diff --git a/tests/test_octodns_provider_googlecloud.py b/tests/test_octodns_provider_googlecloud.py index dacc369..fc7fb03 100644 --- a/tests/test_octodns_provider_googlecloud.py +++ b/tests/test_octodns_provider_googlecloud.py @@ -359,7 +359,8 @@ class TestGoogleCloudProvider(TestCase): "unit.tests.") test_zone = Zone('unit.tests.', []) - provider.populate(test_zone) + exists = provider.populate(test_zone) + self.assertTrue(exists) # test_zone gets fed the same records as zone does, except it's in # the format returned by google API, so after populate they should look @@ -367,7 +368,8 @@ class TestGoogleCloudProvider(TestCase): self.assertEqual(test_zone.records, zone.records) test_zone2 = Zone('nonexistant.zone.', []) - provider.populate(test_zone2, False, False) + exists = provider.populate(test_zone2, False, False) + self.assertFalse(exists) self.assertEqual(len(test_zone2.records), 0, msg="Zone should not get records from wrong domain") From 720e8eb4349d4ee598c452f1345b6b6e46a2d051 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sun, 21 Jan 2018 14:49:20 -0800 Subject: [PATCH 106/141] Implement populate exists for AzureProvider --- octodns/provider/azuredns.py | 7 ++++++- tests/test_octodns_provider_azuredns.py | 6 ++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 1757274..af23ef7 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -322,6 +322,8 @@ class AzureProvider(BaseProvider): :type return: void ''' self.log.debug('populate: name=%s', zone.name) + + exists = False before = len(zone.records) zone_name = zone.name[:len(zone.name) - 1] @@ -331,6 +333,7 @@ class AzureProvider(BaseProvider): _records = set() records = self._dns_client.record_sets.list_by_dns_zone if self._check_zone(zone_name): + exists = True for azrecord in records(self._resource_group, zone_name): if _parse_azure_type(azrecord.type) in self.SUPPORTS: _records.add(azrecord) @@ -344,7 +347,9 @@ class AzureProvider(BaseProvider): record = Record.new(zone, record_name, data, source=self) zone.add_record(record) - self.log.info('populate: found %s records', len(zone.records) - before) + self.log.info('populate: found %s records, exists=%s', + len(zone.records) - before, exists) + return exists def _data_for_A(self, azrecord): return {'values': [ar.ipv4_address for ar in azrecord.arecords]} diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index adf394b..9784945 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -302,7 +302,8 @@ class TestAzureDnsProvider(TestCase): record_list = provider._dns_client.record_sets.list_by_dns_zone record_list.return_value = rs - provider.populate(zone) + exists = provider.populate(zone) + self.assertTrue(exists) self.assertEquals(len(zone.records), 16) @@ -377,6 +378,7 @@ class TestAzureDnsProvider(TestCase): _get = provider._dns_client.zones.get _get.side_effect = CloudError(Mock(status=404), err_msg) - provider.populate(Zone('unit3.test.', [])) + exists = provider.populate(Zone('unit3.test.', [])) + self.assertFalse(exists) self.assertEquals(len(zone.records), 0) From 88ff1729ab42b38c97ea1d81371b8ccb413527cf Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sun, 21 Jan 2018 14:55:53 -0800 Subject: [PATCH 107/141] Implement populate exists for DigitalOceanProvider --- octodns/provider/digitalocean.py | 6 ++++-- tests/test_octodns_provider_digitalocean.py | 2 ++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/octodns/provider/digitalocean.py b/octodns/provider/digitalocean.py index c68e7d6..b1df187 100644 --- a/octodns/provider/digitalocean.py +++ b/octodns/provider/digitalocean.py @@ -232,8 +232,10 @@ class DigitalOceanProvider(BaseProvider): source=self, lenient=lenient) zone.add_record(record) - self.log.info('populate: found %s records', - len(zone.records) - before) + exists = zone.name in self._zone_records + self.log.info('populate: found %s records, exists=%s', + len(zone.records) - before, exists) + return exists def _params_for_multiple(self, record): for value in record.values: diff --git a/tests/test_octodns_provider_digitalocean.py b/tests/test_octodns_provider_digitalocean.py index 5fb47c5..ddc6bc2 100644 --- a/tests/test_octodns_provider_digitalocean.py +++ b/tests/test_octodns_provider_digitalocean.py @@ -165,6 +165,7 @@ class TestDigitalOceanProvider(TestCase): n = len(self.expected.records) - 7 self.assertEquals(n, len(plan.changes)) self.assertEquals(n, provider.apply(plan)) + self.assertFalse(plan.exists) provider._client._request.assert_has_calls([ # created the domain @@ -225,6 +226,7 @@ class TestDigitalOceanProvider(TestCase): })) plan = provider.plan(wanted) + self.assertTrue(plan.exists) self.assertEquals(2, len(plan.changes)) self.assertEquals(2, provider.apply(plan)) # recreate for update, and delete for the 2 parts of the other From 7566250f96a2eb1dc30b5e6a48ab72b7c664dd98 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sun, 21 Jan 2018 14:58:33 -0800 Subject: [PATCH 108/141] Implement populate exists for DnsimpleProvider --- octodns/provider/dnsimple.py | 6 ++++-- tests/test_octodns_provider_dnsimple.py | 2 ++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/octodns/provider/dnsimple.py b/octodns/provider/dnsimple.py index 43b5b9b..304780d 100644 --- a/octodns/provider/dnsimple.py +++ b/octodns/provider/dnsimple.py @@ -272,8 +272,10 @@ class DnsimpleProvider(BaseProvider): source=self, lenient=lenient) zone.add_record(record) - self.log.info('populate: found %s records', - len(zone.records) - before) + exists = zone.name in self._zone_records + self.log.info('populate: found %s records, exists=%s', + len(zone.records) - before, exists) + return exists def _params_for_multiple(self, record): for value in record.values: diff --git a/tests/test_octodns_provider_dnsimple.py b/tests/test_octodns_provider_dnsimple.py index 60dacf1..896425e 100644 --- a/tests/test_octodns_provider_dnsimple.py +++ b/tests/test_octodns_provider_dnsimple.py @@ -133,6 +133,7 @@ class TestDnsimpleProvider(TestCase): n = len(self.expected.records) - 3 self.assertEquals(n, len(plan.changes)) self.assertEquals(n, provider.apply(plan)) + self.assertFalse(plan.exists) provider._client._request.assert_has_calls([ # created the domain @@ -186,6 +187,7 @@ class TestDnsimpleProvider(TestCase): })) plan = provider.plan(wanted) + self.assertTrue(plan.exists) self.assertEquals(2, len(plan.changes)) self.assertEquals(2, provider.apply(plan)) # recreate for update, and deletes for the 2 parts of the other From 449330bbf3b35ee17af77cb6933354b43f37b792 Mon Sep 17 00:00:00 2001 From: Adam Smith Date: Thu, 16 Nov 2017 22:02:08 -0800 Subject: [PATCH 109/141] add DnsMadeEasy Provider --- octodns/provider/dnsmadeeasy.py | 379 ++++++++++++++++++++++++++++++++ 1 file changed, 379 insertions(+) create mode 100644 octodns/provider/dnsmadeeasy.py diff --git a/octodns/provider/dnsmadeeasy.py b/octodns/provider/dnsmadeeasy.py new file mode 100644 index 0000000..dc79561 --- /dev/null +++ b/octodns/provider/dnsmadeeasy.py @@ -0,0 +1,379 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from collections import defaultdict +from requests import Session +from time import strftime, gmtime +import hashlib +import hmac +import logging + +from ..record import Record +from .base import BaseProvider + + +class DnsMadeEasyClientException(Exception): + pass + + +class DnsMadeEasyClientBadRequest(DnsMadeEasyClientException): + + @classmethod + def build_message(self, errors): + return '\n - {}'.format('\n - '.join(errors)) + + def __init__(self, resp): + errors = resp.json()['error'] + super(DnsMadeEasyClientBadRequest, self).__init__( + self.build_message(errors)) + + +class DnsMadeEasyClientUnauthorized(DnsMadeEasyClientException): + + def __init__(self): + super(DnsMadeEasyClientUnauthorized, self).__init__('Unauthorized') + + +class DnsMadeEasyClientForbidden(DnsMadeEasyClientException): + + def __init__(self): + super(DnsMadeEasyClientNotFound, self).__init__('Forbidden') + + +class DnsMadeEasyClientNotFound(DnsMadeEasyClientException): + + def __init__(self): + super(DnsMadeEasyClientNotFound, self).__init__('Not Found') + + +class DnsMadeEasyClient(object): + BASE = 'https://api.dnsmadeeasy.com/V2.0/dns/managed' + + def __init__(self, api_key, secret_key): + self.api_key = api_key + self.secret_key = secret_key + self._sess = Session() + self._domains = None + + def _current_time(self): + return strftime("%a, %d %b %Y %H:%M:%S +0000", gmtime()) + + def _hmac_hash(self, now): + return hmac.new(self.secret_key.encode(), now.encode(), + hashlib.sha1).hexdigest() + + def _request(self, method, path, params=None, data=None): + now = self._current_time() + hmac_hash = self._hmac_hash(now) + + headers = { + 'x-dnsme-apiKey': self.api_key, + 'x-dnsme-hmac': hmac_hash, + 'x-dnsme-requestDate': now + } + self._sess.headers.update(headers) + + url = '{}{}'.format(self.BASE, path) + resp = self._sess.request(method, url, params=params, json=data) + if resp.status_code == 400: + raise DnsMadeEasyClientBadRequest(resp) + if resp.status_code == 401: + raise DnsMadeEasyClientUnauthorized() + if resp.status_code == 403: + raise DnsMadeEasyClientForbidden() + if resp.status_code == 404: + raise DnsMadeEasyClientNotFound() + resp.raise_for_status() + return resp + + @property + def domains(self): + if self._domains is None: + zones = [] + + # has pages in resp, do we need paging? + resp = self._request('GET', '/').json() + zones += resp['data'] + + self._domains = {'{}.'.format(z['name']): z['id'] for z in zones} + + return self._domains + + def domain(self, name): + path = '/id/{}'.format(name) + return self._request('GET', path).json() + + def domain_create(self, name): + self._request('POST', '/', data={'name': name}) + + def records(self, zone_name): + zone_id = self.domains.get(zone_name, False) + path = '/{}/records'.format(zone_id) + ret = [] + + # has pages in resp, do we need paging? + resp = self._request('GET', path).json() + ret += resp['data'] + + # change relative values to absolute + for record in ret: + value = record['value'] + if record['type'] in ['CNAME', 'MX', 'NS', 'SRV']: + if value == '': + record['value'] = zone_name + elif not value.endswith('.'): + record['value'] = '{}.{}'.format(value, zone_name) + + return ret + + def record_create(self, zone_name, params): + zone_id = self.domains.get(zone_name, False) + path = '/{}/records'.format(zone_id) + + self._request('POST', path, data=params) + + def record_delete(self, zone_name, record_id): + zone_id = self.domains.get(zone_name, False) + path = '/{}/records/{}'.format(zone_id, record_id) + self._request('DELETE', path) + + +class DnsMadeEasyProvider(BaseProvider): + ''' + DNSMadeEasy DNS provider using v2.0 API + + dnsmadeeasy: + class: octodns.provider.dnsmadeeasy.DnsMadeEasyProvider + api_key: env/DNSMADEEASY_API_KEY + secret_key: env/DNSMADEEASY_SECRET_KEY + ''' + SUPPORTS_GEO = False + SUPPORTS = set(('A', 'AAAA', 'CAA', 'CNAME', 'MX', + 'NS', 'PTR', 'SPF', 'SRV', 'TXT')) + + def __init__(self, id, api_key, secret_key, *args, **kwargs): + self.log = logging.getLogger('DnsMadeEasyProvider[{}]'.format(id)) + self.log.debug('__init__: id=%s, api_key=***, secret_key=***', id) + super(DnsMadeEasyProvider, self).__init__(id, *args, **kwargs) + self._client = DnsMadeEasyClient(api_key, secret_key) + + self._zone_records = {} + + def _data_for_multiple(self, _type, records): + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': [r['value'] for r in records] + } + + _data_for_A = _data_for_multiple + _data_for_AAAA = _data_for_multiple + _data_for_NS = _data_for_multiple + + def _data_for_CAA(self, _type, records): + values = [] + for record in records: + values.append({ + 'flags': record['issuerCritical'], + 'tag': record['caaType'], + 'value': record['value'][1:-1] + }) + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': values + } + + def _data_for_TXT(self, _type, records): + values = [value['value'].replace(';', '\;') for value in records] + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': values + } + + _data_for_SPF = _data_for_TXT + + def _data_for_MX(self, _type, records): + values = [] + for record in records: + values.append({ + 'preference': record['mxLevel'], + 'exchange': record['value'] + }) + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': values + } + + def _data_for_single(self, _type, records): + record = records[0] + return { + 'ttl': record['ttl'], + 'type': _type, + 'value': record['value'] + } + + _data_for_CNAME = _data_for_single + _data_for_PTR = _data_for_single + + def _data_for_SRV(self, _type, records): + values = [] + for record in records: + values.append({ + 'port': record['port'], + 'priority': record['priority'], + 'target': record['value'], + 'weight': record['weight'] + }) + return { + 'type': _type, + 'ttl': records[0]['ttl'], + 'values': values + } + + def zone_records(self, zone): + if zone.name not in self._zone_records: + try: + self._zone_records[zone.name] = \ + self._client.records(zone.name) + except DnsMadeEasyClientNotFound: + return [] + + return self._zone_records[zone.name] + + def populate(self, zone, target=False, lenient=False): + self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, + target, lenient) + + values = defaultdict(lambda: defaultdict(list)) + for record in self.zone_records(zone): + _type = record['type'] + values[record['name']][record['type']].append(record) + + before = len(zone.records) + for name, types in values.items(): + for _type, records in types.items(): + data_for = getattr(self, '_data_for_{}'.format(_type)) + record = Record.new(zone, name, data_for(_type, records), + source=self, lenient=lenient) + zone.add_record(record) + + self.log.info('populate: found %s records', + len(zone.records) - before) + + def _params_for_multiple(self, record): + for value in record.values: + yield { + 'value': value, + 'name': record.name, + 'ttl': record.ttl, + 'type': record._type + } + + _params_for_A = _params_for_multiple + _params_for_AAAA = _params_for_multiple + + # An A record with this name must exist in this domain for + # this NS record to be valid. Need to handle checking if + # there is an A record before creating NS + _params_for_NS = _params_for_multiple + + def _params_for_single(self, record): + yield { + 'value': record.value, + 'name': record.name, + 'ttl': record.ttl, + 'type': record._type + } + + _params_for_CNAME = _params_for_single + _params_for_PTR = _params_for_single + + def _params_for_MX(self, record): + for value in record.values: + yield { + 'value': value.exchange, + 'name': record.name, + 'mxLevel': value.preference, + 'ttl': record.ttl, + 'type': record._type + } + + def _params_for_SRV(self, record): + for value in record.values: + yield { + 'value': value.target, + 'name': record.name, + 'port': value.port, + 'priority': value.priority, + 'ttl': record.ttl, + 'type': record._type, + 'weight': value.weight + } + + def _params_for_TXT(self, record): + # DNSMadeEasy does not want values escaped + for value in record.chunked_values: + yield { + 'value': value.replace('\;', ';'), + 'name': record.name, + 'ttl': record.ttl, + 'type': record._type + } + + _params_for_SPF = _params_for_TXT + + def _params_for_CAA(self, record): + for value in record.values: + yield { + 'value': value.value, + 'issuerCritical': value.flags, + 'name': record.name, + 'caaType': value.tag, + 'ttl': record.ttl, + 'type': record._type + } + + def _apply_Create(self, change): + new = change.new + params_for = getattr(self, '_params_for_{}'.format(new._type)) + for params in params_for(new): + self._client.record_create(new.zone.name, params) + + def _apply_Update(self, change): + self._apply_Delete(change) + self._apply_Create(change) + + def _apply_Delete(self, change): + existing = change.existing + zone = existing.zone + for record in self.zone_records(zone): + if existing.name == record['name'] and \ + existing._type == record['type']: + self._client.record_delete(zone.name, record['id']) + + def _apply(self, plan): + desired = plan.desired + changes = plan.changes + self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name, + len(changes)) + + domain_name = desired.name[:-1] + try: + self._client.domain(domain_name) + except DnsMadeEasyClientNotFound: + self.log.debug('_apply: no matching zone, creating domain') + self._client.domain_create(domain_name) + + for change in changes: + class_name = change.__class__.__name__ + getattr(self, '_apply_{}'.format(class_name))(change) + + # Clear out the cache if any + self._zone_records.pop(desired.name, None) From ef2ebf71997e5ca37cec05649195c9a03d85fd66 Mon Sep 17 00:00:00 2001 From: Adam Smith Date: Sun, 21 Jan 2018 23:37:01 -0800 Subject: [PATCH 110/141] add exists for zone creation detection --- octodns/provider/dnsmadeeasy.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/octodns/provider/dnsmadeeasy.py b/octodns/provider/dnsmadeeasy.py index dc79561..6ba5ba8 100644 --- a/octodns/provider/dnsmadeeasy.py +++ b/octodns/provider/dnsmadeeasy.py @@ -41,7 +41,7 @@ class DnsMadeEasyClientUnauthorized(DnsMadeEasyClientException): class DnsMadeEasyClientForbidden(DnsMadeEasyClientException): def __init__(self): - super(DnsMadeEasyClientNotFound, self).__init__('Forbidden') + super(DnsMadeEasyClientForbidden, self).__init__('Forbidden') class DnsMadeEasyClientNotFound(DnsMadeEasyClientException): @@ -264,8 +264,10 @@ class DnsMadeEasyProvider(BaseProvider): source=self, lenient=lenient) zone.add_record(record) - self.log.info('populate: found %s records', - len(zone.records) - before) + exists = zone.name in self._zone_records + self.log.info('populate: found %s records, exists=%s', + len(zone.records) - before, exists) + return exists def _params_for_multiple(self, record): for value in record.values: From ff305ca1bbc60018333a71321de14c98d7adb299 Mon Sep 17 00:00:00 2001 From: Eric Vergne Date: Mon, 22 Jan 2018 17:12:06 +0100 Subject: [PATCH 111/141] Implement populate exists for OvhProvider --- octodns/provider/ovh.py | 16 +++++++++++++--- tests/test_octodns_provider_ovh.py | 20 ++++++++++++++++---- 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/octodns/provider/ovh.py b/octodns/provider/ovh.py index 5c2fe0d..93e7594 100644 --- a/octodns/provider/ovh.py +++ b/octodns/provider/ovh.py @@ -11,6 +11,7 @@ import logging from collections import defaultdict import ovh +from ovh import ResourceNotFoundError from octodns.record import Record from .base import BaseProvider @@ -33,6 +34,7 @@ class OvhProvider(BaseProvider): """ SUPPORTS_GEO = False + ZONE_NOT_FOUND_MESSAGE = 'This service does not exist' # This variable is also used in populate method to filter which OVH record # types are supported by octodns @@ -57,7 +59,14 @@ class OvhProvider(BaseProvider): self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, target, lenient) zone_name = zone.name[:-1] - records = self.get_records(zone_name=zone_name) + try: + records = self.get_records(zone_name=zone_name) + exists = True + except ResourceNotFoundError as e: + if e.message != self.ZONE_NOT_FOUND_MESSAGE: + raise + exists = False + records = [] values = defaultdict(lambda: defaultdict(list)) for record in records: @@ -75,8 +84,9 @@ class OvhProvider(BaseProvider): source=self, lenient=lenient) zone.add_record(record) - self.log.info('populate: found %s records', - len(zone.records) - before) + self.log.info('populate: found %s records, exists=%s', + len(zone.records) - before, exists) + return exists def _apply(self, plan): desired = plan.desired diff --git a/tests/test_octodns_provider_ovh.py b/tests/test_octodns_provider_ovh.py index 6a44e25..472f03d 100644 --- a/tests/test_octodns_provider_ovh.py +++ b/tests/test_octodns_provider_ovh.py @@ -8,7 +8,7 @@ from __future__ import absolute_import, division, print_function, \ from unittest import TestCase from mock import patch, call -from ovh import APIError +from ovh import APIError, ResourceNotFoundError, InvalidCredential from octodns.provider.ovh import OvhProvider from octodns.record import Record @@ -307,18 +307,30 @@ class TestOvhProvider(TestCase): with patch.object(provider._client, 'get') as get_mock: zone = Zone('unit.tests.', []) - get_mock.side_effect = APIError('boom') + get_mock.side_effect = ResourceNotFoundError('boom') with self.assertRaises(APIError) as ctx: provider.populate(zone) self.assertEquals(get_mock.side_effect, ctx.exception) - with patch.object(provider._client, 'get') as get_mock: + get_mock.side_effect = InvalidCredential('boom') + with self.assertRaises(APIError) as ctx: + provider.populate(zone) + self.assertEquals(get_mock.side_effect, ctx.exception) + + zone = Zone('unit.tests.', []) + get_mock.side_effect = ResourceNotFoundError('This service does ' + 'not exist') + exists = provider.populate(zone) + self.assertEquals(set(), zone.records) + self.assertFalse(exists) + zone = Zone('unit.tests.', []) get_returns = [[record['id'] for record in self.api_record]] get_returns += self.api_record get_mock.side_effect = get_returns - provider.populate(zone) + exists = provider.populate(zone) self.assertEquals(self.expected, zone.records) + self.assertTrue(exists) @patch('ovh.Client') def test_is_valid_dkim(self, client_mock): From 712d1d86aa88ba837d1b70befe46e2b70bcce3c8 Mon Sep 17 00:00:00 2001 From: Adam Smith Date: Mon, 22 Jan 2018 21:45:21 -0800 Subject: [PATCH 112/141] add DnsMadeEasy to README, sandbox environment toggle --- README.md | 1 + octodns/provider/dnsmadeeasy.py | 23 +++++++++++++++++------ 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 0fff5fa..af8d14f 100644 --- a/README.md +++ b/README.md @@ -152,6 +152,7 @@ The above command pulled the existing data out of Route53 and placed the results | [AzureProvider](/octodns/provider/azuredns.py) | A, AAAA, CNAME, MX, NS, PTR, SRV, TXT | No | | | [CloudflareProvider](/octodns/provider/cloudflare.py) | A, AAAA, ALIAS, CAA, CNAME, MX, NS, SPF, TXT | No | CAA tags restricted | | [DigitalOceanProvider](/octodns/provider/digitalocean.py) | A, AAAA, CAA, CNAME, MX, NS, TXT, SRV | No | CAA tags restricted | +| [DnsMadeEasyProvider](/octodns/provider/dnsmadeeasy.py) | A, AAAA, CAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | CAA tags restricted | | [DnsimpleProvider](/octodns/provider/dnsimple.py) | All | No | CAA tags restricted | | [DynProvider](/octodns/provider/dyn.py) | All | Yes | | | [GoogleCloudProvider](/octodns/provider/googlecloud.py) | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | No | | diff --git a/octodns/provider/dnsmadeeasy.py b/octodns/provider/dnsmadeeasy.py index 6ba5ba8..1ccb566 100644 --- a/octodns/provider/dnsmadeeasy.py +++ b/octodns/provider/dnsmadeeasy.py @@ -51,11 +51,15 @@ class DnsMadeEasyClientNotFound(DnsMadeEasyClientException): class DnsMadeEasyClient(object): - BASE = 'https://api.dnsmadeeasy.com/V2.0/dns/managed' + PRODUCTION = 'https://api.dnsmadeeasy.com/V2.0/dns/managed' + SANDBOX = 'https://api.sandbox.dnsmadeeasy.com/V2.0/dns/managed' - def __init__(self, api_key, secret_key): + def __init__(self, api_key, secret_key, sandbox=False): self.api_key = api_key self.secret_key = secret_key + self._base = self.PRODUCTION + if sandbox: + self._base = self.SANDBOX self._sess = Session() self._domains = None @@ -77,7 +81,7 @@ class DnsMadeEasyClient(object): } self._sess.headers.update(headers) - url = '{}{}'.format(self.BASE, path) + url = '{}{}'.format(self._base, path) resp = self._sess.request(method, url, params=params, json=data) if resp.status_code == 400: raise DnsMadeEasyClientBadRequest(resp) @@ -148,18 +152,25 @@ class DnsMadeEasyProvider(BaseProvider): dnsmadeeasy: class: octodns.provider.dnsmadeeasy.DnsMadeEasyProvider + # Your DnsMadeEasy api key (required) api_key: env/DNSMADEEASY_API_KEY + # Your DnsMadeEasy secret key (required) secret_key: env/DNSMADEEASY_SECRET_KEY + # Whether or not to use Sandbox environment + # (optional, default is false) + sandbox: true ''' SUPPORTS_GEO = False SUPPORTS = set(('A', 'AAAA', 'CAA', 'CNAME', 'MX', 'NS', 'PTR', 'SPF', 'SRV', 'TXT')) - def __init__(self, id, api_key, secret_key, *args, **kwargs): + def __init__(self, id, api_key, secret_key, sandbox=False, + *args, **kwargs): self.log = logging.getLogger('DnsMadeEasyProvider[{}]'.format(id)) - self.log.debug('__init__: id=%s, api_key=***, secret_key=***', id) + self.log.debug('__init__: id=%s, api_key=***, secret_key=***, ' + 'sandbox=%s', id, sandbox) super(DnsMadeEasyProvider, self).__init__(id, *args, **kwargs) - self._client = DnsMadeEasyClient(api_key, secret_key) + self._client = DnsMadeEasyClient(api_key, secret_key, sandbox) self._zone_records = {} From 4dc7c582a35fa0e9b4f951fc86ed854cddc120b5 Mon Sep 17 00:00:00 2001 From: Adam Smith Date: Mon, 22 Jan 2018 22:43:04 -0800 Subject: [PATCH 113/141] add ratelimit_delay parameter --- octodns/provider/dnsmadeeasy.py | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/octodns/provider/dnsmadeeasy.py b/octodns/provider/dnsmadeeasy.py index 1ccb566..8af45a4 100644 --- a/octodns/provider/dnsmadeeasy.py +++ b/octodns/provider/dnsmadeeasy.py @@ -7,7 +7,7 @@ from __future__ import absolute_import, division, print_function, \ from collections import defaultdict from requests import Session -from time import strftime, gmtime +from time import strftime, gmtime, sleep import hashlib import hmac import logging @@ -22,14 +22,10 @@ class DnsMadeEasyClientException(Exception): class DnsMadeEasyClientBadRequest(DnsMadeEasyClientException): - @classmethod - def build_message(self, errors): - return '\n - {}'.format('\n - '.join(errors)) - def __init__(self, resp): errors = resp.json()['error'] super(DnsMadeEasyClientBadRequest, self).__init__( - self.build_message(errors)) + '\n - {}'.format('\n - '.join(errors))) class DnsMadeEasyClientUnauthorized(DnsMadeEasyClientException): @@ -54,13 +50,14 @@ class DnsMadeEasyClient(object): PRODUCTION = 'https://api.dnsmadeeasy.com/V2.0/dns/managed' SANDBOX = 'https://api.sandbox.dnsmadeeasy.com/V2.0/dns/managed' - def __init__(self, api_key, secret_key, sandbox=False): + def __init__(self, api_key, secret_key, sandbox=False, + ratelimit_delay=0.0): self.api_key = api_key self.secret_key = secret_key - self._base = self.PRODUCTION - if sandbox: - self._base = self.SANDBOX + self._base = self.SANDBOX if sandbox else self.PRODUCTION + self.ratelimit_delay = ratelimit_delay self._sess = Session() + self._sess.headers.update({'x-dnsme-apiKey': self.api_key}) self._domains = None def _current_time(self): @@ -75,14 +72,13 @@ class DnsMadeEasyClient(object): hmac_hash = self._hmac_hash(now) headers = { - 'x-dnsme-apiKey': self.api_key, 'x-dnsme-hmac': hmac_hash, 'x-dnsme-requestDate': now } - self._sess.headers.update(headers) url = '{}{}'.format(self._base, path) - resp = self._sess.request(method, url, params=params, json=data) + resp = self._sess.request(method, url, headers=headers, + params=params, json=data) if resp.status_code == 400: raise DnsMadeEasyClientBadRequest(resp) if resp.status_code == 401: @@ -92,6 +88,7 @@ class DnsMadeEasyClient(object): if resp.status_code == 404: raise DnsMadeEasyClientNotFound() resp.raise_for_status() + sleep(self.ratelimit_delay) return resp @property @@ -165,12 +162,13 @@ class DnsMadeEasyProvider(BaseProvider): 'NS', 'PTR', 'SPF', 'SRV', 'TXT')) def __init__(self, id, api_key, secret_key, sandbox=False, - *args, **kwargs): + ratelimit_delay=0.0, *args, **kwargs): self.log = logging.getLogger('DnsMadeEasyProvider[{}]'.format(id)) self.log.debug('__init__: id=%s, api_key=***, secret_key=***, ' 'sandbox=%s', id, sandbox) super(DnsMadeEasyProvider, self).__init__(id, *args, **kwargs) - self._client = DnsMadeEasyClient(api_key, secret_key, sandbox) + self._client = DnsMadeEasyClient(api_key, secret_key, sandbox, + ratelimit_delay) self._zone_records = {} From d8837a14ad5d21470ad1cff847eb5f8039ccd3e2 Mon Sep 17 00:00:00 2001 From: Adam Smith Date: Sat, 27 Jan 2018 21:58:05 -0800 Subject: [PATCH 114/141] add tests for DnsMadeEasy provider --- octodns/provider/dnsmadeeasy.py | 10 +- tests/fixtures/dnsmadeeasy-domains.json | 16 ++ tests/fixtures/dnsmadeeasy-records.json | 312 +++++++++++++++++++++ tests/test_octodns_provider_dnsmadeeasy.py | 202 +++++++++++++ 4 files changed, 531 insertions(+), 9 deletions(-) create mode 100644 tests/fixtures/dnsmadeeasy-domains.json create mode 100644 tests/fixtures/dnsmadeeasy-records.json create mode 100644 tests/test_octodns_provider_dnsmadeeasy.py diff --git a/octodns/provider/dnsmadeeasy.py b/octodns/provider/dnsmadeeasy.py index 8af45a4..550aa0b 100644 --- a/octodns/provider/dnsmadeeasy.py +++ b/octodns/provider/dnsmadeeasy.py @@ -34,12 +34,6 @@ class DnsMadeEasyClientUnauthorized(DnsMadeEasyClientException): super(DnsMadeEasyClientUnauthorized, self).__init__('Unauthorized') -class DnsMadeEasyClientForbidden(DnsMadeEasyClientException): - - def __init__(self): - super(DnsMadeEasyClientForbidden, self).__init__('Forbidden') - - class DnsMadeEasyClientNotFound(DnsMadeEasyClientException): def __init__(self): @@ -81,10 +75,8 @@ class DnsMadeEasyClient(object): params=params, json=data) if resp.status_code == 400: raise DnsMadeEasyClientBadRequest(resp) - if resp.status_code == 401: + if resp.status_code in [401, 403]: raise DnsMadeEasyClientUnauthorized() - if resp.status_code == 403: - raise DnsMadeEasyClientForbidden() if resp.status_code == 404: raise DnsMadeEasyClientNotFound() resp.raise_for_status() diff --git a/tests/fixtures/dnsmadeeasy-domains.json b/tests/fixtures/dnsmadeeasy-domains.json new file mode 100644 index 0000000..de7f7db --- /dev/null +++ b/tests/fixtures/dnsmadeeasy-domains.json @@ -0,0 +1,16 @@ +{ + "totalPages": 1, + "totalRecords": 1, + "data": [{ + "created": 1511740800000, + "folderId": 1990, + "gtdEnabled": false, + "pendingActionId": 0, + "updated": 1511766661574, + "processMulti": false, + "activeThirdParties": [], + "name": "unit.tests", + "id": 123123 + }], + "page": 0 +} \ No newline at end of file diff --git a/tests/fixtures/dnsmadeeasy-records.json b/tests/fixtures/dnsmadeeasy-records.json new file mode 100644 index 0000000..22fbc2f --- /dev/null +++ b/tests/fixtures/dnsmadeeasy-records.json @@ -0,0 +1,312 @@ +{ + "totalPages": 1, + "totalRecords": 21, + "data": [{ + "failover": false, + "monitor": false, + "sourceId": 123123, + "caaType": "issue", + "dynamicDns": false, + "failed": false, + "gtdLocation": "DEFAULT", + "hardLink": false, + "issuerCritical": 0, + "ttl": 3600, + "source": 1, + "name": "", + "value": "\"ca.unit.tests\"", + "id": 11189874, + "type": "CAA" + }, { + "failover": false, + "monitor": false, + "sourceId": 123123, + "dynamicDns": false, + "failed": false, + "gtdLocation": "DEFAULT", + "hardLink": false, + "ttl": 300, + "source": 1, + "name": "", + "value": "1.2.3.4", + "id": 11189875, + "type": "A" + }, { + "failover": false, + "monitor": false, + "sourceId": 123123, + "dynamicDns": false, + "failed": false, + "gtdLocation": "DEFAULT", + "hardLink": false, + "ttl": 300, + "source": 1, + "name": "", + "value": "1.2.3.5", + "id": 11189876, + "type": "A" + }, { + "failover": false, + "monitor": false, + "sourceId": 123123, + "dynamicDns": false, + "failed": false, + "gtdLocation": "DEFAULT", + "hardLink": false, + "ttl": 600, + "weight": 20, + "source": 1, + "name": "_srv._tcp", + "value": "foo-1.unit.tests.", + "id": 11189877, + "priority": 10, + "type": "SRV", + "port": 30 + }, { + "failover": false, + "monitor": false, + "sourceId": 123123, + "dynamicDns": false, + "failed": false, + "gtdLocation": "DEFAULT", + "hardLink": false, + "ttl": 600, + "weight": 20, + "source": 1, + "name": "_srv._tcp", + "value": "foo-2.unit.tests.", + "id": 11189878, + "priority": 12, + "type": "SRV", + "port": 30 + }, { + "failover": false, + "monitor": false, + "sourceId": 123123, + "dynamicDns": false, + "failed": false, + "gtdLocation": "DEFAULT", + "hardLink": false, + "ttl": 600, + "source": 1, + "name": "aaaa", + "value": "2601:644:500:e210:62f8:1dff:feb8:947a", + "id": 11189879, + "type": "AAAA" + }, { + "failover": false, + "monitor": false, + "sourceId": 123123, + "dynamicDns": false, + "failed": false, + "gtdLocation": "DEFAULT", + "hardLink": false, + "ttl": 300, + "source": 1, + "name": "cname", + "value": "", + "id": 11189880, + "type": "CNAME" + }, { + "failover": false, + "monitor": false, + "sourceId": 123123, + "dynamicDns": false, + "failed": false, + "gtdLocation": "DEFAULT", + "hardLink": false, + "ttl": 3600, + "source": 1, + "name": "included", + "value": "", + "id": 11189881, + "type": "CNAME" + }, { + "failover": false, + "monitor": false, + "sourceId": 123123, + "dynamicDns": false, + "failed": false, + "gtdLocation": "DEFAULT", + "hardLink": false, + "mxLevel": 30, + "ttl": 300, + "source": 1, + "name": "mx", + "value": "smtp-3.unit.tests.", + "id": 11189882, + "type": "MX" + }, { + "failover": false, + "monitor": false, + "sourceId": 123123, + "dynamicDns": false, + "failed": false, + "gtdLocation": "DEFAULT", + "hardLink": false, + "mxLevel": 20, + "ttl": 300, + "source": 1, + "name": "mx", + "value": "smtp-2.unit.tests.", + "id": 11189883, + "type": "MX" + }, { + "failover": false, + "monitor": false, + "sourceId": 123123, + "dynamicDns": false, + "failed": false, + "gtdLocation": "DEFAULT", + "hardLink": false, + "mxLevel": 10, + "ttl": 300, + "source": 1, + "name": "mx", + "value": "smtp-4.unit.tests.", + "id": 11189884, + "type": "MX" + }, { + "failover": false, + "monitor": false, + "sourceId": 123123, + "dynamicDns": false, + "failed": false, + "gtdLocation": "DEFAULT", + "hardLink": false, + "mxLevel": 40, + "ttl": 300, + "source": 1, + "name": "mx", + "value": "smtp-1.unit.tests.", + "id": 11189885, + "type": "MX" + }, { + "failover": false, + "monitor": false, + "sourceId": 123123, + "dynamicDns": false, + "failed": false, + "gtdLocation": "DEFAULT", + "hardLink": false, + "ttl": 600, + "source": 1, + "name": "spf", + "value": "\"v=spf1 ip4:192.168.0.1/16-all\"", + "id": 11189886, + "type": "SPF" + }, { + "failover": false, + "monitor": false, + "sourceId": 123123, + "dynamicDns": false, + "failed": false, + "gtdLocation": "DEFAULT", + "hardLink": false, + "ttl": 600, + "source": 1, + "name": "txt", + "value": "\"Bah bah black sheep\"", + "id": 11189887, + "type": "TXT" + }, { + "failover": false, + "monitor": false, + "sourceId": 123123, + "dynamicDns": false, + "failed": false, + "gtdLocation": "DEFAULT", + "hardLink": false, + "ttl": 600, + "source": 1, + "name": "txt", + "value": "\"have you any wool.\"", + "id": 11189888, + "type": "TXT" + }, { + "failover": false, + "monitor": false, + "sourceId": 123123, + "dynamicDns": false, + "failed": false, + "gtdLocation": "DEFAULT", + "hardLink": false, + "ttl": 600, + "source": 1, + "name": "txt", + "value": "\"v=DKIM1;k=rsa;s=email;h=sha256;p=A/kinda+of/long/string+with+numb3rs\"", + "id": 11189889, + "type": "TXT" + }, { + "failover": false, + "monitor": false, + "sourceId": 123123, + "dynamicDns": false, + "failed": false, + "gtdLocation": "DEFAULT", + "hardLink": false, + "ttl": 3600, + "source": 1, + "name": "under", + "value": "ns1.unit.tests.", + "id": 11189890, + "type": "NS" + }, { + "failover": false, + "monitor": false, + "sourceId": 123123, + "dynamicDns": false, + "failed": false, + "gtdLocation": "DEFAULT", + "hardLink": false, + "ttl": 3600, + "source": 1, + "name": "under", + "value": "ns2", + "id": 11189891, + "type": "NS" + }, { + "failover": false, + "monitor": false, + "sourceId": 123123, + "dynamicDns": false, + "failed": false, + "gtdLocation": "DEFAULT", + "hardLink": false, + "ttl": 300, + "source": 1, + "name": "www", + "value": "2.2.3.6", + "id": 11189892, + "type": "A" + }, { + "failover": false, + "monitor": false, + "sourceId": 123123, + "dynamicDns": false, + "failed": false, + "gtdLocation": "DEFAULT", + "hardLink": false, + "ttl": 300, + "source": 1, + "name": "www.sub", + "value": "2.2.3.6", + "id": 11189893, + "type": "A" + }, { + "failover": false, + "monitor": false, + "sourceId": 123123, + "dynamicDns": false, + "failed": false, + "gtdLocation": "DEFAULT", + "hardLink": false, + "ttl": 300, + "source": 1, + "name": "ptr", + "value": "foo.bar.com.", + "id": 11189894, + "type": "PTR" + }], + "page": 0 +} \ No newline at end of file diff --git a/tests/test_octodns_provider_dnsmadeeasy.py b/tests/test_octodns_provider_dnsmadeeasy.py new file mode 100644 index 0000000..576b8f0 --- /dev/null +++ b/tests/test_octodns_provider_dnsmadeeasy.py @@ -0,0 +1,202 @@ +# +# +# + + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from mock import Mock, call +from os.path import dirname, join +from requests import HTTPError +from requests_mock import ANY, mock as requests_mock +from unittest import TestCase + +from octodns.record import Record +from octodns.provider.dnsmadeeasy import DnsMadeEasyClientNotFound, \ + DnsMadeEasyProvider +from octodns.provider.yaml import YamlProvider +from octodns.zone import Zone + +import json + + +class TestDnsMadeEasyProvider(TestCase): + expected = Zone('unit.tests.', []) + source = YamlProvider('test', join(dirname(__file__), 'config')) + source.populate(expected) + + # Our test suite differs a bit, add our NS and remove the simple one + expected.add_record(Record.new(expected, 'under', { + 'ttl': 3600, + 'type': 'NS', + 'values': [ + 'ns1.unit.tests.', + 'ns2.unit.tests.', + ] + })) + for record in list(expected.records): + if record.name == 'sub' and record._type == 'NS': + expected._remove_record(record) + break + + def test_populate(self): + provider = DnsMadeEasyProvider('test', 'api', 'secret') + + # Bad auth + with requests_mock() as mock: + mock.get(ANY, status_code=401, + text='{"error": ["API key not found"]}') + + with self.assertRaises(Exception) as ctx: + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals('Unauthorized', ctx.exception.message) + + # Bad request + with requests_mock() as mock: + mock.get(ANY, status_code=400, + text='{"error": ["Rate limit exceeded"]}') + + with self.assertRaises(Exception) as ctx: + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals('\n - Rate limit exceeded', + ctx.exception.message) + + # General error + with requests_mock() as mock: + mock.get(ANY, status_code=502, text='Things caught fire') + + with self.assertRaises(HTTPError) as ctx: + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals(502, ctx.exception.response.status_code) + + # Non-existant zone doesn't populate anything + with requests_mock() as mock: + mock.get(ANY, status_code=404, + text='') + + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals(set(), zone.records) + + # No diffs == no changes + with requests_mock() as mock: + base = 'https://api.dnsmadeeasy.com/V2.0/dns/managed' + with open('tests/fixtures/dnsmadeeasy-domains.json') as fh: + mock.get('{}{}'.format(base, '/'), text=fh.read()) + with open('tests/fixtures/dnsmadeeasy-records.json') as fh: + mock.get('{}{}'.format(base, '/123123/records'), + text=fh.read()) + + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals(13, len(zone.records)) + changes = self.expected.changes(zone, provider) + self.assertEquals(0, len(changes)) + + # 2nd populate makes no network calls/all from cache + again = Zone('unit.tests.', []) + provider.populate(again) + self.assertEquals(13, len(again.records)) + + # bust the cache + del provider._zone_records[zone.name] + + def test_apply(self): + # Create provider with sandbox enabled + provider = DnsMadeEasyProvider('test', 'api', 'secret', True) + + resp = Mock() + resp.json = Mock() + provider._client._request = Mock(return_value=resp) + + with open('tests/fixtures/dnsmadeeasy-domains.json') as fh: + domains = json.load(fh) + + # non-existant domain, create everything + resp.json.side_effect = [ + DnsMadeEasyClientNotFound, # no zone in populate + DnsMadeEasyClientNotFound, # no domain during apply + domains + ] + plan = provider.plan(self.expected) + + # No root NS, no ignored, no excluded, no unsupported + n = len(self.expected.records) - 5 + self.assertEquals(n, len(plan.changes)) + self.assertEquals(n, provider.apply(plan)) + + provider._client._request.assert_has_calls([ + # created the domain + call('POST', '/', data={'name': 'unit.tests'}), + # get all domains to build the cache + call('GET', '/'), + # created at least one of the record with expected data + call('POST', '/123123/records', data={ + 'name': '_srv._tcp', + 'weight': 20, + 'value': 'foo-1.unit.tests.', + 'priority': 10, + 'ttl': 600, + 'type': 'SRV', + 'port': 30 + }), + ]) + self.assertEquals(25, provider._client._request.call_count) + + provider._client._request.reset_mock() + + # delete 1 and update 1 + provider._client.records = Mock(return_value=[ + { + 'id': 11189897, + 'name': 'www', + 'value': '1.2.3.4', + 'ttl': 300, + 'type': 'A', + }, + { + 'id': 11189898, + 'name': 'www', + 'value': '2.2.3.4', + 'ttl': 300, + 'type': 'A', + }, + { + 'id': 11189899, + 'name': 'ttl', + 'value': '3.2.3.4', + 'ttl': 600, + 'type': 'A', + } + ]) + + # Domain exists, we don't care about return + resp.json.side_effect = ['{}'] + + wanted = Zone('unit.tests.', []) + wanted.add_record(Record.new(wanted, 'ttl', { + 'ttl': 300, + 'type': 'A', + 'value': '3.2.3.4' + })) + + plan = provider.plan(wanted) + self.assertEquals(2, len(plan.changes)) + self.assertEquals(2, provider.apply(plan)) + + # recreate for update, and deletes for the 2 parts of the other + provider._client._request.assert_has_calls([ + call('POST', '/123123/records', data={ + 'value': '3.2.3.4', + 'type': 'A', + 'name': 'ttl', + 'ttl': 300 + }), + call('DELETE', '/123123/records/11189899'), + call('DELETE', '/123123/records/11189897'), + call('DELETE', '/123123/records/11189898') + ], any_order=True) From b5e7af0398404c612093a249510da217efac3c51 Mon Sep 17 00:00:00 2001 From: Paul van Brouwershaven Date: Wed, 31 Jan 2018 09:57:36 +0100 Subject: [PATCH 115/141] Option to handle Cloudflare proxied records This change imports records that are marked as proxied so that they can be synced to other DNS providers as described in [this support acticle](https://support.cloudflare.com/hc/en-us/articles/115000830351-How-to-configure-DNS-for-CNAME-partial-setup-when-managing-DNS-externally). Records that use this functionality will be ignored by this provider and not be synced back to Cloudflare as we don't know the origin record values that would be required. This change does not allow you to enable, disable or configure the CDN itself as that would require a lot of metadata to be handled by OctoDNS. The intention of this change is to allow users to run a multi-DNS provider setup without sending any traffic to their origin directly. See also github/octodns#45 --- octodns/provider/cloudflare.py | 58 ++++++++--- tests/test_octodns_provider_cloudflare.py | 117 ++++++++++++++++++++++ 2 files changed, 162 insertions(+), 13 deletions(-) diff --git a/octodns/provider/cloudflare.py b/octodns/provider/cloudflare.py index 9dfef6d..edce0ff 100644 --- a/octodns/provider/cloudflare.py +++ b/octodns/provider/cloudflare.py @@ -14,14 +14,14 @@ from ..record import Record, Update from .base import BaseProvider -class CloudflareAuthenticationError(Exception): +class CloudflareError(Exception): def __init__(self, data): try: message = data['errors'][0]['message'] except (IndexError, KeyError): message = 'Authentication error' - super(CloudflareAuthenticationError, self).__init__(message) + super(CloudflareError, self).__init__(message) class CloudflareProvider(BaseProvider): @@ -34,6 +34,12 @@ class CloudflareProvider(BaseProvider): email: dns-manager@example.com # The api key (required) token: foo + # Import CDN enabled records as CNAME to {}.cdn.cloudflare.net. Records + # ending at .cdn.cloudflare.net. will be ignored when this provider is + # not used as the source and the cdn option is enabled. + # + # See: https://support.cloudflare.com/hc/en-us/articles/115000830351 + cdn: false ''' SUPPORTS_GEO = False # TODO: support SRV @@ -43,9 +49,10 @@ class CloudflareProvider(BaseProvider): MIN_TTL = 120 TIMEOUT = 15 - def __init__(self, id, email, token, *args, **kwargs): + def __init__(self, id, email, token, cdn=False, *args, **kwargs): self.log = getLogger('CloudflareProvider[{}]'.format(id)) - self.log.debug('__init__: id=%s, email=%s, token=***', id, email) + self.log.debug('__init__: id=%s, email=%s, token=***, cdn=%s', id, + email, cdn) super(CloudflareProvider, self).__init__(id, *args, **kwargs) sess = Session() @@ -53,6 +60,7 @@ class CloudflareProvider(BaseProvider): 'X-Auth-Email': email, 'X-Auth-Key': token, }) + self.cdn = cdn self._sess = sess self._zones = None @@ -65,8 +73,8 @@ class CloudflareProvider(BaseProvider): resp = self._sess.request(method, url, params=params, json=data, timeout=self.TIMEOUT) self.log.debug('_request: status=%d', resp.status_code) - if resp.status_code == 403: - raise CloudflareAuthenticationError(resp.json()) + if resp.status_code == 400 or resp.status_code == 403: + raise CloudflareError(resp.json()) resp.raise_for_status() return resp.json() @@ -88,6 +96,18 @@ class CloudflareProvider(BaseProvider): return self._zones + def _data_for_cdn(self, name, _type, records): + self.log.info('CDN rewrite for %s', records[0]['name']) + _type = "CNAME" + if name == "": + _type = "ALIAS" + + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'value': '{}.cdn.cloudflare.net.'.format(records[0]['name']), + } + def _data_for_multiple(self, _type, records): return { 'ttl': records[0]['ttl'], @@ -170,8 +190,8 @@ class CloudflareProvider(BaseProvider): return self._zone_records[zone.name] def populate(self, zone, target=False, lenient=False): - self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, - target, lenient) + self.log.debug('populate: name=%s, cdn=%s, target=%s, lenient=%s', + zone.name, self.cdn, target, lenient) before = len(zone.records) records = self.zone_records(zone) @@ -186,12 +206,18 @@ class CloudflareProvider(BaseProvider): for name, types in values.items(): for _type, records in types.items(): - # Cloudflare supports ALIAS semantics with root CNAMEs - if _type == 'CNAME' and name == '': - _type = 'ALIAS' + # rewrite Cloudflare proxied records + if self.cdn and records[0]['proxied']: + data = self._data_for_cdn(name, _type, records) + + else: + # Cloudflare supports ALIAS semantics with root CNAMEs + if _type == 'CNAME' and name == '': + _type = 'ALIAS' + + data_for = getattr(self, '_data_for_{}'.format(_type)) + data = data_for(_type, records) - data_for = getattr(self, '_data_for_{}'.format(_type)) - data = data_for(_type, records) record = Record.new(zone, name, data, source=self, lenient=lenient) zone.add_record(record) @@ -250,6 +276,12 @@ class CloudflareProvider(BaseProvider): if _type == 'ALIAS': _type = 'CNAME' + # If this is a record to enable to Cloudflare CDN don't update as we + # don't know the original values. + if (self.cdn and _type == 'CNAME' and + record.value.endswith('.cdn.cloudflare.net.')): + raise StopIteration + contents_for = getattr(self, '_contents_for_{}'.format(_type)) for content in contents_for(record): content.update({ diff --git a/tests/test_octodns_provider_cloudflare.py b/tests/test_octodns_provider_cloudflare.py index 824af9d..f51f805 100644 --- a/tests/test_octodns_provider_cloudflare.py +++ b/tests/test_octodns_provider_cloudflare.py @@ -484,3 +484,120 @@ class TestCloudflareProvider(TestCase): 'ttl': 300, 'type': 'CNAME' }, list(contents)[0]) + + def test_cdn(self): + provider = CloudflareProvider('test', 'email', 'token', True) + + # A CNAME for us to transform to ALIAS + provider.zone_records = Mock(return_value=[ + { + "id": "fc12ab34cd5611334422ab3322997642", + "type": "CNAME", + "name": "cname.unit.tests", + "content": "www.unit.tests", + "proxiable": True, + "proxied": True, + "ttl": 300, + "locked": False, + "zone_id": "ff12ab34cd5611334422ab3322997650", + "zone_name": "unit.tests", + "modified_on": "2017-03-11T18:01:43.420689Z", + "created_on": "2017-03-11T18:01:43.420689Z", + "meta": { + "auto_added": False + } + }, + { + "id": "fc12ab34cd5611334422ab3322997642", + "type": "A", + "name": "a.unit.tests", + "content": "1.1.1.1", + "proxiable": True, + "proxied": True, + "ttl": 300, + "locked": False, + "zone_id": "ff12ab34cd5611334422ab3322997650", + "zone_name": "unit.tests", + "modified_on": "2017-03-11T18:01:43.420689Z", + "created_on": "2017-03-11T18:01:43.420689Z", + "meta": { + "auto_added": False + } + }, + { + "id": "fc12ab34cd5611334422ab3322997642", + "type": "A", + "name": "a.unit.tests", + "content": "1.1.1.2", + "proxiable": True, + "proxied": True, + "ttl": 300, + "locked": False, + "zone_id": "ff12ab34cd5611334422ab3322997650", + "zone_name": "unit.tests", + "modified_on": "2017-03-11T18:01:43.420689Z", + "created_on": "2017-03-11T18:01:43.420689Z", + "meta": { + "auto_added": False + } + }, + ]) + + zone = Zone('unit.tests.', []) + provider.populate(zone) + + # the two A records get merged into one CNAME record poining to the CDN + self.assertEquals(2, len(zone.records)) + + record = list(zone.records)[0] + self.assertEquals('cname', record.name) + self.assertEquals('cname.unit.tests.', record.fqdn) + self.assertEquals('CNAME', record._type) + self.assertEquals('cname.unit.tests.cdn.cloudflare.net.', record.value) + + record = list(zone.records)[1] + self.assertEquals('a', record.name) + self.assertEquals('a.unit.tests.', record.fqdn) + self.assertEquals('CNAME', record._type) + self.assertEquals('a.unit.tests.cdn.cloudflare.net.', record.value) + + # CDN enabled records can't be updated, we don't know the real values + contents = provider._gen_contents(record) + self.assertEquals(0, len(list(contents))) + + def test_cdn_alias(self): + provider = CloudflareProvider('test', 'email', 'token', True) + + # A CNAME for us to transform to ALIAS + provider.zone_records = Mock(return_value=[ + { + "id": "fc12ab34cd5611334422ab3322997642", + "type": "CNAME", + "name": "unit.tests", + "content": "www.unit.tests", + "proxiable": True, + "proxied": True, + "ttl": 300, + "locked": False, + "zone_id": "ff12ab34cd5611334422ab3322997650", + "zone_name": "unit.tests", + "modified_on": "2017-03-11T18:01:43.420689Z", + "created_on": "2017-03-11T18:01:43.420689Z", + "meta": { + "auto_added": False + } + }, + ]) + + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals(1, len(zone.records)) + record = list(zone.records)[0] + self.assertEquals('', record.name) + self.assertEquals('unit.tests.', record.fqdn) + self.assertEquals('ALIAS', record._type) + self.assertEquals('unit.tests.cdn.cloudflare.net.', record.value) + + # CDN enabled records can't be updated, we don't know the real values + contents = provider._gen_contents(record) + self.assertEquals(0, len(list(contents))) From c848860b1f9286c343849d91178539265e037af4 Mon Sep 17 00:00:00 2001 From: Paul van Brouwershaven Date: Fri, 2 Feb 2018 09:33:02 +0100 Subject: [PATCH 116/141] Add SRV support to Cloudflare provider --- README.md | 2 +- octodns/provider/cloudflare.py | 35 ++++++++++-- .../cloudflare-dns_records-page-2.json | 54 ++++++++++++++++++- tests/test_octodns_provider_cloudflare.py | 13 ++--- 4 files changed, 92 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index af8d14f..b99e836 100644 --- a/README.md +++ b/README.md @@ -150,7 +150,7 @@ The above command pulled the existing data out of Route53 and placed the results | Provider | Record Support | GeoDNS Support | Notes | |--|--|--|--| | [AzureProvider](/octodns/provider/azuredns.py) | A, AAAA, CNAME, MX, NS, PTR, SRV, TXT | No | | -| [CloudflareProvider](/octodns/provider/cloudflare.py) | A, AAAA, ALIAS, CAA, CNAME, MX, NS, SPF, TXT | No | CAA tags restricted | +| [CloudflareProvider](/octodns/provider/cloudflare.py) | A, AAAA, ALIAS, CAA, CNAME, MX, NS, SPF, SRV, TXT | No | CAA tags restricted | | [DigitalOceanProvider](/octodns/provider/digitalocean.py) | A, AAAA, CAA, CNAME, MX, NS, TXT, SRV | No | CAA tags restricted | | [DnsMadeEasyProvider](/octodns/provider/dnsmadeeasy.py) | A, AAAA, CAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | CAA tags restricted | | [DnsimpleProvider](/octodns/provider/dnsimple.py) | All | No | CAA tags restricted | diff --git a/octodns/provider/cloudflare.py b/octodns/provider/cloudflare.py index 9dfef6d..f76e0aa 100644 --- a/octodns/provider/cloudflare.py +++ b/octodns/provider/cloudflare.py @@ -36,9 +36,8 @@ class CloudflareProvider(BaseProvider): token: foo ''' SUPPORTS_GEO = False - # TODO: support SRV - SUPPORTS = set(('ALIAS', 'A', 'AAAA', 'CAA', 'CNAME', 'MX', 'NS', 'SPF', - 'TXT')) + SUPPORTS = set(('ALIAS', 'A', 'AAAA', 'CAA', 'CNAME', 'MX', 'NS', 'SRV', + 'SPF', 'TXT')) MIN_TTL = 120 TIMEOUT = 15 @@ -147,6 +146,21 @@ class CloudflareProvider(BaseProvider): 'values': ['{}.'.format(r['content']) for r in records], } + def _data_for_SRV(self, _type, records): + values = [] + for r in records: + values.append({ + 'priority': r['data']['priority'], + 'weight': r['data']['weight'], + 'port': r['data']['port'], + 'target': '{}.'.format(r['data']['target']), + }) + return { + 'type': _type, + 'ttl': records[0]['ttl'], + 'values': values + } + def zone_records(self, zone): if zone.name not in self._zone_records: zone_id = self.zones.get(zone.name, False) @@ -241,6 +255,21 @@ class CloudflareProvider(BaseProvider): 'content': value.exchange } + def _contents_for_SRV(self, record): + service, proto = record.name.split('.', 2) + for value in record.values: + yield { + 'data': { + 'service': service, + 'proto': proto, + 'name': record.zone.name, + 'priority': value.priority, + 'weight': value.weight, + 'port': value.port, + 'target': value.target[:-1], + } + } + def _gen_contents(self, record): name = record.fqdn[:-1] _type = record._type diff --git a/tests/fixtures/cloudflare-dns_records-page-2.json b/tests/fixtures/cloudflare-dns_records-page-2.json index de3d760..558aa2c 100644 --- a/tests/fixtures/cloudflare-dns_records-page-2.json +++ b/tests/fixtures/cloudflare-dns_records-page-2.json @@ -156,14 +156,64 @@ "meta": { "auto_added": false } + }, + { + "id": "fc12ab34cd5611334422ab3322997656", + "type": "SRV", + "name": "_srv._tcp.unit.tests", + "data": { + "service": "_srv", + "proto": "_tcp", + "name": "unit.tests", + "priority": 12, + "weight": 20, + "port": 30, + "target": "foo-2.unit.tests" + }, + "proxiable": true, + "proxied": false, + "ttl": 600, + "locked": false, + "zone_id": "ff12ab34cd5611334422ab3322997650", + "zone_name": "unit.tests", + "modified_on": "2017-03-11T18:01:43.940682Z", + "created_on": "2017-03-11T18:01:43.940682Z", + "meta": { + "auto_added": false + } + }, + { + "id": "fc12ab34cd5611334422ab3322997656", + "type": "SRV", + "name": "_srv._tcp.unit.tests", + "data": { + "service": "_srv", + "proto": "_tcp", + "name": "unit.tests", + "priority": 10, + "weight": 20, + "port": 30, + "target": "foo-1.unit.tests" + }, + "proxiable": true, + "proxied": false, + "ttl": 600, + "locked": false, + "zone_id": "ff12ab34cd5611334422ab3322997650", + "zone_name": "unit.tests", + "modified_on": "2017-03-11T18:01:43.940682Z", + "created_on": "2017-03-11T18:01:43.940682Z", + "meta": { + "auto_added": false + } } ], "result_info": { "page": 2, - "per_page": 10, + "per_page": 11, "total_pages": 2, "count": 9, - "total_count": 19 + "total_count": 21 }, "success": true, "errors": [], diff --git a/tests/test_octodns_provider_cloudflare.py b/tests/test_octodns_provider_cloudflare.py index 824af9d..2a71e8f 100644 --- a/tests/test_octodns_provider_cloudflare.py +++ b/tests/test_octodns_provider_cloudflare.py @@ -119,15 +119,16 @@ class TestCloudflareProvider(TestCase): zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals(11, len(zone.records)) + self.assertEquals(12, len(zone.records)) changes = self.expected.changes(zone, provider) + self.assertEquals(0, len(changes)) # re-populating the same zone/records comes out of cache, no calls again = Zone('unit.tests.', []) provider.populate(again) - self.assertEquals(11, len(again.records)) + self.assertEquals(12, len(again.records)) def test_apply(self): provider = CloudflareProvider('test', 'email', 'token') @@ -141,12 +142,12 @@ class TestCloudflareProvider(TestCase): 'id': 42, } }, # zone create - ] + [None] * 18 # individual record creates + ] + [None] * 20 # individual record creates # non-existant zone, create everything plan = provider.plan(self.expected) - self.assertEquals(11, len(plan.changes)) - self.assertEquals(11, provider.apply(plan)) + self.assertEquals(12, len(plan.changes)) + self.assertEquals(12, provider.apply(plan)) provider._request.assert_has_calls([ # created the domain @@ -171,7 +172,7 @@ class TestCloudflareProvider(TestCase): }), ], True) # expected number of total calls - self.assertEquals(20, provider._request.call_count) + self.assertEquals(22, provider._request.call_count) provider._request.reset_mock() From 8a7145f49f48c05e294b9b0767c35cdc9833e639 Mon Sep 17 00:00:00 2001 From: Paul van Brouwershaven Date: Wed, 7 Feb 2018 14:53:18 +0100 Subject: [PATCH 117/141] Changes according to review --- octodns/provider/cloudflare.py | 33 ++++++++++++++--------- tests/test_octodns_provider_cloudflare.py | 31 +++++++++++++++++---- 2 files changed, 47 insertions(+), 17 deletions(-) diff --git a/octodns/provider/cloudflare.py b/octodns/provider/cloudflare.py index edce0ff..42b42b6 100644 --- a/octodns/provider/cloudflare.py +++ b/octodns/provider/cloudflare.py @@ -15,15 +15,19 @@ from .base import BaseProvider class CloudflareError(Exception): - def __init__(self, data): try: message = data['errors'][0]['message'] except (IndexError, KeyError): - message = 'Authentication error' + message = 'Cloudflare error' super(CloudflareError, self).__init__(message) +class CloudflareAuthenticationError(CloudflareError): + def __init__(self, data): + CloudflareError.__init__(self, data) + + class CloudflareProvider(BaseProvider): ''' Cloudflare DNS provider @@ -53,7 +57,7 @@ class CloudflareProvider(BaseProvider): self.log = getLogger('CloudflareProvider[{}]'.format(id)) self.log.debug('__init__: id=%s, email=%s, token=***, cdn=%s', id, email, cdn) - super(CloudflareProvider, self).__init__(id, *args, **kwargs) + super(CloudflareProvider, self).__init__(id, cdn, *args, **kwargs) sess = Session() sess.headers.update({ @@ -73,8 +77,11 @@ class CloudflareProvider(BaseProvider): resp = self._sess.request(method, url, params=params, json=data, timeout=self.TIMEOUT) self.log.debug('_request: status=%d', resp.status_code) - if resp.status_code == 400 or resp.status_code == 403: + if resp.status_code == 400: raise CloudflareError(resp.json()) + if resp.status_code == 403: + raise CloudflareAuthenticationError(resp.json()) + resp.raise_for_status() return resp.json() @@ -190,8 +197,8 @@ class CloudflareProvider(BaseProvider): return self._zone_records[zone.name] def populate(self, zone, target=False, lenient=False): - self.log.debug('populate: name=%s, cdn=%s, target=%s, lenient=%s', - zone.name, self.cdn, target, lenient) + self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, + target, lenient) before = len(zone.records) records = self.zone_records(zone) @@ -232,6 +239,14 @@ class CloudflareProvider(BaseProvider): new['ttl'] = max(120, new['ttl']) if new == existing: return False + + # If this is a record to enable to Cloudflare CDN don't update as + # we don't know the original values. + if (hasattr(change.new, '_type') and (change.new._type == 'CNAME' or + change.new._type == 'ALIAS') and + change.new.value.endswith('.cdn.cloudflare.net.')): + return False + return True def _contents_for_multiple(self, record): @@ -276,12 +291,6 @@ class CloudflareProvider(BaseProvider): if _type == 'ALIAS': _type = 'CNAME' - # If this is a record to enable to Cloudflare CDN don't update as we - # don't know the original values. - if (self.cdn and _type == 'CNAME' and - record.value.endswith('.cdn.cloudflare.net.')): - raise StopIteration - contents_for = getattr(self, '_contents_for_{}'.format(_type)) for content in contents_for(record): content.update({ diff --git a/tests/test_octodns_provider_cloudflare.py b/tests/test_octodns_provider_cloudflare.py index f51f805..a8477eb 100644 --- a/tests/test_octodns_provider_cloudflare.py +++ b/tests/test_octodns_provider_cloudflare.py @@ -62,7 +62,7 @@ class TestCloudflareProvider(TestCase): with self.assertRaises(Exception) as ctx: zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals('Authentication error', ctx.exception.message) + self.assertEquals('Cloudflare error', ctx.exception.message) # General error with requests_mock() as mock: @@ -562,8 +562,21 @@ class TestCloudflareProvider(TestCase): self.assertEquals('a.unit.tests.cdn.cloudflare.net.', record.value) # CDN enabled records can't be updated, we don't know the real values - contents = provider._gen_contents(record) - self.assertEquals(0, len(list(contents))) + # never point a Cloudflare record to itsself. + wanted = Zone('unit.tests.', []) + wanted.add_record(Record.new(wanted, 'cname', { + 'ttl': 300, + 'type': 'CNAME', + 'value': 'change.unit.tests.cdn.cloudflare.net.' + })) + wanted.add_record(Record.new(wanted, 'new', { + 'ttl': 300, + 'type': 'CNAME', + 'value': 'new.unit.tests.cdn.cloudflare.net.' + })) + + plan = provider.plan(wanted) + self.assertEquals(1, len(plan.changes)) def test_cdn_alias(self): provider = CloudflareProvider('test', 'email', 'token', True) @@ -599,5 +612,13 @@ class TestCloudflareProvider(TestCase): self.assertEquals('unit.tests.cdn.cloudflare.net.', record.value) # CDN enabled records can't be updated, we don't know the real values - contents = provider._gen_contents(record) - self.assertEquals(0, len(list(contents))) + # never point a Cloudflare record to itsself. + wanted = Zone('unit.tests.', []) + wanted.add_record(Record.new(wanted, '', { + 'ttl': 300, + 'type': 'ALIAS', + 'value': 'change.unit.tests.cdn.cloudflare.net.' + })) + + plan = provider.plan(wanted) + self.assertEquals(False, hasattr(plan, 'changes')) From c4179ef0e8c78c5dd9d6ebb242a162400fbb6337 Mon Sep 17 00:00:00 2001 From: Paul van Brouwershaven Date: Wed, 7 Feb 2018 17:35:19 +0100 Subject: [PATCH 118/141] Allow proxied records with the same name --- octodns/provider/cloudflare.py | 21 ++++++++-- tests/test_octodns_provider_cloudflare.py | 49 ++++++++++++++++++++++- 2 files changed, 65 insertions(+), 5 deletions(-) diff --git a/octodns/provider/cloudflare.py b/octodns/provider/cloudflare.py index 42b42b6..3c1f64e 100644 --- a/octodns/provider/cloudflare.py +++ b/octodns/provider/cloudflare.py @@ -227,6 +227,15 @@ class CloudflareProvider(BaseProvider): record = Record.new(zone, name, data, source=self, lenient=lenient) + + # only one rewrite is needed for names where the proxy is + # enabled at multiple records with a different type but + # the same name + if (self.cdn and records[0]['proxied'] and + record in zone._records[name]): + self.log.info('CDN rewrite %s already in zone', name) + continue + zone.add_record(record) self.log.info('populate: found %s records', @@ -240,12 +249,18 @@ class CloudflareProvider(BaseProvider): if new == existing: return False - # If this is a record to enable to Cloudflare CDN don't update as + # If this is a record to enable Cloudflare CDN don't update as # we don't know the original values. - if (hasattr(change.new, '_type') and (change.new._type == 'CNAME' or - change.new._type == 'ALIAS') and + if (hasattr(change.new, '_type') and + (change.new._type == 'CNAME' or + change.new._type == 'ALIAS') and change.new.value.endswith('.cdn.cloudflare.net.')): return False + if (hasattr(change.existing, '_type') and + (change.existing._type == 'CNAME' or + change.existing._type == 'ALIAS') and + change.existing.value.endswith('.cdn.cloudflare.net.')): + return False return True diff --git a/tests/test_octodns_provider_cloudflare.py b/tests/test_octodns_provider_cloudflare.py index a8477eb..8d07b68 100644 --- a/tests/test_octodns_provider_cloudflare.py +++ b/tests/test_octodns_provider_cloudflare.py @@ -541,21 +541,61 @@ class TestCloudflareProvider(TestCase): "auto_added": False } }, + { + "id": "fc12ab34cd5611334422ab3322997642", + "type": "A", + "name": "multi.unit.tests", + "content": "1.1.1.3", + "proxiable": True, + "proxied": True, + "ttl": 300, + "locked": False, + "zone_id": "ff12ab34cd5611334422ab3322997650", + "zone_name": "unit.tests", + "modified_on": "2017-03-11T18:01:43.420689Z", + "created_on": "2017-03-11T18:01:43.420689Z", + "meta": { + "auto_added": False + } + }, + { + "id": "fc12ab34cd5611334422ab3322997642", + "type": "AAAA", + "name": "multi.unit.tests", + "content": "::1", + "proxiable": True, + "proxied": True, + "ttl": 300, + "locked": False, + "zone_id": "ff12ab34cd5611334422ab3322997650", + "zone_name": "unit.tests", + "modified_on": "2017-03-11T18:01:43.420689Z", + "created_on": "2017-03-11T18:01:43.420689Z", + "meta": { + "auto_added": False + } + }, ]) zone = Zone('unit.tests.', []) provider.populate(zone) # the two A records get merged into one CNAME record poining to the CDN - self.assertEquals(2, len(zone.records)) + self.assertEquals(3, len(zone.records)) record = list(zone.records)[0] + self.assertEquals('multi', record.name) + self.assertEquals('multi.unit.tests.', record.fqdn) + self.assertEquals('CNAME', record._type) + self.assertEquals('multi.unit.tests.cdn.cloudflare.net.', record.value) + + record = list(zone.records)[1] self.assertEquals('cname', record.name) self.assertEquals('cname.unit.tests.', record.fqdn) self.assertEquals('CNAME', record._type) self.assertEquals('cname.unit.tests.cdn.cloudflare.net.', record.value) - record = list(zone.records)[1] + record = list(zone.records)[2] self.assertEquals('a', record.name) self.assertEquals('a.unit.tests.', record.fqdn) self.assertEquals('CNAME', record._type) @@ -574,6 +614,11 @@ class TestCloudflareProvider(TestCase): 'type': 'CNAME', 'value': 'new.unit.tests.cdn.cloudflare.net.' })) + wanted.add_record(Record.new(wanted, 'created', { + 'ttl': 300, + 'type': 'CNAME', + 'value': 'www.unit.tests.' + })) plan = provider.plan(wanted) self.assertEquals(1, len(plan.changes)) From 6f0b0ddb08c927bb2d65038330d7fb503a4b70ad Mon Sep 17 00:00:00 2001 From: Paul van Brouwershaven Date: Thu, 8 Feb 2018 08:30:27 +0100 Subject: [PATCH 119/141] Test different exception types --- tests/test_octodns_provider_cloudflare.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/test_octodns_provider_cloudflare.py b/tests/test_octodns_provider_cloudflare.py index 8d07b68..e406f94 100644 --- a/tests/test_octodns_provider_cloudflare.py +++ b/tests/test_octodns_provider_cloudflare.py @@ -42,6 +42,20 @@ class TestCloudflareProvider(TestCase): def test_populate(self): provider = CloudflareProvider('test', 'email', 'token') + # Bad requests + with requests_mock() as mock: + mock.get(ANY, status_code=400, + text='{"success":false,"errors":[{"code":1101,' + '"message":"request was invalid"}],' + '"messages":[],"result":null}') + + with self.assertRaises(Exception) as ctx: + zone = Zone('unit.tests.', []) + provider.populate(zone) + + self.assertEquals('CloudflareError', type(ctx.exception).__name__) + self.assertEquals('request was invalid', ctx.exception.message) + # Bad auth with requests_mock() as mock: mock.get(ANY, status_code=403, @@ -52,6 +66,8 @@ class TestCloudflareProvider(TestCase): with self.assertRaises(Exception) as ctx: zone = Zone('unit.tests.', []) provider.populate(zone) + self.assertEquals('CloudflareAuthenticationError', + type(ctx.exception).__name__) self.assertEquals('Unknown X-Auth-Key or X-Auth-Email', ctx.exception.message) @@ -62,6 +78,8 @@ class TestCloudflareProvider(TestCase): with self.assertRaises(Exception) as ctx: zone = Zone('unit.tests.', []) provider.populate(zone) + self.assertEquals('CloudflareAuthenticationError', + type(ctx.exception).__name__) self.assertEquals('Cloudflare error', ctx.exception.message) # General error From c6634b3ccc41162e3f07c552fb786431aa0882bc Mon Sep 17 00:00:00 2001 From: Paul van Brouwershaven Date: Fri, 9 Feb 2018 13:44:58 +0100 Subject: [PATCH 120/141] Simplify _include_change check --- octodns/provider/cloudflare.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/octodns/provider/cloudflare.py b/octodns/provider/cloudflare.py index 3c1f64e..a9264e9 100644 --- a/octodns/provider/cloudflare.py +++ b/octodns/provider/cloudflare.py @@ -251,15 +251,8 @@ class CloudflareProvider(BaseProvider): # If this is a record to enable Cloudflare CDN don't update as # we don't know the original values. - if (hasattr(change.new, '_type') and - (change.new._type == 'CNAME' or - change.new._type == 'ALIAS') and - change.new.value.endswith('.cdn.cloudflare.net.')): - return False - if (hasattr(change.existing, '_type') and - (change.existing._type == 'CNAME' or - change.existing._type == 'ALIAS') and - change.existing.value.endswith('.cdn.cloudflare.net.')): + if (change.record._type in ('ALIAS', 'CNAME') and + change.record.value.endswith('.cdn.cloudflare.net.')): return False return True From 093398ff94c5f51ab2c3aa2e57031fe6eda6a3dd Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sun, 11 Feb 2018 17:01:38 -0800 Subject: [PATCH 121/141] Support for SSHFP ECDSA (3) and SHA-256 (2) RFC 6594 & RFC 7479 --- octodns/record.py | 4 ++-- tests/test_octodns_record.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/octodns/record.py b/octodns/record.py index 728c187..b608ef6 100644 --- a/octodns/record.py +++ b/octodns/record.py @@ -678,8 +678,8 @@ class PtrRecord(_ValueMixin, Record): class SshfpValue(object): - VALID_ALGORITHMS = (1, 2) - VALID_FINGERPRINT_TYPES = (1,) + VALID_ALGORITHMS = (1, 2, 3) + VALID_FINGERPRINT_TYPES = (1, 2) @classmethod def _validate_value(cls, value): diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index 0ba54de..9f3dc33 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -1360,7 +1360,7 @@ class TestRecordValidation(TestCase): 'ttl': 600, 'value': { 'algorithm': 'nope', - 'fingerprint_type': 1, + 'fingerprint_type': 2, 'fingerprint': 'bf6b6825d2977c511a475bbefb88aad54a92ac73' } }) @@ -1386,7 +1386,7 @@ class TestRecordValidation(TestCase): 'type': 'SSHFP', 'ttl': 600, 'value': { - 'algorithm': 1, + 'algorithm': 2, 'fingerprint': 'bf6b6825d2977c511a475bbefb88aad54a92ac73' } }) @@ -1398,7 +1398,7 @@ class TestRecordValidation(TestCase): 'type': 'SSHFP', 'ttl': 600, 'value': { - 'algorithm': 1, + 'algorithm': 3, 'fingerprint_type': 'yeeah', 'fingerprint': 'bf6b6825d2977c511a475bbefb88aad54a92ac73' } From 20d9ce7991704f43d34ebf4fcf69fa4ed6c01928 Mon Sep 17 00:00:00 2001 From: trnsnt Date: Mon, 12 Feb 2018 17:31:27 +0100 Subject: [PATCH 122/141] Fix SRV and SSHFP record for OVH provider --- octodns/provider/ovh.py | 17 +++++++++-------- tests/test_octodns_provider_ovh.py | 12 ++++++------ 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/octodns/provider/ovh.py b/octodns/provider/ovh.py index 5c2fe0d..4c10646 100644 --- a/octodns/provider/ovh.py +++ b/octodns/provider/ovh.py @@ -259,10 +259,11 @@ class OvhProvider(BaseProvider): def _params_for_SRV(record): for value in record.values: yield { - 'subDomain': '{} {} {} {}'.format(value.priority, - value.weight, value.port, - value.target), - 'target': record.name, + 'target': '{} {} {} {}'.format(value.priority, + value.weight, + value.port, + value.target), + 'subDomain': record.name, 'ttl': record.ttl, 'fieldType': record._type } @@ -271,10 +272,10 @@ class OvhProvider(BaseProvider): def _params_for_SSHFP(record): for value in record.values: yield { - 'subDomain': '{} {} {}'.format(value.algorithm, - value.fingerprint_type, - value.fingerprint), - 'target': record.name, + 'target': '{} {} {}'.format(value.algorithm, + value.fingerprint_type, + value.fingerprint), + 'subDomain': record.name, 'ttl': record.ttl, 'fieldType': record._type } diff --git a/tests/test_octodns_provider_ovh.py b/tests/test_octodns_provider_ovh.py index 6a44e25..9404f9e 100644 --- a/tests/test_octodns_provider_ovh.py +++ b/tests/test_octodns_provider_ovh.py @@ -378,11 +378,11 @@ class TestOvhProvider(TestCase): call(u'/domain/zone/unit.tests/record', fieldType=u'A', subDomain=u'', target=u'1.2.3.4', ttl=100), call(u'/domain/zone/unit.tests/record', fieldType=u'SRV', - subDomain=u'10 20 30 foo-1.unit.tests.', - target='_srv._tcp', ttl=800), + subDomain='_srv._tcp', + target=u'10 20 30 foo-1.unit.tests.', ttl=800), call(u'/domain/zone/unit.tests/record', fieldType=u'SRV', - subDomain=u'40 50 60 foo-2.unit.tests.', - target='_srv._tcp', ttl=800), + subDomain='_srv._tcp', + target=u'40 50 60 foo-2.unit.tests.', ttl=800), call(u'/domain/zone/unit.tests/record', fieldType=u'PTR', subDomain='4', target=u'unit.tests.', ttl=900), call(u'/domain/zone/unit.tests/record', fieldType=u'NS', @@ -390,8 +390,8 @@ class TestOvhProvider(TestCase): call(u'/domain/zone/unit.tests/record', fieldType=u'NS', subDomain='www3', target=u'ns4.unit.tests.', ttl=700), call(u'/domain/zone/unit.tests/record', - fieldType=u'SSHFP', target=u'', ttl=1100, - subDomain=u'1 1 bf6b6825d2977c511a475bbefb88a' + fieldType=u'SSHFP', subDomain=u'', ttl=1100, + target=u'1 1 bf6b6825d2977c511a475bbefb88a' u'ad54' u'a92ac73', ), From 2e17176442e8d6384b85db2553a2944a0fe1263a Mon Sep 17 00:00:00 2001 From: Steve Coursen Date: Tue, 13 Feb 2018 09:14:05 -0500 Subject: [PATCH 123/141] Move if branch around to avoid creating unnecessary filter chains --- octodns/provider/ns1.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 5ea68b6..36ea742 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -233,11 +233,11 @@ class Ns1Provider(BaseProvider): }, ) params['filters'] = [] - if len(params['answers']) > 1: - params['filters'].append( - {"filter": "shuffle", "config": {}} - ) if has_country: + if len(params['answers']) > 1: + params['filters'].append( + {"filter": "shuffle", "config": {}} + ) params['filters'].append( {"filter": "geotarget_country", "config": {}} ) From f62f82496696683112932ca9ee76fb9b426af18d Mon Sep 17 00:00:00 2001 From: Masaki Tagawa Date: Tue, 13 Feb 2018 23:43:30 +0900 Subject: [PATCH 124/141] Escape unescaped semicolons coming out of Google Cloud DNS --- octodns/provider/googlecloud.py | 8 ++++++-- tests/test_octodns_provider_googlecloud.py | 10 ++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/octodns/provider/googlecloud.py b/octodns/provider/googlecloud.py index 6ca0794..5ea43ac 100644 --- a/octodns/provider/googlecloud.py +++ b/octodns/provider/googlecloud.py @@ -9,6 +9,7 @@ import shlex import time from logging import getLogger from uuid import uuid4 +import re from google.cloud import dns @@ -269,12 +270,15 @@ class GoogleCloudProvider(BaseProvider): _data_for_PTR = _data_for_CNAME + _fix_semicolons = re.compile(r'(? 1: return { - 'values': gcloud_record.rrdatas} + 'values': [self._fix_semicolons.sub('\;', rr) + for rr in gcloud_record.rrdatas]} return { - 'value': gcloud_record.rrdatas[0]} + 'value': self._fix_semicolons.sub('\;', gcloud_record.rrdatas[0])} def _data_for_SRV(self, gcloud_record): return {'values': [{ diff --git a/tests/test_octodns_provider_googlecloud.py b/tests/test_octodns_provider_googlecloud.py index adc2112..79fda7f 100644 --- a/tests/test_octodns_provider_googlecloud.py +++ b/tests/test_octodns_provider_googlecloud.py @@ -427,3 +427,13 @@ class TestGoogleCloudProvider(TestCase): mock_zone.create.assert_called() provider.gcloud_client.zone.assert_called() + + def test_semicolon_fixup(self): + provider = self._get_provider() + + self.assertEquals({ + 'values': ['abcd\\; ef\\;g', 'hij\\; klm\\;n'] + }, provider._data_for_TXT( + DummyResourceRecordSet( + 'unit.tests.', 'TXT', 0, ['abcd; ef;g', 'hij\\; klm\\;n']) + )) From 2a16e988e051a02677b86db830a3c3322887bcc5 Mon Sep 17 00:00:00 2001 From: Masaki Tagawa Date: Wed, 14 Feb 2018 01:16:09 +0900 Subject: [PATCH 125/141] Generate the zone name following the spec of Google Cloud DNS Zone name must begin with a letter, end with a letter or digit, and only contain lowercase letters, digits or dashes, and be 63 characters or less. For instance, a reverse zone of IPv6 may violate the spec on the first character and the length of the zone name. --- octodns/provider/googlecloud.py | 7 +++++-- tests/test_octodns_provider_googlecloud.py | 15 +++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/octodns/provider/googlecloud.py b/octodns/provider/googlecloud.py index 6ca0794..b8f2b97 100644 --- a/octodns/provider/googlecloud.py +++ b/octodns/provider/googlecloud.py @@ -127,9 +127,12 @@ class GoogleCloudProvider(BaseProvider): :type return: new google.cloud.dns.ManagedZone """ # Zone name must begin with a letter, end with a letter or digit, - # and only contain lowercase letters, digits or dashes + # and only contain lowercase letters, digits or dashes, + # and be 63 characters or less zone_name = '{}-{}'.format( - dns_name[:-1].replace('.', '-'), uuid4().hex) + dns_name.replace('.', '-'), uuid4().hex)[:63] + if not zone_name[:1].isalpha(): + zone_name = 'zone-{}'.format(zone_name[:58]) gcloud_zone = self.gcloud_client.zone( name=zone_name, diff --git a/tests/test_octodns_provider_googlecloud.py b/tests/test_octodns_provider_googlecloud.py index adc2112..6c6c2aa 100644 --- a/tests/test_octodns_provider_googlecloud.py +++ b/tests/test_octodns_provider_googlecloud.py @@ -427,3 +427,18 @@ class TestGoogleCloudProvider(TestCase): mock_zone.create.assert_called() provider.gcloud_client.zone.assert_called() + + def test__create_zone_ip6_arpa(self): + def _create_dummy_zone(name, dns_name): + return DummyGoogleCloudZone(name=name, dns_name=dns_name) + + provider = self._get_provider() + + provider.gcloud_client = Mock() + provider.gcloud_client.zone = Mock(side_effect=_create_dummy_zone) + + mock_zone = \ + provider._create_gcloud_zone('0.0.0.0.8.b.d.0.1.0.0.2.ip6.arpa') + + self.assertRegexpMatches(mock_zone.name, '^[a-z][a-z0-9-]*[a-z0-9]$') + self.assertEqual(len(mock_zone.name), 63) From c6aae7b9b385f4991f7ecdb7d66e8510e5dd7cb5 Mon Sep 17 00:00:00 2001 From: Masaki Tagawa Date: Wed, 14 Feb 2018 06:24:57 +0900 Subject: [PATCH 126/141] Always prepend a legal name to the zone name --- octodns/provider/googlecloud.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/octodns/provider/googlecloud.py b/octodns/provider/googlecloud.py index b8f2b97..97c8706 100644 --- a/octodns/provider/googlecloud.py +++ b/octodns/provider/googlecloud.py @@ -129,10 +129,8 @@ class GoogleCloudProvider(BaseProvider): # Zone name must begin with a letter, end with a letter or digit, # and only contain lowercase letters, digits or dashes, # and be 63 characters or less - zone_name = '{}-{}'.format( + zone_name = 'zone-{}-{}'.format( dns_name.replace('.', '-'), uuid4().hex)[:63] - if not zone_name[:1].isalpha(): - zone_name = 'zone-{}'.format(zone_name[:58]) gcloud_zone = self.gcloud_client.zone( name=zone_name, From eb97b43d2876c33b82fcf0c4765c33a5738403ea Mon Sep 17 00:00:00 2001 From: Steve Coursen Date: Thu, 15 Feb 2018 10:49:43 -0500 Subject: [PATCH 127/141] length of answers check is unnecessary --- octodns/provider/ns1.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 36ea742..80797d8 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -234,10 +234,9 @@ class Ns1Provider(BaseProvider): ) params['filters'] = [] if has_country: - if len(params['answers']) > 1: - params['filters'].append( - {"filter": "shuffle", "config": {}} - ) + params['filters'].append( + {"filter": "shuffle", "config": {}} + ) params['filters'].append( {"filter": "geotarget_country", "config": {}} ) From 743af639891326506b4f22b9715cac3e2d5031db Mon Sep 17 00:00:00 2001 From: Hirotaka Nakajima Date: Sat, 17 Feb 2018 23:00:31 +0900 Subject: [PATCH 128/141] Remove unnecessary argument "cdn" --- octodns/provider/cloudflare.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/cloudflare.py b/octodns/provider/cloudflare.py index 3d47629..8df146d 100644 --- a/octodns/provider/cloudflare.py +++ b/octodns/provider/cloudflare.py @@ -56,7 +56,7 @@ class CloudflareProvider(BaseProvider): self.log = getLogger('CloudflareProvider[{}]'.format(id)) self.log.debug('__init__: id=%s, email=%s, token=***, cdn=%s', id, email, cdn) - super(CloudflareProvider, self).__init__(id, cdn, *args, **kwargs) + super(CloudflareProvider, self).__init__(id, *args, **kwargs) sess = Session() sess.headers.update({ From 7215d80230e4c598eb8caedf429dc142b10b1523 Mon Sep 17 00:00:00 2001 From: Masaki Tagawa Date: Sun, 18 Feb 2018 10:36:02 +0900 Subject: [PATCH 129/141] PEP8 --- tests/test_octodns_provider_googlecloud.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_octodns_provider_googlecloud.py b/tests/test_octodns_provider_googlecloud.py index 3ad87cb..c2677af 100644 --- a/tests/test_octodns_provider_googlecloud.py +++ b/tests/test_octodns_provider_googlecloud.py @@ -451,4 +451,4 @@ class TestGoogleCloudProvider(TestCase): }, provider._data_for_TXT( DummyResourceRecordSet( 'unit.tests.', 'TXT', 0, ['abcd; ef;g', 'hij\\; klm\\;n']) - )) \ No newline at end of file + )) From 886a26bc6f0ffd78bc0d41bbf88e9fc26e686b21 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 19 Feb 2018 11:30:35 -0800 Subject: [PATCH 130/141] Extract _record_for from populate, use round trip in _apply_Update This will ensure that we have exactly the same logic/behavior across the board when turning records into content and prevent the :-( hack that was in here before. It was missing the max(min, ttl) bit we throw everything else through and this makes that consistent. Most importantly it'll prevent us from having to fix bugs or make improvements in multiple code paths in the future. --- octodns/provider/cloudflare.py | 56 ++++++++++++++++------------------ 1 file changed, 27 insertions(+), 29 deletions(-) diff --git a/octodns/provider/cloudflare.py b/octodns/provider/cloudflare.py index 8df146d..0d058dc 100644 --- a/octodns/provider/cloudflare.py +++ b/octodns/provider/cloudflare.py @@ -210,6 +210,20 @@ class CloudflareProvider(BaseProvider): return self._zone_records[zone.name] + def _record_for(self, zone, name, _type, records, lenient): + # rewrite Cloudflare proxied records + if self.cdn and records[0]['proxied']: + data = self._data_for_cdn(name, _type, records) + else: + # Cloudflare supports ALIAS semantics with root CNAMEs + if _type == 'CNAME' and name == '': + _type = 'ALIAS' + + data_for = getattr(self, '_data_for_{}'.format(_type)) + data = data_for(_type, records) + + return Record.new(zone, name, data, source=self, lenient=lenient) + def populate(self, zone, target=False, lenient=False): self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, target, lenient) @@ -226,21 +240,8 @@ class CloudflareProvider(BaseProvider): for name, types in values.items(): for _type, records in types.items(): - - # rewrite Cloudflare proxied records - if self.cdn and records[0]['proxied']: - data = self._data_for_cdn(name, _type, records) - - else: - # Cloudflare supports ALIAS semantics with root CNAMEs - if _type == 'CNAME' and name == '': - _type = 'ALIAS' - - data_for = getattr(self, '_data_for_{}'.format(_type)) - data = data_for(_type, records) - - record = Record.new(zone, name, data, source=self, - lenient=lenient) + record = self._record_for(zone, name, _type, records, + lenient) # only one rewrite is needed for names where the proxy is # enabled at multiple records with a different type but @@ -374,10 +375,6 @@ class CloudflareProvider(BaseProvider): for c in self._gen_contents(change.new) } - # We need a list of keys to consider for diffs, use the first content - # before we muck with anything - keys = existing_contents.values()[0].keys() - # Find the things we need to add adds = [] for k, content in new_contents.items(): @@ -387,22 +384,23 @@ class CloudflareProvider(BaseProvider): except KeyError: adds.append(content) - zone_id = self.zones[change.new.zone.name] + zone = change.new.zone + zone_id = self.zones[zone.name] # Find things we need to remove name = change.new.fqdn[:-1] + hostname = zone.hostname_from_fqdn(name) _type = change.new._type # OK, work through each record from the zone - for record in self.zone_records(change.new.zone): + for record in self.zone_records(zone): if name == record['name'] and _type == record['type']: - # This is match for our name and type, we need to look at - # contents now, build a dict of the relevant keys and vals - content = {} - for k in keys: - content[k] = record[k] - # :-( - if _type in ('CNAME', 'MX', 'NS'): - content['content'] += '.' + + # Round trip the single value through a record to contents flow + # to get a consistent _gen_contents result that matches what + # went in to new_contents + r = self._record_for(zone, hostname, _type, [record], True) + content = self._gen_contents(r).next() + # If the hash of that dict isn't in new this record isn't # needed if self._hash_content(content) not in new_contents: From 4b44ab14b1f0a52f1051c67656d6e3dd6f0ba903 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 19 Feb 2018 11:40:32 -0800 Subject: [PATCH 131/141] Use MIN_TTL, not 120 literal --- octodns/provider/cloudflare.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/cloudflare.py b/octodns/provider/cloudflare.py index 0d058dc..eb08952 100644 --- a/octodns/provider/cloudflare.py +++ b/octodns/provider/cloudflare.py @@ -260,7 +260,7 @@ class CloudflareProvider(BaseProvider): if isinstance(change, Update): existing = change.existing.data new = change.new.data - new['ttl'] = max(120, new['ttl']) + new['ttl'] = max(self.MIN_TTL, new['ttl']) if new == existing: return False From 29bc1f3af3cb68c0c8c851c8169f842e81a749a6 Mon Sep 17 00:00:00 2001 From: Sergei Shmanko Date: Thu, 22 Feb 2018 11:36:25 +0200 Subject: [PATCH 132/141] fix logging for update/delete_pcent_threshold --- octodns/provider/base.py | 3 ++- octodns/provider/plan.py | 12 ++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/octodns/provider/base.py b/octodns/provider/base.py index 2d4680f..3f8a5b8 100644 --- a/octodns/provider/base.py +++ b/octodns/provider/base.py @@ -17,7 +17,8 @@ class BaseProvider(BaseSource): delete_pcent_threshold=Plan.MAX_SAFE_DELETE_PCENT): super(BaseProvider, self).__init__(id) self.log.debug('__init__: id=%s, apply_disabled=%s, ' - 'update_pcent_threshold=%d, delete_pcent_threshold=%d', + 'update_pcent_threshold=%.2f' + 'delete_pcent_threshold=%.2f', id, apply_disabled, update_pcent_threshold, diff --git a/octodns/provider/plan.py b/octodns/provider/plan.py index 3e86826..9613809 100644 --- a/octodns/provider/plan.py +++ b/octodns/provider/plan.py @@ -60,17 +60,17 @@ class Plan(object): delete_pcent = self.change_counts['Delete'] / existing_record_count if update_pcent > self.update_pcent_threshold: - raise UnsafePlan('Too many updates, {} is over {} percent' + raise UnsafePlan('Too many updates, {:.2f} is over {:.2f} %' '({}/{})'.format( - update_pcent, - self.MAX_SAFE_UPDATE_PCENT * 100, + update_pcent * 100, + self.update_pcent_threshold * 100, self.change_counts['Update'], existing_record_count)) if delete_pcent > self.delete_pcent_threshold: - raise UnsafePlan('Too many deletes, {} is over {} percent' + raise UnsafePlan('Too many deletes, {:.2f} is over {:.2f} %' '({}/{})'.format( - delete_pcent, - self.MAX_SAFE_DELETE_PCENT * 100, + delete_pcent * 100, + self.delete_pcent_threshold * 100, self.change_counts['Delete'], existing_record_count)) From 543b1c9dbdd0810aad851adf53f99fc4b6cc72d8 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 24 Feb 2018 09:10:57 -0800 Subject: [PATCH 133/141] Fix handling of Cloudflare ALIAS updates --- octodns/provider/cloudflare.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/octodns/provider/cloudflare.py b/octodns/provider/cloudflare.py index eb08952..688b10e 100644 --- a/octodns/provider/cloudflare.py +++ b/octodns/provider/cloudflare.py @@ -388,17 +388,19 @@ class CloudflareProvider(BaseProvider): zone_id = self.zones[zone.name] # Find things we need to remove - name = change.new.fqdn[:-1] - hostname = zone.hostname_from_fqdn(name) + hostname = zone.hostname_from_fqdn(change.new.fqdn[:-1]) _type = change.new._type # OK, work through each record from the zone for record in self.zone_records(zone): - if name == record['name'] and _type == record['type']: + name = zone.hostname_from_fqdn(record['name']) + # Use the _record_for so that we include all of standard + # converstion logic + r = self._record_for(zone, name, record['type'], [record], True) + if hostname == r.name and _type == r._type: # Round trip the single value through a record to contents flow # to get a consistent _gen_contents result that matches what # went in to new_contents - r = self._record_for(zone, hostname, _type, [record], True) content = self._gen_contents(r).next() # If the hash of that dict isn't in new this record isn't From 9f2b65ec83aae71220ef0367d864ab444f3ed8b3 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 24 Feb 2018 09:19:23 -0800 Subject: [PATCH 134/141] Change str() to unicode() to avoid encoding problems --- README.md | 1 + octodns/cmds/report.py | 6 +++--- octodns/provider/base.py | 2 +- octodns/provider/ns1.py | 4 ++-- octodns/provider/plan.py | 16 ++++++++-------- octodns/record.py | 5 +++-- octodns/zone.py | 2 +- tests/test_octodns_provider_base.py | 14 +++++++------- 8 files changed, 26 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index b99e836..0abf9c6 100644 --- a/README.md +++ b/README.md @@ -169,6 +169,7 @@ The above command pulled the existing data out of Route53 and placed the results * ALIAS support varies a lot from provider to provider care should be taken to verify that your needs are met in detail. * Dyn's UI doesn't allow editing or view of TTL, but the API accepts and stores the value provided, this value does not appear to be used when served * Dnsimple's uses the configured TTL when serving things through the ALIAS, there's also a secondary TXT record created alongside the ALIAS that octoDNS ignores +* octoDNS itself supports non-ASCII character sets, but in testing Cloudflare is the only provider where that is currently functional end-to-end. Others have failures either in the client libraries or API calls ## Custom Sources and Providers diff --git a/octodns/cmds/report.py b/octodns/cmds/report.py index 06a4484..2b32e77 100755 --- a/octodns/cmds/report.py +++ b/octodns/cmds/report.py @@ -65,7 +65,7 @@ def main(): resolver = AsyncResolver(configure=False, num_workers=int(args.num_workers)) if not ip_addr_re.match(server): - server = str(query(server, 'A')[0]) + server = unicode(query(server, 'A')[0]) log.info('server=%s', server) resolver.nameservers = [server] resolver.lifetime = int(args.timeout) @@ -81,12 +81,12 @@ def main(): stdout.write(',') stdout.write(record._type) stdout.write(',') - stdout.write(str(record.ttl)) + stdout.write(unicode(record.ttl)) compare = {} for future in futures: stdout.write(',') try: - answers = [str(r) for r in future.result()] + answers = [unicode(r) for r in future.result()] except (NoAnswer, NoNameservers): answers = ['*no answer*'] except NXDOMAIN: diff --git a/octodns/provider/base.py b/octodns/provider/base.py index 3f8a5b8..6f67f1c 100644 --- a/octodns/provider/base.py +++ b/octodns/provider/base.py @@ -62,7 +62,7 @@ class BaseProvider(BaseSource): extra = self._extra_changes(existing, changes) if extra: self.log.info('plan: extra changes\n %s', '\n ' - .join([str(c) for c in extra])) + .join([unicode(c) for c in extra])) changes += extra if changes: diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 80797d8..d214062 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -75,9 +75,9 @@ class Ns1Provider(BaseProvider): else: values.extend(answer['answer']) codes.append([]) - values = [str(x) for x in values] + values = [unicode(x) for x in values] geo = OrderedDict( - {str(k): [str(x) for x in v] for k, v in geo.items()} + {unicode(k): [unicode(x) for x in v] for k, v in geo.items()} ) data['values'] = values data['geo'] = geo diff --git a/octodns/provider/plan.py b/octodns/provider/plan.py index 9613809..5944d6e 100644 --- a/octodns/provider/plan.py +++ b/octodns/provider/plan.py @@ -140,11 +140,11 @@ class PlanLogger(_PlanOutput): def _value_stringifier(record, sep): try: - values = [str(v) for v in record.values] + values = [unicode(v) for v in record.values] except AttributeError: values = [record.value] for code, gv in sorted(getattr(record, 'geo', {}).items()): - vs = ', '.join([str(v) for v in gv.values]) + vs = ', '.join([unicode(v) for v in gv.values]) values.append('{}: {}'.format(code, vs)) return sep.join(values) @@ -181,7 +181,7 @@ class PlanMarkdown(_PlanOutput): fh.write(' | ') # TTL if existing: - fh.write(str(existing.ttl)) + fh.write(unicode(existing.ttl)) fh.write(' | ') fh.write(_value_stringifier(existing, '; ')) fh.write(' | |\n') @@ -189,7 +189,7 @@ class PlanMarkdown(_PlanOutput): fh.write('| | | | ') if new: - fh.write(str(new.ttl)) + fh.write(unicode(new.ttl)) fh.write(' | ') fh.write(_value_stringifier(new, '; ')) fh.write(' | ') @@ -197,7 +197,7 @@ class PlanMarkdown(_PlanOutput): fh.write(' |\n') fh.write('\nSummary: ') - fh.write(str(plan)) + fh.write(unicode(plan)) fh.write('\n\n') else: fh.write('## No changes were planned\n') @@ -243,7 +243,7 @@ class PlanHtml(_PlanOutput): # TTL if existing: fh.write(' ') - fh.write(str(existing.ttl)) + fh.write(unicode(existing.ttl)) fh.write('\n ') fh.write(_value_stringifier(existing, '
')) fh.write('\n \n \n') @@ -252,7 +252,7 @@ class PlanHtml(_PlanOutput): if new: fh.write(' ') - fh.write(str(new.ttl)) + fh.write(unicode(new.ttl)) fh.write('\n ') fh.write(_value_stringifier(new, '
')) fh.write('\n ') @@ -260,7 +260,7 @@ class PlanHtml(_PlanOutput): fh.write('\n \n') fh.write(' \n Summary: ') - fh.write(str(plan)) + fh.write(unicode(plan)) fh.write('\n \n\n') else: fh.write('No changes were planned') diff --git a/octodns/record.py b/octodns/record.py index b608ef6..82ee191 100644 --- a/octodns/record.py +++ b/octodns/record.py @@ -122,7 +122,7 @@ class Record(object): self.__class__.__name__, name) self.zone = zone # force everything lower-case just to be safe - self.name = str(name).lower() if name else name + self.name = unicode(name).lower() if name else name self.source = source self.ttl = int(data['ttl']) @@ -274,7 +274,8 @@ class _ValuesMixin(object): return ret def __repr__(self): - values = "['{}']".format("', '".join([str(v) for v in self.values])) + values = "['{}']".format("', '".join([unicode(v) + for v in self.values])) return '<{} {} {}, {}, {}>'.format(self.__class__.__name__, self._type, self.ttl, self.fqdn, values) diff --git a/octodns/zone.py b/octodns/zone.py index bbc38d0..a8a91ca 100644 --- a/octodns/zone.py +++ b/octodns/zone.py @@ -38,7 +38,7 @@ class Zone(object): raise Exception('Invalid zone name {}, missing ending dot' .format(name)) # Force everyting to lowercase just to be safe - self.name = str(name).lower() if name else name + self.name = unicode(name).lower() if name else name self.sub_zones = sub_zones # We're grouping by node, it allows us to efficently search for # duplicates and detect when CNAMEs co-exist with other records diff --git a/tests/test_octodns_provider_base.py b/tests/test_octodns_provider_base.py index 472b008..855efaf 100644 --- a/tests/test_octodns_provider_base.py +++ b/tests/test_octodns_provider_base.py @@ -177,7 +177,7 @@ class TestBaseProvider(TestCase): }) for i in range(int(Plan.MIN_EXISTING_RECORDS)): - zone.add_record(Record.new(zone, str(i), { + zone.add_record(Record.new(zone, unicode(i), { 'ttl': 60, 'type': 'A', 'value': '2.3.4.5' @@ -208,7 +208,7 @@ class TestBaseProvider(TestCase): }) for i in range(int(Plan.MIN_EXISTING_RECORDS)): - zone.add_record(Record.new(zone, str(i), { + zone.add_record(Record.new(zone, unicode(i), { 'ttl': 60, 'type': 'A', 'value': '2.3.4.5' @@ -234,7 +234,7 @@ class TestBaseProvider(TestCase): }) for i in range(int(Plan.MIN_EXISTING_RECORDS)): - zone.add_record(Record.new(zone, str(i), { + zone.add_record(Record.new(zone, unicode(i), { 'ttl': 60, 'type': 'A', 'value': '2.3.4.5' @@ -256,7 +256,7 @@ class TestBaseProvider(TestCase): }) for i in range(int(Plan.MIN_EXISTING_RECORDS)): - zone.add_record(Record.new(zone, str(i), { + zone.add_record(Record.new(zone, unicode(i), { 'ttl': 60, 'type': 'A', 'value': '2.3.4.5' @@ -282,7 +282,7 @@ class TestBaseProvider(TestCase): }) for i in range(int(Plan.MIN_EXISTING_RECORDS)): - zone.add_record(Record.new(zone, str(i), { + zone.add_record(Record.new(zone, unicode(i), { 'ttl': 60, 'type': 'A', 'value': '2.3.4.5' @@ -305,7 +305,7 @@ class TestBaseProvider(TestCase): }) for i in range(int(Plan.MIN_EXISTING_RECORDS)): - zone.add_record(Record.new(zone, str(i), { + zone.add_record(Record.new(zone, unicode(i), { 'ttl': 60, 'type': 'A', 'value': '2.3.4.5' @@ -333,7 +333,7 @@ class TestBaseProvider(TestCase): }) for i in range(int(Plan.MIN_EXISTING_RECORDS)): - zone.add_record(Record.new(zone, str(i), { + zone.add_record(Record.new(zone, unicode(i), { 'ttl': 60, 'type': 'A', 'value': '2.3.4.5' From f5c17638a4a2fa4ad596fa7f45ba92e87f71e83e Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 24 Feb 2018 09:27:43 -0800 Subject: [PATCH 135/141] Remove Rackspace's _as_unicode, no longer necessary --- octodns/provider/rackspace.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py index 12b2c54..e370315 100644 --- a/octodns/provider/rackspace.py +++ b/octodns/provider/rackspace.py @@ -130,17 +130,9 @@ class RackspaceProvider(BaseProvider): def _delete(self, path, data=None): return self._request('DELETE', path, data=data) - @staticmethod - def _as_unicode(s, codec): - if not isinstance(s, unicode): - return unicode(s, codec) - return s - @classmethod def _key_for_record(cls, rs_record): - return cls._as_unicode(rs_record['type'], 'ascii'), \ - cls._as_unicode(rs_record['name'], 'utf-8'), \ - cls._as_unicode(rs_record['data'], 'utf-8') + return rs_record['type'], rs_record['name'], rs_record['data'] def _data_for_multiple(self, rrset): return { From 48f804e3291204c6bfbd2bc795526189d53c35a3 Mon Sep 17 00:00:00 2001 From: Zophren Date: Tue, 27 Feb 2018 18:05:34 +0100 Subject: [PATCH 136/141] Update Readme - OVH provider capabilities Add DKIM to provider capabilities --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0abf9c6..26539f9 100644 --- a/README.md +++ b/README.md @@ -157,7 +157,7 @@ The above command pulled the existing data out of Route53 and placed the results | [DynProvider](/octodns/provider/dyn.py) | All | Yes | | | [GoogleCloudProvider](/octodns/provider/googlecloud.py) | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | No | | | [Ns1Provider](/octodns/provider/ns1.py) | All | Yes | No health checking for GeoDNS | -| [OVH](/octodns/provider/ovh.py) | A, AAAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, SSHFP, TXT | No | | +| [OVH](/octodns/provider/ovh.py) | A, AAAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, SSHFP, TXT, DKIM | No | | | [PowerDnsProvider](/octodns/provider/powerdns.py) | All | No | | | [Rackspace](/octodns/provider/rackspace.py) | A, AAAA, ALIAS, CNAME, MX, NS, PTR, SPF, TXT | No | | | [Route53](/octodns/provider/route53.py) | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | Yes | | From 5d2ba2e715f770a4ddd90a2b5ca93563a3cdf785 Mon Sep 17 00:00:00 2001 From: Josh Soref Date: Tue, 27 Feb 2018 12:09:47 -0500 Subject: [PATCH 137/141] Spelling (#214) * spelling: ancillary * spelling: antarctica * spelling: australia * spelling: authentication * spelling: continental * spelling: constructor * spelling: conversion * spelling: creation * spelling: doesn't * spelling: easily * spelling: efficiently * spelling: equivalent * spelling: essentially * spelling: everything * spelling: exactly * spelling: be * spelling: expensive * spelling: supports * spelling: healthcheck * spelling: immediately * spelling: ignored * spelling: invocation * spelling: itself * spelling: leftovers * spelling: missing * spelling: natural * spelling: nonexistent * spelling: peculiarities * spelling: pointing This change hit a line length limitation, so I'm wrapping it and adding a period which appears to match local style... * spelling: quicker * spelling: response * spelling: requested * spelling: redirect * spelling: traffic * spelling: unknown * spelling: uploaded * spelling: useful * spelling: separately * spelling: zone --- CHANGELOG.md | 2 +- docs/records.md | 8 ++++---- octodns/cmds/sync.py | 2 +- octodns/manager.py | 2 +- octodns/provider/azuredns.py | 4 ++-- octodns/provider/base.py | 4 ++-- octodns/provider/cloudflare.py | 2 +- octodns/provider/digitalocean.py | 2 +- octodns/provider/dnsimple.py | 2 +- octodns/provider/dyn.py | 14 +++++++------- octodns/provider/powerdns.py | 8 ++++---- octodns/provider/route53.py | 6 +++--- octodns/record.py | 2 +- octodns/source/base.py | 2 +- octodns/zone.py | 4 ++-- script/release | 2 +- tests/test_octodns_provider_base.py | 6 +++--- tests/test_octodns_provider_cloudflare.py | 7 ++++--- tests/test_octodns_provider_dyn.py | 16 ++++++++-------- tests/test_octodns_provider_googlecloud.py | 10 +++++----- tests/test_octodns_provider_ovh.py | 6 +++--- tests/test_octodns_provider_powerdns.py | 4 ++-- tests/test_octodns_provider_route53.py | 6 +++--- tests/test_octodns_record.py | 4 ++-- 24 files changed, 63 insertions(+), 62 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6777ea9..3d8b441 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -60,7 +60,7 @@ better in the future :fingers_crossed: #### Miscellaneous -* Use a 3rd party lib for nautrual sorting of keys, rather than my old +* Use a 3rd party lib for natural sorting of keys, rather than my old implementation. Sorting can be disabled in the YamlProvider with `enforce_order: False`. * Semi-colon/escaping fixes and improvements. diff --git a/docs/records.md b/docs/records.md index 82d9a37..75f4d0b 100644 --- a/docs/records.md +++ b/docs/records.md @@ -53,11 +53,11 @@ The geo labels breakdown based on: 1. - 'AF': 14, # Continental Africa - - 'AN': 17, # Continental Antartica - - 'AS': 15, # Contentinal Asia - - 'EU': 13, # Contentinal Europe + - 'AN': 17, # Continental Antarctica + - 'AS': 15, # Continental Asia + - 'EU': 13, # Continental Europe - 'NA': 11, # Continental North America - - 'OC': 16, # Contentinal Austrailia/Oceania + - 'OC': 16, # Continental Australia/Oceania - 'SA': 12, # Continental South America 2. ISO Country Code https://en.wikipedia.org/wiki/ISO_3166-2 diff --git a/octodns/cmds/sync.py b/octodns/cmds/sync.py index 4dd3e87..60793e7 100755 --- a/octodns/cmds/sync.py +++ b/octodns/cmds/sync.py @@ -26,7 +26,7 @@ def main(): help='Limit sync to the specified zone(s)') # --sources isn't an option here b/c filtering sources out would be super - # dangerous since you could eaily end up with an empty zone and delete + # dangerous since you could easily end up with an empty zone and delete # everything, or even just part of things when there are multiple sources parser.add_argument('--target', default=[], action='append', diff --git a/octodns/manager.py b/octodns/manager.py index d4debf6..c9c6a13 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -51,7 +51,7 @@ class MakeThreadFuture(object): class MainThreadExecutor(object): ''' - Dummy executor that runs things on the main thread during the involcation + Dummy executor that runs things on the main thread during the invocation of submit, but still returns a future object with the result. This allows code to be written to handle async, even in the case where we don't want to use multiple threads/workers and would prefer that things flow as if diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 1757274..65160cc 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -39,7 +39,7 @@ class _AzureRecord(object): } def __init__(self, resource_group, record, delete=False): - '''Contructor for _AzureRecord. + '''Constructor for _AzureRecord. Notes on Azure records: An Azure record set has the form RecordSet(name=<...>, type=<...>, arecords=[...], aaaa_records, ..) @@ -222,7 +222,7 @@ class AzureProvider(BaseProvider): azuredns: class: octodns.provider.azuredns.AzureProvider client_id: env/AZURE_APPLICATION_ID - key: env/AZURE_AUTHENICATION_KEY + key: env/AZURE_AUTHENTICATION_KEY directory_id: env/AZURE_DIRECTORY_ID sub_id: env/AZURE_SUBSCRIPTION_ID resource_group: 'TestResource1' diff --git a/octodns/provider/base.py b/octodns/provider/base.py index 6f67f1c..4e99651 100644 --- a/octodns/provider/base.py +++ b/octodns/provider/base.py @@ -30,14 +30,14 @@ class BaseProvider(BaseSource): def _include_change(self, change): ''' An opportunity for providers to filter out false positives due to - pecularities in their implementation. E.g. minimum TTLs. + peculiarities in their implementation. E.g. minimum TTLs. ''' return True def _extra_changes(self, existing, changes): ''' An opportunity for providers to add extra changes to the plan that are - necessary to update ancilary record data or configure the zone. E.g. + necessary to update ancillary record data or configure the zone. E.g. base NS records. ''' return [] diff --git a/octodns/provider/cloudflare.py b/octodns/provider/cloudflare.py index 688b10e..b66fcfa 100644 --- a/octodns/provider/cloudflare.py +++ b/octodns/provider/cloudflare.py @@ -394,7 +394,7 @@ class CloudflareProvider(BaseProvider): for record in self.zone_records(zone): name = zone.hostname_from_fqdn(record['name']) # Use the _record_for so that we include all of standard - # converstion logic + # conversion logic r = self._record_for(zone, name, record['type'], [record], True) if hostname == r.name and _type == r._type: diff --git a/octodns/provider/digitalocean.py b/octodns/provider/digitalocean.py index c68e7d6..5de8aee 100644 --- a/octodns/provider/digitalocean.py +++ b/octodns/provider/digitalocean.py @@ -56,7 +56,7 @@ class DigitalOceanClient(object): self._request('POST', '/domains', data={'name': name, 'ip_address': '192.0.2.1'}) - # After the zone is created, immeadiately delete the record + # After the zone is created, immediately delete the record records = self.records(name) for record in records: if record['name'] == '' and record['type'] == 'A': diff --git a/octodns/provider/dnsimple.py b/octodns/provider/dnsimple.py index 43b5b9b..ca95706 100644 --- a/octodns/provider/dnsimple.py +++ b/octodns/provider/dnsimple.py @@ -160,7 +160,7 @@ class DnsimpleProvider(BaseProvider): record['content'].split(' ', 5) except ValueError: # their api will let you create invalid records, this - # essnetially handles that by ignoring them for values + # essentially handles that by ignoring them for values # purposes. That will cause updates to happen to delete them if # they shouldn't exist or update them if they're wrong continue diff --git a/octodns/provider/dyn.py b/octodns/provider/dyn.py index 010fd31..bcd8308 100644 --- a/octodns/provider/dyn.py +++ b/octodns/provider/dyn.py @@ -40,7 +40,7 @@ class _CachingDynZone(DynZone): cls.log.debug('get: fetched') except DynectGetError: if not create: - cls.log.debug("get: does't exist") + cls.log.debug("get: doesn't exist") return None # this value shouldn't really matter, it's not tied to # whois or anything @@ -129,11 +129,11 @@ class DynProvider(BaseProvider): REGION_CODES = { 'NA': 11, # Continental North America 'SA': 12, # Continental South America - 'EU': 13, # Contentinal Europe + 'EU': 13, # Continental Europe 'AF': 14, # Continental Africa - 'AS': 15, # Contentinal Asia - 'OC': 16, # Contentinal Austrailia/Oceania - 'AN': 17, # Continental Antartica + 'AS': 15, # Continental Asia + 'OC': 16, # Continental Australia/Oceania + 'AN': 17, # Continental Antarctica } _sess_create_lock = Lock() @@ -166,7 +166,7 @@ class DynProvider(BaseProvider): if DynectSession.get_session() is None: # We need to create a new session for this thread and DynectSession # creation is not thread-safe so we have to do the locking. If we - # don't and multiple sessions start creattion before the the first + # don't and multiple sessions start creation before the the first # has finished (long time b/c it makes http calls) the subsequent # creates will blow away DynectSession._instances, potentially # multiple times if there are multiple creates in flight. Only the @@ -291,7 +291,7 @@ class DynProvider(BaseProvider): try: fqdn, _type = td.label.split(':', 1) except ValueError as e: - self.log.warn("Failed to load TraficDirector '%s': %s", + self.log.warn("Failed to load TrafficDirector '%s': %s", td.label, e.message) continue tds[fqdn][_type] = td diff --git a/octodns/provider/powerdns.py b/octodns/provider/powerdns.py index 4527f8e..aafb64e 100644 --- a/octodns/provider/powerdns.py +++ b/octodns/provider/powerdns.py @@ -178,7 +178,7 @@ class PowerDnsBaseProvider(BaseProvider): raise Exception('PowerDNS unauthorized host={}' .format(self.host)) elif e.response.status_code == 422: - # 422 means powerdns doesn't know anything about the requsted + # 422 means powerdns doesn't know anything about the requested # domain. We'll just ignore it here and leave the zone # untouched. pass @@ -294,8 +294,8 @@ class PowerDnsBaseProvider(BaseProvider): return [] # sorting mostly to make things deterministic for testing, but in - # theory it let us find what we're after quickier (though sorting would - # ve more exepensive.) + # theory it let us find what we're after quicker (though sorting would + # be more expensive.) for record in sorted(existing.records): if record == ns: # We've found the top-level NS record, return any changes @@ -341,7 +341,7 @@ class PowerDnsBaseProvider(BaseProvider): e.response.text) raise self.log.info('_apply: creating zone=%s', desired.name) - # 422 means powerdns doesn't know anything about the requsted + # 422 means powerdns doesn't know anything about the requested # domain. We'll try to create it with the correct records instead # of update. Hopefully all the mods are creates :-) data = { diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index 5bda074..cf9c3c4 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -61,7 +61,7 @@ class _Route53Record(object): # NOTE: we're using __hash__ and __cmp__ methods that consider # _Route53Records equivalent if they have the same class, fqdn, and _type. - # Values are ignored. This is usful when computing diffs/changes. + # Values are ignored. This is useful when computing diffs/changes. def __hash__(self): 'sub-classes should never use this method' @@ -679,7 +679,7 @@ class Route53Provider(BaseProvider): .get('CountryCode', False) == '*': # it's a default record continue - # we expect a healtcheck now + # we expect a healthcheck now try: health_check_id = rrset['HealthCheckId'] caller_ref = \ @@ -730,7 +730,7 @@ class Route53Provider(BaseProvider): batch_rs_count) # send the batch self._really_apply(batch, zone_id) - # start a new batch with the lefovers + # start a new batch with the leftovers batch = mods batch_rs_count = mods_rs_count diff --git a/octodns/record.py b/octodns/record.py index 82ee191..6d1e25b 100644 --- a/octodns/record.py +++ b/octodns/record.py @@ -151,7 +151,7 @@ class Record(object): # NOTE: we're using __hash__ and __cmp__ methods that consider Records # equivalent if they have the same name & _type. Values are ignored. This - # is usful when computing diffs/changes. + # is useful when computing diffs/changes. def __hash__(self): return '{}:{}'.format(self.name, self._type).__hash__() diff --git a/octodns/source/base.py b/octodns/source/base.py index 4ace09f..15455b8 100644 --- a/octodns/source/base.py +++ b/octodns/source/base.py @@ -30,7 +30,7 @@ class BaseSource(object): When `lenient` is True the populate call may skip record validation and do a "best effort" load of data. That will allow through some common, but not best practices stuff that we otherwise would reject. E.g. no - trailing . or mising escapes for ;. + trailing . or missing escapes for ;. ''' raise NotImplementedError('Abstract base class, populate method ' 'missing') diff --git a/octodns/zone.py b/octodns/zone.py index a8a91ca..bed3a59 100644 --- a/octodns/zone.py +++ b/octodns/zone.py @@ -37,10 +37,10 @@ class Zone(object): if not name[-1] == '.': raise Exception('Invalid zone name {}, missing ending dot' .format(name)) - # Force everyting to lowercase just to be safe + # Force everything to lowercase just to be safe self.name = unicode(name).lower() if name else name self.sub_zones = sub_zones - # We're grouping by node, it allows us to efficently search for + # We're grouping by node, it allows us to efficiently search for # duplicates and detect when CNAMEs co-exist with other records self._records = defaultdict(set) # optional leading . to match empty hostname diff --git a/script/release b/script/release index d8fabf2..bcc0ba3 100755 --- a/script/release +++ b/script/release @@ -11,4 +11,4 @@ git tag -s v$VERSION -m "Release $VERSION" git push origin v$VERSION echo "Tagged and pushed v$VERSION" python setup.py sdist upload -echo "Updloaded $VERSION" +echo "Uploaded $VERSION" diff --git a/tests/test_octodns_provider_base.py b/tests/test_octodns_provider_base.py index 855efaf..22544e1 100644 --- a/tests/test_octodns_provider_base.py +++ b/tests/test_octodns_provider_base.py @@ -63,14 +63,14 @@ class TestBaseProvider(TestCase): zone = Zone('unit.tests.', []) with self.assertRaises(NotImplementedError) as ctx: - HasSupportsGeo('hassupportesgeo').populate(zone) + HasSupportsGeo('hassupportsgeo').populate(zone) self.assertEquals('Abstract base class, SUPPORTS property missing', ctx.exception.message) class HasSupports(HasSupportsGeo): SUPPORTS = set(('A',)) with self.assertRaises(NotImplementedError) as ctx: - HasSupports('hassupportes').populate(zone) + HasSupports('hassupports').populate(zone) self.assertEquals('Abstract base class, populate method missing', ctx.exception.message) @@ -94,7 +94,7 @@ class TestBaseProvider(TestCase): 'value': '1.2.3.4' })) - self.assertTrue(HasSupports('hassupportesgeo') + self.assertTrue(HasSupports('hassupportsgeo') .supports(list(zone.records)[0])) plan = HasPopulate('haspopulate').plan(zone) diff --git a/tests/test_octodns_provider_cloudflare.py b/tests/test_octodns_provider_cloudflare.py index 7bb1b6a..36a6d35 100644 --- a/tests/test_octodns_provider_cloudflare.py +++ b/tests/test_octodns_provider_cloudflare.py @@ -599,7 +599,8 @@ class TestCloudflareProvider(TestCase): zone = Zone('unit.tests.', []) provider.populate(zone) - # the two A records get merged into one CNAME record poining to the CDN + # the two A records get merged into one CNAME record pointing to + # the CDN. self.assertEquals(3, len(zone.records)) record = list(zone.records)[0] @@ -621,7 +622,7 @@ class TestCloudflareProvider(TestCase): self.assertEquals('a.unit.tests.cdn.cloudflare.net.', record.value) # CDN enabled records can't be updated, we don't know the real values - # never point a Cloudflare record to itsself. + # never point a Cloudflare record to itself. wanted = Zone('unit.tests.', []) wanted.add_record(Record.new(wanted, 'cname', { 'ttl': 300, @@ -676,7 +677,7 @@ class TestCloudflareProvider(TestCase): self.assertEquals('unit.tests.cdn.cloudflare.net.', record.value) # CDN enabled records can't be updated, we don't know the real values - # never point a Cloudflare record to itsself. + # never point a Cloudflare record to itself. wanted = Zone('unit.tests.', []) wanted.add_record(Record.new(wanted, '', { 'ttl': 300, diff --git a/tests/test_octodns_provider_dyn.py b/tests/test_octodns_provider_dyn.py index 4415347..d1be606 100644 --- a/tests/test_octodns_provider_dyn.py +++ b/tests/test_octodns_provider_dyn.py @@ -491,7 +491,7 @@ class TestDynProviderGeo(TestCase): traffic_director_response = loads(fh.read()) @property - def traffic_directors_reponse(self): + def traffic_directors_response(self): return { 'data': [{ 'active': 'Y', @@ -607,7 +607,7 @@ class TestDynProviderGeo(TestCase): mock.side_effect = [{'data': []}] self.assertEquals({}, provider.traffic_directors) - # a supported td and an ingored one + # a supported td and an ignored one response = { 'data': [{ 'active': 'Y', @@ -650,7 +650,7 @@ class TestDynProviderGeo(TestCase): set(tds.keys())) self.assertEquals(['A'], tds['unit.tests.'].keys()) self.assertEquals(['A'], tds['geo.unit.tests.'].keys()) - provider.log.warn.assert_called_with("Failed to load TraficDirector " + provider.log.warn.assert_called_with("Failed to load TrafficDirector " "'%s': %s", 'something else', 'need more than 1 value to ' 'unpack') @@ -758,7 +758,7 @@ class TestDynProviderGeo(TestCase): # only traffic director mock.side_effect = [ # get traffic directors - self.traffic_directors_reponse, + self.traffic_directors_response, # get traffic director self.traffic_director_response, # get zone @@ -809,7 +809,7 @@ class TestDynProviderGeo(TestCase): # both traffic director and regular, regular is ignored mock.side_effect = [ # get traffic directors - self.traffic_directors_reponse, + self.traffic_directors_response, # get traffic director self.traffic_director_response, # get zone @@ -859,7 +859,7 @@ class TestDynProviderGeo(TestCase): # busted traffic director mock.side_effect = [ # get traffic directors - self.traffic_directors_reponse, + self.traffic_directors_response, # get traffic director busted_traffic_director_response, # get zone @@ -932,14 +932,14 @@ class TestDynProviderGeo(TestCase): provider = DynProvider('test', 'cust', 'user', 'pass', traffic_directors_enabled=True) - # will be tested seperately + # will be tested separately provider._mod_rulesets = MagicMock() mock.side_effect = [ # create traffic director self.traffic_director_response, # get traffic directors - self.traffic_directors_reponse + self.traffic_directors_response ] provider._mod_geo_Create(None, Create(self.geo_record)) # td now lives in cache diff --git a/tests/test_octodns_provider_googlecloud.py b/tests/test_octodns_provider_googlecloud.py index 6c6c2aa..127cb53 100644 --- a/tests/test_octodns_provider_googlecloud.py +++ b/tests/test_octodns_provider_googlecloud.py @@ -361,10 +361,10 @@ class TestGoogleCloudProvider(TestCase): # test_zone gets fed the same records as zone does, except it's in # the format returned by google API, so after populate they should look - # excactly the same. + # exactly the same. self.assertEqual(test_zone.records, zone.records) - test_zone2 = Zone('nonexistant.zone.', []) + test_zone2 = Zone('nonexistent.zone.', []) provider.populate(test_zone2, False, False) self.assertEqual(len(test_zone2.records), 0, @@ -401,8 +401,8 @@ class TestGoogleCloudProvider(TestCase): provider.gcloud_client.list_zones = Mock( return_value=DummyIterator([])) - self.assertIsNone(provider.gcloud_zones.get("nonexistant.xone"), - msg="Check that nonexistant zones return None when" + self.assertIsNone(provider.gcloud_zones.get("nonexistent.zone"), + msg="Check that nonexistent zones return None when" "there's no create=True flag") def test__get_rrsets(self): @@ -423,7 +423,7 @@ class TestGoogleCloudProvider(TestCase): provider.gcloud_client.list_zones = Mock( return_value=DummyIterator([])) - mock_zone = provider._create_gcloud_zone("nonexistant.zone.mock") + mock_zone = provider._create_gcloud_zone("nonexistent.zone.mock") mock_zone.create.assert_called() provider.gcloud_client.zone.assert_called() diff --git a/tests/test_octodns_provider_ovh.py b/tests/test_octodns_provider_ovh.py index 9404f9e..3e7affe 100644 --- a/tests/test_octodns_provider_ovh.py +++ b/tests/test_octodns_provider_ovh.py @@ -199,14 +199,14 @@ class TestOvhProvider(TestCase): api_record.append({ 'fieldType': 'SPF', 'ttl': 1000, - 'target': 'v=spf1 include:unit.texts.rerirect ~all', + 'target': 'v=spf1 include:unit.texts.redirect ~all', 'subDomain': '', 'id': 13 }) expected.add(Record.new(zone, '', { 'ttl': 1000, 'type': 'SPF', - 'value': 'v=spf1 include:unit.texts.rerirect ~all' + 'value': 'v=spf1 include:unit.texts.redirect ~all' })) # SSHFP @@ -404,7 +404,7 @@ class TestOvhProvider(TestCase): call(u'/domain/zone/unit.tests/record', fieldType=u'SPF', subDomain=u'', ttl=1000, target=u'v=spf1 include:unit.texts.' - u'rerirect ~all', + u'redirect ~all', ), call(u'/domain/zone/unit.tests/record', fieldType=u'A', subDomain='sub', target=u'1.2.3.4', ttl=200), diff --git a/tests/test_octodns_provider_powerdns.py b/tests/test_octodns_provider_powerdns.py index 22ccdd6..5d89c3e 100644 --- a/tests/test_octodns_provider_powerdns.py +++ b/tests/test_octodns_provider_powerdns.py @@ -100,7 +100,7 @@ class TestPowerDnsProvider(TestCase): # No existing records -> creates for every record in expected with requests_mock() as mock: mock.get(ANY, status_code=200, text=EMPTY_TEXT) - # post 201, is reponse to the create with data + # post 201, is response to the create with data mock.patch(ANY, status_code=201, text=assert_rrsets_callback) plan = provider.plan(expected) @@ -118,7 +118,7 @@ class TestPowerDnsProvider(TestCase): mock.get(ANY, status_code=422, text='') # patch 422's, unknown zone mock.patch(ANY, status_code=422, text=dumps(not_found)) - # post 201, is reponse to the create with data + # post 201, is response to the create with data mock.post(ANY, status_code=201, text=assert_rrsets_callback) plan = provider.plan(expected) diff --git a/tests/test_octodns_provider_route53.py b/tests/test_octodns_provider_route53.py index 1cd4548..c2afcd2 100644 --- a/tests/test_octodns_provider_route53.py +++ b/tests/test_octodns_provider_route53.py @@ -331,9 +331,9 @@ class TestRoute53Provider(TestCase): stubber.assert_no_pending_responses() # Populate a zone that doesn't exist - noexist = Zone('does.not.exist.', []) - provider.populate(noexist) - self.assertEquals(set(), noexist.records) + nonexistent = Zone('does.not.exist.', []) + provider.populate(nonexistent) + self.assertEquals(set(), nonexistent.records) def test_sync(self): provider, stubber = self._get_stubbed_provider() diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index 9f3dc33..56502a0 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -430,7 +430,7 @@ class TestRecord(TestCase): self.assertEqual(change.new, other) # full sorting - # equivilent + # equivalent b_naptr_value = b.values[0] self.assertEquals(0, b_naptr_value.__cmp__(b_naptr_value)) # by order @@ -710,7 +710,7 @@ class TestRecord(TestCase): Record.new(self.zone, 'unknown', {}) self.assertTrue('missing type' in ctx.exception.message) - # Unkown type + # Unknown type with self.assertRaises(Exception) as ctx: Record.new(self.zone, 'unknown', { 'type': 'XXX', From 876c09dcc07efb721f3ac08262d23bb2d9861a4e Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 3 Mar 2018 10:12:34 -0800 Subject: [PATCH 138/141] Flesh out UT for new Plan.exists messaging --- tests/test_octodns_plan.py | 45 ++++++++++++++++++++++++++++---------- 1 file changed, 33 insertions(+), 12 deletions(-) diff --git a/tests/test_octodns_plan.py b/tests/test_octodns_plan.py index 91dd948..7d849be 100644 --- a/tests/test_octodns_plan.py +++ b/tests/test_octodns_plan.py @@ -16,15 +16,6 @@ from octodns.zone import Zone from helpers import SimpleProvider -class TestPlanLogger(TestCase): - - def test_invalid_level(self): - with self.assertRaises(Exception) as ctx: - PlanLogger('invalid', 'not-a-level') - self.assertEquals('Unsupported level: not-a-level', - ctx.exception.message) - - simple = SimpleProvider() zone = Zone('unit.tests.', []) existing = Record.new(zone, 'a', { @@ -48,15 +39,45 @@ create = Create(Record.new(zone, 'b', { 'type': 'CNAME', 'value': 'foo.unit.tests.' }, simple)) +create2 = Create(Record.new(zone, 'c', { + 'ttl': 60, + 'type': 'CNAME', + 'value': 'foo.unit.tests.' +})) update = Update(existing, new) delete = Delete(new) -changes = [create, delete, update] +changes = [create, create2, delete, update] plans = [ (simple, Plan(zone, zone, changes, True)), - (simple, Plan(zone, zone, changes, True)), + (simple, Plan(zone, zone, changes, False)), ] +class TestPlanLogger(TestCase): + + def test_invalid_level(self): + with self.assertRaises(Exception) as ctx: + PlanLogger('invalid', 'not-a-level') + self.assertEquals('Unsupported level: not-a-level', + ctx.exception.message) + + def test_create(self): + + class MockLogger(object): + + def __init__(self): + self.out = StringIO() + + def log(self, level, msg): + self.out.write(msg) + + log = MockLogger() + PlanLogger('logger').run(log, plans) + out = log.out.getvalue() + self.assertTrue('Summary: Creates=2, Updates=1, ' + 'Deletes=1, Existing Records=0' in out) + + class TestPlanHtml(TestCase): log = getLogger('TestPlanHtml') @@ -69,7 +90,7 @@ class TestPlanHtml(TestCase): out = StringIO() PlanHtml('html').run(plans, fh=out) out = out.getvalue() - self.assertTrue(' Summary: Creates=1, Updates=1, ' + self.assertTrue(' Summary: Creates=2, Updates=1, ' 'Deletes=1, Existing Records=0' in out) From 80adb22a4ba235fb685d273dfed9ffe4770af2ef Mon Sep 17 00:00:00 2001 From: Michael Vermaes Date: Sun, 4 Mar 2018 14:18:21 +0800 Subject: [PATCH 139/141] Check Route 53 records against all supported types --- octodns/provider/route53.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index 3333f4e..9d34d25 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -461,7 +461,7 @@ class Route53Provider(BaseProvider): record_name = zone.hostname_from_fqdn(rrset['Name']) record_name = _octal_replace(record_name) record_type = rrset['Type'] - if record_type == 'SOA': + if record_type not in self.SUPPORTS: continue data = getattr(self, '_data_for_{}'.format(record_type))(rrset) records[record_name][record_type].append(data) From 8c1fe707e8493230835fffbb02088b1b6cd6b48d Mon Sep 17 00:00:00 2001 From: Michael McAllister Date: Sun, 4 Mar 2018 18:02:51 +1100 Subject: [PATCH 140/141] ISSUE #26 Skip Alias recordset for Route53 Provider --- octodns/provider/route53.py | 6 ++++++ tests/test_octodns_provider_route53.py | 8 ++++++++ 2 files changed, 14 insertions(+) diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index 3333f4e..d206320 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -463,6 +463,12 @@ class Route53Provider(BaseProvider): record_type = rrset['Type'] if record_type == 'SOA': continue + if 'AliasTarget' in rrset: + # Alias records are Route53 specific and are not + # portable, so we need to skip them + self.log.warning("%s is an Alias record. Skipping..." + % rrset['Name']) + continue data = getattr(self, '_data_for_{}'.format(record_type))(rrset) records[record_name][record_type].append(data) diff --git a/tests/test_octodns_provider_route53.py b/tests/test_octodns_provider_route53.py index 3839a16..f4fc99f 100644 --- a/tests/test_octodns_provider_route53.py +++ b/tests/test_octodns_provider_route53.py @@ -313,6 +313,14 @@ class TestRoute53Provider(TestCase): 'Value': '0 issue "ca.unit.tests"', }], 'TTL': 69, + }, { + 'AliasTarget': { + 'HostedZoneId': 'Z119WBBTVP5WFX', + 'EvaluateTargetHealth': False, + 'DNSName': 'unit.tests.' + }, + 'Type': 'A', + 'Name': 'alias.unit.tests.' }], 'IsTruncated': False, 'MaxItems': '100', From 29b6f5a88682c43b63467e2a7b894acf2844083c Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Thu, 15 Mar 2018 06:22:40 -0700 Subject: [PATCH 141/141] Unsorted GeoValue.values can result in false diffs --- octodns/record.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/record.py b/octodns/record.py index 6d1e25b..0ae1335 100644 --- a/octodns/record.py +++ b/octodns/record.py @@ -184,7 +184,7 @@ class GeoValue(object): self.continent_code = match.group('continent_code') self.country_code = match.group('country_code') self.subdivision_code = match.group('subdivision_code') - self.values = values + self.values = sorted(values) @property def parents(self):