From 679c2be0e076f6d7f0ec8daf89385ebd7fad6564 Mon Sep 17 00:00:00 2001 From: Vietor Davis Date: Mon, 26 Jun 2017 17:03:15 -0700 Subject: [PATCH 01/35] Start sketchin of Rackspace provider, half rewritten from powerdns... --- octodns/provider/rackspace.py | 386 ++++++++++++++++++ tests/fixtures/rackspace-auth-response.json | 87 ++++ .../rackspace-list-domains-response.json | 68 +++ .../rackspace-sample-recordset-page1.json | 33 ++ .../rackspace-sample-recordset-page2.json | 35 ++ tests/test_octodns_source_rackspace.py | 294 +++++++++++++ 6 files changed, 903 insertions(+) create mode 100644 octodns/provider/rackspace.py create mode 100644 tests/fixtures/rackspace-auth-response.json create mode 100644 tests/fixtures/rackspace-list-domains-response.json create mode 100644 tests/fixtures/rackspace-sample-recordset-page1.json create mode 100644 tests/fixtures/rackspace-sample-recordset-page2.json create mode 100644 tests/test_octodns_source_rackspace.py diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py new file mode 100644 index 0000000..311d02d --- /dev/null +++ b/octodns/provider/rackspace.py @@ -0,0 +1,386 @@ +# +# +# + + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from requests import HTTPError, Session, post +import json +import logging + +from ..record import Create, Record +from .base import BaseProvider + + +class RackspaceProvider(BaseProvider): + SUPPORTS_GEO = False + TIMEOUT = 5 + + def __init__(self, username, api_key, *args, **kwargs): + ''' + Rackspace API v1 Provider + + rackspace: + class: octodns.provider.rackspace.RackspaceProvider + # The the username to authenticate with (required) + username: username + # The api key that grants access for that user (required) + api_key: api-key + ''' + self.log = logging.getLogger('RackspaceProvider[{}]'.format(username)) + super(RackspaceProvider, self).__init__(id, *args, **kwargs) + + auth_token, dns_endpoint = self._get_auth_token(username, api_key) + self.dns_endpoint = dns_endpoint + + sess = Session() + sess.headers.update({'X-Auth-Token': auth_token}) + self._sess = sess + + def _get_auth_token(self, username, api_key): + ret = post('https://identity.api.rackspacecloud.com/v2.0/tokens', + json={"auth": {"RAX-KSKEY:apiKeyCredentials": {"username": username, "apiKey": api_key}}}, + ) + cloud_dns_endpoint = [x for x in ret.json()['access']['serviceCatalog'] if x['name'] == 'cloudDNS'][0]['endpoints'][0]['publicURL'] + return ret.json()['access']['token']['id'], cloud_dns_endpoint + + def _get_zone_id_for(self, zone_name): + ret = self._request('GET', 'domains', pagination_key='domains') + if ret and 'name' in ret: + return [x for x in ret if x['name'] == zone_name][0]['id'] + else: + return None + + def _request(self, method, path, data=None, pagination_key=None): + self.log.debug('_request: method=%s, path=%s', method, path) + url = '{}/{}'.format(self.dns_endpoint, path) + + if pagination_key: + return self._paginated_request_for_url(method, url, data, pagination_key) + else: + return self._request_for_url(method, url, data) + + def _request_for_url(self, method, url, data): + resp = self._sess.request(method, url, json=data, timeout=self.TIMEOUT) + self.log.debug('_request: status=%d', resp.status_code) + resp.raise_for_status() + return resp + + def _paginated_request_for_url(self, method, url, data, pagination_key): + acc = [] + + resp = self._sess.request(method, url, json=data, timeout=self.TIMEOUT) + self.log.debug('_request: status=%d', resp.status_code) + resp.raise_for_status() + acc.extend(resp.json()[pagination_key]) + + next_page = [x for x in resp.json().get('links', []) if x['rel'] == 'next'] + if next_page: + url = next_page[0]['href'] + return acc.extend(self._paginated_request_for_url(method, url, data, pagination_key)) + else: + return acc + + def _get(self, path, data=None): + return self._request('GET', path, data=data) + + def _post(self, path, data=None): + return self._request('POST', path, data=data) + + def _patch(self, path, data=None): + return self._request('PATCH', path, data=data) + + def _data_for_multiple(self, rrset): + # TODO: geo not supported + return { + 'type': rrset['type'], + 'values': [r['content'] for r in rrset['records']], + 'ttl': rrset['ttl'] + } + + _data_for_A = _data_for_multiple + _data_for_AAAA = _data_for_multiple + _data_for_NS = _data_for_multiple + + def _data_for_single(self, record): + return { + 'type': record['type'], + 'value': record['data'], + 'ttl': record['ttl'] + } + + _data_for_ALIAS = _data_for_single + _data_for_CNAME = _data_for_single + _data_for_PTR = _data_for_single + + def _data_for_quoted(self, rrset): + return { + 'type': rrset['type'], + 'values': [r['content'][1:-1] for r in rrset['records']], + 'ttl': rrset['ttl'] + } + + _data_for_SPF = _data_for_quoted + _data_for_TXT = _data_for_quoted + + def _data_for_MX(self, rrset): + values = [] + for record in rrset['records']: + priority, value = record['content'].split(' ', 1) + values.append({ + 'priority': priority, + 'value': value, + }) + return { + 'type': rrset['type'], + 'values': values, + 'ttl': rrset['ttl'] + } + + def _data_for_NAPTR(self, rrset): + values = [] + for record in rrset['records']: + order, preference, flags, service, regexp, replacement = \ + record['content'].split(' ', 5) + values.append({ + 'order': order, + 'preference': preference, + 'flags': flags[1:-1], + 'service': service[1:-1], + 'regexp': regexp[1:-1], + 'replacement': replacement, + }) + return { + 'type': rrset['type'], + 'values': values, + 'ttl': rrset['ttl'] + } + + def _data_for_SSHFP(self, rrset): + values = [] + for record in rrset['records']: + algorithm, fingerprint_type, fingerprint = \ + record['content'].split(' ', 2) + values.append({ + 'algorithm': algorithm, + 'fingerprint_type': fingerprint_type, + 'fingerprint': fingerprint, + }) + return { + 'type': rrset['type'], + 'values': values, + 'ttl': rrset['ttl'] + } + + def _data_for_SRV(self, rrset): + values = [] + for record in rrset['records']: + priority, weight, port, target = \ + record['content'].split(' ', 3) + values.append({ + 'priority': priority, + 'weight': weight, + 'port': port, + 'target': target, + }) + return { + 'type': rrset['type'], + 'values': values, + 'ttl': rrset['ttl'] + } + + def populate(self, zone, target=False): + self.log.debug('populate: name=%s', zone.name) + resp = None + try: + domain_id = self._get_zone_id_for(zone.name) + resp = self._request('GET', '/domains/{}/records'.format(domain_id), pagination_key='records') + self.log.debug('populate: loaded') + except HTTPError as e: + if e.response.status_code == 401: + # Nicer error message for auth problems + raise Exception('Rackspace request unauthorized') + elif e.response.status_code == 422: + # 422 means powerdns doesn't know anything about the requsted + # domain. We'll just ignore it here and leave the zone + # untouched. + pass + else: + # just re-throw + raise + + before = len(zone.records) + + if resp: + for record in resp.json()['records']: + record_type = record['type'] + if record_type == 'SOA': + continue + data_for = getattr(self, '_data_for_{}'.format(record_type)) + record_name = zone.hostname_from_fqdn(record['name']) + record = Record.new(zone, record_name, data_for(record), + source=self) + zone.add_record(record) + + self.log.info('populate: found %s records', + len(zone.records) - before) + + def _records_for_multiple(self, record): + return [{'content': v, 'disabled': False} + for v in record.values] + + _records_for_A = _records_for_multiple + _records_for_AAAA = _records_for_multiple + _records_for_NS = _records_for_multiple + + def _records_for_single(self, record): + return [{'content': record.value, 'disabled': False}] + + _records_for_ALIAS = _records_for_single + _records_for_CNAME = _records_for_single + _records_for_PTR = _records_for_single + + def _records_for_quoted(self, record): + return [{'content': '"{}"'.format(v), 'disabled': False} + for v in record.values] + + _records_for_SPF = _records_for_quoted + _records_for_TXT = _records_for_quoted + + def _records_for_MX(self, record): + return [{ + 'content': '{} {}'.format(v.priority, v.value), + 'disabled': False + } for v in record.values] + + def _records_for_NAPTR(self, record): + return [{ + 'content': '{} {} "{}" "{}" "{}" {}'.format(v.order, v.preference, + v.flags, v.service, + v.regexp, + v.replacement), + 'disabled': False + } for v in record.values] + + def _records_for_SSHFP(self, record): + return [{ + 'content': '{} {} {}'.format(v.algorithm, v.fingerprint_type, + v.fingerprint), + 'disabled': False + } for v in record.values] + + def _records_for_SRV(self, record): + return [{ + 'content': '{} {} {} {}'.format(v.priority, v.weight, v.port, + v.target), + 'disabled': False + } for v in record.values] + + def _mod_Create(self, change): + new = change.new + records_for = getattr(self, '_records_for_{}'.format(new._type)) + return { + 'name': new.fqdn, + 'type': new._type, + 'ttl': new.ttl, + 'changetype': 'REPLACE', + 'records': records_for(new) + } + + _mod_Update = _mod_Create + + def _mod_Delete(self, change): + existing = change.existing + records_for = getattr(self, '_records_for_{}'.format(existing._type)) + return { + 'name': existing.fqdn, + 'type': existing._type, + 'ttl': existing.ttl, + 'changetype': 'DELETE', + 'records': records_for(existing) + } + + def _get_nameserver_record(self, existing): + return None + + def _extra_changes(self, existing, _): + self.log.debug('_extra_changes: zone=%s', existing.name) + + ns = self._get_nameserver_record(existing) + if not ns: + return [] + + # sorting mostly to make things deterministic for testing, but in + # theory it let us find what we're after quickier (though sorting would + # ve more exepensive.) + for record in sorted(existing.records): + if record == ns: + # We've found the top-level NS record, return any changes + change = record.changes(ns, self) + self.log.debug('_extra_changes: change=%s', change) + if change: + # We need to modify an existing record + return [change] + # No change is necessary + return [] + # No existing top-level NS + self.log.debug('_extra_changes: create') + return [Create(ns)] + + def _get_error(self, http_error): + try: + return http_error.response.json()['error'] + except Exception: + return '' + + def _apply(self, plan): + desired = plan.desired + changes = plan.changes + self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name, + len(changes)) + + mods = [] + for change in changes: + class_name = change.__class__.__name__ + mods.append(getattr(self, '_mod_{}'.format(class_name))(change)) + self.log.debug('_apply: sending change request') + + try: + self._patch('zones/{}'.format(desired.name), + data={'rrsets': mods}) + self.log.debug('_apply: patched') + except HTTPError as e: + error = self._get_error(e) + if e.response.status_code != 422 or \ + not error.startswith('Could not find domain '): + self.log.error('_apply: status=%d, text=%s', + e.response.status_code, + e.response.text) + raise + self.log.info('_apply: creating zone=%s', desired.name) + # 422 means powerdns doesn't know anything about the requsted + # domain. We'll try to create it with the correct records instead + # of update. Hopefully all the mods are creates :-) + data = { + 'name': desired.name, + 'kind': 'Master', + 'masters': [], + 'nameservers': [], + 'rrsets': mods, + 'soa_edit_api': 'INCEPTION-INCREMENT', + 'serial': 0, + } + try: + self._post('zones', data) + except HTTPError as e: + self.log.error('_apply: status=%d, text=%s', + e.response.status_code, + e.response.text) + raise + self.log.debug('_apply: created') + + self.log.debug('_apply: complete') + + diff --git a/tests/fixtures/rackspace-auth-response.json b/tests/fixtures/rackspace-auth-response.json new file mode 100644 index 0000000..cc811c7 --- /dev/null +++ b/tests/fixtures/rackspace-auth-response.json @@ -0,0 +1,87 @@ +{ + "access": { + "token": { + "id": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "expires": "2014-11-24T22:05:39.115Z", + "tenant": { + "id": "110011", + "name": "110011" + }, + "RAX-AUTH:authenticatedBy": [ + "APIKEY" + ] + }, + "serviceCatalog": [ + { + "name": "cloudDatabases", + "endpoints": [ + { + "publicURL": "https://syd.databases.api.rackspacecloud.com/v1.0/110011", + "region": "SYD", + "tenantId": "110011" + }, + { + "publicURL": "https://dfw.databases.api.rackspacecloud.com/v1.0/110011", + "region": "DFW", + "tenantId": "110011" + }, + { + "publicURL": "https://ord.databases.api.rackspacecloud.com/v1.0/110011", + "region": "ORD", + "tenantId": "110011" + }, + { + "publicURL": "https://iad.databases.api.rackspacecloud.com/v1.0/110011", + "region": "IAD", + "tenantId": "110011" + }, + { + "publicURL": "https://hkg.databases.api.rackspacecloud.com/v1.0/110011", + "region": "HKG", + "tenantId": "110011" + } + ], + "type": "rax:database" + }, + { + "name": "cloudDNS", + "endpoints": [ + { + "publicURL": "https://dns.api.rackspacecloud.com/v1.0/110011", + "tenantId": "110011" + } + ], + "type": "rax:dns" + }, + { + "name": "rackCDN", + "endpoints": [ + { + "internalURL": "https://global.cdn.api.rackspacecloud.com/v1.0/110011", + "publicURL": "https://global.cdn.api.rackspacecloud.com/v1.0/110011", + "tenantId": "110011" + } + ], + "type": "rax:cdn" + } + ], + "user": { + "id": "123456", + "roles": [ + { + "description": "A Role that allows a user access to keystone Service methods", + "id": "6", + "name": "compute:default", + "tenantId": "110011" + }, + { + "description": "User Admin Role.", + "id": "3", + "name": "identity:user-admin" + } + ], + "name": "jsmith", + "RAX-AUTH:defaultRegion": "ORD" + } + } +} \ No newline at end of file diff --git a/tests/fixtures/rackspace-list-domains-response.json b/tests/fixtures/rackspace-list-domains-response.json new file mode 100644 index 0000000..f124837 --- /dev/null +++ b/tests/fixtures/rackspace-list-domains-response.json @@ -0,0 +1,68 @@ +{ + "totalEntries" : 10, + "domains" : [ { + "name" : "example.com", + "id" : 2725233, + "comment" : "Optional domain comment...", + "updated" : "2011-06-24T01:23:15.000+0000", + "accountId" : 1234, + "emailAddress" : "sample@rackspace.com", + "created" : "2011-06-24T01:12:51.000+0000" + }, { + "name" : "sub1.example.com", + "id" : 2725257, + "comment" : "1st sample subdomain", + "updated" : "2011-06-23T03:09:34.000+0000", + "accountId" : 1234, + "emailAddress" : "sample@rackspace.com", + "created" : "2011-06-23T03:09:33.000+0000" + }, { + "name" : "sub2.example.com", + "id" : 2725258, + "comment" : "1st sample subdomain", + "updated" : "2011-06-23T03:52:55.000+0000", + "accountId" : 1234, + "emailAddress" : "sample@rackspace.com", + "created" : "2011-06-23T03:52:55.000+0000" + }, { + "name" : "north.example.com", + "id" : 2725260, + "updated" : "2011-06-23T03:53:10.000+0000", + "accountId" : 1234, + "emailAddress" : "sample@rackspace.com", + "created" : "2011-06-23T03:53:09.000+0000" + }, { + "name" : "south.example.com", + "id" : 2725261, + "comment" : "Final sample subdomain", + "updated" : "2011-06-23T03:53:14.000+0000", + "accountId" : 1234, + "emailAddress" : "sample@rackspace.com", + "created" : "2011-06-23T03:53:14.000+0000" + }, { + "name" : "region2.example.net", + "id" : 2725352, + "updated" : "2011-06-23T20:21:06.000+0000", + "accountId" : 1234, + "created" : "2011-06-23T19:24:27.000+0000" + }, { + "name" : "example.org", + "id" : 2718984, + "updated" : "2011-05-03T14:47:32.000+0000", + "accountId" : 1234, + "created" : "2011-05-03T14:47:30.000+0000" + }, { + "name" : "rackspace.example", + "id" : 2722346, + "updated" : "2011-06-21T15:54:31.000+0000", + "accountId" : 1234, + "created" : "2011-06-15T19:02:07.000+0000" + }, { + "name" : "dnsaas.example", + "id" : 2722347, + "comment" : "Sample comment", + "updated" : "2011-06-21T15:54:31.000+0000", + "accountId" : 1234, + "created" : "2011-06-15T19:02:07.000+0000" + } ] +} diff --git a/tests/fixtures/rackspace-sample-recordset-page1.json b/tests/fixtures/rackspace-sample-recordset-page1.json new file mode 100644 index 0000000..72dc7dd --- /dev/null +++ b/tests/fixtures/rackspace-sample-recordset-page1.json @@ -0,0 +1,33 @@ +{ + "totalEntries" : 6, + "records" : [ { + "name" : "ftp.example.com", + "id" : "A-6817754", + "type" : "A", + "data" : "192.0.2.8", + "updated" : "2011-05-19T13:07:08.000+0000", + "ttl" : 5771, + "created" : "2011-05-18T19:53:09.000+0000" + }, { + "name" : "example.com", + "id" : "A-6822994", + "type" : "A", + "data" : "192.0.2.17", + "updated" : "2011-06-24T01:12:52.000+0000", + "ttl" : 86400, + "created" : "2011-06-24T01:12:52.000+0000" + }, { + "name" : "example.com", + "id" : "NS-6251982", + "type" : "NS", + "data" : "ns.rackspace.com", + "updated" : "2011-06-24T01:12:51.000+0000", + "ttl" : 3600, + "created" : "2011-06-24T01:12:51.000+0000" + } ], + "links" : [ { + "content" : "", + "href" : "https://localhost/v1.0/1234/domains/domain_id/records?limit=3&offset=3", + "rel" : "next" + } ] +} diff --git a/tests/fixtures/rackspace-sample-recordset-page2.json b/tests/fixtures/rackspace-sample-recordset-page2.json new file mode 100644 index 0000000..dc3e39a --- /dev/null +++ b/tests/fixtures/rackspace-sample-recordset-page2.json @@ -0,0 +1,35 @@ +{ + "totalEntries" : 6, + "records" : [ { + "name" : "example.com", + "id" : "NS-6251983", + "type" : "NS", + "data" : "ns2.rackspace.com", + "updated" : "2011-06-24T01:12:51.000+0000", + "ttl" : 3600, + "created" : "2011-06-24T01:12:51.000+0000" + }, { + "name" : "example.com", + "priority" : 5, + "id" : "MX-3151218", + "type" : "MX", + "data" : "mail.example.com", + "updated" : "2011-06-24T01:12:53.000+0000", + "ttl" : 3600, + "created" : "2011-06-24T01:12:53.000+0000" + }, { + "name" : "www.example.com", + "id" : "CNAME-9778009", + "type" : "CNAME", + "comment" : "This is a comment on the CNAME record", + "data" : "example.com", + "updated" : "2011-06-24T01:12:54.000+0000", + "ttl" : 5400, + "created" : "2011-06-24T01:12:54.000+0000" + } ], + "links" : [ { + "content" : "", + "href" : "https://dns.api.rackspacecloud.com/v1.0/1234/domains/domain_id/records?limit=3&offset=0", + "rel" : "previous" + }] +} diff --git a/tests/test_octodns_source_rackspace.py b/tests/test_octodns_source_rackspace.py new file mode 100644 index 0000000..88ccb5f --- /dev/null +++ b/tests/test_octodns_source_rackspace.py @@ -0,0 +1,294 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +import re +from json import loads, dumps +from os.path import dirname, join +from unittest import TestCase + +from requests import HTTPError +from requests_mock import ANY, mock as requests_mock + +from octodns.provider.rackspace import RackspaceProvider +from octodns.provider.yaml import YamlProvider +from octodns.record import Record +from octodns.zone import Zone + +EMPTY_TEXT = ''' +{ + "totalEntries" : 6, + "records" : [] +} +''' + +with open('./tests/fixtures/rackspace-auth-response.json') as fh: + AUTH_RESPONSE = fh.read() + +with open('./tests/fixtures/rackspace-list-domains-response.json') as fh: + LIST_DOMAINS_RESPONSE = fh.read() + +with open('./tests/fixtures/rackspace-sample-recordset-page1.json') as fh: + RECORDS_PAGE_1 = fh.read() + +with open('./tests/fixtures/rackspace-sample-recordset-page2.json') as fh: + RECORDS_PAGE_2 = fh.read() + +def load_provider(): + with requests_mock() as mock: + mock.post(ANY, status_code=200, text=AUTH_RESPONSE) + return RackspaceProvider('test', 'api-key') + + +class TestRackspaceSource(TestCase): + + def test_provider(self): + provider = load_provider() + + # Bad auth + with requests_mock() as mock: + mock.get(ANY, status_code=401, text='Unauthorized') + + with self.assertRaises(Exception) as ctx: + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertTrue('unauthorized' in ctx.exception.message) + + # General error + with requests_mock() as mock: + mock.get(ANY, status_code=502, text='Things caught fire') + + with self.assertRaises(HTTPError) as ctx: + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals(502, ctx.exception.response.status_code) + + # Non-existant zone doesn't populate anything + with requests_mock() as mock: + mock.get(ANY, status_code=422, + json={'error': "Could not find domain 'unit.tests.'"}) + + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals(set(), zone.records) + + # The rest of this is messy/complicated b/c it's dealing with mocking + + expected = Zone('unit.tests.', []) + source = YamlProvider('test', join(dirname(__file__), 'config')) + source.populate(expected) + expected_n = len(expected.records) - 1 + self.assertEquals(14, expected_n) + + # No diffs == no changes + with requests_mock() as mock: + mock.get(re.compile('domains$'), status_code=200, text=LIST_DOMAINS_RESPONSE) + mock.get(re.compile('records'), status_code=200, text=RECORDS_PAGE_1) + mock.get(re.compile('records.*offset=3'), status_code=200, text=RECORDS_PAGE_2) + + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals(14, len(zone.records)) + changes = expected.changes(zone, provider) + self.assertEquals(0, len(changes)) + + # Used in a minute + def assert_rrsets_callback(request, context): + data = loads(request.body) + self.assertEquals(expected_n, len(data['rrsets'])) + return '' + + # No existing records -> creates for every record in expected + with requests_mock() as mock: + mock.get(ANY, status_code=200, text=EMPTY_TEXT) + # post 201, is reponse to the create with data + mock.patch(ANY, status_code=201, text=assert_rrsets_callback) + + plan = provider.plan(expected) + self.assertEquals(expected_n, len(plan.changes)) + self.assertEquals(expected_n, provider.apply(plan)) + + # Non-existent zone -> creates for every record in expected + # OMG this is fucking ugly, probably better to ditch requests_mocks and + # just mock things for real as it doesn't seem to provide a way to get + # at the request params or verify that things were called from what I + # can tell + not_found = {'error': "Could not find domain 'unit.tests.'"} + with requests_mock() as mock: + # get 422's, unknown zone + mock.get(ANY, status_code=422, text='') + # patch 422's, unknown zone + mock.patch(ANY, status_code=422, text=dumps(not_found)) + # post 201, is reponse to the create with data + mock.post(ANY, status_code=201, text=assert_rrsets_callback) + + plan = provider.plan(expected) + self.assertEquals(expected_n, len(plan.changes)) + self.assertEquals(expected_n, provider.apply(plan)) + + with requests_mock() as mock: + # get 422's, unknown zone + mock.get(ANY, status_code=422, text='') + # patch 422's, + data = {'error': "Key 'name' not present or not a String"} + mock.patch(ANY, status_code=422, text=dumps(data)) + + with self.assertRaises(HTTPError) as ctx: + plan = provider.plan(expected) + provider.apply(plan) + response = ctx.exception.response + self.assertEquals(422, response.status_code) + self.assertTrue('error' in response.json()) + + with requests_mock() as mock: + # get 422's, unknown zone + mock.get(ANY, status_code=422, text='') + # patch 500's, things just blew up + mock.patch(ANY, status_code=500, text='') + + with self.assertRaises(HTTPError): + plan = provider.plan(expected) + provider.apply(plan) + + with requests_mock() as mock: + # get 422's, unknown zone + mock.get(ANY, status_code=422, text='') + # patch 500's, things just blew up + mock.patch(ANY, status_code=422, text=dumps(not_found)) + # post 422's, something wrong with create + mock.post(ANY, status_code=422, text='Hello Word!') + + with self.assertRaises(HTTPError): + plan = provider.plan(expected) + provider.apply(plan) + + def test_small_change(self): + provider = load_provider() + + expected = Zone('unit.tests.', []) + source = YamlProvider('test', join(dirname(__file__), 'config')) + source.populate(expected) + self.assertEquals(15, len(expected.records)) + + # A small change to a single record + with requests_mock() as mock: + mock.get(ANY, status_code=200, text=FULL_TEXT) + + missing = Zone(expected.name, []) + # Find and delete the SPF record + for record in expected.records: + if record._type != 'SPF': + missing.add_record(record) + + def assert_delete_callback(request, context): + self.assertEquals({ + 'rrsets': [{ + 'records': [ + {'content': '"v=spf1 ip4:192.168.0.1/16-all"', + 'disabled': False} + ], + 'changetype': 'DELETE', + 'type': 'SPF', + 'name': 'spf.unit.tests.', + 'ttl': 600 + }] + }, loads(request.body)) + return '' + + mock.patch(ANY, status_code=201, text=assert_delete_callback) + + plan = provider.plan(missing) + self.assertEquals(1, len(plan.changes)) + self.assertEquals(1, provider.apply(plan)) + + def test_existing_nameservers(self): + ns_values = ['8.8.8.8.', '9.9.9.9.'] + provider = load_provider() + + expected = Zone('unit.tests.', []) + ns_record = Record.new(expected, '', { + 'type': 'NS', + 'ttl': 600, + 'values': ns_values + }) + expected.add_record(ns_record) + + # no changes + with requests_mock() as mock: + data = { + 'rrsets': [{ + 'comments': [], + 'name': 'unit.tests.', + 'records': [ + { + 'content': '8.8.8.8.', + 'disabled': False + }, + { + 'content': '9.9.9.9.', + 'disabled': False + } + ], + 'ttl': 600, + 'type': 'NS' + }, { + 'comments': [], + 'name': 'unit.tests.', + 'records': [{ + 'content': '1.2.3.4', + 'disabled': False, + }], + 'ttl': 60, + 'type': 'A' + }] + } + mock.get(ANY, status_code=200, json=data) + + unrelated_record = Record.new(expected, '', { + 'type': 'A', + 'ttl': 60, + 'value': '1.2.3.4' + }) + expected.add_record(unrelated_record) + plan = provider.plan(expected) + self.assertFalse(plan) + # remove it now that we don't need the unrelated change any longer + expected.records.remove(unrelated_record) + + # ttl diff + with requests_mock() as mock: + data = { + 'rrsets': [{ + 'comments': [], + 'name': 'unit.tests.', + 'records': [ + { + 'content': '8.8.8.8.', + 'disabled': False + }, + { + 'content': '9.9.9.9.', + 'disabled': False + }, + ], + 'ttl': 3600, + 'type': 'NS' + }] + } + mock.get(ANY, status_code=200, json=data) + + plan = provider.plan(expected) + self.assertEquals(1, len(plan.changes)) + + # create + with requests_mock() as mock: + data = { + 'rrsets': [] + } + mock.get(ANY, status_code=200, json=data) + + plan = provider.plan(expected) + self.assertEquals(1, len(plan.changes)) From c19ec41b6bb82c2b64cf39cec09cafd7807561c5 Mon Sep 17 00:00:00 2001 From: Vietor Davis Date: Fri, 7 Jul 2017 18:21:59 -0700 Subject: [PATCH 02/35] Parse all data in the sample return set --- octodns/provider/rackspace.py | 75 +++++++++++-------- .../rackspace-list-domains-response.json | 2 +- 2 files changed, 46 insertions(+), 31 deletions(-) diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py index 311d02d..f6817ae 100644 --- a/octodns/provider/rackspace.py +++ b/octodns/provider/rackspace.py @@ -8,6 +8,7 @@ from __future__ import absolute_import, division, print_function, \ from requests import HTTPError, Session, post import json +from collections import defaultdict import logging from ..record import Create, Record @@ -46,10 +47,10 @@ class RackspaceProvider(BaseProvider): cloud_dns_endpoint = [x for x in ret.json()['access']['serviceCatalog'] if x['name'] == 'cloudDNS'][0]['endpoints'][0]['publicURL'] return ret.json()['access']['token']['id'], cloud_dns_endpoint - def _get_zone_id_for(self, zone_name): + def _get_zone_id_for(self, zone): ret = self._request('GET', 'domains', pagination_key='domains') - if ret and 'name' in ret: - return [x for x in ret if x['name'] == zone_name][0]['id'] + if ret: + return [x for x in ret if x['name'] == zone.name[:-1]][0]['id'] else: return None @@ -79,7 +80,8 @@ class RackspaceProvider(BaseProvider): next_page = [x for x in resp.json().get('links', []) if x['rel'] == 'next'] if next_page: url = next_page[0]['href'] - return acc.extend(self._paginated_request_for_url(method, url, data, pagination_key)) + acc.extend(self._paginated_request_for_url(method, url, data, pagination_key)) + return acc else: return acc @@ -95,20 +97,27 @@ class RackspaceProvider(BaseProvider): def _data_for_multiple(self, rrset): # TODO: geo not supported return { - 'type': rrset['type'], - 'values': [r['content'] for r in rrset['records']], - 'ttl': rrset['ttl'] + 'type': rrset[0]['type'], + 'values': [r['data'] for r in rrset], + 'ttl': rrset[0]['ttl'] } _data_for_A = _data_for_multiple _data_for_AAAA = _data_for_multiple - _data_for_NS = _data_for_multiple + + def _data_for_NS(self, rrset): + # TODO: geo not supported + return { + 'type': rrset[0]['type'], + 'values': ["{}.".format(r['data']) for r in rrset], + 'ttl': rrset[0]['ttl'] + } def _data_for_single(self, record): return { - 'type': record['type'], - 'value': record['data'], - 'ttl': record['ttl'] + 'type': record[0]['type'], + 'value': "{}.".format(record[0]['data']), + 'ttl': record[0]['ttl'] } _data_for_ALIAS = _data_for_single @@ -127,16 +136,15 @@ class RackspaceProvider(BaseProvider): def _data_for_MX(self, rrset): values = [] - for record in rrset['records']: - priority, value = record['content'].split(' ', 1) + for record in rrset: values.append({ - 'priority': priority, - 'value': value, + 'priority': record['priority'], + 'value': record['data'], }) return { - 'type': rrset['type'], + 'type': rrset[0]['type'], 'values': values, - 'ttl': rrset['ttl'] + 'ttl': rrset[0]['ttl'] } def _data_for_NAPTR(self, rrset): @@ -193,10 +201,10 @@ class RackspaceProvider(BaseProvider): def populate(self, zone, target=False): self.log.debug('populate: name=%s', zone.name) - resp = None + resp_data = None try: - domain_id = self._get_zone_id_for(zone.name) - resp = self._request('GET', '/domains/{}/records'.format(domain_id), pagination_key='records') + domain_id = self._get_zone_id_for(zone) + resp_data = self._request('GET', '/domains/{}/records'.format(domain_id), pagination_key='records') self.log.debug('populate: loaded') except HTTPError as e: if e.response.status_code == 401: @@ -213,20 +221,27 @@ class RackspaceProvider(BaseProvider): before = len(zone.records) - if resp: - for record in resp.json()['records']: - record_type = record['type'] - if record_type == 'SOA': - continue - data_for = getattr(self, '_data_for_{}'.format(record_type)) - record_name = zone.hostname_from_fqdn(record['name']) - record = Record.new(zone, record_name, data_for(record), - source=self) - zone.add_record(record) + if resp_data: + records = self._group_records(resp_data) + for record_type, records_of_type in records.items(): + for raw_record_name, record_set in records_of_type.items(): + if record_type == 'SOA': + continue + data_for = getattr(self, '_data_for_{}'.format(record_type)) + record_name = zone.hostname_from_fqdn(raw_record_name) + record = Record.new(zone, record_name, data_for(record_set), + source=self) + zone.add_record(record) self.log.info('populate: found %s records', len(zone.records) - before) + def _group_records(self, all_records): + records = defaultdict(lambda: defaultdict(list)) + for record in all_records: + records[record['type']][record['name']].append(record) + return records + def _records_for_multiple(self, record): return [{'content': v, 'disabled': False} for v in record.values] diff --git a/tests/fixtures/rackspace-list-domains-response.json b/tests/fixtures/rackspace-list-domains-response.json index f124837..725641a 100644 --- a/tests/fixtures/rackspace-list-domains-response.json +++ b/tests/fixtures/rackspace-list-domains-response.json @@ -58,7 +58,7 @@ "accountId" : 1234, "created" : "2011-06-15T19:02:07.000+0000" }, { - "name" : "dnsaas.example", + "name" : "unit.tests", "id" : 2722347, "comment" : "Sample comment", "updated" : "2011-06-21T15:54:31.000+0000", From 21b3ffb509391d4aa37187b34254eb0f745097f8 Mon Sep 17 00:00:00 2001 From: Vietor Davis Date: Fri, 7 Jul 2017 18:37:04 -0700 Subject: [PATCH 03/35] Minor test updates for rackspace --- tests/test_octodns_source_rackspace.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_octodns_source_rackspace.py b/tests/test_octodns_source_rackspace.py index 88ccb5f..44ec1eb 100644 --- a/tests/test_octodns_source_rackspace.py +++ b/tests/test_octodns_source_rackspace.py @@ -91,9 +91,9 @@ class TestRackspaceSource(TestCase): zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals(14, len(zone.records)) + self.assertEquals(5, len(zone.records)) changes = expected.changes(zone, provider) - self.assertEquals(0, len(changes)) + self.assertEquals(18, len(changes)) # Used in a minute def assert_rrsets_callback(request, context): @@ -103,7 +103,8 @@ class TestRackspaceSource(TestCase): # No existing records -> creates for every record in expected with requests_mock() as mock: - mock.get(ANY, status_code=200, text=EMPTY_TEXT) + mock.get(re.compile('domains$'), status_code=200, text=LIST_DOMAINS_RESPONSE) + mock.get(re.compile('records'), status_code=200, text=EMPTY_TEXT) # post 201, is reponse to the create with data mock.patch(ANY, status_code=201, text=assert_rrsets_callback) From 0579ff6f2dda0be69197ce3cf4a9bf899159ffb0 Mon Sep 17 00:00:00 2001 From: Terrence Cole Date: Wed, 12 Jul 2017 16:34:22 -0700 Subject: [PATCH 04/35] Working push for A records. --- octodns/provider/rackspace.py | 173 ++-- ...sample-recordset-existing-nameservers.json | 29 + tests/test_octodns_source_rackspace.py | 801 ++++++++++++++---- 3 files changed, 755 insertions(+), 248 deletions(-) create mode 100644 tests/fixtures/rackspace-sample-recordset-existing-nameservers.json diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py index f6817ae..eef20d9 100644 --- a/octodns/provider/rackspace.py +++ b/octodns/provider/rackspace.py @@ -1,13 +1,10 @@ # # # - - from __future__ import absolute_import, division, print_function, \ unicode_literals from requests import HTTPError, Session, post -import json from collections import defaultdict import logging @@ -40,6 +37,10 @@ class RackspaceProvider(BaseProvider): sess.headers.update({'X-Auth-Token': auth_token}) self._sess = sess + # Map record type, name, and data to an id when populating so that + # we can find the id for update and delete operations. + self._id_map = {} + def _get_auth_token(self, username, api_key): ret = post('https://identity.api.rackspacecloud.com/v2.0/tokens', json={"auth": {"RAX-KSKEY:apiKeyCredentials": {"username": username, "apiKey": api_key}}}, @@ -91,9 +92,15 @@ class RackspaceProvider(BaseProvider): def _post(self, path, data=None): return self._request('POST', path, data=data) + def _put(self, path, data=None): + return self._request('PUT', path, data=data) + def _patch(self, path, data=None): return self._request('PATCH', path, data=data) + def _delete(self, path, data=None): + return self._request('DELETE', path, data=data) + def _data_for_multiple(self, rrset): # TODO: geo not supported return { @@ -204,7 +211,7 @@ class RackspaceProvider(BaseProvider): resp_data = None try: domain_id = self._get_zone_id_for(zone) - resp_data = self._request('GET', '/domains/{}/records'.format(domain_id), pagination_key='records') + resp_data = self._request('GET', 'domains/{}/records'.format(domain_id), pagination_key='records') self.log.debug('populate: loaded') except HTTPError as e: if e.response.status_code == 401: @@ -224,9 +231,9 @@ class RackspaceProvider(BaseProvider): if resp_data: records = self._group_records(resp_data) for record_type, records_of_type in records.items(): + if record_type == 'SOA': + continue for raw_record_name, record_set in records_of_type.items(): - if record_type == 'SOA': - continue data_for = getattr(self, '_data_for_{}'.format(record_type)) record_name = zone.hostname_from_fqdn(raw_record_name) record = Record.new(zone, record_name, data_for(record_set), @@ -239,6 +246,7 @@ class RackspaceProvider(BaseProvider): def _group_records(self, all_records): records = defaultdict(lambda: defaultdict(list)) for record in all_records: + self._id_map[(record['type'], record['name'], record['data'])] = record['id'] records[record['type']][record['name']].append(record) return records @@ -294,61 +302,45 @@ class RackspaceProvider(BaseProvider): } for v in record.values] def _mod_Create(self, change): - new = change.new - records_for = getattr(self, '_records_for_{}'.format(new._type)) - return { - 'name': new.fqdn, - 'type': new._type, - 'ttl': new.ttl, - 'changetype': 'REPLACE', - 'records': records_for(new) - } - - _mod_Update = _mod_Create + out = [] + for value in change.new.values: + out.append({ + 'name': change.new.fqdn, + 'type': change.new._type, + 'data': value, + 'ttl': change.new.ttl, + }) + return out + + def _mod_Update(self, change): + # A reduction in number of values in an update record needs + # to get upgraded into a Delete change for the removed values. + deleted_values = set(change.existing.values) - set(change.new.values) + delete_out = self._delete_given_change_values(change, deleted_values) + + update_out = [] + for value in change.new.values: + key = (change.existing._type, change.existing.fqdn, value) + rsid = self._id_map[key] + update_out.append({ + 'id': rsid, + 'name': change.new.fqdn, + 'data': value, + 'ttl': change.new.ttl, + }) + return update_out, delete_out def _mod_Delete(self, change): - existing = change.existing - records_for = getattr(self, '_records_for_{}'.format(existing._type)) - return { - 'name': existing.fqdn, - 'type': existing._type, - 'ttl': existing.ttl, - 'changetype': 'DELETE', - 'records': records_for(existing) - } + return self._delete_given_change_values(change, change.existing.values) - def _get_nameserver_record(self, existing): - return None - - def _extra_changes(self, existing, _): - self.log.debug('_extra_changes: zone=%s', existing.name) - - ns = self._get_nameserver_record(existing) - if not ns: - return [] - - # sorting mostly to make things deterministic for testing, but in - # theory it let us find what we're after quickier (though sorting would - # ve more exepensive.) - for record in sorted(existing.records): - if record == ns: - # We've found the top-level NS record, return any changes - change = record.changes(ns, self) - self.log.debug('_extra_changes: change=%s', change) - if change: - # We need to modify an existing record - return [change] - # No change is necessary - return [] - # No existing top-level NS - self.log.debug('_extra_changes: create') - return [Create(ns)] - - def _get_error(self, http_error): - try: - return http_error.response.json()['error'] - except Exception: - return '' + def _delete_given_change_values(self, change, values): + out = [] + for value in values: + key = (change.existing._type, change.existing.fqdn, value) + rsid = self._id_map[key] + out.append('id=' + rsid) + del self._id_map[key] + return out def _apply(self, plan): desired = plan.desired @@ -356,46 +348,29 @@ class RackspaceProvider(BaseProvider): self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name, len(changes)) - mods = [] + domain_id = self._get_zone_id_for(desired) + creates = [] + updates = [] + deletes = [] for change in changes: - class_name = change.__class__.__name__ - mods.append(getattr(self, '_mod_{}'.format(class_name))(change)) - self.log.debug('_apply: sending change request') - - try: - self._patch('zones/{}'.format(desired.name), - data={'rrsets': mods}) - self.log.debug('_apply: patched') - except HTTPError as e: - error = self._get_error(e) - if e.response.status_code != 422 or \ - not error.startswith('Could not find domain '): - self.log.error('_apply: status=%d, text=%s', - e.response.status_code, - e.response.text) - raise - self.log.info('_apply: creating zone=%s', desired.name) - # 422 means powerdns doesn't know anything about the requsted - # domain. We'll try to create it with the correct records instead - # of update. Hopefully all the mods are creates :-) - data = { - 'name': desired.name, - 'kind': 'Master', - 'masters': [], - 'nameservers': [], - 'rrsets': mods, - 'soa_edit_api': 'INCEPTION-INCREMENT', - 'serial': 0, - } - try: - self._post('zones', data) - except HTTPError as e: - self.log.error('_apply: status=%d, text=%s', - e.response.status_code, - e.response.text) - raise - self.log.debug('_apply: created') - - self.log.debug('_apply: complete') - + if change.__class__.__name__ == 'Create': + creates += self._mod_Create(change) + elif change.__class__.__name__ == 'Update': + add_updates, add_deletes = self._mod_Update(change) + updates += add_updates + deletes += add_deletes + elif change.__class__.__name__ == 'Delete': + deletes += self._mod_Delete(change) + + if creates: + data = {"records": sorted(creates, key=lambda v: v['name'])} + self._post('domains/{}/records'.format(domain_id), data=data) + + if updates: + data = {"records": sorted(updates, key=lambda v: v['name'])} + self._put('domains/{}/records'.format(domain_id), data=data) + + if deletes: + params = "&".join(sorted(deletes)) + self._delete('domains/{}/records?{}'.format(domain_id, params)) diff --git a/tests/fixtures/rackspace-sample-recordset-existing-nameservers.json b/tests/fixtures/rackspace-sample-recordset-existing-nameservers.json new file mode 100644 index 0000000..034aa44 --- /dev/null +++ b/tests/fixtures/rackspace-sample-recordset-existing-nameservers.json @@ -0,0 +1,29 @@ +{ + "totalEntries" : 3, + "records" : [{ + "name" : "unit.tests.", + "id" : "A-6822995", + "type" : "A", + "data" : "1.2.3.4", + "updated" : "2011-06-24T01:12:53.000+0000", + "ttl" : 60, + "created" : "2011-06-24T01:12:53.000+0000" + }, { + "name" : "unit.tests.", + "id" : "NS-454454", + "type" : "NS", + "data" : "8.8.8.8.", + "updated" : "2011-06-24T01:12:51.000+0000", + "ttl" : 600, + "created" : "2011-06-24T01:12:51.000+0000" + }, { + "name" : "unit.tests.", + "id" : "NS-454455", + "type" : "NS", + "data" : "9.9.9.9.", + "updated" : "2011-06-24T01:12:52.000+0000", + "ttl" : 600, + "created" : "2011-06-24T01:12:52.000+0000" + }], + "links" : [] +} diff --git a/tests/test_octodns_source_rackspace.py b/tests/test_octodns_source_rackspace.py index 44ec1eb..4c53e95 100644 --- a/tests/test_octodns_source_rackspace.py +++ b/tests/test_octodns_source_rackspace.py @@ -5,10 +5,11 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals +import json import re -from json import loads, dumps from os.path import dirname, join from unittest import TestCase +from urlparse import urlparse from requests import HTTPError from requests_mock import ANY, mock as requests_mock @@ -18,9 +19,11 @@ from octodns.provider.yaml import YamlProvider from octodns.record import Record from octodns.zone import Zone +from pprint import pprint + EMPTY_TEXT = ''' { - "totalEntries" : 6, + "totalEntries" : 0, "records" : [] } ''' @@ -37,51 +40,66 @@ with open('./tests/fixtures/rackspace-sample-recordset-page1.json') as fh: with open('./tests/fixtures/rackspace-sample-recordset-page2.json') as fh: RECORDS_PAGE_2 = fh.read() -def load_provider(): - with requests_mock() as mock: - mock.post(ANY, status_code=200, text=AUTH_RESPONSE) - return RackspaceProvider('test', 'api-key') - - -class TestRackspaceSource(TestCase): +with open('./tests/fixtures/rackspace-sample-recordset-existing-nameservers.json') as fh: + RECORDS_EXISTING_NAMESERVERS = fh.read() - def test_provider(self): - provider = load_provider() +class TestRackspaceProvider(TestCase): + def setUp(self): + with requests_mock() as mock: + mock.post(ANY, status_code=200, text=AUTH_RESPONSE) + self.provider = RackspaceProvider('test', 'api-key') + self.assertTrue(mock.called_once) - # Bad auth + def test_bad_auth(self): with requests_mock() as mock: mock.get(ANY, status_code=401, text='Unauthorized') with self.assertRaises(Exception) as ctx: zone = Zone('unit.tests.', []) - provider.populate(zone) + self.provider.populate(zone) self.assertTrue('unauthorized' in ctx.exception.message) + self.assertTrue(mock.called_once) - # General error + def test_server_error(self): with requests_mock() as mock: mock.get(ANY, status_code=502, text='Things caught fire') with self.assertRaises(HTTPError) as ctx: zone = Zone('unit.tests.', []) - provider.populate(zone) + self.provider.populate(zone) self.assertEquals(502, ctx.exception.response.status_code) + self.assertTrue(mock.called_once) - # Non-existant zone doesn't populate anything + def test_nonexistent_zone(self): + # Non-existent zone doesn't populate anything with requests_mock() as mock: mock.get(ANY, status_code=422, json={'error': "Could not find domain 'unit.tests.'"}) zone = Zone('unit.tests.', []) - provider.populate(zone) + self.provider.populate(zone) self.assertEquals(set(), zone.records) + self.assertTrue(mock.called_once) - # The rest of this is messy/complicated b/c it's dealing with mocking + def test_multipage_populate(self): + with requests_mock() as mock: + mock.get(re.compile('domains$'), status_code=200, text=LIST_DOMAINS_RESPONSE) + mock.get(re.compile('records'), status_code=200, text=RECORDS_PAGE_1) + mock.get(re.compile('records.*offset=3'), status_code=200, text=RECORDS_PAGE_2) + zone = Zone('unit.tests.', []) + self.provider.populate(zone) + self.assertEquals(5, len(zone.records)) + + def _load_full_config(self): expected = Zone('unit.tests.', []) source = YamlProvider('test', join(dirname(__file__), 'config')) source.populate(expected) - expected_n = len(expected.records) - 1 - self.assertEquals(14, expected_n) + self.assertEquals(15, len(expected.records)) + return expected + + def test_changes_are_formatted_correctly(self): + expected = self._load_full_config() # No diffs == no changes with requests_mock() as mock: @@ -90,27 +108,551 @@ class TestRackspaceSource(TestCase): mock.get(re.compile('records.*offset=3'), status_code=200, text=RECORDS_PAGE_2) zone = Zone('unit.tests.', []) - provider.populate(zone) - self.assertEquals(5, len(zone.records)) - changes = expected.changes(zone, provider) + self.provider.populate(zone) + changes = expected.changes(zone, self.provider) self.assertEquals(18, len(changes)) + def test_plan_disappearing_ns_records(self): + expected = Zone('unit.tests.', []) + expected.add_record(Record.new(expected, '', { + 'type': 'NS', + 'ttl': 600, + 'values': ['8.8.8.8.', '9.9.9.9.'] + })) + expected.add_record(Record.new(expected, 'sub', { + 'type': 'NS', + 'ttl': 600, + 'values': ['8.8.8.8.', '9.9.9.9.'] + })) + with requests_mock() as mock: + mock.get(re.compile('domains$'), status_code=200, text=LIST_DOMAINS_RESPONSE) + mock.get(re.compile('records'), status_code=200, text=EMPTY_TEXT) + + plan = self.provider.plan(expected) + self.assertTrue(mock.called) + + # OctoDNS does not propagate top-level NS records. + self.assertEquals(1, len(plan.changes)) + + def _test_apply_with_data(self, data): + expected = Zone('unit.tests.', []) + for record in data.OtherRecords: + expected.add_record(Record.new(expected, record['subdomain'], record['data'])) + + with requests_mock() as list_mock: + list_mock.get(re.compile('domains$'), status_code=200, text=LIST_DOMAINS_RESPONSE) + list_mock.get(re.compile('records'), status_code=200, json=data.OwnRecords) + plan = self.provider.plan(expected) + self.assertTrue(list_mock.called) + if not data.ExpectChanges: + self.assertFalse(plan) + return + + with requests_mock() as mock: + called = set() + + def make_assert_sending_right_body(expected): + def _assert_sending_right_body(request, _context): + called.add(request.method) + if request.method != 'DELETE': + self.assertEqual(request.headers['content-type'], 'application/json') + self.assertDictEqual(expected, json.loads(request.body)) + else: + parts = urlparse(request.url) + self.assertEqual(expected, parts.query) + return '' + return _assert_sending_right_body + + mock.get(re.compile('domains$'), status_code=200, text=LIST_DOMAINS_RESPONSE) + mock.post(re.compile('domains/.*/records$'), status_code=202, + text=make_assert_sending_right_body(data.ExpectedAdditions)) + mock.delete(re.compile('domains/.*/records?.*'), status_code=202, + text=make_assert_sending_right_body(data.ExpectedDeletions)) + mock.put(re.compile('domains/.*/records$'), status_code=202, + text=make_assert_sending_right_body(data.ExpectedUpdates)) + + self.provider.apply(plan) + self.assertTrue(data.ExpectedAdditions is None or "POST" in called) + self.assertTrue(data.ExpectedDeletions is None or "DELETE" in called) + self.assertTrue(data.ExpectedUpdates is None or "PUT" in called) + + def test_apply_no_change_empty(self): + class TestData(object): + OtherRecords = [] + OwnRecords = { + "totalEntries": 0, + "records": [] + } + ExpectChanges = False + ExpectedAdditions = None + ExpectedDeletions = None + ExpectedUpdates = None + return self._test_apply_with_data(TestData) + + def test_apply_no_change_a_records(self): + class TestData(object): + OtherRecords = [ + { + "subdomain": '', + "data": { + 'type': 'A', + 'ttl': 60, + 'values': ['1.2.3.4', '1.2.3.5', '1.2.3.6'] + } + } + ] + OwnRecords = { + "totalEntries": 3, + "records": [{ + "name": "unit.tests.", + "id": "A-111111", + "type": "A", + "data": "1.2.3.4", + "ttl": 60 + }, { + "name": "unit.tests.", + "id": "A-222222", + "type": "A", + "data": "1.2.3.5", + "ttl": 60 + }, { + "name": "unit.tests.", + "id": "A-333333", + "type": "A", + "data": "1.2.3.6", + "ttl": 60 + }] + } + ExpectChanges = False + ExpectedAdditions = None + ExpectedDeletions = None + ExpectedUpdates = None + return self._test_apply_with_data(TestData) + + def test_apply_no_change_a_records_cross_zone(self): + class TestData(object): + OtherRecords = [ + { + "subdomain": 'foo', + "data": { + 'type': 'A', + 'ttl': 60, + 'value': '1.2.3.4' + } + }, + { + "subdomain": 'bar', + "data": { + 'type': 'A', + 'ttl': 60, + 'value': '1.2.3.4' + } + } + ] + OwnRecords = { + "totalEntries": 3, + "records": [{ + "name": "foo.unit.tests.", + "id": "A-111111", + "type": "A", + "data": "1.2.3.4", + "ttl": 60 + }, { + "name": "bar.unit.tests.", + "id": "A-222222", + "type": "A", + "data": "1.2.3.4", + "ttl": 60 + }] + } + ExpectChanges = False + ExpectedAdditions = None + ExpectedDeletions = None + ExpectedUpdates = None + return self._test_apply_with_data(TestData) + + def test_apply_one_addition(self): + class TestData(object): + OtherRecords = [ + { + "subdomain": '', + "data": { + 'type': 'A', + 'ttl': 60, + 'value': '1.2.3.4' + } + } + ] + OwnRecords = { + "totalEntries": 0, + "records": [] + } + ExpectChanges = True + ExpectedAdditions = { + "records": [{ + "name": "unit.tests.", + "type": "A", + "data": "1.2.3.4", + "ttl": 60 + }] + } + ExpectedDeletions = None + ExpectedUpdates = None + return self._test_apply_with_data(TestData) + + def test_apply_multiple_additions_exploding(self): + class TestData(object): + OtherRecords = [ + { + "subdomain": '', + "data": { + 'type': 'A', + 'ttl': 60, + 'values': ['1.2.3.4', '1.2.3.5', '1.2.3.6'] + } + } + ] + OwnRecords = { + "totalEntries": 0, + "records": [] + } + ExpectChanges = True + ExpectedAdditions = { + "records": [{ + "name": "unit.tests.", + "type": "A", + "data": "1.2.3.4", + "ttl": 60 + }, { + "name": "unit.tests.", + "type": "A", + "data": "1.2.3.5", + "ttl": 60 + }, { + "name": "unit.tests.", + "type": "A", + "data": "1.2.3.6", + "ttl": 60 + }] + } + ExpectedDeletions = None + ExpectedUpdates = None + return self._test_apply_with_data(TestData) + + def test_apply_multiple_additions_namespaced(self): + class TestData(object): + OtherRecords = [{ + "subdomain": 'foo', + "data": { + 'type': 'A', + 'ttl': 60, + 'value': '1.2.3.4' + } + }, { + "subdomain": 'bar', + "data": { + 'type': 'A', + 'ttl': 60, + 'value': '1.2.3.4' + } + }] + OwnRecords = { + "totalEntries": 0, + "records": [] + } + ExpectChanges = True + ExpectedAdditions = { + "records": [{ + "name": "bar.unit.tests.", + "type": "A", + "data": "1.2.3.4", + "ttl": 60 + }, { + "name": "foo.unit.tests.", + "type": "A", + "data": "1.2.3.4", + "ttl": 60 + }] + } + ExpectedDeletions = None + ExpectedUpdates = None + return self._test_apply_with_data(TestData) + + def test_apply_single_deletion(self): + class TestData(object): + OtherRecords = [] + OwnRecords = { + "totalEntries": 1, + "records": [{ + "name": "unit.tests.", + "id": "A-111111", + "type": "A", + "data": "1.2.3.4", + "ttl": 60 + }] + } + ExpectChanges = True + ExpectedAdditions = None + ExpectedDeletions = "id=A-111111" + ExpectedUpdates = None + return self._test_apply_with_data(TestData) + + def test_apply_multiple_deletions(self): + class TestData(object): + OtherRecords = [ + { + "subdomain": '', + "data": { + 'type': 'A', + 'ttl': 60, + 'value': '1.2.3.5' + } + } + ] + OwnRecords = { + "totalEntries": 3, + "records": [{ + "name": "unit.tests.", + "id": "A-111111", + "type": "A", + "data": "1.2.3.4", + "ttl": 60 + }, { + "name": "unit.tests.", + "id": "A-222222", + "type": "A", + "data": "1.2.3.5", + "ttl": 60 + }, { + "name": "unit.tests.", + "id": "A-333333", + "type": "A", + "data": "1.2.3.6", + "ttl": 60 + }] + } + ExpectChanges = True + ExpectedAdditions = None + ExpectedDeletions = "id=A-111111&id=A-333333" + ExpectedUpdates = { + "records": [{ + "name": "unit.tests.", + "id": "A-222222", + "data": "1.2.3.5", + "ttl": 60 + }] + } + return self._test_apply_with_data(TestData) + + def test_apply_multiple_deletions_cross_zone(self): + class TestData(object): + OtherRecords = [ + { + "subdomain": '', + "data": { + 'type': 'A', + 'ttl': 60, + 'value': '1.2.3.4' + } + } + ] + OwnRecords = { + "totalEntries": 3, + "records": [{ + "name": "unit.tests.", + "id": "A-111111", + "type": "A", + "data": "1.2.3.4", + "ttl": 60 + }, { + "name": "foo.unit.tests.", + "id": "A-222222", + "type": "A", + "data": "1.2.3.5", + "ttl": 60 + }, { + "name": "bar.unit.tests.", + "id": "A-333333", + "type": "A", + "data": "1.2.3.6", + "ttl": 60 + }] + } + ExpectChanges = True + ExpectedAdditions = None + ExpectedDeletions = "id=A-222222&id=A-333333" + ExpectedUpdates = None + return self._test_apply_with_data(TestData) + + def test_apply_single_update(self): + class TestData(object): + OtherRecords = [ + { + "subdomain": '', + "data": { + 'type': 'A', + 'ttl': 3600, + 'value': '1.2.3.4' + } + } + ] + OwnRecords = { + "totalEntries": 1, + "records": [{ + "name": "unit.tests.", + "id": "A-111111", + "type": "A", + "data": "1.2.3.4", + "ttl": 60 + }] + } + ExpectChanges = True + ExpectedAdditions = None + ExpectedDeletions = None + ExpectedUpdates = { + "records": [{ + "name": "unit.tests.", + "id": "A-111111", + "data": "1.2.3.4", + "ttl": 3600 + }] + } + return self._test_apply_with_data(TestData) + + def test_apply_multiple_updates(self): + class TestData(object): + OtherRecords = [ + { + "subdomain": '', + "data": { + 'type': 'A', + 'ttl': 3600, + 'values': ['1.2.3.4', '1.2.3.5', '1.2.3.6'] + } + } + ] + OwnRecords = { + "totalEntries": 3, + "records": [{ + "name": "unit.tests.", + "id": "A-111111", + "type": "A", + "data": "1.2.3.4", + "ttl": 60 + }, { + "name": "unit.tests.", + "id": "A-222222", + "type": "A", + "data": "1.2.3.5", + "ttl": 60 + }, { + "name": "unit.tests.", + "id": "A-333333", + "type": "A", + "data": "1.2.3.6", + "ttl": 60 + }] + } + ExpectChanges = True + ExpectedAdditions = None + ExpectedDeletions = None + ExpectedUpdates = { + "records": [{ + "name": "unit.tests.", + "id": "A-111111", + "data": "1.2.3.4", + "ttl": 3600 + }, { + "name": "unit.tests.", + "id": "A-222222", + "data": "1.2.3.5", + "ttl": 3600 + }, { + "name": "unit.tests.", + "id": "A-333333", + "data": "1.2.3.6", + "ttl": 3600 + }] + } + return self._test_apply_with_data(TestData) + + def test_apply_multiple_updates_cross_zone(self): + class TestData(object): + OtherRecords = [ + { + "subdomain": 'foo', + "data": { + 'type': 'A', + 'ttl': 3600, + 'value': '1.2.3.4' + } + }, + { + "subdomain": 'bar', + "data": { + 'type': 'A', + 'ttl': 3600, + 'value': '1.2.3.4' + } + } + ] + OwnRecords = { + "totalEntries": 2, + "records": [{ + "name": "foo.unit.tests.", + "id": "A-111111", + "type": "A", + "data": "1.2.3.4", + "ttl": 60 + }, { + "name": "bar.unit.tests.", + "id": "A-222222", + "type": "A", + "data": "1.2.3.4", + "ttl": 60 + }] + } + ExpectChanges = True + ExpectedAdditions = None + ExpectedDeletions = None + ExpectedUpdates = { + "records": [{ + "name": "bar.unit.tests.", + "id": "A-222222", + "data": "1.2.3.4", + "ttl": 3600 + }, { + "name": "foo.unit.tests.", + "id": "A-111111", + "data": "1.2.3.4", + "ttl": 3600 + }] + } + return self._test_apply_with_data(TestData) + + def test_provider(self): + expected = self._load_full_config() + + # No existing records -> creates for every record in expected + with requests_mock() as mock: + mock.get(re.compile('domains$'), status_code=200, text=LIST_DOMAINS_RESPONSE) + mock.get(re.compile('records'), status_code=200, text=EMPTY_TEXT) + + plan = self.provider.plan(expected) + self.assertTrue(mock.called) + self.assertEquals(len(expected.records), len(plan.changes)) + # Used in a minute def assert_rrsets_callback(request, context): data = loads(request.body) self.assertEquals(expected_n, len(data['rrsets'])) return '' - # No existing records -> creates for every record in expected with requests_mock() as mock: - mock.get(re.compile('domains$'), status_code=200, text=LIST_DOMAINS_RESPONSE) - mock.get(re.compile('records'), status_code=200, text=EMPTY_TEXT) - # post 201, is reponse to the create with data + # post 201, is response to the create with data mock.patch(ANY, status_code=201, text=assert_rrsets_callback) - plan = provider.plan(expected) - self.assertEquals(expected_n, len(plan.changes)) - self.assertEquals(expected_n, provider.apply(plan)) + self.assertEquals(expected_n, self.provider.apply(plan)) # Non-existent zone -> creates for every record in expected # OMG this is fucking ugly, probably better to ditch requests_mocks and @@ -123,12 +665,12 @@ class TestRackspaceSource(TestCase): mock.get(ANY, status_code=422, text='') # patch 422's, unknown zone mock.patch(ANY, status_code=422, text=dumps(not_found)) - # post 201, is reponse to the create with data + # post 201, is response to the create with data mock.post(ANY, status_code=201, text=assert_rrsets_callback) - plan = provider.plan(expected) + plan = self.provider.plan(expected) self.assertEquals(expected_n, len(plan.changes)) - self.assertEquals(expected_n, provider.apply(plan)) + self.assertEquals(expected_n, self.provider.apply(plan)) with requests_mock() as mock: # get 422's, unknown zone @@ -138,8 +680,8 @@ class TestRackspaceSource(TestCase): mock.patch(ANY, status_code=422, text=dumps(data)) with self.assertRaises(HTTPError) as ctx: - plan = provider.plan(expected) - provider.apply(plan) + plan = self.provider.plan(expected) + self.provider.apply(plan) response = ctx.exception.response self.assertEquals(422, response.status_code) self.assertTrue('error' in response.json()) @@ -151,8 +693,8 @@ class TestRackspaceSource(TestCase): mock.patch(ANY, status_code=500, text='') with self.assertRaises(HTTPError): - plan = provider.plan(expected) - provider.apply(plan) + plan = self.provider.plan(expected) + self.provider.apply(plan) with requests_mock() as mock: # get 422's, unknown zone @@ -163,133 +705,94 @@ class TestRackspaceSource(TestCase): mock.post(ANY, status_code=422, text='Hello Word!') with self.assertRaises(HTTPError): - plan = provider.plan(expected) - provider.apply(plan) - - def test_small_change(self): - provider = load_provider() + plan = self.provider.plan(expected) + self.provider.apply(plan) + def test_plan_no_changes(self): expected = Zone('unit.tests.', []) - source = YamlProvider('test', join(dirname(__file__), 'config')) - source.populate(expected) - self.assertEquals(15, len(expected.records)) + expected.add_record(Record.new(expected, '', { + 'type': 'NS', + 'ttl': 600, + 'values': ['8.8.8.8.', '9.9.9.9.'] + })) + expected.add_record(Record.new(expected, '', { + 'type': 'A', + 'ttl': 60, + 'value': '1.2.3.4' + })) - # A small change to a single record with requests_mock() as mock: - mock.get(ANY, status_code=200, text=FULL_TEXT) - - missing = Zone(expected.name, []) - # Find and delete the SPF record - for record in expected.records: - if record._type != 'SPF': - missing.add_record(record) - - def assert_delete_callback(request, context): - self.assertEquals({ - 'rrsets': [{ - 'records': [ - {'content': '"v=spf1 ip4:192.168.0.1/16-all"', - 'disabled': False} - ], - 'changetype': 'DELETE', - 'type': 'SPF', - 'name': 'spf.unit.tests.', - 'ttl': 600 - }] - }, loads(request.body)) - return '' - - mock.patch(ANY, status_code=201, text=assert_delete_callback) - - plan = provider.plan(missing) - self.assertEquals(1, len(plan.changes)) - self.assertEquals(1, provider.apply(plan)) + mock.get(re.compile('domains/.*/records'), status_code=200, text=RECORDS_EXISTING_NAMESERVERS) + mock.get(re.compile('domains$'), status_code=200, text=LIST_DOMAINS_RESPONSE) - def test_existing_nameservers(self): - ns_values = ['8.8.8.8.', '9.9.9.9.'] - provider = load_provider() + plan = self.provider.plan(expected) + self.assertTrue(mock.called) + self.assertFalse(plan) + + def test_plan_remove_a_record(self): expected = Zone('unit.tests.', []) - ns_record = Record.new(expected, '', { + expected.add_record(Record.new(expected, '', { 'type': 'NS', 'ttl': 600, - 'values': ns_values - }) - expected.add_record(ns_record) + 'values': ['8.8.8.8.', '9.9.9.9.'] + })) - # no changes with requests_mock() as mock: - data = { - 'rrsets': [{ - 'comments': [], - 'name': 'unit.tests.', - 'records': [ - { - 'content': '8.8.8.8.', - 'disabled': False - }, - { - 'content': '9.9.9.9.', - 'disabled': False - } - ], - 'ttl': 600, - 'type': 'NS' - }, { - 'comments': [], - 'name': 'unit.tests.', - 'records': [{ - 'content': '1.2.3.4', - 'disabled': False, - }], - 'ttl': 60, - 'type': 'A' - }] - } - mock.get(ANY, status_code=200, json=data) - - unrelated_record = Record.new(expected, '', { - 'type': 'A', - 'ttl': 60, - 'value': '1.2.3.4' - }) - expected.add_record(unrelated_record) - plan = provider.plan(expected) - self.assertFalse(plan) - # remove it now that we don't need the unrelated change any longer - expected.records.remove(unrelated_record) + mock.get(re.compile('domains/.*/records'), status_code=200, text=RECORDS_EXISTING_NAMESERVERS) + mock.get(re.compile('domains$'), status_code=200, text=LIST_DOMAINS_RESPONSE) + + plan = self.provider.plan(expected) + self.assertTrue(mock.called) + self.assertEquals(1, len(plan.changes)) + self.assertEqual(plan.changes[0].existing.ttl, 60) + self.assertEqual(plan.changes[0].existing.values[0], '1.2.3.4') + + def test_plan_create_a_record(self): + expected = Zone('unit.tests.', []) + expected.add_record(Record.new(expected, '', { + 'type': 'NS', + 'ttl': 600, + 'values': ['8.8.8.8.', '9.9.9.9.'] + })) + expected.add_record(Record.new(expected, '', { + 'type': 'A', + 'ttl': 60, + 'values': ['1.2.3.4', '1.2.3.5'] + })) - # ttl diff with requests_mock() as mock: - data = { - 'rrsets': [{ - 'comments': [], - 'name': 'unit.tests.', - 'records': [ - { - 'content': '8.8.8.8.', - 'disabled': False - }, - { - 'content': '9.9.9.9.', - 'disabled': False - }, - ], - 'ttl': 3600, - 'type': 'NS' - }] - } - mock.get(ANY, status_code=200, json=data) + mock.get(re.compile('domains/.*/records'), status_code=200, text=RECORDS_EXISTING_NAMESERVERS) + mock.get(re.compile('domains$'), status_code=200, text=LIST_DOMAINS_RESPONSE) - plan = provider.plan(expected) + plan = self.provider.plan(expected) + self.assertTrue(mock.called) self.assertEquals(1, len(plan.changes)) + self.assertEqual(plan.changes[0].new.ttl, 60) + self.assertEqual(plan.changes[0].new.values[0], '1.2.3.4') + self.assertEqual(plan.changes[0].new.values[1], '1.2.3.5') + + def test_plan_change_ttl(self): + expected = Zone('unit.tests.', []) + expected.add_record(Record.new(expected, '', { + 'type': 'NS', + 'ttl': 600, + 'values': ['8.8.8.8.', '9.9.9.9.'] + })) + expected.add_record(Record.new(expected, '', { + 'type': 'A', + 'ttl': 86400, + 'value': '1.2.3.4' + })) - # create with requests_mock() as mock: - data = { - 'rrsets': [] - } - mock.get(ANY, status_code=200, json=data) + mock.get(re.compile('domains/.*/records'), status_code=200, text=RECORDS_EXISTING_NAMESERVERS) + mock.get(re.compile('domains$'), status_code=200, text=LIST_DOMAINS_RESPONSE) - plan = provider.plan(expected) - self.assertEquals(1, len(plan.changes)) + plan = self.provider.plan(expected) + + self.assertTrue(mock.called) + self.assertEqual(1, len(plan.changes)) + self.assertEqual(plan.changes[0].existing.ttl, 60) + self.assertEqual(plan.changes[0].new.ttl, 86400) + self.assertEqual(plan.changes[0].new.values[0], '1.2.3.4') From 823423054fa21b8a3b7d968bfa2def206912137d Mon Sep 17 00:00:00 2001 From: Terrence Cole Date: Wed, 12 Jul 2017 16:35:39 -0700 Subject: [PATCH 05/35] Rename the test file to reflect the new functionality. --- ...dns_source_rackspace.py => test_octodns_provider_rackspace.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/{test_octodns_source_rackspace.py => test_octodns_provider_rackspace.py} (100%) diff --git a/tests/test_octodns_source_rackspace.py b/tests/test_octodns_provider_rackspace.py similarity index 100% rename from tests/test_octodns_source_rackspace.py rename to tests/test_octodns_provider_rackspace.py From 01f8431d7440a4fa45eb76e40a2201e4ec7b7f7f Mon Sep 17 00:00:00 2001 From: Terrence Cole Date: Thu, 13 Jul 2017 11:44:09 -0700 Subject: [PATCH 06/35] Make formatting consistent and improve record type support. --- octodns/provider/rackspace.py | 299 ++++++++++-------- ...sample-recordset-existing-nameservers.json | 6 +- tests/test_octodns_provider_rackspace.py | 152 ++++++--- 3 files changed, 281 insertions(+), 176 deletions(-) diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py index eef20d9..110412b 100644 --- a/octodns/provider/rackspace.py +++ b/octodns/provider/rackspace.py @@ -12,6 +12,33 @@ from ..record import Create, Record from .base import BaseProvider +def add_trailing_dot(s): + assert s + assert s[-1] != '.' + return s + '.' + + +def remove_trailing_dot(s): + assert s + assert s[-1] == '.' + return s[:-1] + + +def strip_quotes(s): + assert s + assert len(s) > 2 + assert s[0] == '"' + assert s[-1] == '"' + return s[1:-1] + + +def add_quotes(s): + assert s + assert s[0] != '"' + assert s[-1] != '"' + return '"{}"'.format(s) + + class RackspaceProvider(BaseProvider): SUPPORTS_GEO = False TIMEOUT = 5 @@ -101,6 +128,10 @@ class RackspaceProvider(BaseProvider): def _delete(self, path, data=None): return self._request('DELETE', path, data=data) + @staticmethod + def _key_for_record(rs_record): + return rs_record['type'], rs_record['name'], rs_record['data'] + def _data_for_multiple(self, rrset): # TODO: geo not supported return { @@ -116,14 +147,14 @@ class RackspaceProvider(BaseProvider): # TODO: geo not supported return { 'type': rrset[0]['type'], - 'values': ["{}.".format(r['data']) for r in rrset], + 'values': [add_trailing_dot(r['data']) for r in rrset], 'ttl': rrset[0]['ttl'] } def _data_for_single(self, record): return { 'type': record[0]['type'], - 'value': "{}.".format(record[0]['data']), + 'value': add_trailing_dot(record[0]['data']), 'ttl': record[0]['ttl'] } @@ -134,7 +165,7 @@ class RackspaceProvider(BaseProvider): def _data_for_quoted(self, rrset): return { 'type': rrset['type'], - 'values': [r['content'][1:-1] for r in rrset['records']], + 'values': [strip_quotes(r['content']) for r in rrset['records']], 'ttl': rrset['ttl'] } @@ -146,7 +177,7 @@ class RackspaceProvider(BaseProvider): for record in rrset: values.append({ 'priority': record['priority'], - 'value': record['data'], + 'value': add_trailing_dot(record['data']), }) return { 'type': rrset[0]['type'], @@ -155,56 +186,59 @@ class RackspaceProvider(BaseProvider): } def _data_for_NAPTR(self, rrset): - values = [] - for record in rrset['records']: - order, preference, flags, service, regexp, replacement = \ - record['content'].split(' ', 5) - values.append({ - 'order': order, - 'preference': preference, - 'flags': flags[1:-1], - 'service': service[1:-1], - 'regexp': regexp[1:-1], - 'replacement': replacement, - }) - return { - 'type': rrset['type'], - 'values': values, - 'ttl': rrset['ttl'] - } + raise NotImplementedError("Missing support for reading NAPTR records") + # values = [] + # for record in rrset['records']: + # order, preference, flags, service, regexp, replacement = \ + # record['content'].split(' ', 5) + # values.append({ + # 'order': order, + # 'preference': preference, + # 'flags': flags[1:-1], + # 'service': service[1:-1], + # 'regexp': regexp[1:-1], + # 'replacement': replacement, + # }) + # return { + # 'type': rrset['type'], + # 'values': values, + # 'ttl': rrset['ttl'] + # } def _data_for_SSHFP(self, rrset): - values = [] - for record in rrset['records']: - algorithm, fingerprint_type, fingerprint = \ - record['content'].split(' ', 2) - values.append({ - 'algorithm': algorithm, - 'fingerprint_type': fingerprint_type, - 'fingerprint': fingerprint, - }) - return { - 'type': rrset['type'], - 'values': values, - 'ttl': rrset['ttl'] - } + raise NotImplementedError("Missing support for reading SSHFP records") + # values = [] + # for record in rrset['records']: + # algorithm, fingerprint_type, fingerprint = \ + # record['content'].split(' ', 2) + # values.append({ + # 'algorithm': algorithm, + # 'fingerprint_type': fingerprint_type, + # 'fingerprint': fingerprint, + # }) + # return { + # 'type': rrset['type'], + # 'values': values, + # 'ttl': rrset['ttl'] + # } def _data_for_SRV(self, rrset): - values = [] - for record in rrset['records']: - priority, weight, port, target = \ - record['content'].split(' ', 3) - values.append({ - 'priority': priority, - 'weight': weight, - 'port': port, - 'target': target, - }) - return { - 'type': rrset['type'], - 'values': values, - 'ttl': rrset['ttl'] - } + raise NotImplementedError("Missing support for reading SRV records") + # values = [] + # for record in rrset['records']: + # priority, weight, port, target = \ + # record['content'].split(' ', 3) + # values.append({ + # 'priority': priority, + # 'weight': weight, + # 'port': port, + # 'target': target, + # }) + # return { + # 'type': rrset['type'], + # 'values': values, + # 'ttl': rrset['ttl'] + # } def populate(self, zone, target=False): self.log.debug('populate: name=%s', zone.name) @@ -217,14 +251,10 @@ class RackspaceProvider(BaseProvider): if e.response.status_code == 401: # Nicer error message for auth problems raise Exception('Rackspace request unauthorized') - elif e.response.status_code == 422: - # 422 means powerdns doesn't know anything about the requsted - # domain. We'll just ignore it here and leave the zone - # untouched. - pass - else: - # just re-throw - raise + elif e.response.status_code == 404: + # Zone not found leaves the zone empty instead of failing. + return + raise before = len(zone.records) @@ -246,70 +276,82 @@ class RackspaceProvider(BaseProvider): def _group_records(self, all_records): records = defaultdict(lambda: defaultdict(list)) for record in all_records: - self._id_map[(record['type'], record['name'], record['data'])] = record['id'] + self._id_map[self._key_for_record(record)] = record['id'] records[record['type']][record['name']].append(record) return records - def _records_for_multiple(self, record): - return [{'content': v, 'disabled': False} - for v in record.values] - - _records_for_A = _records_for_multiple - _records_for_AAAA = _records_for_multiple - _records_for_NS = _records_for_multiple - - def _records_for_single(self, record): - return [{'content': record.value, 'disabled': False}] - - _records_for_ALIAS = _records_for_single - _records_for_CNAME = _records_for_single - _records_for_PTR = _records_for_single - - def _records_for_quoted(self, record): - return [{'content': '"{}"'.format(v), 'disabled': False} - for v in record.values] - - _records_for_SPF = _records_for_quoted - _records_for_TXT = _records_for_quoted - - def _records_for_MX(self, record): - return [{ - 'content': '{} {}'.format(v.priority, v.value), - 'disabled': False - } for v in record.values] - - def _records_for_NAPTR(self, record): - return [{ - 'content': '{} {} "{}" "{}" "{}" {}'.format(v.order, v.preference, - v.flags, v.service, - v.regexp, - v.replacement), - 'disabled': False - } for v in record.values] - - def _records_for_SSHFP(self, record): - return [{ - 'content': '{} {} {}'.format(v.algorithm, v.fingerprint_type, - v.fingerprint), - 'disabled': False - } for v in record.values] - - def _records_for_SRV(self, record): - return [{ - 'content': '{} {} {} {}'.format(v.priority, v.weight, v.port, - v.target), - 'disabled': False - } for v in record.values] + @staticmethod + def _record_for_ip(record, value): + return { + 'name': record.fqdn, + 'type': record._type, + 'data': value, + 'ttl': max(record.ttl, 300), + } + _record_for_A = _record_for_ip + _record_for_AAAA = _record_for_ip + + @staticmethod + def _record_for_named(record, value): + return { + 'name': record.fqdn, + 'type': record._type, + 'data': remove_trailing_dot(value), + 'ttl': max(record.ttl, 300), + } + _record_for_NS = _record_for_named + _record_for_ALIAS = _record_for_named + _record_for_CNAME = _record_for_named + _record_for_PTR = _record_for_named + + @staticmethod + def _record_for_quoted(record, value): + return { + 'name': record.fqdn, + 'type': record._type, + 'data': add_quotes(value), + 'ttl': max(record.ttl, 300), + } + _record_for_SPF = _record_for_quoted + _record_for_TXT = _record_for_quoted + + @staticmethod + def _record_for_MX(record, value): + return { + 'name': record.fqdn, + 'type': record._type, + 'data': remove_trailing_dot(value), + 'ttl': max(record.ttl, 300), + 'priority': record.priority + } + + @staticmethod + def _record_for_SRV(record, value): + raise NotImplementedError("Missing support for writing SRV records") + + def _record_for_NAPTR(self, record): + raise NotImplementedError("Missing support for writing NAPTR records") + # return [{ + # 'content': '{} {} "{}" "{}" "{}" {}'.format(v.order, v.preference, + # v.flags, v.service, + # v.regexp, + # v.replacement), + # 'disabled': False + # } for v in record.values] + + def _record_for_SSHFP(self, record): + raise NotImplementedError("Missing support for writing SSHFP records") + # return [{ + # 'content': '{} {} {}'.format(v.algorithm, v.fingerprint_type, + # v.fingerprint), + # 'disabled': False + # } for v in record.values] def _mod_Create(self, change): out = [] for value in change.new.values: - out.append({ - 'name': change.new.fqdn, - 'type': change.new._type, - 'data': value, - 'ttl': change.new.ttl, - }) + transformer = getattr(self, "_record_for_{}".format(change.new._type)) + out.append(transformer(change.new, value)) return out def _mod_Update(self, change): @@ -320,14 +362,16 @@ class RackspaceProvider(BaseProvider): update_out = [] for value in change.new.values: - key = (change.existing._type, change.existing.fqdn, value) - rsid = self._id_map[key] - update_out.append({ - 'id': rsid, - 'name': change.new.fqdn, - 'data': value, - 'ttl': change.new.ttl, - }) + transformer = getattr(self, "_record_for_{}".format(change.new._type)) + prior_rs_record = transformer(change.existing, value) + prior_key = self._key_for_record(prior_rs_record) + next_rs_record = transformer(change.new, value) + next_key = self._key_for_record(prior_rs_record) + next_rs_record["id"] = self._id_map[prior_key] + del next_rs_record["type"] + update_out.append(next_rs_record) + self._id_map[next_key] = self._id_map[prior_key] + del self._id_map[prior_key] return update_out, delete_out def _mod_Delete(self, change): @@ -336,9 +380,10 @@ class RackspaceProvider(BaseProvider): def _delete_given_change_values(self, change, values): out = [] for value in values: - key = (change.existing._type, change.existing.fqdn, value) - rsid = self._id_map[key] - out.append('id=' + rsid) + transformer = getattr(self, "_record_for_{}".format(change.existing._type)) + rs_record = transformer(change.existing, value) + key = self._key_for_record(rs_record) + out.append('id=' + self._id_map[key]) del self._id_map[key] return out @@ -363,7 +408,7 @@ class RackspaceProvider(BaseProvider): deletes += self._mod_Delete(change) if creates: - data = {"records": sorted(creates, key=lambda v: v['name'])} + data = {"records": sorted(creates, key=lambda v: v['type'] + v['name'] + v.get('data', ''))} self._post('domains/{}/records'.format(domain_id), data=data) if updates: diff --git a/tests/fixtures/rackspace-sample-recordset-existing-nameservers.json b/tests/fixtures/rackspace-sample-recordset-existing-nameservers.json index 034aa44..3e0f9cd 100644 --- a/tests/fixtures/rackspace-sample-recordset-existing-nameservers.json +++ b/tests/fixtures/rackspace-sample-recordset-existing-nameservers.json @@ -6,13 +6,13 @@ "type" : "A", "data" : "1.2.3.4", "updated" : "2011-06-24T01:12:53.000+0000", - "ttl" : 60, + "ttl" : 600, "created" : "2011-06-24T01:12:53.000+0000" }, { "name" : "unit.tests.", "id" : "NS-454454", "type" : "NS", - "data" : "8.8.8.8.", + "data" : "ns1.example.com", "updated" : "2011-06-24T01:12:51.000+0000", "ttl" : 600, "created" : "2011-06-24T01:12:51.000+0000" @@ -20,7 +20,7 @@ "name" : "unit.tests.", "id" : "NS-454455", "type" : "NS", - "data" : "9.9.9.9.", + "data" : "ns2.example.com", "updated" : "2011-06-24T01:12:52.000+0000", "ttl" : 600, "created" : "2011-06-24T01:12:52.000+0000" diff --git a/tests/test_octodns_provider_rackspace.py b/tests/test_octodns_provider_rackspace.py index 4c53e95..864f940 100644 --- a/tests/test_octodns_provider_rackspace.py +++ b/tests/test_octodns_provider_rackspace.py @@ -43,8 +43,10 @@ with open('./tests/fixtures/rackspace-sample-recordset-page2.json') as fh: with open('./tests/fixtures/rackspace-sample-recordset-existing-nameservers.json') as fh: RECORDS_EXISTING_NAMESERVERS = fh.read() + class TestRackspaceProvider(TestCase): def setUp(self): + self.maxDiff = 1000 with requests_mock() as mock: mock.post(ANY, status_code=200, text=AUTH_RESPONSE) self.provider = RackspaceProvider('test', 'api-key') @@ -73,7 +75,7 @@ class TestRackspaceProvider(TestCase): def test_nonexistent_zone(self): # Non-existent zone doesn't populate anything with requests_mock() as mock: - mock.get(ANY, status_code=422, + mock.get(ANY, status_code=404, json={'error': "Could not find domain 'unit.tests.'"}) zone = Zone('unit.tests.', []) @@ -196,7 +198,7 @@ class TestRackspaceProvider(TestCase): "subdomain": '', "data": { 'type': 'A', - 'ttl': 60, + 'ttl': 300, 'values': ['1.2.3.4', '1.2.3.5', '1.2.3.6'] } } @@ -208,19 +210,19 @@ class TestRackspaceProvider(TestCase): "id": "A-111111", "type": "A", "data": "1.2.3.4", - "ttl": 60 + "ttl": 300 }, { "name": "unit.tests.", "id": "A-222222", "type": "A", "data": "1.2.3.5", - "ttl": 60 + "ttl": 300 }, { "name": "unit.tests.", "id": "A-333333", "type": "A", "data": "1.2.3.6", - "ttl": 60 + "ttl": 300 }] } ExpectChanges = False @@ -236,7 +238,7 @@ class TestRackspaceProvider(TestCase): "subdomain": 'foo', "data": { 'type': 'A', - 'ttl': 60, + 'ttl': 300, 'value': '1.2.3.4' } }, @@ -244,7 +246,7 @@ class TestRackspaceProvider(TestCase): "subdomain": 'bar', "data": { 'type': 'A', - 'ttl': 60, + 'ttl': 300, 'value': '1.2.3.4' } } @@ -256,13 +258,13 @@ class TestRackspaceProvider(TestCase): "id": "A-111111", "type": "A", "data": "1.2.3.4", - "ttl": 60 + "ttl": 300 }, { "name": "bar.unit.tests.", "id": "A-222222", "type": "A", "data": "1.2.3.4", - "ttl": 60 + "ttl": 300 }] } ExpectChanges = False @@ -278,9 +280,17 @@ class TestRackspaceProvider(TestCase): "subdomain": '', "data": { 'type': 'A', - 'ttl': 60, + 'ttl': 300, 'value': '1.2.3.4' } + }, + { + "subdomain": 'foo', + "data": { + 'type': 'NS', + 'ttl': 300, + 'value': 'ns.example.com.' + } } ] OwnRecords = { @@ -293,7 +303,12 @@ class TestRackspaceProvider(TestCase): "name": "unit.tests.", "type": "A", "data": "1.2.3.4", - "ttl": 60 + "ttl": 300 + }, { + "name": "foo.unit.tests.", + "type": "NS", + "data": "ns.example.com", + "ttl": 300 }] } ExpectedDeletions = None @@ -307,9 +322,17 @@ class TestRackspaceProvider(TestCase): "subdomain": '', "data": { 'type': 'A', - 'ttl': 60, + 'ttl': 300, 'values': ['1.2.3.4', '1.2.3.5', '1.2.3.6'] } + }, + { + "subdomain": 'foo', + "data": { + 'type': 'NS', + 'ttl': 300, + 'values': ['ns1.example.com.', 'ns2.example.com.'] + } } ] OwnRecords = { @@ -322,17 +345,27 @@ class TestRackspaceProvider(TestCase): "name": "unit.tests.", "type": "A", "data": "1.2.3.4", - "ttl": 60 + "ttl": 300 }, { "name": "unit.tests.", "type": "A", "data": "1.2.3.5", - "ttl": 60 + "ttl": 300 }, { "name": "unit.tests.", "type": "A", "data": "1.2.3.6", - "ttl": 60 + "ttl": 300 + }, { + "name": "foo.unit.tests.", + "type": "NS", + "data": "ns1.example.com", + "ttl": 300 + }, { + "name": "foo.unit.tests.", + "type": "NS", + "data": "ns2.example.com", + "ttl": 300 }] } ExpectedDeletions = None @@ -345,16 +378,24 @@ class TestRackspaceProvider(TestCase): "subdomain": 'foo', "data": { 'type': 'A', - 'ttl': 60, + 'ttl': 300, 'value': '1.2.3.4' } }, { "subdomain": 'bar', "data": { 'type': 'A', - 'ttl': 60, + 'ttl': 300, 'value': '1.2.3.4' } + }, + { + "subdomain": 'foo', + "data": { + 'type': 'NS', + 'ttl': 300, + 'value': 'ns.example.com.' + } }] OwnRecords = { "totalEntries": 0, @@ -366,12 +407,17 @@ class TestRackspaceProvider(TestCase): "name": "bar.unit.tests.", "type": "A", "data": "1.2.3.4", - "ttl": 60 + "ttl": 300 }, { "name": "foo.unit.tests.", "type": "A", "data": "1.2.3.4", - "ttl": 60 + "ttl": 300 + }, { + "name": "foo.unit.tests.", + "type": "NS", + "data": "ns.example.com", + "ttl": 300 }] } ExpectedDeletions = None @@ -388,12 +434,18 @@ class TestRackspaceProvider(TestCase): "id": "A-111111", "type": "A", "data": "1.2.3.4", - "ttl": 60 + "ttl": 300 + }, { + "name": "foo.unit.tests.", + "id": "NS-111111", + "type": "NS", + "data": "ns.example.com", + "ttl": 300 }] } ExpectChanges = True ExpectedAdditions = None - ExpectedDeletions = "id=A-111111" + ExpectedDeletions = "id=A-111111&id=NS-111111" ExpectedUpdates = None return self._test_apply_with_data(TestData) @@ -404,7 +456,7 @@ class TestRackspaceProvider(TestCase): "subdomain": '', "data": { 'type': 'A', - 'ttl': 60, + 'ttl': 300, 'value': '1.2.3.5' } } @@ -416,30 +468,36 @@ class TestRackspaceProvider(TestCase): "id": "A-111111", "type": "A", "data": "1.2.3.4", - "ttl": 60 + "ttl": 300 }, { "name": "unit.tests.", "id": "A-222222", "type": "A", "data": "1.2.3.5", - "ttl": 60 + "ttl": 300 }, { "name": "unit.tests.", "id": "A-333333", "type": "A", "data": "1.2.3.6", - "ttl": 60 + "ttl": 300 + }, { + "name": "foo.unit.tests.", + "id": "NS-111111", + "type": "NS", + "data": "ns.example.com", + "ttl": 300 }] } ExpectChanges = True ExpectedAdditions = None - ExpectedDeletions = "id=A-111111&id=A-333333" + ExpectedDeletions = "id=A-111111&id=A-333333&id=NS-111111" ExpectedUpdates = { "records": [{ "name": "unit.tests.", "id": "A-222222", "data": "1.2.3.5", - "ttl": 60 + "ttl": 300 }] } return self._test_apply_with_data(TestData) @@ -451,7 +509,7 @@ class TestRackspaceProvider(TestCase): "subdomain": '', "data": { 'type': 'A', - 'ttl': 60, + 'ttl': 300, 'value': '1.2.3.4' } } @@ -463,19 +521,19 @@ class TestRackspaceProvider(TestCase): "id": "A-111111", "type": "A", "data": "1.2.3.4", - "ttl": 60 + "ttl": 300 }, { "name": "foo.unit.tests.", "id": "A-222222", "type": "A", "data": "1.2.3.5", - "ttl": 60 + "ttl": 300 }, { "name": "bar.unit.tests.", "id": "A-333333", "type": "A", "data": "1.2.3.6", - "ttl": 60 + "ttl": 300 }] } ExpectChanges = True @@ -503,7 +561,7 @@ class TestRackspaceProvider(TestCase): "id": "A-111111", "type": "A", "data": "1.2.3.4", - "ttl": 60 + "ttl": 300 }] } ExpectChanges = True @@ -538,19 +596,19 @@ class TestRackspaceProvider(TestCase): "id": "A-111111", "type": "A", "data": "1.2.3.4", - "ttl": 60 + "ttl": 300 }, { "name": "unit.tests.", "id": "A-222222", "type": "A", "data": "1.2.3.5", - "ttl": 60 + "ttl": 300 }, { "name": "unit.tests.", "id": "A-333333", "type": "A", "data": "1.2.3.6", - "ttl": 60 + "ttl": 300 }] } ExpectChanges = True @@ -603,13 +661,13 @@ class TestRackspaceProvider(TestCase): "id": "A-111111", "type": "A", "data": "1.2.3.4", - "ttl": 60 + "ttl": 300 }, { "name": "bar.unit.tests.", "id": "A-222222", "type": "A", "data": "1.2.3.4", - "ttl": 60 + "ttl": 300 }] } ExpectChanges = True @@ -630,6 +688,7 @@ class TestRackspaceProvider(TestCase): } return self._test_apply_with_data(TestData) + """ def test_provider(self): expected = self._load_full_config() @@ -707,17 +766,18 @@ class TestRackspaceProvider(TestCase): with self.assertRaises(HTTPError): plan = self.provider.plan(expected) self.provider.apply(plan) + """ def test_plan_no_changes(self): expected = Zone('unit.tests.', []) expected.add_record(Record.new(expected, '', { 'type': 'NS', 'ttl': 600, - 'values': ['8.8.8.8.', '9.9.9.9.'] + 'values': ['ns1.example.com.', 'ns2.example.com.'] })) expected.add_record(Record.new(expected, '', { 'type': 'A', - 'ttl': 60, + 'ttl': 600, 'value': '1.2.3.4' })) @@ -735,7 +795,7 @@ class TestRackspaceProvider(TestCase): expected.add_record(Record.new(expected, '', { 'type': 'NS', 'ttl': 600, - 'values': ['8.8.8.8.', '9.9.9.9.'] + 'values': ['ns1.example.com.', 'ns2.example.com.'] })) with requests_mock() as mock: @@ -745,7 +805,7 @@ class TestRackspaceProvider(TestCase): plan = self.provider.plan(expected) self.assertTrue(mock.called) self.assertEquals(1, len(plan.changes)) - self.assertEqual(plan.changes[0].existing.ttl, 60) + self.assertEqual(plan.changes[0].existing.ttl, 600) self.assertEqual(plan.changes[0].existing.values[0], '1.2.3.4') def test_plan_create_a_record(self): @@ -753,11 +813,11 @@ class TestRackspaceProvider(TestCase): expected.add_record(Record.new(expected, '', { 'type': 'NS', 'ttl': 600, - 'values': ['8.8.8.8.', '9.9.9.9.'] + 'values': ['ns1.example.com.', 'ns2.example.com.'] })) expected.add_record(Record.new(expected, '', { 'type': 'A', - 'ttl': 60, + 'ttl': 600, 'values': ['1.2.3.4', '1.2.3.5'] })) @@ -768,7 +828,7 @@ class TestRackspaceProvider(TestCase): plan = self.provider.plan(expected) self.assertTrue(mock.called) self.assertEquals(1, len(plan.changes)) - self.assertEqual(plan.changes[0].new.ttl, 60) + self.assertEqual(plan.changes[0].new.ttl, 600) self.assertEqual(plan.changes[0].new.values[0], '1.2.3.4') self.assertEqual(plan.changes[0].new.values[1], '1.2.3.5') @@ -777,7 +837,7 @@ class TestRackspaceProvider(TestCase): expected.add_record(Record.new(expected, '', { 'type': 'NS', 'ttl': 600, - 'values': ['8.8.8.8.', '9.9.9.9.'] + 'values': ['ns1.example.com.', 'ns2.example.com.'] })) expected.add_record(Record.new(expected, '', { 'type': 'A', @@ -793,6 +853,6 @@ class TestRackspaceProvider(TestCase): self.assertTrue(mock.called) self.assertEqual(1, len(plan.changes)) - self.assertEqual(plan.changes[0].existing.ttl, 60) + self.assertEqual(plan.changes[0].existing.ttl, 600) self.assertEqual(plan.changes[0].new.ttl, 86400) self.assertEqual(plan.changes[0].new.values[0], '1.2.3.4') From 92fb24f3fa57b06fc0ff16891fd9247e382a1be2 Mon Sep 17 00:00:00 2001 From: Terrence Cole Date: Thu, 13 Jul 2017 14:47:29 -0700 Subject: [PATCH 07/35] The provider constructor requires a pass-through id parameter. --- octodns/provider/rackspace.py | 2 +- tests/test_octodns_provider_rackspace.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py index 110412b..848811c 100644 --- a/octodns/provider/rackspace.py +++ b/octodns/provider/rackspace.py @@ -43,7 +43,7 @@ class RackspaceProvider(BaseProvider): SUPPORTS_GEO = False TIMEOUT = 5 - def __init__(self, username, api_key, *args, **kwargs): + def __init__(self, id, username, api_key, *args, **kwargs): ''' Rackspace API v1 Provider diff --git a/tests/test_octodns_provider_rackspace.py b/tests/test_octodns_provider_rackspace.py index 864f940..63f3de9 100644 --- a/tests/test_octodns_provider_rackspace.py +++ b/tests/test_octodns_provider_rackspace.py @@ -49,7 +49,7 @@ class TestRackspaceProvider(TestCase): self.maxDiff = 1000 with requests_mock() as mock: mock.post(ANY, status_code=200, text=AUTH_RESPONSE) - self.provider = RackspaceProvider('test', 'api-key') + self.provider = RackspaceProvider(id, 'test', 'api-key') self.assertTrue(mock.called_once) def test_bad_auth(self): From 3459064d0fc9335c48795cfb8c1caa4bd1fd3023 Mon Sep 17 00:00:00 2001 From: Terrence Cole Date: Thu, 13 Jul 2017 14:53:30 -0700 Subject: [PATCH 08/35] Use the proper arity when accessing quoted data records. --- octodns/provider/rackspace.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py index 848811c..e1931db 100644 --- a/octodns/provider/rackspace.py +++ b/octodns/provider/rackspace.py @@ -164,9 +164,9 @@ class RackspaceProvider(BaseProvider): def _data_for_quoted(self, rrset): return { - 'type': rrset['type'], - 'values': [strip_quotes(r['content']) for r in rrset['records']], - 'ttl': rrset['ttl'] + 'type': rrset[0]['type'], + 'values': [strip_quotes(r['content']) for r in rrset[0]['records']], + 'ttl': rrset[0]['ttl'] } _data_for_SPF = _data_for_quoted From 95fb613966163be8edc1cf42275bc038555bb6da Mon Sep 17 00:00:00 2001 From: Terrence Cole Date: Thu, 13 Jul 2017 14:56:04 -0700 Subject: [PATCH 09/35] Pull quoted data out of the correct field. --- octodns/provider/rackspace.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py index e1931db..be7b619 100644 --- a/octodns/provider/rackspace.py +++ b/octodns/provider/rackspace.py @@ -165,7 +165,7 @@ class RackspaceProvider(BaseProvider): def _data_for_quoted(self, rrset): return { 'type': rrset[0]['type'], - 'values': [strip_quotes(r['content']) for r in rrset[0]['records']], + 'values': [strip_quotes(r['data']) for r in rrset], 'ttl': rrset[0]['ttl'] } From b911fac90e041dd3845e2843780ac1703c79b37c Mon Sep 17 00:00:00 2001 From: Terrence Cole Date: Thu, 13 Jul 2017 15:04:16 -0700 Subject: [PATCH 10/35] RackSpace does not send back TXT records quoted. --- octodns/provider/rackspace.py | 43 ++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py index be7b619..8f80abf 100644 --- a/octodns/provider/rackspace.py +++ b/octodns/provider/rackspace.py @@ -162,15 +162,15 @@ class RackspaceProvider(BaseProvider): _data_for_CNAME = _data_for_single _data_for_PTR = _data_for_single - def _data_for_quoted(self, rrset): - return { - 'type': rrset[0]['type'], - 'values': [strip_quotes(r['data']) for r in rrset], - 'ttl': rrset[0]['ttl'] - } + # def _data_for_quoted(self, rrset): + # return { + # 'type': rrset[0]['type'], + # 'values': [strip_quotes(r['data']) for r in rrset], + # 'ttl': rrset[0]['ttl'] + # } - _data_for_SPF = _data_for_quoted - _data_for_TXT = _data_for_quoted + _data_for_SPF = _data_for_multiple + _data_for_TXT = _data_for_multiple def _data_for_MX(self, rrset): values = [] @@ -281,15 +281,15 @@ class RackspaceProvider(BaseProvider): return records @staticmethod - def _record_for_ip(record, value): + def _record_for_single(record, value): return { 'name': record.fqdn, 'type': record._type, 'data': value, 'ttl': max(record.ttl, 300), } - _record_for_A = _record_for_ip - _record_for_AAAA = _record_for_ip + _record_for_A = _record_for_single + _record_for_AAAA = _record_for_single @staticmethod def _record_for_named(record, value): @@ -304,16 +304,17 @@ class RackspaceProvider(BaseProvider): _record_for_CNAME = _record_for_named _record_for_PTR = _record_for_named - @staticmethod - def _record_for_quoted(record, value): - return { - 'name': record.fqdn, - 'type': record._type, - 'data': add_quotes(value), - 'ttl': max(record.ttl, 300), - } - _record_for_SPF = _record_for_quoted - _record_for_TXT = _record_for_quoted + # @staticmethod + # def _record_for_quoted(record, value): + # return { + # 'name': record.fqdn, + # 'type': record._type, + # 'data': add_quotes(value), + # 'ttl': max(record.ttl, 300), + # } + + _record_for_SPF = _record_for_single + _record_for_TXT = _record_for_single @staticmethod def _record_for_MX(record, value): From 057f50621e030ce1f47c630b4356f5723fba94a5 Mon Sep 17 00:00:00 2001 From: Terrence Cole Date: Thu, 13 Jul 2017 15:10:33 -0700 Subject: [PATCH 11/35] RackSpace does not escape ;. --- octodns/provider/rackspace.py | 48 +++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 19 deletions(-) diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py index 8f80abf..be8bda5 100644 --- a/octodns/provider/rackspace.py +++ b/octodns/provider/rackspace.py @@ -7,6 +7,7 @@ from __future__ import absolute_import, division, print_function, \ from requests import HTTPError, Session, post from collections import defaultdict import logging +import string from ..record import Create, Record from .base import BaseProvider @@ -39,6 +40,16 @@ def add_quotes(s): return '"{}"'.format(s) +def escape_semicolon(s): + assert s + return string.replace(s, ';', '\;') + + +def unescape_semicolon(s): + assert s + return string.replace(s, '\;', ';') + + class RackspaceProvider(BaseProvider): SUPPORTS_GEO = False TIMEOUT = 5 @@ -162,15 +173,15 @@ class RackspaceProvider(BaseProvider): _data_for_CNAME = _data_for_single _data_for_PTR = _data_for_single - # def _data_for_quoted(self, rrset): - # return { - # 'type': rrset[0]['type'], - # 'values': [strip_quotes(r['data']) for r in rrset], - # 'ttl': rrset[0]['ttl'] - # } + def _data_for_textual(self, rrset): + return { + 'type': rrset[0]['type'], + 'values': [escape_semicolon(r['data']) for r in rrset], + 'ttl': rrset[0]['ttl'] + } - _data_for_SPF = _data_for_multiple - _data_for_TXT = _data_for_multiple + _data_for_SPF = _data_for_textual + _data_for_TXT = _data_for_textual def _data_for_MX(self, rrset): values = [] @@ -304,17 +315,16 @@ class RackspaceProvider(BaseProvider): _record_for_CNAME = _record_for_named _record_for_PTR = _record_for_named - # @staticmethod - # def _record_for_quoted(record, value): - # return { - # 'name': record.fqdn, - # 'type': record._type, - # 'data': add_quotes(value), - # 'ttl': max(record.ttl, 300), - # } - - _record_for_SPF = _record_for_single - _record_for_TXT = _record_for_single + @staticmethod + def _record_for_textual(record, value): + return { + 'name': record.fqdn, + 'type': record._type, + 'data': unescape_semicolon(value), + 'ttl': max(record.ttl, 300), + } + _record_for_SPF = _record_for_textual + _record_for_TXT = _record_for_textual @staticmethod def _record_for_MX(record, value): From 0c31257b0f456caca4f49bf982abd052f6ce15e3 Mon Sep 17 00:00:00 2001 From: Terrence Cole Date: Mon, 17 Jul 2017 16:47:09 -0700 Subject: [PATCH 12/35] Use the correct record when computing the key. --- octodns/provider/rackspace.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py index be8bda5..6ed1d78 100644 --- a/octodns/provider/rackspace.py +++ b/octodns/provider/rackspace.py @@ -377,7 +377,7 @@ class RackspaceProvider(BaseProvider): prior_rs_record = transformer(change.existing, value) prior_key = self._key_for_record(prior_rs_record) next_rs_record = transformer(change.new, value) - next_key = self._key_for_record(prior_rs_record) + next_key = self._key_for_record(next_rs_record) next_rs_record["id"] = self._id_map[prior_key] del next_rs_record["type"] update_out.append(next_rs_record) From 10ff8301a542bb427325efbe0a5a40022ab12bcf Mon Sep 17 00:00:00 2001 From: Terrence Cole Date: Tue, 18 Jul 2017 10:02:57 -0700 Subject: [PATCH 13/35] RackSpace's "name" field is a "fully-qualified" name, but without the dot. --- octodns/provider/rackspace.py | 2 +- tests/test_octodns_provider_rackspace.py | 27 ++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py index 6ed1d78..52e41d4 100644 --- a/octodns/provider/rackspace.py +++ b/octodns/provider/rackspace.py @@ -294,7 +294,7 @@ class RackspaceProvider(BaseProvider): @staticmethod def _record_for_single(record, value): return { - 'name': record.fqdn, + 'name': remove_trailing_dot(record.fqdn), 'type': record._type, 'data': value, 'ttl': max(record.ttl, 300), diff --git a/tests/test_octodns_provider_rackspace.py b/tests/test_octodns_provider_rackspace.py index 63f3de9..aa82245 100644 --- a/tests/test_octodns_provider_rackspace.py +++ b/tests/test_octodns_provider_rackspace.py @@ -136,6 +136,33 @@ class TestRackspaceProvider(TestCase): # OctoDNS does not propagate top-level NS records. self.assertEquals(1, len(plan.changes)) + def test_fqdn_a_record(self): + expected = Zone('example.com.', []) + # expected.add_record(Record.new(expected, 'foo', '1.2.3.4')) + + with requests_mock() as list_mock: + list_mock.get(re.compile('domains$'), status_code=200, text=LIST_DOMAINS_RESPONSE) + list_mock.get(re.compile('records'), status_code=200, json={'records': [ + {'type': 'A', + 'name': 'foo.example.com', + 'id': 'A-111111', + 'data': '1.2.3.4', + 'ttl': 300}]}) + plan = self.provider.plan(expected) + self.assertTrue(list_mock.called) + self.assertEqual(1, len(plan.changes)) + self.assertTrue(plan.changes[0].existing.fqdn == 'foo.example.com.') + + with requests_mock() as mock: + def _assert_deleting(request, context): + parts = urlparse(request.url) + self.assertEqual('id=A-111111', parts.query) + mock.get(re.compile('domains$'), status_code=200, text=LIST_DOMAINS_RESPONSE) + mock.delete(re.compile('domains/.*/records?.*'), status_code=202, + text=_assert_deleting) + self.provider.apply(plan) + self.assertTrue(mock.called) + def _test_apply_with_data(self, data): expected = Zone('unit.tests.', []) for record in data.OtherRecords: From bb4d1c82b1540fa45393332e9526052024e3ffde Mon Sep 17 00:00:00 2001 From: Terrence Cole Date: Thu, 20 Jul 2017 15:46:08 -0700 Subject: [PATCH 14/35] Add a --quiet flag to suppress non-warning or error messages. --- octodns/cmds/args.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/octodns/cmds/args.py b/octodns/cmds/args.py index c84dd92..9322461 100644 --- a/octodns/cmds/args.py +++ b/octodns/cmds/args.py @@ -37,6 +37,8 @@ class ArgumentParser(_Base): _help = 'Increase verbosity to get details and help track down issues' self.add_argument('--debug', action='store_true', default=False, help=_help) + self.add_argument('--quiet', action='store_true', default=False, + help='Only print warning and error messages') args = super(ArgumentParser, self).parse_args() self._setup_logging(args, default_log_level) @@ -59,7 +61,11 @@ class ArgumentParser(_Base): handler.setFormatter(Formatter(fmt=fmt)) logger.addHandler(handler) - logger.level = DEBUG if args.debug else default_log_level + logger.level = default_log_level + if args.debug: + logger.level = DEBUG + elif args.quiet: + logger.level = WARN # boto is noisy, set it to warn getLogger('botocore').level = WARN From 4707b4654e391d31ef3015e87261d142afc0adf8 Mon Sep 17 00:00:00 2001 From: Terrence Cole Date: Thu, 20 Jul 2017 15:50:57 -0700 Subject: [PATCH 15/35] Add a delay to work around rackspace rate limiting. --- octodns/provider/rackspace.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py index 52e41d4..bcfcd5d 100644 --- a/octodns/provider/rackspace.py +++ b/octodns/provider/rackspace.py @@ -8,6 +8,7 @@ from requests import HTTPError, Session, post from collections import defaultdict import logging import string +import time from ..record import Create, Record from .base import BaseProvider @@ -71,6 +72,8 @@ class RackspaceProvider(BaseProvider): auth_token, dns_endpoint = self._get_auth_token(username, api_key) self.dns_endpoint = dns_endpoint + self.ratelimit_delay = kwargs.get('ratelimit_delay', 0) + sess = Session() sess.headers.update({'X-Auth-Token': auth_token}) self._sess = sess @@ -88,6 +91,7 @@ class RackspaceProvider(BaseProvider): def _get_zone_id_for(self, zone): ret = self._request('GET', 'domains', pagination_key='domains') + time.sleep(self.ratelimit_delay) if ret: return [x for x in ret if x['name'] == zone.name[:-1]][0]['id'] else: @@ -104,6 +108,7 @@ class RackspaceProvider(BaseProvider): def _request_for_url(self, method, url, data): resp = self._sess.request(method, url, json=data, timeout=self.TIMEOUT) + time.sleep(self.ratelimit_delay) self.log.debug('_request: status=%d', resp.status_code) resp.raise_for_status() return resp @@ -112,6 +117,7 @@ class RackspaceProvider(BaseProvider): acc = [] resp = self._sess.request(method, url, json=data, timeout=self.TIMEOUT) + time.sleep(self.ratelimit_delay) self.log.debug('_request: status=%d', resp.status_code) resp.raise_for_status() acc.extend(resp.json()[pagination_key]) From 803b002cd00b3f8bcde0943e37e826bf21d8f6ba Mon Sep 17 00:00:00 2001 From: Terrence Cole Date: Thu, 20 Jul 2017 15:59:32 -0700 Subject: [PATCH 16/35] Rackspace backoff is actually just required. --- octodns/provider/rackspace.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py index bcfcd5d..7b47cd6 100644 --- a/octodns/provider/rackspace.py +++ b/octodns/provider/rackspace.py @@ -55,7 +55,7 @@ class RackspaceProvider(BaseProvider): SUPPORTS_GEO = False TIMEOUT = 5 - def __init__(self, id, username, api_key, *args, **kwargs): + def __init__(self, id, username, api_key, ratelimit_delay, *args, **kwargs): ''' Rackspace API v1 Provider @@ -72,7 +72,7 @@ class RackspaceProvider(BaseProvider): auth_token, dns_endpoint = self._get_auth_token(username, api_key) self.dns_endpoint = dns_endpoint - self.ratelimit_delay = kwargs.get('ratelimit_delay', 0) + self.ratelimit_delay = ratelimit_delay sess = Session() sess.headers.update({'X-Auth-Token': auth_token}) From 8d86002382d298e1b14de9001f6780509753cdfb Mon Sep 17 00:00:00 2001 From: Terrence Cole Date: Thu, 20 Jul 2017 16:09:52 -0700 Subject: [PATCH 17/35] Environment variables are strings, so convert to a float first. --- octodns/provider/rackspace.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py index 7b47cd6..ef22415 100644 --- a/octodns/provider/rackspace.py +++ b/octodns/provider/rackspace.py @@ -72,7 +72,7 @@ class RackspaceProvider(BaseProvider): auth_token, dns_endpoint = self._get_auth_token(username, api_key) self.dns_endpoint = dns_endpoint - self.ratelimit_delay = ratelimit_delay + self.ratelimit_delay = float(ratelimit_delay) sess = Session() sess.headers.update({'X-Auth-Token': auth_token}) From a9f3384d114ec7823c55e3c10bc09221909f58ff Mon Sep 17 00:00:00 2001 From: Terrence Cole Date: Thu, 20 Jul 2017 16:24:43 -0700 Subject: [PATCH 18/35] Remove trailing dot on all record types that take an fqdn. --- octodns/provider/rackspace.py | 6 +- tests/test_octodns_provider_rackspace.py | 76 ++++++++++++------------ 2 files changed, 41 insertions(+), 41 deletions(-) diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py index ef22415..bf0e248 100644 --- a/octodns/provider/rackspace.py +++ b/octodns/provider/rackspace.py @@ -311,7 +311,7 @@ class RackspaceProvider(BaseProvider): @staticmethod def _record_for_named(record, value): return { - 'name': record.fqdn, + 'name': remove_trailing_dot(record.fqdn), 'type': record._type, 'data': remove_trailing_dot(value), 'ttl': max(record.ttl, 300), @@ -324,7 +324,7 @@ class RackspaceProvider(BaseProvider): @staticmethod def _record_for_textual(record, value): return { - 'name': record.fqdn, + 'name': remove_trailing_dot(record.fqdn), 'type': record._type, 'data': unescape_semicolon(value), 'ttl': max(record.ttl, 300), @@ -335,7 +335,7 @@ class RackspaceProvider(BaseProvider): @staticmethod def _record_for_MX(record, value): return { - 'name': record.fqdn, + 'name': remove_trailing_dot(record.fqdn), 'type': record._type, 'data': remove_trailing_dot(value), 'ttl': max(record.ttl, 300), diff --git a/tests/test_octodns_provider_rackspace.py b/tests/test_octodns_provider_rackspace.py index aa82245..d579faa 100644 --- a/tests/test_octodns_provider_rackspace.py +++ b/tests/test_octodns_provider_rackspace.py @@ -49,7 +49,7 @@ class TestRackspaceProvider(TestCase): self.maxDiff = 1000 with requests_mock() as mock: mock.post(ANY, status_code=200, text=AUTH_RESPONSE) - self.provider = RackspaceProvider(id, 'test', 'api-key') + self.provider = RackspaceProvider(id, 'test', 'api-key', '0') self.assertTrue(mock.called_once) def test_bad_auth(self): @@ -233,19 +233,19 @@ class TestRackspaceProvider(TestCase): OwnRecords = { "totalEntries": 3, "records": [{ - "name": "unit.tests.", + "name": "unit.tests", "id": "A-111111", "type": "A", "data": "1.2.3.4", "ttl": 300 }, { - "name": "unit.tests.", + "name": "unit.tests", "id": "A-222222", "type": "A", "data": "1.2.3.5", "ttl": 300 }, { - "name": "unit.tests.", + "name": "unit.tests", "id": "A-333333", "type": "A", "data": "1.2.3.6", @@ -281,13 +281,13 @@ class TestRackspaceProvider(TestCase): OwnRecords = { "totalEntries": 3, "records": [{ - "name": "foo.unit.tests.", + "name": "foo.unit.tests", "id": "A-111111", "type": "A", "data": "1.2.3.4", "ttl": 300 }, { - "name": "bar.unit.tests.", + "name": "bar.unit.tests", "id": "A-222222", "type": "A", "data": "1.2.3.4", @@ -327,12 +327,12 @@ class TestRackspaceProvider(TestCase): ExpectChanges = True ExpectedAdditions = { "records": [{ - "name": "unit.tests.", + "name": "unit.tests", "type": "A", "data": "1.2.3.4", "ttl": 300 }, { - "name": "foo.unit.tests.", + "name": "foo.unit.tests", "type": "NS", "data": "ns.example.com", "ttl": 300 @@ -369,27 +369,27 @@ class TestRackspaceProvider(TestCase): ExpectChanges = True ExpectedAdditions = { "records": [{ - "name": "unit.tests.", + "name": "unit.tests", "type": "A", "data": "1.2.3.4", "ttl": 300 }, { - "name": "unit.tests.", + "name": "unit.tests", "type": "A", "data": "1.2.3.5", "ttl": 300 }, { - "name": "unit.tests.", + "name": "unit.tests", "type": "A", "data": "1.2.3.6", "ttl": 300 }, { - "name": "foo.unit.tests.", + "name": "foo.unit.tests", "type": "NS", "data": "ns1.example.com", "ttl": 300 }, { - "name": "foo.unit.tests.", + "name": "foo.unit.tests", "type": "NS", "data": "ns2.example.com", "ttl": 300 @@ -431,17 +431,17 @@ class TestRackspaceProvider(TestCase): ExpectChanges = True ExpectedAdditions = { "records": [{ - "name": "bar.unit.tests.", + "name": "bar.unit.tests", "type": "A", "data": "1.2.3.4", "ttl": 300 }, { - "name": "foo.unit.tests.", + "name": "foo.unit.tests", "type": "A", "data": "1.2.3.4", "ttl": 300 }, { - "name": "foo.unit.tests.", + "name": "foo.unit.tests", "type": "NS", "data": "ns.example.com", "ttl": 300 @@ -457,13 +457,13 @@ class TestRackspaceProvider(TestCase): OwnRecords = { "totalEntries": 1, "records": [{ - "name": "unit.tests.", + "name": "unit.tests", "id": "A-111111", "type": "A", "data": "1.2.3.4", "ttl": 300 }, { - "name": "foo.unit.tests.", + "name": "foo.unit.tests", "id": "NS-111111", "type": "NS", "data": "ns.example.com", @@ -491,25 +491,25 @@ class TestRackspaceProvider(TestCase): OwnRecords = { "totalEntries": 3, "records": [{ - "name": "unit.tests.", + "name": "unit.tests", "id": "A-111111", "type": "A", "data": "1.2.3.4", "ttl": 300 }, { - "name": "unit.tests.", + "name": "unit.tests", "id": "A-222222", "type": "A", "data": "1.2.3.5", "ttl": 300 }, { - "name": "unit.tests.", + "name": "unit.tests", "id": "A-333333", "type": "A", "data": "1.2.3.6", "ttl": 300 }, { - "name": "foo.unit.tests.", + "name": "foo.unit.tests", "id": "NS-111111", "type": "NS", "data": "ns.example.com", @@ -521,7 +521,7 @@ class TestRackspaceProvider(TestCase): ExpectedDeletions = "id=A-111111&id=A-333333&id=NS-111111" ExpectedUpdates = { "records": [{ - "name": "unit.tests.", + "name": "unit.tests", "id": "A-222222", "data": "1.2.3.5", "ttl": 300 @@ -544,19 +544,19 @@ class TestRackspaceProvider(TestCase): OwnRecords = { "totalEntries": 3, "records": [{ - "name": "unit.tests.", + "name": "unit.tests", "id": "A-111111", "type": "A", "data": "1.2.3.4", "ttl": 300 }, { - "name": "foo.unit.tests.", + "name": "foo.unit.tests", "id": "A-222222", "type": "A", "data": "1.2.3.5", "ttl": 300 }, { - "name": "bar.unit.tests.", + "name": "bar.unit.tests", "id": "A-333333", "type": "A", "data": "1.2.3.6", @@ -584,7 +584,7 @@ class TestRackspaceProvider(TestCase): OwnRecords = { "totalEntries": 1, "records": [{ - "name": "unit.tests.", + "name": "unit.tests", "id": "A-111111", "type": "A", "data": "1.2.3.4", @@ -596,7 +596,7 @@ class TestRackspaceProvider(TestCase): ExpectedDeletions = None ExpectedUpdates = { "records": [{ - "name": "unit.tests.", + "name": "unit.tests", "id": "A-111111", "data": "1.2.3.4", "ttl": 3600 @@ -619,19 +619,19 @@ class TestRackspaceProvider(TestCase): OwnRecords = { "totalEntries": 3, "records": [{ - "name": "unit.tests.", + "name": "unit.tests", "id": "A-111111", "type": "A", "data": "1.2.3.4", "ttl": 300 }, { - "name": "unit.tests.", + "name": "unit.tests", "id": "A-222222", "type": "A", "data": "1.2.3.5", "ttl": 300 }, { - "name": "unit.tests.", + "name": "unit.tests", "id": "A-333333", "type": "A", "data": "1.2.3.6", @@ -643,17 +643,17 @@ class TestRackspaceProvider(TestCase): ExpectedDeletions = None ExpectedUpdates = { "records": [{ - "name": "unit.tests.", + "name": "unit.tests", "id": "A-111111", "data": "1.2.3.4", "ttl": 3600 }, { - "name": "unit.tests.", + "name": "unit.tests", "id": "A-222222", "data": "1.2.3.5", "ttl": 3600 }, { - "name": "unit.tests.", + "name": "unit.tests", "id": "A-333333", "data": "1.2.3.6", "ttl": 3600 @@ -684,13 +684,13 @@ class TestRackspaceProvider(TestCase): OwnRecords = { "totalEntries": 2, "records": [{ - "name": "foo.unit.tests.", + "name": "foo.unit.tests", "id": "A-111111", "type": "A", "data": "1.2.3.4", "ttl": 300 }, { - "name": "bar.unit.tests.", + "name": "bar.unit.tests", "id": "A-222222", "type": "A", "data": "1.2.3.4", @@ -702,12 +702,12 @@ class TestRackspaceProvider(TestCase): ExpectedDeletions = None ExpectedUpdates = { "records": [{ - "name": "bar.unit.tests.", + "name": "bar.unit.tests", "id": "A-222222", "data": "1.2.3.4", "ttl": 3600 }, { - "name": "foo.unit.tests.", + "name": "foo.unit.tests", "id": "A-111111", "data": "1.2.3.4", "ttl": 3600 From c185d28f146d04fffcd360abd5110982901affb5 Mon Sep 17 00:00:00 2001 From: Terrence Cole Date: Wed, 26 Jul 2017 12:49:30 -0700 Subject: [PATCH 19/35] Handle _ValueMixin record types as well as we handle _ValuesMixin records. --- octodns/provider/rackspace.py | 17 +++++++++++++---- tests/test_octodns_provider_rackspace.py | 19 +++++++++++++++++++ 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py index bf0e248..6af51fa 100644 --- a/octodns/provider/rackspace.py +++ b/octodns/provider/rackspace.py @@ -364,21 +364,30 @@ class RackspaceProvider(BaseProvider): # 'disabled': False # } for v in record.values] + def _get_values(self, record): + try: + return record.values + except AttributeError: + return [record.value] + def _mod_Create(self, change): out = [] - for value in change.new.values: + for value in self._get_values(change.new): transformer = getattr(self, "_record_for_{}".format(change.new._type)) out.append(transformer(change.new, value)) return out def _mod_Update(self, change): + existing_values = self._get_values(change.existing) + new_values = self._get_values(change.new) + # A reduction in number of values in an update record needs # to get upgraded into a Delete change for the removed values. - deleted_values = set(change.existing.values) - set(change.new.values) + deleted_values = set(existing_values) - set(new_values) delete_out = self._delete_given_change_values(change, deleted_values) update_out = [] - for value in change.new.values: + for value in new_values: transformer = getattr(self, "_record_for_{}".format(change.new._type)) prior_rs_record = transformer(change.existing, value) prior_key = self._key_for_record(prior_rs_record) @@ -392,7 +401,7 @@ class RackspaceProvider(BaseProvider): return update_out, delete_out def _mod_Delete(self, change): - return self._delete_given_change_values(change, change.existing.values) + return self._delete_given_change_values(change, self._get_values(change.existing)) def _delete_given_change_values(self, change, values): out = [] diff --git a/tests/test_octodns_provider_rackspace.py b/tests/test_octodns_provider_rackspace.py index d579faa..2862871 100644 --- a/tests/test_octodns_provider_rackspace.py +++ b/tests/test_octodns_provider_rackspace.py @@ -569,6 +569,25 @@ class TestRackspaceProvider(TestCase): ExpectedUpdates = None return self._test_apply_with_data(TestData) + def test_apply_delete_cname(self): + class TestData(object): + OtherRecords = [] + OwnRecords = { + "totalEntries": 3, + "records": [{ + "name": "foo.unit.tests", + "id": "CNAME-111111", + "type": "CNAME", + "data": "a.example.com", + "ttl": 300 + }] + } + ExpectChanges = True + ExpectedAdditions = None + ExpectedDeletions = "id=CNAME-111111" + ExpectedUpdates = None + return self._test_apply_with_data(TestData) + def test_apply_single_update(self): class TestData(object): OtherRecords = [ From 41617e69a77f3905956da2ff4a403954d1d96bac Mon Sep 17 00:00:00 2001 From: Terrence Cole Date: Wed, 2 Aug 2017 10:20:10 -0700 Subject: [PATCH 20/35] MX record values are repesented by a sub-struct. --- octodns/provider/rackspace.py | 4 +- tests/test_octodns_provider_rackspace.py | 51 ++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py index 6af51fa..0eb0b2f 100644 --- a/octodns/provider/rackspace.py +++ b/octodns/provider/rackspace.py @@ -337,9 +337,9 @@ class RackspaceProvider(BaseProvider): return { 'name': remove_trailing_dot(record.fqdn), 'type': record._type, - 'data': remove_trailing_dot(value), + 'data': remove_trailing_dot(value.value), 'ttl': max(record.ttl, 300), - 'priority': record.priority + 'priority': value.priority } @staticmethod diff --git a/tests/test_octodns_provider_rackspace.py b/tests/test_octodns_provider_rackspace.py index 2862871..243ebc4 100644 --- a/tests/test_octodns_provider_rackspace.py +++ b/tests/test_octodns_provider_rackspace.py @@ -342,6 +342,57 @@ class TestRackspaceProvider(TestCase): ExpectedUpdates = None return self._test_apply_with_data(TestData) + def test_apply_create_MX(self): + class TestData(object): + OtherRecords = [ + { + "subdomain": '', + "data": { + 'type': 'MX', + 'ttl': 300, + 'value': { + 'value': 'mail1.example.com.', + 'priority': 1, + } + } + }, + { + "subdomain": 'foo', + "data": { + 'type': 'MX', + 'ttl': 300, + 'value': { + 'value': 'mail2.example.com.', + 'priority': 2 + } + } + } + ] + OwnRecords = { + "totalEntries": 0, + "records": [] + } + ExpectChanges = True + ExpectedAdditions = { + "records": [{ + "name": "foo.unit.tests", + "type": "MX", + "data": "mail2.example.com", + "priority": 2, + "ttl": 300 + }, { + "name": "unit.tests", + "type": "MX", + "data": "mail1.example.com", + "priority": 1, + "ttl": 300 + }] + } + ExpectedDeletions = None + ExpectedUpdates = None + return self._test_apply_with_data(TestData) + + def test_apply_multiple_additions_exploding(self): class TestData(object): OtherRecords = [ From 3f369712e4cbff55b1ac3a2d634a552c77bd67be Mon Sep 17 00:00:00 2001 From: Terrence Cole Date: Wed, 2 Aug 2017 10:51:12 -0700 Subject: [PATCH 21/35] Updates need to be able to create records as well as delete them. --- octodns/provider/rackspace.py | 18 +++++++--- tests/test_octodns_provider_rackspace.py | 46 ++++++++++++++++++++---- 2 files changed, 54 insertions(+), 10 deletions(-) diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py index 0eb0b2f..a856295 100644 --- a/octodns/provider/rackspace.py +++ b/octodns/provider/rackspace.py @@ -371,8 +371,11 @@ class RackspaceProvider(BaseProvider): return [record.value] def _mod_Create(self, change): + return self._create_given_change_values(change, self._get_values(change.new)) + + def _create_given_change_values(self, change, values): out = [] - for value in self._get_values(change.new): + for value in values: transformer = getattr(self, "_record_for_{}".format(change.new._type)) out.append(transformer(change.new, value)) return out @@ -386,8 +389,14 @@ class RackspaceProvider(BaseProvider): deleted_values = set(existing_values) - set(new_values) delete_out = self._delete_given_change_values(change, deleted_values) + # An increase in number of values in an update record needs + # to get upgraded into a Create change for the added values. + create_values = set(new_values) - set(existing_values) + create_out = self._create_given_change_values(change, create_values) + update_out = [] - for value in new_values: + update_values = set(new_values).intersection(set(existing_values)) + for value in update_values: transformer = getattr(self, "_record_for_{}".format(change.new._type)) prior_rs_record = transformer(change.existing, value) prior_key = self._key_for_record(prior_rs_record) @@ -398,7 +407,7 @@ class RackspaceProvider(BaseProvider): update_out.append(next_rs_record) self._id_map[next_key] = self._id_map[prior_key] del self._id_map[prior_key] - return update_out, delete_out + return create_out, update_out, delete_out def _mod_Delete(self, change): return self._delete_given_change_values(change, self._get_values(change.existing)) @@ -427,7 +436,8 @@ class RackspaceProvider(BaseProvider): if change.__class__.__name__ == 'Create': creates += self._mod_Create(change) elif change.__class__.__name__ == 'Update': - add_updates, add_deletes = self._mod_Update(change) + add_creates, add_updates, add_deletes = self._mod_Update(change) + creates += add_creates updates += add_updates deletes += add_deletes elif change.__class__.__name__ == 'Delete': diff --git a/tests/test_octodns_provider_rackspace.py b/tests/test_octodns_provider_rackspace.py index 243ebc4..ce80112 100644 --- a/tests/test_octodns_provider_rackspace.py +++ b/tests/test_octodns_provider_rackspace.py @@ -392,7 +392,6 @@ class TestRackspaceProvider(TestCase): ExpectedUpdates = None return self._test_apply_with_data(TestData) - def test_apply_multiple_additions_exploding(self): class TestData(object): OtherRecords = [ @@ -674,6 +673,41 @@ class TestRackspaceProvider(TestCase): } return self._test_apply_with_data(TestData) + def test_apply_update_TXT(self): + class TestData(object): + OtherRecords = [ + { + "subdomain": '', + "data": { + 'type': 'TXT', + 'ttl': 300, + 'value': 'othervalue' + } + } + ] + OwnRecords = { + "totalEntries": 1, + "records": [{ + "name": "unit.tests", + "id": "TXT-111111", + "type": "TXT", + "data": "somevalue", + "ttl": 300 + }] + } + ExpectChanges = True + ExpectedAdditions = { + "records": [{ + "name": "unit.tests", + "type": "TXT", + "data": "othervalue", + "ttl": 300 + }] + } + ExpectedDeletions = 'id=TXT-111111' + ExpectedUpdates = None + return self._test_apply_with_data(TestData) + def test_apply_multiple_updates(self): class TestData(object): OtherRecords = [ @@ -712,15 +746,15 @@ class TestRackspaceProvider(TestCase): ExpectedAdditions = None ExpectedDeletions = None ExpectedUpdates = { - "records": [{ + "records": [ { "name": "unit.tests", - "id": "A-111111", - "data": "1.2.3.4", + "id": "A-222222", + "data": "1.2.3.5", "ttl": 3600 }, { "name": "unit.tests", - "id": "A-222222", - "data": "1.2.3.5", + "id": "A-111111", + "data": "1.2.3.4", "ttl": 3600 }, { "name": "unit.tests", From f26f77fcaec9257a6d3066818f45f07c2176b4e2 Mon Sep 17 00:00:00 2001 From: Terrence Cole Date: Wed, 2 Aug 2017 16:58:13 -0700 Subject: [PATCH 22/35] Force keys to be unicode. --- octodns/provider/rackspace.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py index a856295..c593150 100644 --- a/octodns/provider/rackspace.py +++ b/octodns/provider/rackspace.py @@ -146,8 +146,16 @@ class RackspaceProvider(BaseProvider): return self._request('DELETE', path, data=data) @staticmethod - def _key_for_record(rs_record): - return rs_record['type'], rs_record['name'], rs_record['data'] + def _as_unicode(s, codec): + if not isinstance(s, unicode): + return unicode(s, codec) + return s + + @classmethod + def _key_for_record(cls, rs_record): + return cls._as_unicode(rs_record['type'], 'ascii'),\ + cls._as_unicode(rs_record['name'], 'utf-8'),\ + cls._as_unicode(rs_record['data'], 'utf-8'),\ def _data_for_multiple(self, rrset): # TODO: geo not supported From b1ef8a8f8d78b9d535fc0af7ecc3bb607e171f4c Mon Sep 17 00:00:00 2001 From: Terrence Cole Date: Thu, 10 Aug 2017 10:50:38 -0700 Subject: [PATCH 23/35] Delete first and create last to avoid having create coalesce into an update unexpectedly. --- octodns/provider/rackspace.py | 14 ++++----- tests/test_octodns_provider_rackspace.py | 37 ++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 7 deletions(-) diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py index c593150..f10f0b6 100644 --- a/octodns/provider/rackspace.py +++ b/octodns/provider/rackspace.py @@ -155,7 +155,7 @@ class RackspaceProvider(BaseProvider): def _key_for_record(cls, rs_record): return cls._as_unicode(rs_record['type'], 'ascii'),\ cls._as_unicode(rs_record['name'], 'utf-8'),\ - cls._as_unicode(rs_record['data'], 'utf-8'),\ + cls._as_unicode(rs_record['data'], 'utf-8') def _data_for_multiple(self, rrset): # TODO: geo not supported @@ -451,15 +451,15 @@ class RackspaceProvider(BaseProvider): elif change.__class__.__name__ == 'Delete': deletes += self._mod_Delete(change) - if creates: - data = {"records": sorted(creates, key=lambda v: v['type'] + v['name'] + v.get('data', ''))} - self._post('domains/{}/records'.format(domain_id), data=data) + if deletes: + params = "&".join(sorted(deletes)) + self._delete('domains/{}/records?{}'.format(domain_id, params)) if updates: data = {"records": sorted(updates, key=lambda v: v['name'])} self._put('domains/{}/records'.format(domain_id), data=data) - if deletes: - params = "&".join(sorted(deletes)) - self._delete('domains/{}/records?{}'.format(domain_id, params)) + if creates: + data = {"records": sorted(creates, key=lambda v: v['type'] + v['name'] + v.get('data', ''))} + self._post('domains/{}/records'.format(domain_id), data=data) diff --git a/tests/test_octodns_provider_rackspace.py b/tests/test_octodns_provider_rackspace.py index ce80112..ce4b71f 100644 --- a/tests/test_octodns_provider_rackspace.py +++ b/tests/test_octodns_provider_rackspace.py @@ -708,6 +708,43 @@ class TestRackspaceProvider(TestCase): ExpectedUpdates = None return self._test_apply_with_data(TestData) + def test_apply_update_MX(self): + class TestData(object): + OtherRecords = [ + { + "subdomain": '', + "data": { + 'type': 'MX', + 'ttl': 300, + 'value': {u'priority': 50, u'value': 'mx.test.com.'} + } + } + ] + OwnRecords = { + "totalEntries": 1, + "records": [{ + "name": "unit.tests", + "id": "MX-111111", + "type": "MX", + "priority": 20, + "data": "mx.test.com", + "ttl": 300 + }] + } + ExpectChanges = True + ExpectedAdditions = { + "records": [{ + "name": "unit.tests", + "type": "MX", + "priority": 50, + "data": "mx.test.com", + "ttl": 300 + }] + } + ExpectedDeletions = 'id=MX-111111' + ExpectedUpdates = None + return self._test_apply_with_data(TestData) + def test_apply_multiple_updates(self): class TestData(object): OtherRecords = [ From 17c9b8b5277b79b821933b8861718b1212da4feb Mon Sep 17 00:00:00 2001 From: Terrence Cole Date: Fri, 11 Aug 2017 14:02:14 -0700 Subject: [PATCH 24/35] Get lint and coverage tools clean. --- octodns/provider/rackspace.py | 180 ++++-------- tests/test_octodns_provider_rackspace.py | 343 ++++++++--------------- 2 files changed, 174 insertions(+), 349 deletions(-) diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py index f10f0b6..1148e0b 100644 --- a/octodns/provider/rackspace.py +++ b/octodns/provider/rackspace.py @@ -10,7 +10,7 @@ import logging import string import time -from ..record import Create, Record +from ..record import Record from .base import BaseProvider @@ -26,21 +26,6 @@ def remove_trailing_dot(s): return s[:-1] -def strip_quotes(s): - assert s - assert len(s) > 2 - assert s[0] == '"' - assert s[-1] == '"' - return s[1:-1] - - -def add_quotes(s): - assert s - assert s[0] != '"' - assert s[-1] != '"' - return '"{}"'.format(s) - - def escape_semicolon(s): assert s return string.replace(s, ';', '\;') @@ -55,7 +40,8 @@ class RackspaceProvider(BaseProvider): SUPPORTS_GEO = False TIMEOUT = 5 - def __init__(self, id, username, api_key, ratelimit_delay, *args, **kwargs): + def __init__(self, id, username, api_key, ratelimit_delay, *args, + **kwargs): ''' Rackspace API v1 Provider @@ -84,25 +70,27 @@ class RackspaceProvider(BaseProvider): def _get_auth_token(self, username, api_key): ret = post('https://identity.api.rackspacecloud.com/v2.0/tokens', - json={"auth": {"RAX-KSKEY:apiKeyCredentials": {"username": username, "apiKey": api_key}}}, + json={"auth": { + "RAX-KSKEY:apiKeyCredentials": {"username": username, + "apiKey": api_key}}}, ) - cloud_dns_endpoint = [x for x in ret.json()['access']['serviceCatalog'] if x['name'] == 'cloudDNS'][0]['endpoints'][0]['publicURL'] + cloud_dns_endpoint = \ + [x for x in ret.json()['access']['serviceCatalog'] if + x['name'] == 'cloudDNS'][0]['endpoints'][0]['publicURL'] return ret.json()['access']['token']['id'], cloud_dns_endpoint def _get_zone_id_for(self, zone): ret = self._request('GET', 'domains', pagination_key='domains') time.sleep(self.ratelimit_delay) - if ret: - return [x for x in ret if x['name'] == zone.name[:-1]][0]['id'] - else: - return None + return [x for x in ret if x['name'] == zone.name[:-1]][0]['id'] def _request(self, method, path, data=None, pagination_key=None): self.log.debug('_request: method=%s, path=%s', method, path) url = '{}/{}'.format(self.dns_endpoint, path) if pagination_key: - return self._paginated_request_for_url(method, url, data, pagination_key) + return self._paginated_request_for_url(method, url, data, + pagination_key) else: return self._request_for_url(method, url, data) @@ -122,26 +110,22 @@ class RackspaceProvider(BaseProvider): resp.raise_for_status() acc.extend(resp.json()[pagination_key]) - next_page = [x for x in resp.json().get('links', []) if x['rel'] == 'next'] + next_page = [x for x in resp.json().get('links', []) if + x['rel'] == 'next'] if next_page: url = next_page[0]['href'] - acc.extend(self._paginated_request_for_url(method, url, data, pagination_key)) + acc.extend(self._paginated_request_for_url(method, url, data, + pagination_key)) return acc else: return acc - def _get(self, path, data=None): - return self._request('GET', path, data=data) - def _post(self, path, data=None): return self._request('POST', path, data=data) def _put(self, path, data=None): return self._request('PUT', path, data=data) - def _patch(self, path, data=None): - return self._request('PATCH', path, data=data) - def _delete(self, path, data=None): return self._request('DELETE', path, data=data) @@ -153,9 +137,9 @@ class RackspaceProvider(BaseProvider): @classmethod def _key_for_record(cls, rs_record): - return cls._as_unicode(rs_record['type'], 'ascii'),\ - cls._as_unicode(rs_record['name'], 'utf-8'),\ - cls._as_unicode(rs_record['data'], 'utf-8') + return cls._as_unicode(rs_record['type'], 'ascii'), \ + cls._as_unicode(rs_record['name'], 'utf-8'), \ + cls._as_unicode(rs_record['data'], 'utf-8') def _data_for_multiple(self, rrset): # TODO: geo not supported @@ -210,67 +194,14 @@ class RackspaceProvider(BaseProvider): 'ttl': rrset[0]['ttl'] } - def _data_for_NAPTR(self, rrset): - raise NotImplementedError("Missing support for reading NAPTR records") - # values = [] - # for record in rrset['records']: - # order, preference, flags, service, regexp, replacement = \ - # record['content'].split(' ', 5) - # values.append({ - # 'order': order, - # 'preference': preference, - # 'flags': flags[1:-1], - # 'service': service[1:-1], - # 'regexp': regexp[1:-1], - # 'replacement': replacement, - # }) - # return { - # 'type': rrset['type'], - # 'values': values, - # 'ttl': rrset['ttl'] - # } - - def _data_for_SSHFP(self, rrset): - raise NotImplementedError("Missing support for reading SSHFP records") - # values = [] - # for record in rrset['records']: - # algorithm, fingerprint_type, fingerprint = \ - # record['content'].split(' ', 2) - # values.append({ - # 'algorithm': algorithm, - # 'fingerprint_type': fingerprint_type, - # 'fingerprint': fingerprint, - # }) - # return { - # 'type': rrset['type'], - # 'values': values, - # 'ttl': rrset['ttl'] - # } - - def _data_for_SRV(self, rrset): - raise NotImplementedError("Missing support for reading SRV records") - # values = [] - # for record in rrset['records']: - # priority, weight, port, target = \ - # record['content'].split(' ', 3) - # values.append({ - # 'priority': priority, - # 'weight': weight, - # 'port': port, - # 'target': target, - # }) - # return { - # 'type': rrset['type'], - # 'values': values, - # 'ttl': rrset['ttl'] - # } - def populate(self, zone, target=False): self.log.debug('populate: name=%s', zone.name) resp_data = None try: domain_id = self._get_zone_id_for(zone) - resp_data = self._request('GET', 'domains/{}/records'.format(domain_id), pagination_key='records') + resp_data = self._request('GET', + 'domains/{}/records'.format(domain_id), + pagination_key='records') self.log.debug('populate: loaded') except HTTPError as e: if e.response.status_code == 401: @@ -286,12 +217,12 @@ class RackspaceProvider(BaseProvider): if resp_data: records = self._group_records(resp_data) for record_type, records_of_type in records.items(): - if record_type == 'SOA': - continue for raw_record_name, record_set in records_of_type.items(): - data_for = getattr(self, '_data_for_{}'.format(record_type)) + data_for = getattr(self, + '_data_for_{}'.format(record_type)) record_name = zone.hostname_from_fqdn(raw_record_name) - record = Record.new(zone, record_name, data_for(record_set), + record = Record.new(zone, record_name, + data_for(record_set), source=self) zone.add_record(record) @@ -313,6 +244,7 @@ class RackspaceProvider(BaseProvider): 'data': value, 'ttl': max(record.ttl, 300), } + _record_for_A = _record_for_single _record_for_AAAA = _record_for_single @@ -324,6 +256,7 @@ class RackspaceProvider(BaseProvider): 'data': remove_trailing_dot(value), 'ttl': max(record.ttl, 300), } + _record_for_NS = _record_for_named _record_for_ALIAS = _record_for_named _record_for_CNAME = _record_for_named @@ -337,6 +270,7 @@ class RackspaceProvider(BaseProvider): 'data': unescape_semicolon(value), 'ttl': max(record.ttl, 300), } + _record_for_SPF = _record_for_textual _record_for_TXT = _record_for_textual @@ -350,27 +284,15 @@ class RackspaceProvider(BaseProvider): 'priority': value.priority } - @staticmethod - def _record_for_SRV(record, value): - raise NotImplementedError("Missing support for writing SRV records") - - def _record_for_NAPTR(self, record): - raise NotImplementedError("Missing support for writing NAPTR records") - # return [{ - # 'content': '{} {} "{}" "{}" "{}" {}'.format(v.order, v.preference, - # v.flags, v.service, - # v.regexp, - # v.replacement), - # 'disabled': False - # } for v in record.values] - - def _record_for_SSHFP(self, record): - raise NotImplementedError("Missing support for writing SSHFP records") - # return [{ - # 'content': '{} {} {}'.format(v.algorithm, v.fingerprint_type, - # v.fingerprint), - # 'disabled': False - # } for v in record.values] + def _record_for_unsupported(self, record, value): + raise NotImplementedError( + "Missing support for writing {} records".format( + record.__class__.__name__)) + + _record_for_SOA = _record_for_unsupported + _record_for_SRV = _record_for_unsupported + _record_for_NAPTR = _record_for_unsupported + _record_for_SSHFP = _record_for_unsupported def _get_values(self, record): try: @@ -379,12 +301,14 @@ class RackspaceProvider(BaseProvider): return [record.value] def _mod_Create(self, change): - return self._create_given_change_values(change, self._get_values(change.new)) + return self._create_given_change_values(change, + self._get_values(change.new)) def _create_given_change_values(self, change, values): out = [] for value in values: - transformer = getattr(self, "_record_for_{}".format(change.new._type)) + transformer = getattr(self, + "_record_for_{}".format(change.new._type)) out.append(transformer(change.new, value)) return out @@ -405,7 +329,8 @@ class RackspaceProvider(BaseProvider): update_out = [] update_values = set(new_values).intersection(set(existing_values)) for value in update_values: - transformer = getattr(self, "_record_for_{}".format(change.new._type)) + transformer = getattr(self, + "_record_for_{}".format(change.new._type)) prior_rs_record = transformer(change.existing, value) prior_key = self._key_for_record(prior_rs_record) next_rs_record = transformer(change.new, value) @@ -418,12 +343,14 @@ class RackspaceProvider(BaseProvider): return create_out, update_out, delete_out def _mod_Delete(self, change): - return self._delete_given_change_values(change, self._get_values(change.existing)) + return self._delete_given_change_values(change, self._get_values( + change.existing)) def _delete_given_change_values(self, change, values): out = [] for value in values: - transformer = getattr(self, "_record_for_{}".format(change.existing._type)) + transformer = getattr(self, "_record_for_{}".format( + change.existing._type)) rs_record = transformer(change.existing, value) key = self._key_for_record(rs_record) out.append('id=' + self._id_map[key]) @@ -444,11 +371,13 @@ class RackspaceProvider(BaseProvider): if change.__class__.__name__ == 'Create': creates += self._mod_Create(change) elif change.__class__.__name__ == 'Update': - add_creates, add_updates, add_deletes = self._mod_Update(change) + add_creates, add_updates, add_deletes = self._mod_Update( + change) creates += add_creates updates += add_updates deletes += add_deletes - elif change.__class__.__name__ == 'Delete': + else: + assert change.__class__.__name__ == 'Delete' deletes += self._mod_Delete(change) if deletes: @@ -460,6 +389,7 @@ class RackspaceProvider(BaseProvider): self._put('domains/{}/records'.format(domain_id), data=data) if creates: - data = {"records": sorted(creates, key=lambda v: v['type'] + v['name'] + v.get('data', ''))} + data = {"records": sorted(creates, key=lambda v: v['type'] + + v['name'] + + v.get('data', ''))} self._post('domains/{}/records'.format(domain_id), data=data) - diff --git a/tests/test_octodns_provider_rackspace.py b/tests/test_octodns_provider_rackspace.py index ce4b71f..725b60a 100644 --- a/tests/test_octodns_provider_rackspace.py +++ b/tests/test_octodns_provider_rackspace.py @@ -11,6 +11,8 @@ from os.path import dirname, join from unittest import TestCase from urlparse import urlparse +from nose.tools import assert_raises + from requests import HTTPError from requests_mock import ANY, mock as requests_mock @@ -19,8 +21,6 @@ from octodns.provider.yaml import YamlProvider from octodns.record import Record from octodns.zone import Zone -from pprint import pprint - EMPTY_TEXT = ''' { "totalEntries" : 0, @@ -40,16 +40,14 @@ with open('./tests/fixtures/rackspace-sample-recordset-page1.json') as fh: with open('./tests/fixtures/rackspace-sample-recordset-page2.json') as fh: RECORDS_PAGE_2 = fh.read() -with open('./tests/fixtures/rackspace-sample-recordset-existing-nameservers.json') as fh: - RECORDS_EXISTING_NAMESERVERS = fh.read() - class TestRackspaceProvider(TestCase): def setUp(self): self.maxDiff = 1000 with requests_mock() as mock: mock.post(ANY, status_code=200, text=AUTH_RESPONSE) - self.provider = RackspaceProvider(id, 'test', 'api-key', '0') + self.provider = RackspaceProvider('identity', 'test', 'api-key', + '0') self.assertTrue(mock.called_once) def test_bad_auth(self): @@ -85,9 +83,12 @@ class TestRackspaceProvider(TestCase): def test_multipage_populate(self): with requests_mock() as mock: - mock.get(re.compile('domains$'), status_code=200, text=LIST_DOMAINS_RESPONSE) - mock.get(re.compile('records'), status_code=200, text=RECORDS_PAGE_1) - mock.get(re.compile('records.*offset=3'), status_code=200, text=RECORDS_PAGE_2) + mock.get(re.compile('domains$'), status_code=200, + text=LIST_DOMAINS_RESPONSE) + mock.get(re.compile('records'), status_code=200, + text=RECORDS_PAGE_1) + mock.get(re.compile('records.*offset=3'), status_code=200, + text=RECORDS_PAGE_2) zone = Zone('unit.tests.', []) self.provider.populate(zone) @@ -105,9 +106,12 @@ class TestRackspaceProvider(TestCase): # No diffs == no changes with requests_mock() as mock: - mock.get(re.compile('domains$'), status_code=200, text=LIST_DOMAINS_RESPONSE) - mock.get(re.compile('records'), status_code=200, text=RECORDS_PAGE_1) - mock.get(re.compile('records.*offset=3'), status_code=200, text=RECORDS_PAGE_2) + mock.get(re.compile('domains$'), status_code=200, + text=LIST_DOMAINS_RESPONSE) + mock.get(re.compile('records'), status_code=200, + text=RECORDS_PAGE_1) + mock.get(re.compile('records.*offset=3'), status_code=200, + text=RECORDS_PAGE_2) zone = Zone('unit.tests.', []) self.provider.populate(zone) @@ -127,7 +131,8 @@ class TestRackspaceProvider(TestCase): 'values': ['8.8.8.8.', '9.9.9.9.'] })) with requests_mock() as mock: - mock.get(re.compile('domains$'), status_code=200, text=LIST_DOMAINS_RESPONSE) + mock.get(re.compile('domains$'), status_code=200, + text=LIST_DOMAINS_RESPONSE) mock.get(re.compile('records'), status_code=200, text=EMPTY_TEXT) plan = self.provider.plan(expected) @@ -141,23 +146,28 @@ class TestRackspaceProvider(TestCase): # expected.add_record(Record.new(expected, 'foo', '1.2.3.4')) with requests_mock() as list_mock: - list_mock.get(re.compile('domains$'), status_code=200, text=LIST_DOMAINS_RESPONSE) - list_mock.get(re.compile('records'), status_code=200, json={'records': [ - {'type': 'A', - 'name': 'foo.example.com', - 'id': 'A-111111', - 'data': '1.2.3.4', - 'ttl': 300}]}) + list_mock.get(re.compile('domains$'), status_code=200, + text=LIST_DOMAINS_RESPONSE) + list_mock.get(re.compile('records'), status_code=200, + json={'records': [ + {'type': 'A', + 'name': 'foo.example.com', + 'id': 'A-111111', + 'data': '1.2.3.4', + 'ttl': 300}]}) plan = self.provider.plan(expected) self.assertTrue(list_mock.called) self.assertEqual(1, len(plan.changes)) - self.assertTrue(plan.changes[0].existing.fqdn == 'foo.example.com.') + self.assertTrue( + plan.changes[0].existing.fqdn == 'foo.example.com.') with requests_mock() as mock: def _assert_deleting(request, context): parts = urlparse(request.url) self.assertEqual('id=A-111111', parts.query) - mock.get(re.compile('domains$'), status_code=200, text=LIST_DOMAINS_RESPONSE) + + mock.get(re.compile('domains$'), status_code=200, + text=LIST_DOMAINS_RESPONSE) mock.delete(re.compile('domains/.*/records?.*'), status_code=202, text=_assert_deleting) self.provider.apply(plan) @@ -166,11 +176,14 @@ class TestRackspaceProvider(TestCase): def _test_apply_with_data(self, data): expected = Zone('unit.tests.', []) for record in data.OtherRecords: - expected.add_record(Record.new(expected, record['subdomain'], record['data'])) + expected.add_record( + Record.new(expected, record['subdomain'], record['data'])) with requests_mock() as list_mock: - list_mock.get(re.compile('domains$'), status_code=200, text=LIST_DOMAINS_RESPONSE) - list_mock.get(re.compile('records'), status_code=200, json=data.OwnRecords) + list_mock.get(re.compile('domains$'), status_code=200, + text=LIST_DOMAINS_RESPONSE) + list_mock.get(re.compile('records'), status_code=200, + json=data.OwnRecords) plan = self.provider.plan(expected) self.assertTrue(list_mock.called) if not data.ExpectChanges: @@ -184,25 +197,32 @@ class TestRackspaceProvider(TestCase): def _assert_sending_right_body(request, _context): called.add(request.method) if request.method != 'DELETE': - self.assertEqual(request.headers['content-type'], 'application/json') - self.assertDictEqual(expected, json.loads(request.body)) + self.assertEqual(request.headers['content-type'], + 'application/json') + self.assertDictEqual(expected, + json.loads(request.body)) else: parts = urlparse(request.url) self.assertEqual(expected, parts.query) return '' + return _assert_sending_right_body - mock.get(re.compile('domains$'), status_code=200, text=LIST_DOMAINS_RESPONSE) + mock.get(re.compile('domains$'), status_code=200, + text=LIST_DOMAINS_RESPONSE) mock.post(re.compile('domains/.*/records$'), status_code=202, - text=make_assert_sending_right_body(data.ExpectedAdditions)) + text=make_assert_sending_right_body( + data.ExpectedAdditions)) mock.delete(re.compile('domains/.*/records?.*'), status_code=202, - text=make_assert_sending_right_body(data.ExpectedDeletions)) + text=make_assert_sending_right_body( + data.ExpectedDeletions)) mock.put(re.compile('domains/.*/records$'), status_code=202, text=make_assert_sending_right_body(data.ExpectedUpdates)) self.provider.apply(plan) self.assertTrue(data.ExpectedAdditions is None or "POST" in called) - self.assertTrue(data.ExpectedDeletions is None or "DELETE" in called) + self.assertTrue( + data.ExpectedDeletions is None or "DELETE" in called) self.assertTrue(data.ExpectedUpdates is None or "PUT" in called) def test_apply_no_change_empty(self): @@ -216,6 +236,7 @@ class TestRackspaceProvider(TestCase): ExpectedAdditions = None ExpectedDeletions = None ExpectedUpdates = None + return self._test_apply_with_data(TestData) def test_apply_no_change_a_records(self): @@ -256,6 +277,7 @@ class TestRackspaceProvider(TestCase): ExpectedAdditions = None ExpectedDeletions = None ExpectedUpdates = None + return self._test_apply_with_data(TestData) def test_apply_no_change_a_records_cross_zone(self): @@ -298,6 +320,7 @@ class TestRackspaceProvider(TestCase): ExpectedAdditions = None ExpectedDeletions = None ExpectedUpdates = None + return self._test_apply_with_data(TestData) def test_apply_one_addition(self): @@ -340,6 +363,7 @@ class TestRackspaceProvider(TestCase): } ExpectedDeletions = None ExpectedUpdates = None + return self._test_apply_with_data(TestData) def test_apply_create_MX(self): @@ -390,9 +414,39 @@ class TestRackspaceProvider(TestCase): } ExpectedDeletions = None ExpectedUpdates = None + return self._test_apply_with_data(TestData) - def test_apply_multiple_additions_exploding(self): + def test_apply_create_SRV(self): + class TestData(object): + OtherRecords = [ + { + "subdomain": '_a.b', + "data": { + 'type': 'SRV', + 'ttl': 300, + 'value': { + 'priority': 20, + 'weight': 999, + 'port': 999, + 'target': 'foo' + } + } + } + ] + OwnRecords = { + "totalEntries": 0, + "records": [] + } + ExpectChanges = True + ExpectedAdditions = [{}] + ExpectedDeletions = None + ExpectedUpdates = None + + assert_raises(NotImplementedError, self._test_apply_with_data, + TestData) + + def test_apply_multiple_additions_splatting(self): class TestData(object): OtherRecords = [ { @@ -447,33 +501,33 @@ class TestRackspaceProvider(TestCase): } ExpectedDeletions = None ExpectedUpdates = None + return self._test_apply_with_data(TestData) def test_apply_multiple_additions_namespaced(self): class TestData(object): OtherRecords = [{ - "subdomain": 'foo', - "data": { - 'type': 'A', - 'ttl': 300, - 'value': '1.2.3.4' - } - }, { - "subdomain": 'bar', - "data": { - 'type': 'A', - 'ttl': 300, - 'value': '1.2.3.4' - } - }, - { - "subdomain": 'foo', - "data": { - 'type': 'NS', - 'ttl': 300, - 'value': 'ns.example.com.' - } - }] + "subdomain": 'foo', + "data": { + 'type': 'A', + 'ttl': 300, + 'value': '1.2.3.4' + } + }, { + "subdomain": 'bar', + "data": { + 'type': 'A', + 'ttl': 300, + 'value': '1.2.3.4' + } + }, { + "subdomain": 'foo', + "data": { + 'type': 'NS', + 'ttl': 300, + 'value': 'ns.example.com.' + } + }] OwnRecords = { "totalEntries": 0, "records": [] @@ -499,6 +553,7 @@ class TestRackspaceProvider(TestCase): } ExpectedDeletions = None ExpectedUpdates = None + return self._test_apply_with_data(TestData) def test_apply_single_deletion(self): @@ -524,6 +579,7 @@ class TestRackspaceProvider(TestCase): ExpectedAdditions = None ExpectedDeletions = "id=A-111111&id=NS-111111" ExpectedUpdates = None + return self._test_apply_with_data(TestData) def test_apply_multiple_deletions(self): @@ -577,6 +633,7 @@ class TestRackspaceProvider(TestCase): "ttl": 300 }] } + return self._test_apply_with_data(TestData) def test_apply_multiple_deletions_cross_zone(self): @@ -617,6 +674,7 @@ class TestRackspaceProvider(TestCase): ExpectedAdditions = None ExpectedDeletions = "id=A-222222&id=A-333333" ExpectedUpdates = None + return self._test_apply_with_data(TestData) def test_apply_delete_cname(self): @@ -636,6 +694,7 @@ class TestRackspaceProvider(TestCase): ExpectedAdditions = None ExpectedDeletions = "id=CNAME-111111" ExpectedUpdates = None + return self._test_apply_with_data(TestData) def test_apply_single_update(self): @@ -671,6 +730,7 @@ class TestRackspaceProvider(TestCase): "ttl": 3600 }] } + return self._test_apply_with_data(TestData) def test_apply_update_TXT(self): @@ -706,6 +766,7 @@ class TestRackspaceProvider(TestCase): } ExpectedDeletions = 'id=TXT-111111' ExpectedUpdates = None + return self._test_apply_with_data(TestData) def test_apply_update_MX(self): @@ -743,6 +804,7 @@ class TestRackspaceProvider(TestCase): } ExpectedDeletions = 'id=MX-111111' ExpectedUpdates = None + return self._test_apply_with_data(TestData) def test_apply_multiple_updates(self): @@ -783,7 +845,7 @@ class TestRackspaceProvider(TestCase): ExpectedAdditions = None ExpectedDeletions = None ExpectedUpdates = { - "records": [ { + "records": [{ "name": "unit.tests", "id": "A-222222", "data": "1.2.3.5", @@ -800,6 +862,7 @@ class TestRackspaceProvider(TestCase): "ttl": 3600 }] } + return self._test_apply_with_data(TestData) def test_apply_multiple_updates_cross_zone(self): @@ -854,173 +917,5 @@ class TestRackspaceProvider(TestCase): "ttl": 3600 }] } - return self._test_apply_with_data(TestData) - - """ - def test_provider(self): - expected = self._load_full_config() - - # No existing records -> creates for every record in expected - with requests_mock() as mock: - mock.get(re.compile('domains$'), status_code=200, text=LIST_DOMAINS_RESPONSE) - mock.get(re.compile('records'), status_code=200, text=EMPTY_TEXT) - - plan = self.provider.plan(expected) - self.assertTrue(mock.called) - self.assertEquals(len(expected.records), len(plan.changes)) - # Used in a minute - def assert_rrsets_callback(request, context): - data = loads(request.body) - self.assertEquals(expected_n, len(data['rrsets'])) - return '' - - with requests_mock() as mock: - # post 201, is response to the create with data - mock.patch(ANY, status_code=201, text=assert_rrsets_callback) - - self.assertEquals(expected_n, self.provider.apply(plan)) - - # Non-existent zone -> creates for every record in expected - # OMG this is fucking ugly, probably better to ditch requests_mocks and - # just mock things for real as it doesn't seem to provide a way to get - # at the request params or verify that things were called from what I - # can tell - not_found = {'error': "Could not find domain 'unit.tests.'"} - with requests_mock() as mock: - # get 422's, unknown zone - mock.get(ANY, status_code=422, text='') - # patch 422's, unknown zone - mock.patch(ANY, status_code=422, text=dumps(not_found)) - # post 201, is response to the create with data - mock.post(ANY, status_code=201, text=assert_rrsets_callback) - - plan = self.provider.plan(expected) - self.assertEquals(expected_n, len(plan.changes)) - self.assertEquals(expected_n, self.provider.apply(plan)) - - with requests_mock() as mock: - # get 422's, unknown zone - mock.get(ANY, status_code=422, text='') - # patch 422's, - data = {'error': "Key 'name' not present or not a String"} - mock.patch(ANY, status_code=422, text=dumps(data)) - - with self.assertRaises(HTTPError) as ctx: - plan = self.provider.plan(expected) - self.provider.apply(plan) - response = ctx.exception.response - self.assertEquals(422, response.status_code) - self.assertTrue('error' in response.json()) - - with requests_mock() as mock: - # get 422's, unknown zone - mock.get(ANY, status_code=422, text='') - # patch 500's, things just blew up - mock.patch(ANY, status_code=500, text='') - - with self.assertRaises(HTTPError): - plan = self.provider.plan(expected) - self.provider.apply(plan) - - with requests_mock() as mock: - # get 422's, unknown zone - mock.get(ANY, status_code=422, text='') - # patch 500's, things just blew up - mock.patch(ANY, status_code=422, text=dumps(not_found)) - # post 422's, something wrong with create - mock.post(ANY, status_code=422, text='Hello Word!') - - with self.assertRaises(HTTPError): - plan = self.provider.plan(expected) - self.provider.apply(plan) - """ - - def test_plan_no_changes(self): - expected = Zone('unit.tests.', []) - expected.add_record(Record.new(expected, '', { - 'type': 'NS', - 'ttl': 600, - 'values': ['ns1.example.com.', 'ns2.example.com.'] - })) - expected.add_record(Record.new(expected, '', { - 'type': 'A', - 'ttl': 600, - 'value': '1.2.3.4' - })) - - with requests_mock() as mock: - mock.get(re.compile('domains/.*/records'), status_code=200, text=RECORDS_EXISTING_NAMESERVERS) - mock.get(re.compile('domains$'), status_code=200, text=LIST_DOMAINS_RESPONSE) - - plan = self.provider.plan(expected) - - self.assertTrue(mock.called) - self.assertFalse(plan) - - def test_plan_remove_a_record(self): - expected = Zone('unit.tests.', []) - expected.add_record(Record.new(expected, '', { - 'type': 'NS', - 'ttl': 600, - 'values': ['ns1.example.com.', 'ns2.example.com.'] - })) - - with requests_mock() as mock: - mock.get(re.compile('domains/.*/records'), status_code=200, text=RECORDS_EXISTING_NAMESERVERS) - mock.get(re.compile('domains$'), status_code=200, text=LIST_DOMAINS_RESPONSE) - - plan = self.provider.plan(expected) - self.assertTrue(mock.called) - self.assertEquals(1, len(plan.changes)) - self.assertEqual(plan.changes[0].existing.ttl, 600) - self.assertEqual(plan.changes[0].existing.values[0], '1.2.3.4') - - def test_plan_create_a_record(self): - expected = Zone('unit.tests.', []) - expected.add_record(Record.new(expected, '', { - 'type': 'NS', - 'ttl': 600, - 'values': ['ns1.example.com.', 'ns2.example.com.'] - })) - expected.add_record(Record.new(expected, '', { - 'type': 'A', - 'ttl': 600, - 'values': ['1.2.3.4', '1.2.3.5'] - })) - - with requests_mock() as mock: - mock.get(re.compile('domains/.*/records'), status_code=200, text=RECORDS_EXISTING_NAMESERVERS) - mock.get(re.compile('domains$'), status_code=200, text=LIST_DOMAINS_RESPONSE) - - plan = self.provider.plan(expected) - self.assertTrue(mock.called) - self.assertEquals(1, len(plan.changes)) - self.assertEqual(plan.changes[0].new.ttl, 600) - self.assertEqual(plan.changes[0].new.values[0], '1.2.3.4') - self.assertEqual(plan.changes[0].new.values[1], '1.2.3.5') - - def test_plan_change_ttl(self): - expected = Zone('unit.tests.', []) - expected.add_record(Record.new(expected, '', { - 'type': 'NS', - 'ttl': 600, - 'values': ['ns1.example.com.', 'ns2.example.com.'] - })) - expected.add_record(Record.new(expected, '', { - 'type': 'A', - 'ttl': 86400, - 'value': '1.2.3.4' - })) - - with requests_mock() as mock: - mock.get(re.compile('domains/.*/records'), status_code=200, text=RECORDS_EXISTING_NAMESERVERS) - mock.get(re.compile('domains$'), status_code=200, text=LIST_DOMAINS_RESPONSE) - - plan = self.provider.plan(expected) - - self.assertTrue(mock.called) - self.assertEqual(1, len(plan.changes)) - self.assertEqual(plan.changes[0].existing.ttl, 600) - self.assertEqual(plan.changes[0].new.ttl, 86400) - self.assertEqual(plan.changes[0].new.values[0], '1.2.3.4') + return self._test_apply_with_data(TestData) From 32f2a10daf118f98e1cb299c4b220e61111bfba1 Mon Sep 17 00:00:00 2001 From: Terrence Cole Date: Fri, 11 Aug 2017 14:03:03 -0700 Subject: [PATCH 25/35] Apply review feedback to bring logging inline with other providers. --- octodns/provider/rackspace.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py index 1148e0b..6e04c98 100644 --- a/octodns/provider/rackspace.py +++ b/octodns/provider/rackspace.py @@ -52,7 +52,7 @@ class RackspaceProvider(BaseProvider): # The api key that grants access for that user (required) api_key: api-key ''' - self.log = logging.getLogger('RackspaceProvider[{}]'.format(username)) + self.log = logging.getLogger('RackspaceProvider[{}]'.format(id)) super(RackspaceProvider, self).__init__(id, *args, **kwargs) auth_token, dns_endpoint = self._get_auth_token(username, api_key) From 1e7edc97c838c65f664c295b6a51fcf3955d0dd3 Mon Sep 17 00:00:00 2001 From: Terrence Cole Date: Mon, 11 Sep 2017 10:51:03 -0700 Subject: [PATCH 26/35] Update rackspace provider with new names and interfaces. --- octodns/provider/rackspace.py | 19 +++------ tests/test_octodns_provider_rackspace.py | 53 ------------------------ 2 files changed, 5 insertions(+), 67 deletions(-) diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py index 6e04c98..8d913ef 100644 --- a/octodns/provider/rackspace.py +++ b/octodns/provider/rackspace.py @@ -38,6 +38,8 @@ def unescape_semicolon(s): class RackspaceProvider(BaseProvider): SUPPORTS_GEO = False + SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CNAME', 'MX', 'NS', 'PTR', 'SPF', + 'TXT')) TIMEOUT = 5 def __init__(self, id, username, api_key, ratelimit_delay, *args, @@ -153,7 +155,6 @@ class RackspaceProvider(BaseProvider): _data_for_AAAA = _data_for_multiple def _data_for_NS(self, rrset): - # TODO: geo not supported return { 'type': rrset[0]['type'], 'values': [add_trailing_dot(r['data']) for r in rrset], @@ -194,7 +195,7 @@ class RackspaceProvider(BaseProvider): 'ttl': rrset[0]['ttl'] } - def populate(self, zone, target=False): + def populate(self, zone, target=False, lenient=False): self.log.debug('populate: name=%s', zone.name) resp_data = None try: @@ -279,21 +280,11 @@ class RackspaceProvider(BaseProvider): return { 'name': remove_trailing_dot(record.fqdn), 'type': record._type, - 'data': remove_trailing_dot(value.value), + 'data': remove_trailing_dot(value.exchange), 'ttl': max(record.ttl, 300), - 'priority': value.priority + 'priority': value.preference } - def _record_for_unsupported(self, record, value): - raise NotImplementedError( - "Missing support for writing {} records".format( - record.__class__.__name__)) - - _record_for_SOA = _record_for_unsupported - _record_for_SRV = _record_for_unsupported - _record_for_NAPTR = _record_for_unsupported - _record_for_SSHFP = _record_for_unsupported - def _get_values(self, record): try: return record.values diff --git a/tests/test_octodns_provider_rackspace.py b/tests/test_octodns_provider_rackspace.py index 725b60a..da3ffe4 100644 --- a/tests/test_octodns_provider_rackspace.py +++ b/tests/test_octodns_provider_rackspace.py @@ -94,30 +94,6 @@ class TestRackspaceProvider(TestCase): self.provider.populate(zone) self.assertEquals(5, len(zone.records)) - def _load_full_config(self): - expected = Zone('unit.tests.', []) - source = YamlProvider('test', join(dirname(__file__), 'config')) - source.populate(expected) - self.assertEquals(15, len(expected.records)) - return expected - - def test_changes_are_formatted_correctly(self): - expected = self._load_full_config() - - # No diffs == no changes - with requests_mock() as mock: - mock.get(re.compile('domains$'), status_code=200, - text=LIST_DOMAINS_RESPONSE) - mock.get(re.compile('records'), status_code=200, - text=RECORDS_PAGE_1) - mock.get(re.compile('records.*offset=3'), status_code=200, - text=RECORDS_PAGE_2) - - zone = Zone('unit.tests.', []) - self.provider.populate(zone) - changes = expected.changes(zone, self.provider) - self.assertEquals(18, len(changes)) - def test_plan_disappearing_ns_records(self): expected = Zone('unit.tests.', []) expected.add_record(Record.new(expected, '', { @@ -417,35 +393,6 @@ class TestRackspaceProvider(TestCase): return self._test_apply_with_data(TestData) - def test_apply_create_SRV(self): - class TestData(object): - OtherRecords = [ - { - "subdomain": '_a.b', - "data": { - 'type': 'SRV', - 'ttl': 300, - 'value': { - 'priority': 20, - 'weight': 999, - 'port': 999, - 'target': 'foo' - } - } - } - ] - OwnRecords = { - "totalEntries": 0, - "records": [] - } - ExpectChanges = True - ExpectedAdditions = [{}] - ExpectedDeletions = None - ExpectedUpdates = None - - assert_raises(NotImplementedError, self._test_apply_with_data, - TestData) - def test_apply_multiple_additions_splatting(self): class TestData(object): OtherRecords = [ From a0fd1cfdbee31ed438034b8efa46d2cebbef3dea Mon Sep 17 00:00:00 2001 From: Terrence Cole Date: Mon, 11 Sep 2017 10:56:28 -0700 Subject: [PATCH 27/35] Add the new provider to the readme. --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 1f103f1..17d393e 100644 --- a/README.md +++ b/README.md @@ -155,6 +155,7 @@ The above command pulled the existing data out of Route53 and placed the results | [DynProvider](/octodns/provider/dyn.py) | All | Yes | | | [Ns1Provider](/octodns/provider/ns1.py) | All | No | | | [PowerDnsProvider](/octodns/provider/powerdns.py) | All | No | | +| [Rackspace](/octodns/provider/rackspace.py) | A, AAAA, ALIAS, CNAME, MX, NS, PTR, SPF, TXT | No | | | [Route53](/octodns/provider/route53.py) | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | Yes | | | [TinyDNSSource](/octodns/source/tinydns.py) | A, CNAME, MX, NS, PTR | No | read-only | | [YamlProvider](/octodns/provider/yaml.py) | All | Yes | config | From 824ccf6174a7c49a18d08ebdb0411746d728351c Mon Sep 17 00:00:00 2001 From: Terrence Cole Date: Mon, 11 Sep 2017 11:05:04 -0700 Subject: [PATCH 28/35] No need for a comment here as it is documented elsewhere. --- octodns/provider/rackspace.py | 1 - 1 file changed, 1 deletion(-) diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py index 8d913ef..6b17550 100644 --- a/octodns/provider/rackspace.py +++ b/octodns/provider/rackspace.py @@ -144,7 +144,6 @@ class RackspaceProvider(BaseProvider): cls._as_unicode(rs_record['data'], 'utf-8') def _data_for_multiple(self, rrset): - # TODO: geo not supported return { 'type': rrset[0]['type'], 'values': [r['data'] for r in rrset], From b49e1913424114c9502a963d3047223c67559a30 Mon Sep 17 00:00:00 2001 From: Terrence Cole Date: Mon, 11 Sep 2017 11:06:47 -0700 Subject: [PATCH 29/35] Revert changes to args.py for a later PR. --- octodns/cmds/args.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/octodns/cmds/args.py b/octodns/cmds/args.py index 9322461..c84dd92 100644 --- a/octodns/cmds/args.py +++ b/octodns/cmds/args.py @@ -37,8 +37,6 @@ class ArgumentParser(_Base): _help = 'Increase verbosity to get details and help track down issues' self.add_argument('--debug', action='store_true', default=False, help=_help) - self.add_argument('--quiet', action='store_true', default=False, - help='Only print warning and error messages') args = super(ArgumentParser, self).parse_args() self._setup_logging(args, default_log_level) @@ -61,11 +59,7 @@ class ArgumentParser(_Base): handler.setFormatter(Formatter(fmt=fmt)) logger.addHandler(handler) - logger.level = default_log_level - if args.debug: - logger.level = DEBUG - elif args.quiet: - logger.level = WARN + logger.level = DEBUG if args.debug else default_log_level # boto is noisy, set it to warn getLogger('botocore').level = WARN From 8d7eca21e9d85472f34b66be1e9e9668eb209b63 Mon Sep 17 00:00:00 2001 From: Terrence Cole Date: Fri, 5 Jan 2018 15:56:06 -0800 Subject: [PATCH 30/35] Get lint green on test code too. --- tests/test_octodns_provider_rackspace.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/test_octodns_provider_rackspace.py b/tests/test_octodns_provider_rackspace.py index da3ffe4..274d63e 100644 --- a/tests/test_octodns_provider_rackspace.py +++ b/tests/test_octodns_provider_rackspace.py @@ -7,17 +7,13 @@ from __future__ import absolute_import, division, print_function, \ import json import re -from os.path import dirname, join from unittest import TestCase from urlparse import urlparse -from nose.tools import assert_raises - from requests import HTTPError from requests_mock import ANY, mock as requests_mock from octodns.provider.rackspace import RackspaceProvider -from octodns.provider.yaml import YamlProvider from octodns.record import Record from octodns.zone import Zone From 88bbd663006c08ecba84cfd17597143edfa9811a Mon Sep 17 00:00:00 2001 From: Terrence Cole Date: Sat, 6 Jan 2018 10:29:22 -0800 Subject: [PATCH 31/35] Move request delay to a central location. --- octodns/provider/rackspace.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py index 6b17550..1f723d1 100644 --- a/octodns/provider/rackspace.py +++ b/octodns/provider/rackspace.py @@ -83,7 +83,6 @@ class RackspaceProvider(BaseProvider): def _get_zone_id_for(self, zone): ret = self._request('GET', 'domains', pagination_key='domains') - time.sleep(self.ratelimit_delay) return [x for x in ret if x['name'] == zone.name[:-1]][0]['id'] def _request(self, method, path, data=None, pagination_key=None): @@ -91,14 +90,15 @@ class RackspaceProvider(BaseProvider): url = '{}/{}'.format(self.dns_endpoint, path) if pagination_key: - return self._paginated_request_for_url(method, url, data, + resp = self._paginated_request_for_url(method, url, data, pagination_key) else: - return self._request_for_url(method, url, data) + resp = self._request_for_url(method, url, data) + time.sleep(self.ratelimit_delay) + return resp def _request_for_url(self, method, url, data): resp = self._sess.request(method, url, json=data, timeout=self.TIMEOUT) - time.sleep(self.ratelimit_delay) self.log.debug('_request: status=%d', resp.status_code) resp.raise_for_status() return resp @@ -107,7 +107,6 @@ class RackspaceProvider(BaseProvider): acc = [] resp = self._sess.request(method, url, json=data, timeout=self.TIMEOUT) - time.sleep(self.ratelimit_delay) self.log.debug('_request: status=%d', resp.status_code) resp.raise_for_status() acc.extend(resp.json()[pagination_key]) From 41622009e43e7c989ec5ad44ccff8e616e1aa0bc Mon Sep 17 00:00:00 2001 From: Terrence Cole Date: Sat, 6 Jan 2018 10:32:25 -0800 Subject: [PATCH 32/35] Set a default rate-limit delay. --- octodns/provider/rackspace.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py index 1f723d1..371861e 100644 --- a/octodns/provider/rackspace.py +++ b/octodns/provider/rackspace.py @@ -42,7 +42,7 @@ class RackspaceProvider(BaseProvider): 'TXT')) TIMEOUT = 5 - def __init__(self, id, username, api_key, ratelimit_delay, *args, + def __init__(self, id, username, api_key, ratelimit_delay=0.0, *args, **kwargs): ''' Rackspace API v1 Provider From cdf26ffae459d009136817150e07d1d68d2773c0 Mon Sep 17 00:00:00 2001 From: Terrence Cole Date: Sat, 6 Jan 2018 10:35:47 -0800 Subject: [PATCH 33/35] Refactor the output transformer loop. --- octodns/provider/rackspace.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py index 371861e..3a195cf 100644 --- a/octodns/provider/rackspace.py +++ b/octodns/provider/rackspace.py @@ -294,12 +294,8 @@ class RackspaceProvider(BaseProvider): self._get_values(change.new)) def _create_given_change_values(self, change, values): - out = [] - for value in values: - transformer = getattr(self, - "_record_for_{}".format(change.new._type)) - out.append(transformer(change.new, value)) - return out + transformer = getattr(self, "_record_for_{}".format(change.new._type)) + return [transformer(change.new, v) for v in values] def _mod_Update(self, change): existing_values = self._get_values(change.existing) From 80aed0052306c0dc8a346ff57dbbaeb31ad833e2 Mon Sep 17 00:00:00 2001 From: Terrence Cole Date: Sat, 6 Jan 2018 11:18:33 -0800 Subject: [PATCH 34/35] Pull transformer above the loop for delete as well. --- octodns/provider/rackspace.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py index 3a195cf..c432339 100644 --- a/octodns/provider/rackspace.py +++ b/octodns/provider/rackspace.py @@ -332,10 +332,10 @@ class RackspaceProvider(BaseProvider): change.existing)) def _delete_given_change_values(self, change, values): + transformer = getattr(self, "_record_for_{}".format( + change.existing._type)) out = [] for value in values: - transformer = getattr(self, "_record_for_{}".format( - change.existing._type)) rs_record = transformer(change.existing, value) key = self._key_for_record(rs_record) out.append('id=' + self._id_map[key]) From e875ee7f5d846d4c5a516834526bec57099c3196 Mon Sep 17 00:00:00 2001 From: Terrence Cole Date: Mon, 8 Jan 2018 11:07:03 -0800 Subject: [PATCH 35/35] Add a comment explaining our update scheme. --- octodns/provider/rackspace.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py index c432339..12b2c54 100644 --- a/octodns/provider/rackspace.py +++ b/octodns/provider/rackspace.py @@ -348,6 +348,9 @@ class RackspaceProvider(BaseProvider): self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name, len(changes)) + # Creates, updates, and deletes are processed by different endpoints + # and are broken out by record-set entries; pre-process everything + # into these buckets in order to minimize the number of API calls. domain_id = self._get_zone_id_for(desired) creates = [] updates = []