From a9c6d8c6baa46d2b72753478575b5342c0c56019 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 29 May 2017 08:26:38 -0700 Subject: [PATCH 01/64] Rework _Route53Record to avoid a bunch of hacks They were working around the lack of class hierarchy, this addresses that by adding 2 child classes. It gets rid of a bunch of ugly default params and if-this junk in the main class that was trying to deal with plain & geo records. Also as a positive side effect it gets rid of some hacks that lived in Route53Provider dealing with the fact that the old setup couldn't detect going to/from a GeoDNS record correctly. --- octodns/provider/route53.py | 237 +++++++++++++++---------- tests/test_octodns_provider_route53.py | 102 ++++++++--- 2 files changed, 221 insertions(+), 118 deletions(-) diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index 3849561..c42a394 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -16,27 +16,71 @@ from ..record import Record, Update from .base import BaseProvider +octal_re = re.compile(r'\\(\d\d\d)') + + +def _octal_replace(s): + # See http://docs.aws.amazon.com/Route53/latest/DeveloperGuide/ + # DomainNameFormat.html + return octal_re.sub(lambda m: chr(int(m.group(1), 8)), s) + + class _Route53Record(object): - def __init__(self, fqdn, _type, ttl, record=None, values=None, geo=None, - health_check_id=None): - self.fqdn = fqdn - self._type = _type - self.ttl = ttl - # From here on things are a little ugly, it works, but would be nice to - # clean up someday. - if record: - values_for = getattr(self, '_values_for_{}'.format(self._type)) - self.values = values_for(record) + @classmethod + def new(self, provider, record, creating): + ret = set() + if getattr(record, 'geo', False): + ret.add(_Route53GeoDefault(provider, record, creating)) + for ident, geo in record.geo.items(): + ret.add(_Route53GeoRecord(provider, record, ident, geo, + creating)) else: - self.values = values - self.geo = geo - self.health_check_id = health_check_id - self.is_geo_default = False + ret.add(_Route53Record(provider, record, creating)) + return ret - @property - def _geo_code(self): - return getattr(self.geo, 'code', '') + def __init__(self, provider, record, creating): + self.fqdn = record.fqdn + self._type = record._type + self.ttl = record.ttl + + values_for = getattr(self, '_values_for_{}'.format(self._type)) + self.values = values_for(record) + + def mod(self, action): + return { + 'Action': action, + 'ResourceRecordSet': { + 'Name': self.fqdn, + 'ResourceRecords': [{'Value': v} for v in self.values], + 'TTL': self.ttl, + 'Type': self._type, + } + } + + # NOTE: we're using __hash__ and __cmp__ methods that consider + # _Route53Records equivalent if they have the same class, fqdn, and _type. + # Values are ignored. This is usful when computing diffs/changes. + + def __hash__(self): + 'sub-classes should never use this method' + return '{}:{}'.format(self.fqdn, self._type).__hash__() + + def __cmp__(self, other): + '''sub-classes should call up to this and return its value if non-zero. + When it's zero they should compute their own __cmp__''' + if self.__class__ != other.__class__: + return cmp(self.__class__, other.__class__) + elif self.fqdn != other.fqdn: + return cmp(self.fqdn, other.fqdn) + elif self._type != other._type: + return cmp(self._type, other._type) + # We're ignoring ttl, it's not an actual differentiator + return 0 + + def __repr__(self): + return '_Route53Record<{} {} {} {}>'.format(self.fqdn, self._type, + self.ttl, self.values) def _values_for_values(self, record): return record.values @@ -75,68 +119,91 @@ class _Route53Record(object): v.target) for v in record.values] + +class _Route53GeoDefault(_Route53Record): + + def mod(self, action): + return { + 'Action': action, + 'ResourceRecordSet': { + 'Name': self.fqdn, + 'GeoLocation': { + 'CountryCode': '*' + }, + 'ResourceRecords': [{'Value': v} for v in self.values], + 'SetIdentifier': 'default', + 'TTL': self.ttl, + 'Type': self._type, + } + } + + def __hash__(self): + return '{}:{}:default'.format(self.fqdn, self._type).__hash__() + + def __repr__(self): + return '_Route53GeoDefault<{} {} {} {}>'.format(self.fqdn, self._type, + self.ttl, self.values) + + +class _Route53GeoRecord(_Route53Record): + + def __init__(self, provider, record, ident, geo, creating): + super(_Route53GeoRecord, self).__init__(provider, record, creating) + self.geo = geo + + self.health_check_id = provider.get_health_check_id(record, ident, + geo, creating) + def mod(self, action): + geo = self.geo rrset = { 'Name': self.fqdn, - 'Type': self._type, + 'GeoLocation': { + 'CountryCode': '*' + }, + 'ResourceRecords': [{'Value': v} for v in geo.values], + 'SetIdentifier': geo.code, 'TTL': self.ttl, - 'ResourceRecords': [{'Value': v} for v in self.values], + 'Type': self._type, } - if self.is_geo_default: + + if self.health_check_id: + rrset['HealthCheckId'] = self.health_check_id + + if geo.subdivision_code: rrset['GeoLocation'] = { - 'CountryCode': '*' + 'CountryCode': geo.country_code, + 'SubdivisionCode': geo.subdivision_code + } + elif geo.country_code: + rrset['GeoLocation'] = { + 'CountryCode': geo.country_code + } + else: + rrset['GeoLocation'] = { + 'ContinentCode': geo.continent_code } - rrset['SetIdentifier'] = 'default' - elif self.geo: - geo = self.geo - rrset['SetIdentifier'] = geo.code - if self.health_check_id: - rrset['HealthCheckId'] = self.health_check_id - if geo.subdivision_code: - rrset['GeoLocation'] = { - 'CountryCode': geo.country_code, - 'SubdivisionCode': geo.subdivision_code - } - elif geo.country_code: - rrset['GeoLocation'] = { - 'CountryCode': geo.country_code - } - else: - rrset['GeoLocation'] = { - 'ContinentCode': geo.continent_code - } return { 'Action': action, 'ResourceRecordSet': rrset, } - # NOTE: we're using __hash__ and __cmp__ methods that consider - # _Route53Records equivalent if they have the same fqdn, _type, and - # geo.ident. Values are ignored. This is usful when computing - # diffs/changes. - def __hash__(self): return '{}:{}:{}'.format(self.fqdn, self._type, - self._geo_code).__hash__() + self.geo.code).__hash__() def __cmp__(self, other): - return 0 if (self.fqdn == other.fqdn and - self._type == other._type and - self._geo_code == other._geo_code) else 1 + ret = super(_Route53GeoRecord, self).__cmp__(other) + if ret != 0: + return ret + return cmp(self.geo.code, other.geo.code) def __repr__(self): - return '_Route53Record<{} {:>5} {:8} {}>' \ - .format(self.fqdn, self._type, self._geo_code, self.values) - - -octal_re = re.compile(r'\\(\d\d\d)') - - -def _octal_replace(s): - # See http://docs.aws.amazon.com/Route53/latest/DeveloperGuide/ - # DomainNameFormat.html - return octal_re.sub(lambda m: chr(int(m.group(1), 8)), s) + return '_Route53GeoRecord<{} {} {} {} {}>'.format(self.fqdn, + self._type, self.ttl, + self.geo.code, + self.values) class Route53Provider(BaseProvider): @@ -388,7 +455,7 @@ class Route53Provider(BaseProvider): def _gen_mods(self, action, records): ''' - Turns `_Route53Record`s in to `change_resource_record_sets` `Changes` + Turns `_Route53*`s in to `change_resource_record_sets` `Changes` ''' return [r.mod(action) for r in records] @@ -417,14 +484,14 @@ class Route53Provider(BaseProvider): # We've got a cached version use it return self._health_checks - def _get_health_check_id(self, record, ident, geo, create): + def get_health_check_id(self, record, ident, geo, create): # fqdn & the first value are special, we use them to match up health # checks to their records. Route53 health checks check a single ip and # we're going to assume that ips are interchangeable to avoid # health-checking each one independently fqdn = record.fqdn first_value = geo.values[0] - self.log.debug('_get_health_check_id: fqdn=%s, type=%s, geo=%s, ' + self.log.debug('get_health_check_id: fqdn=%s, type=%s, geo=%s, ' 'first_value=%s', fqdn, record._type, ident, first_value) @@ -470,7 +537,7 @@ class Route53Provider(BaseProvider): # store the new health check so that we'll be able to find it in the # future self._health_checks[id] = health_check - self.log.info('_get_health_check_id: created id=%s, host=%s, ' + self.log.info('get_health_check_id: created id=%s, host=%s, ' 'first_value=%s', id, host, first_value) return id @@ -479,8 +546,9 @@ class Route53Provider(BaseProvider): # Find the health checks we're using for the new route53 records in_use = set() for r in new: - if r.health_check_id: - in_use.add(r.health_check_id) + hc_id = getattr(r, 'health_check_id', False) + if hc_id: + in_use.add(hc_id) self.log.debug('_gc_health_checks: in_use=%s', in_use) # Now we need to run through ALL the health checks looking for those # that apply to this record, deleting any that do and are no longer in @@ -499,23 +567,9 @@ class Route53Provider(BaseProvider): def _gen_records(self, record, creating=False): ''' - Turns an octodns.Record into one or more `_Route53Record`s + Turns an octodns.Record into one or more `_Route53*`s ''' - records = set() - base = _Route53Record(record.fqdn, record._type, record.ttl, - record=record) - records.add(base) - if getattr(record, 'geo', False): - base.is_geo_default = True - for ident, geo in record.geo.items(): - health_check_id = self._get_health_check_id(record, ident, geo, - creating) - records.add(_Route53Record(record.fqdn, record._type, - record.ttl, values=geo.values, - geo=geo, - health_check_id=health_check_id)) - - return records + return _Route53Record.new(self, record, creating) def _mod_Create(self, change): # New is the stuff that needs to be created @@ -541,24 +595,11 @@ class Route53Provider(BaseProvider): # things that haven't actually changed, but that's for another day. # We can't use set math here b/c we won't be able to control which of # the two objects will be in the result and we need to ensure it's the - # new one and we have to include some special handling when converting - # to/from a GEO enabled record + # new one. upserts = set() - existing_records = {r: r for r in existing_records} for new_record in new_records: - try: - existing_record = existing_records[new_record] - if new_record.is_geo_default != existing_record.is_geo_default: - # going from normal to geo or geo to normal, need a delete - # and create - deletes.add(existing_record) - creates.add(new_record) - else: - # just an update - upserts.add(new_record) - except KeyError: - # Completely new record, ignore - pass + if new_record in existing_records: + upserts.add(new_record) return self._gen_mods('DELETE', deletes) + \ self._gen_mods('CREATE', creates) + \ diff --git a/tests/test_octodns_provider_route53.py b/tests/test_octodns_provider_route53.py index e1a73f8..36b3070 100644 --- a/tests/test_octodns_provider_route53.py +++ b/tests/test_octodns_provider_route53.py @@ -11,8 +11,8 @@ from unittest import TestCase from mock import patch from octodns.record import Create, Delete, Record, Update -from octodns.provider.route53 import _Route53Record, Route53Provider, \ - _octal_replace +from octodns.provider.route53 import Route53Provider, _Route53GeoDefault, \ + _Route53GeoRecord, _Route53Record, _octal_replace from octodns.zone import Zone from helpers import GeoProvider @@ -522,21 +522,21 @@ class TestRoute53Provider(TestCase): 'Changes': [{ 'Action': 'DELETE', 'ResourceRecordSet': { - 'GeoLocation': {'ContinentCode': 'OC'}, + 'GeoLocation': {'CountryCode': '*'}, 'Name': 'simple.unit.tests.', - 'ResourceRecords': [{'Value': '3.2.3.4'}, - {'Value': '4.2.3.4'}], - 'SetIdentifier': 'OC', + 'ResourceRecords': [{'Value': '1.2.3.4'}, + {'Value': '2.2.3.4'}], + 'SetIdentifier': 'default', 'TTL': 61, 'Type': 'A'} }, { 'Action': 'DELETE', 'ResourceRecordSet': { - 'GeoLocation': {'CountryCode': '*'}, + 'GeoLocation': {'ContinentCode': 'OC'}, 'Name': 'simple.unit.tests.', - 'ResourceRecords': [{'Value': '1.2.3.4'}, - {'Value': '2.2.3.4'}], - 'SetIdentifier': 'default', + 'ResourceRecords': [{'Value': '3.2.3.4'}, + {'Value': '4.2.3.4'}], + 'SetIdentifier': 'OC', 'TTL': 61, 'Type': 'A'} }, { @@ -694,8 +694,7 @@ class TestRoute53Provider(TestCase): 'AF': ['4.2.3.4'], } }) - id = provider._get_health_check_id(record, 'AF', record.geo['AF'], - True) + id = provider.get_health_check_id(record, 'AF', record.geo['AF'], True) self.assertEquals('42', id) def test_health_check_create(self): @@ -765,13 +764,12 @@ class TestRoute53Provider(TestCase): }) # if not allowed to create returns none - id = provider._get_health_check_id(record, 'AF', record.geo['AF'], - False) + id = provider.get_health_check_id(record, 'AF', record.geo['AF'], + False) self.assertFalse(id) # when allowed to create we do - id = provider._get_health_check_id(record, 'AF', record.geo['AF'], - True) + id = provider.get_health_check_id(record, 'AF', record.geo['AF'], True) self.assertEquals('42', id) stubber.assert_no_pending_responses() @@ -1106,10 +1104,6 @@ class TestRoute53Provider(TestCase): self.assertEquals(0, len(extra)) stubber.assert_no_pending_responses() - def test_route_53_record(self): - # Just make sure it doesn't blow up - _Route53Record('foo.unit.tests.', 'A', 30).__repr__() - def _get_test_plan(self, max_changes): provider = Route53Provider('test', 'abc', '123', max_changes) @@ -1217,3 +1211,71 @@ class TestRoute53Provider(TestCase): with self.assertRaises(Exception) as ctx: provider.apply(plan) self.assertTrue('modifications' in ctx.exception.message) + + +class TestRoute53Records(TestCase): + + def test_route53_record(self): + existing = Zone('unit.tests.', []) + record_a = Record.new(existing, '', { + 'geo': { + 'NA-US': ['2.2.2.2', '3.3.3.3'], + 'OC': ['4.4.4.4', '5.5.5.5'] + }, + 'ttl': 99, + 'type': 'A', + 'values': ['9.9.9.9'] + }) + a = _Route53Record(None, record_a, False) + self.assertEquals(a, a) + b = _Route53Record(None, Record.new(existing, '', + {'ttl': 32, 'type': 'A', + 'values': ['8.8.8.8', + '1.1.1.1']}), + False) + self.assertEquals(b, b) + c = _Route53Record(None, Record.new(existing, 'other', + {'ttl': 99, 'type': 'A', + 'values': ['9.9.9.9']}), + False) + self.assertEquals(c, c) + d = _Route53Record(None, Record.new(existing, '', + {'ttl': 42, 'type': 'CNAME', + 'value': 'foo.bar.'}), + False) + self.assertEquals(d, d) + + # Same fqdn & type is same record + self.assertEquals(a, b) + # Same name & different type is not the same + self.assertNotEquals(a, d) + # Different name & same type is not the same + self.assertNotEquals(a, c) + + # Same everything, different class is not the same + e = _Route53GeoDefault(None, record_a, False) + self.assertNotEquals(a, e) + + class DummyProvider(object): + + def get_health_check_id(self, *args, **kwargs): + return None + + provider = DummyProvider() + f = _Route53GeoRecord(provider, record_a, 'NA-US', + record_a.geo['NA-US'], False) + self.assertEquals(f, f) + g = _Route53GeoRecord(provider, record_a, 'OC', + record_a.geo['OC'], False) + self.assertEquals(g, g) + + # Geo and non-geo are not the same, using Geo as primary to get it's + # __cmp__ + self.assertNotEquals(f, a) + # Same everything, different geo's is not the same + self.assertNotEquals(f, g) + + # Make sure it doesn't blow up + a.__repr__() + e.__repr__() + f.__repr__() From 0e20c076b002ef87f3a7d12b2c791d55b6e4c05c Mon Sep 17 00:00:00 2001 From: Heesu Hwang Date: Fri, 16 Jun 2017 14:20:36 -0700 Subject: [PATCH 02/64] First skeleton of Azure DNS Provider class --- octodns/provider/azuredns.py | 62 ++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 octodns/provider/azuredns.py diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py new file mode 100644 index 0000000..17e93b9 --- /dev/null +++ b/octodns/provider/azuredns.py @@ -0,0 +1,62 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from azure.common.credentials import ServicePrincipalCredentials +from azure.mgmt.dns import DnsManagementClient + + +from collections import defaultdict +# from incf.countryutils.transformations import cca_to_ctca2 TODO: add geo sup. +import logging +import re + +from ..record import Record, Update +from .base import BaseProvider + +class AzureProvider(BaseProvider): + ''' + Azure DNS Provider + + azure.py: + class: octodns.provider.azure.AzureProvider + # Current support of authentication of access to Azure services only + # includes using a Service Principal: + # https://docs.microsoft.com/en-us/azure/azure-resource-manager/ + # resource-group-create-service-principal-portal + # The Azure Active Directory Application ID (referred to client ID) req: + client_id: + # Authentication Key Value req: + key: + # Directory ID (referred to tenant ID) req: + directory_id: + # Subscription ID req: + sub_id: + # Resource Group name req: + resource_group: + + testing: test authentication vars located in /home/t-hehwan/vars.txt + ''' + + # TODO. Will add support as project progresses. + SUPPORTS_GEO = False + + def __init__(self, id, client_id, key, directory_id, sub_id, \ + resource_group, *args, **kwargs): + self.log = logging.getLogger('AzureProvider[{}]'.format(id)) + self.log.debug('__init__: id=%s, client_id=%s, ' + 'key=***, directory_id:%s', id, client_id, directory_id) + super(AzureProvider, self).__init__(id, *args, **kwargs) + + credentials = ServicePrincipalCredentials( + client_id = client_id, secret = key, tenant = directory_id + ) + self._dns_client = DnsManagementClient(credentials, sub_id) + self._resource_group = resource_group + + + def _apply(self, plan): + From 386ada34f0750922a930fc0660f668d3552d6aae Mon Sep 17 00:00:00 2001 From: Heesu Hwang Date: Mon, 19 Jun 2017 17:04:25 -0700 Subject: [PATCH 03/64] Added onto azuredns.py. Still completing code skeleton --- octodns/provider/azuredns.py | 116 +++++++++++++++++++++++++++++++---- 1 file changed, 103 insertions(+), 13 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 17e93b9..8e1b992 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -7,7 +7,7 @@ from __future__ import absolute_import, division, print_function, \ from azure.common.credentials import ServicePrincipalCredentials from azure.mgmt.dns import DnsManagementClient - +from azure.mgmt.dns.models import * from collections import defaultdict # from incf.countryutils.transformations import cca_to_ctca2 TODO: add geo sup. @@ -17,6 +17,23 @@ import re from ..record import Record, Update from .base import BaseProvider +# Only made for A records. will have to adjust for more generic params types +class _AzureRecord(object): + def __init__(self, resource_group_name, record, values=None) + self.resource_group_name = resource_group_name + self.zone_name = record.zone.name + self.relative_record_set_name = record.name + self.record_type = record._type + + type_name = '{}records'.format(self.record_type) + class_name = '{}'.format(self.record_type).capitalize() + + 'Record'.format(self.record_type) + _values = [record._process_values] + self.params = {'ttl':record.ttl or 1800, \ + type_name:[eval(class_name)(value) for value in _values] or []} + + + class AzureProvider(BaseProvider): ''' Azure DNS Provider @@ -44,19 +61,92 @@ class AzureProvider(BaseProvider): # TODO. Will add support as project progresses. SUPPORTS_GEO = False - def __init__(self, id, client_id, key, directory_id, sub_id, \ - resource_group, *args, **kwargs): - self.log = logging.getLogger('AzureProvider[{}]'.format(id)) - self.log.debug('__init__: id=%s, client_id=%s, ' + def __init__(self, id, client_id, key, directory_id, sub_id, \ + resource_group, *args, **kwargs): + self.log = logging.getLogger('AzureProvider[{}]'.format(id)) + self.log.debug('__init__: id=%s, client_id=%s, ' 'key=***, directory_id:%s', id, client_id, directory_id) - super(AzureProvider, self).__init__(id, *args, **kwargs) + super(AzureProvider, self).__init__(id, *args, **kwargs) - credentials = ServicePrincipalCredentials( - client_id = client_id, secret = key, tenant = directory_id - ) - self._dns_client = DnsManagementClient(credentials, sub_id) - self._resource_group = resource_group + credentials = ServicePrincipalCredentials( + client_id = client_id, secret = key, tenant = directory_id + ) + self._dns_client = DnsManagementClient(credentials, sub_id) + self._resource_group = resource_group + + + self._azure_zones = None + self._azure_records = {} # this is populated through populate() + # TODO: health checks a la route53. - def _apply(self, plan): - + + + # TODO: add support for all types. First skeleton: add A. + def supports(self, record): + return record._type == 'A' + + @property + def azure_zones(self): + # TODO: return zones. will be created by populate() + + # Given a zone name, returns the zone id. If DNE, creates it. + def _get_zone_id(self, name): + + + def populate(self, zone, target): + self._azure_records = {} + + for record in zone.records: + + + def _apply_Create(self, change): + new = change.new + ar = self._get_azure_record(new) + + create = self._dns_client.record_sets.create_or_update + create(ar.resource_group_name, ar.zone_name, ar.relative_record_set_name \ + ar.record_type, ar.params) + + # type plan: Plan class from .base + def _apply(self, plan): + desired = plan.desired + changes = plan.changes + self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name, + len(changes)) + + + # validate that the zone exists. function creates zone if DNE. + self._get_zone_id(desired.name) + + + + # Some parsing bits to call _mod_Create or _mod_Delete. + # changes is a list of Delete and Create objects. + for change in changes: + class_name = change.__class__.__name__ + getattr(self, '_apply_{}'.format(class_name))(change) + + + + # ********** + # Figuring out what object plan is. + + # self._executor = ThreadPoolExecutor(max_workers) + # futures.append(self._executor.submit(self._populate_and_plan, + # zone_name, sources, targets)) + # plans = [p for f in futures for p in f.results()] + # type of plans[0] == type of one output of _populate_and_plan + + # for target, plan in plans: + # apply(plan) + + + # type(target) == BaseProvider + # type(plan) == Plan() + + # Plan(existing, desired, changes) + # existing.type == desired.type == Zone(desired.name, desired.sub_zones) + # Zone(name, sub_zones) (str and set of strs) + # changes.type = [Delete/Create] + \ No newline at end of file From db35ffe72e1ad0e28b3779f53d2da45d561a1d6c Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 19 Jun 2017 22:17:48 -0700 Subject: [PATCH 04/64] Replace my custom natrual sorting with natsort module Better to use something real/tested and less likely buggy/limited. --- octodns/yaml.py | 21 ++++----------------- requirements.txt | 1 + setup.py | 1 + 3 files changed, 6 insertions(+), 17 deletions(-) diff --git a/octodns/yaml.py b/octodns/yaml.py index 2cab58c..d4ab541 100644 --- a/octodns/yaml.py +++ b/octodns/yaml.py @@ -5,25 +5,12 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals +from natsort import natsort_keygen from yaml import SafeDumper, SafeLoader, load, dump from yaml.constructor import ConstructorError -import re -# zero-padded sort, simplified version of -# https://www.xormedia.com/natural-sort-order-with-zero-padding/ -_pad_re = re.compile('\d+') - - -def _zero_pad(match): - return '{:04d}'.format(int(match.group(0))) - - -def _zero_padded_numbers(s): - try: - int(s) - except ValueError: - return _pad_re.sub(lambda d: _zero_pad(d), s) +_natsort_key = natsort_keygen() # Found http://stackoverflow.com/a/21912744 which guided me on how to hook in @@ -34,7 +21,7 @@ class SortEnforcingLoader(SafeLoader): self.flatten_mapping(node) ret = self.construct_pairs(node) keys = [d[0] for d in ret] - if keys != sorted(keys, key=_zero_padded_numbers): + if keys != sorted(keys, key=_natsort_key): raise ConstructorError(None, None, "keys out of order: {}" .format(', '.join(keys)), node.start_mark) return dict(ret) @@ -59,7 +46,7 @@ class SortingDumper(SafeDumper): def _representer(self, data): data = data.items() - data.sort(key=lambda d: _zero_padded_numbers(d[0])) + data.sort(key=lambda d: _natsort_key(d[0])) return self.represent_mapping(self.DEFAULT_MAPPING_TAG, data) diff --git a/requirements.txt b/requirements.txt index efd7577..b10ca4c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,6 +10,7 @@ futures==3.0.5 incf.countryutils==1.0 ipaddress==1.0.18 jmespath==0.9.0 +natsort==5.0.3 nsone==0.9.10 python-dateutil==2.6.0 requests==2.13.0 diff --git a/setup.py b/setup.py index ebb4092..f2b901d 100644 --- a/setup.py +++ b/setup.py @@ -34,6 +34,7 @@ setup( 'futures>=3.0.5', 'incf.countryutils>=1.0', 'ipaddress>=1.0.18', + 'natsort>=5.0.3', 'python-dateutil>=2.6.0', 'requests>=2.13.0' ], From 4a7ce9e833cbb6344837491fdc98cd1752e7baf7 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 19 Jun 2017 22:36:08 -0700 Subject: [PATCH 05/64] Bake in the existing, but less than great hex sorting behavior --- tests/test_octodns_yaml.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/test_octodns_yaml.py b/tests/test_octodns_yaml.py index 9c3cec5..0f454b3 100644 --- a/tests/test_octodns_yaml.py +++ b/tests/test_octodns_yaml.py @@ -59,3 +59,12 @@ class TestYaml(TestCase): }, buf) self.assertEquals("---\n'*.1.1': 42\n'*.2.1': 44\n'*.11.1': 43\n", buf.getvalue()) + + # hex sorting isn't ideal, not treated as hex, this make sure we don't + # change the behavior + buf = StringIO() + safe_dump({ + '45a03129': 42, + '45a0392a': 43, + }, buf) + self.assertEquals("---\n45a0392a: 43\n45a03129: 42\n", buf.getvalue()) From 046cde43b20871072ef57528b33654d44ee5dbe6 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 19 Jun 2017 22:44:34 -0700 Subject: [PATCH 06/64] Make sorting enforcement optional with YamlProvider --- octodns/provider/yaml.py | 14 ++++++++++---- tests/test_octodns_provider_yaml.py | 6 ++++++ 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/octodns/provider/yaml.py b/octodns/provider/yaml.py index 1089f02..7b2d209 100644 --- a/octodns/provider/yaml.py +++ b/octodns/provider/yaml.py @@ -26,16 +26,22 @@ class YamlProvider(BaseProvider): # The ttl to use for records when not specified in the data # (optional, default 3600) default_ttl: 3600 + # Whether or not to enforce sorting order on the yaml config + # (optional, default True) + enforce_order: True ''' SUPPORTS_GEO = True - def __init__(self, id, directory, default_ttl=3600, *args, **kwargs): + def __init__(self, id, directory, default_ttl=3600, enforce_order=True, + *args, **kwargs): self.log = logging.getLogger('YamlProvider[{}]'.format(id)) - self.log.debug('__init__: id=%s, directory=%s, default_ttl=%d', id, - directory, default_ttl) + self.log.debug('__init__: id=%s, directory=%s, default_ttl=%d, ' + 'enforce_order=%d', id, directory, default_ttl, + enforce_order) super(YamlProvider, self).__init__(id, *args, **kwargs) self.directory = directory self.default_ttl = default_ttl + self.enforce_order = enforce_order def populate(self, zone, target=False): self.log.debug('populate: zone=%s, target=%s', zone.name, target) @@ -47,7 +53,7 @@ class YamlProvider(BaseProvider): before = len(zone.records) filename = join(self.directory, '{}yaml'.format(zone.name)) with open(filename, 'r') as fh: - yaml_data = safe_load(fh) + yaml_data = safe_load(fh, enforce_order=self.enforce_order) if yaml_data: for name, data in yaml_data.items(): if not isinstance(data, list): diff --git a/tests/test_octodns_provider_yaml.py b/tests/test_octodns_provider_yaml.py index 05c5248..9438f01 100644 --- a/tests/test_octodns_provider_yaml.py +++ b/tests/test_octodns_provider_yaml.py @@ -100,6 +100,12 @@ class TestYamlProvider(TestCase): with self.assertRaises(ConstructorError): source.populate(zone) + source = YamlProvider('test', join(dirname(__file__), 'config'), + enforce_order=False) + # no exception + source.populate(zone) + self.assertEqual(2, len(zone.records)) + def test_subzone_handling(self): source = YamlProvider('test', join(dirname(__file__), 'config')) From ae9dd97f16e06152d27f28098b56a0f91eb476a5 Mon Sep 17 00:00:00 2001 From: Heesu Hwang Date: Tue, 20 Jun 2017 14:43:51 -0700 Subject: [PATCH 07/64] Filled out skeleton. Starting Testing --- .gitignore | 3 ++ MakeFile | 5 +++ octodns/provider/azuredns.py | 66 ++++++++++++++++++++++++++++++------ 3 files changed, 64 insertions(+), 10 deletions(-) create mode 100644 MakeFile diff --git a/.gitignore b/.gitignore index 842a688..eca95c9 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,6 @@ nosetests.xml octodns.egg-info/ output/ tmp/ +Makefile +build/ +config/ \ No newline at end of file diff --git a/MakeFile b/MakeFile new file mode 100644 index 0000000..53fc947 --- /dev/null +++ b/MakeFile @@ -0,0 +1,5 @@ +local-rebuild: + sudo rm -r build + sudo rm -r octodns.egg-info/ + sudo python setup.py build -q + sudo python setup.py install -q \ No newline at end of file diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 8e1b992..2c55d7f 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -19,14 +19,14 @@ from .base import BaseProvider # Only made for A records. will have to adjust for more generic params types class _AzureRecord(object): - def __init__(self, resource_group_name, record, values=None) + def __init__(self, resource_group_name, record, values=None): self.resource_group_name = resource_group_name self.zone_name = record.zone.name self.relative_record_set_name = record.name self.record_type = record._type type_name = '{}records'.format(self.record_type) - class_name = '{}'.format(self.record_type).capitalize() + + class_name = '{}'.format(self.record_type).capitalize() + \ 'Record'.format(self.record_type) _values = [record._process_values] self.params = {'ttl':record.ttl or 1800, \ @@ -75,30 +75,76 @@ class AzureProvider(BaseProvider): self._resource_group = resource_group - self._azure_zones = None - self._azure_records = {} # this is populated through populate() + self._azure_zones = None # will be a dictionary. key: name. val: id. + self._azure_records = None # will be dict by octodns record, az record + self._supported_types = ['A'] + # TODO: health checks a la route53. # TODO: add support for all types. First skeleton: add A. def supports(self, record): - return record._type == 'A' + # TODO: possibly refactor + return record._type in self._supported_types @property def azure_zones(self): - # TODO: return zones. will be created by populate() + if self._azure_zones is None: + self.log.debug('azure_zones: loading') + zones = {} + for zone in self._dns_client.zones.list(): + zones[zone['name']] = zone['id'] + self._azure_zones = zones + return self._azure_zones # Given a zone name, returns the zone id. If DNE, creates it. - def _get_zone_id(self, name): - + def _get_zone_id(self, name, create=False): + self.log.debug('_get_zone_id: name=%s', name) + if name in self.azure_zones: + id = self.azure_zones[name] + self.log.debug('_get_zone_id: id=%s', id) + return id + if create: + #TODO + return None + # Create a dictionary of record objects by zone and octodns record names + # TODO: add geo parsing def populate(self, zone, target): - self._azure_records = {} + self.log.debug('populate: name=%s', zone.name) + before = len(zone.records) - for record in zone.records: + zone_id = self._get_zone_id(zone.name) + if zone_id: + records = defaultdict(list) + for type in self._supported_types: + for azrecord in self.dns_client.record_sets.list_by_type(self._resource_group, zone.name, type): + record_name = azrecord.name + data = getattr(self, '_data_for_{}'.format(type))(type, azrecord) + record = Record.new(zone, record_name, data, source=self) + zone.add_record(record) + self._azure_records[record] = _AzureRecord(self._resource_group, record, record.data) + + self.log.info('populate: found %s records', len(zone.records)-before) + + # might not need + def _get_type(azrecord): + azrecord['type'].split('/')[-1] + + def _data_for_A(self, type, azrecord): + return { + 'type': type + 'ttl': azrecord['ttl'], + 'values': [ar.ipv4_address for ar in azrecord.arecords] + } + def _get_azure_record(record): + try: + return self._azure_records[record] + except: + raise def _apply_Create(self, change): new = change.new From f48ef28688cc7be2211e42d4a39614fbb4cccb40 Mon Sep 17 00:00:00 2001 From: Heesu Hwang Date: Tue, 20 Jun 2017 14:55:59 -0700 Subject: [PATCH 08/64] Added shell script --- MakeFile | 5 ----- octodns/provider/azuredns.py | 3 +++ rb.txt | 7 +++++++ 3 files changed, 10 insertions(+), 5 deletions(-) delete mode 100644 MakeFile create mode 100755 rb.txt diff --git a/MakeFile b/MakeFile deleted file mode 100644 index 53fc947..0000000 --- a/MakeFile +++ /dev/null @@ -1,5 +0,0 @@ -local-rebuild: - sudo rm -r build - sudo rm -r octodns.egg-info/ - sudo python setup.py build -q - sudo python setup.py install -q \ No newline at end of file diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 2c55d7f..da348c1 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -17,6 +17,9 @@ import re from ..record import Record, Update from .base import BaseProvider + +#TODO: changes made to master include adding /build, Makefile to .gitignore and +# making Makefile. # Only made for A records. will have to adjust for more generic params types class _AzureRecord(object): def __init__(self, resource_group_name, record, values=None): diff --git a/rb.txt b/rb.txt new file mode 100755 index 0000000..30d33a1 --- /dev/null +++ b/rb.txt @@ -0,0 +1,7 @@ +#!/bin/bash +#script to rebuild octodns quickly + +sudo rm -r /home/t-hehwan/GitHub/octodns/build +sudo rm -r /home/t-hehwan/GitHub/octodns/octodns.egg-info +sudo python /home/t-hehwan/GitHub/octodns/setup.py build -q +sudo python /home/t-hehwan/GitHub/octodns/setup.py install -q From f5bce43e1054fe310dc77e379797149360a93326 Mon Sep 17 00:00:00 2001 From: Heesu Hwang Date: Tue, 20 Jun 2017 15:54:35 -0700 Subject: [PATCH 09/64] Testing AzureProvider. TODO: resolve 'Exception: Unknown provider class: octodns.provider.azure.AzureProvider' --- octodns/provider/azuredns.py | 85 ++++++++++++++++++------------------ rb.txt | 5 ++- 2 files changed, 45 insertions(+), 45 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index da348c1..f1b342a 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -4,7 +4,7 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals - + from azure.common.credentials import ServicePrincipalCredentials from azure.mgmt.dns import DnsManagementClient from azure.mgmt.dns.models import * @@ -13,10 +13,13 @@ from collections import defaultdict # from incf.countryutils.transformations import cca_to_ctca2 TODO: add geo sup. import logging import re +from ..record import Record, Up -from ..record import Record, Update from .base import BaseProvider +class A(BaseProvider): + def __init__(self): + pass #TODO: changes made to master include adding /build, Makefile to .gitignore and # making Makefile. @@ -38,60 +41,56 @@ class _AzureRecord(object): class AzureProvider(BaseProvider): - ''' - Azure DNS Provider - - azure.py: - class: octodns.provider.azure.AzureProvider - # Current support of authentication of access to Azure services only - # includes using a Service Principal: - # https://docs.microsoft.com/en-us/azure/azure-resource-manager/ - # resource-group-create-service-principal-portal - # The Azure Active Directory Application ID (referred to client ID) req: - client_id: - # Authentication Key Value req: - key: - # Directory ID (referred to tenant ID) req: - directory_id: - # Subscription ID req: - sub_id: - # Resource Group name req: - resource_group: - - testing: test authentication vars located in /home/t-hehwan/vars.txt - ''' - - # TODO. Will add support as project progresses. - SUPPORTS_GEO = False - - def __init__(self, id, client_id, key, directory_id, sub_id, \ - resource_group, *args, **kwargs): + ''' + Azure DNS Provider + + azure.py: + class: octodns.provider.azure.AzureProvider + # Current support of authentication of access to Azure services only + # includes using a Service Principal: + # https://docs.microsoft.com/en-us/azure/azure-resource-manager/ + # resource-group-create-service-principal-portal + # The Azure Active Directory Application ID (referred to client ID) req: + client_id: + # Authentication Key Value req: + key: + # Directory ID (referred to tenant ID) req: + directory_id: + # Subscription ID req: + sub_id: + # Resource Group name req: + resource_group: + + testing: test authentication vars located in /home/t-hehwan/vars.txt + ''' + SUPPORTS_GEO = False # TODO. Will add support as project progresses. + + def __init__(self, id, client_id, key, directory_id, sub_id, resource_group, *args, **kwargs): self.log = logging.getLogger('AzureProvider[{}]'.format(id)) self.log.debug('__init__: id=%s, client_id=%s, ' - 'key=***, directory_id:%s', id, client_id, directory_id) + 'key=***, directory_id:%s', id, client_id, directory_id) super(AzureProvider, self).__init__(id, *args, **kwargs) - + credentials = ServicePrincipalCredentials( - client_id = client_id, secret = key, tenant = directory_id + client_id = client_id, secret = key, tenant = directory_id ) self._dns_client = DnsManagementClient(credentials, sub_id) self._resource_group = resource_group - self._azure_zones = None # will be a dictionary. key: name. val: id. self._azure_records = None # will be dict by octodns record, az record - + self._supported_types = ['A'] + + # TODO: health checks a la route53. + + - # TODO: health checks a la route53. - - - # TODO: add support for all types. First skeleton: add A. def supports(self, record): # TODO: possibly refactor return record._type in self._supported_types - + @property def azure_zones(self): if self._azure_zones is None: @@ -110,6 +109,7 @@ class AzureProvider(BaseProvider): self.log.debug('_get_zone_id: id=%s', id) return id if create: + raise Exception #TODO return None @@ -138,7 +138,7 @@ class AzureProvider(BaseProvider): def _data_for_A(self, type, azrecord): return { - 'type': type + 'type': type, 'ttl': azrecord['ttl'], 'values': [ar.ipv4_address for ar in azrecord.arecords] } @@ -154,8 +154,7 @@ class AzureProvider(BaseProvider): ar = self._get_azure_record(new) create = self._dns_client.record_sets.create_or_update - create(ar.resource_group_name, ar.zone_name, ar.relative_record_set_name \ - ar.record_type, ar.params) + create(ar.resource_group_name, ar.zone_name, ar.relative_record_set_name, ar.record_type, ar.params) # type plan: Plan class from .base def _apply(self, plan): diff --git a/rb.txt b/rb.txt index 30d33a1..d6a7949 100755 --- a/rb.txt +++ b/rb.txt @@ -3,5 +3,6 @@ sudo rm -r /home/t-hehwan/GitHub/octodns/build sudo rm -r /home/t-hehwan/GitHub/octodns/octodns.egg-info -sudo python /home/t-hehwan/GitHub/octodns/setup.py build -q -sudo python /home/t-hehwan/GitHub/octodns/setup.py install -q +sudo python /home/t-hehwan/GitHub/octodns/setup.py -q build +sudo python /home/t-hehwan/GitHub/octodns/setup.py -q install +octodns-sync --config-file=./config/production.yaml \ No newline at end of file From 92828ce1c6a99c6d29fe2b223cc9e938a404fae1 Mon Sep 17 00:00:00 2001 From: Heesu Hwang Date: Wed, 21 Jun 2017 14:21:14 -0700 Subject: [PATCH 10/64] Successfully able to add A records. TODO: check against live server to remove records not listed in config --- octodns/provider/azuredns.py | 85 +++++++++++++++++++++++++----------- 1 file changed, 60 insertions(+), 25 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index f1b342a..3a442e8 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -4,6 +4,7 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals +import sys from azure.common.credentials import ServicePrincipalCredentials from azure.mgmt.dns import DnsManagementClient @@ -13,32 +14,41 @@ from collections import defaultdict # from incf.countryutils.transformations import cca_to_ctca2 TODO: add geo sup. import logging import re -from ..record import Record, Up - +from ..record import Record, Update from .base import BaseProvider -class A(BaseProvider): - def __init__(self): - pass #TODO: changes made to master include adding /build, Makefile to .gitignore and # making Makefile. # Only made for A records. will have to adjust for more generic params types class _AzureRecord(object): - def __init__(self, resource_group_name, record, values=None): - self.resource_group_name = resource_group_name - self.zone_name = record.zone.name - self.relative_record_set_name = record.name + def __init__(self, resource_group, record, values=None): + self.resource_group = resource_group + self.zone_name = record.zone.name[0:len(record.zone.name)-1] # strips last period + self.relative_record_set_name = record.name or '@' self.record_type = record._type - type_name = '{}records'.format(self.record_type) + type_name = '{}records'.format(self.record_type).lower() class_name = '{}'.format(self.record_type).capitalize() + \ 'Record'.format(self.record_type) - _values = [record._process_values] - self.params = {'ttl':record.ttl or 1800, \ - type_name:[eval(class_name)(value) for value in _values] or []} + data = values or record.data #This should fail if it gets to record.data? It only returns ttl. TODO + print('findme: ',file=sys.stderr) + print(data, file=sys.stderr) + print('\n',file=sys.stderr) + + #depending on mult values or not + self.params = {} + try: + self.params = {'ttl':record.ttl or 1800, \ + type_name:[eval(class_name)(ip) for ip in data['values']] or []} + except KeyError: # means that doesn't have multiple values but single value + self.params = {'ttl':record.ttl or 1800, \ + type_name:[eval(class_name)(data['value'])] or []} + + # ar = _AzureRecord(self._resource_group, new, new.data) + class AzureProvider(BaseProvider): ''' @@ -96,8 +106,8 @@ class AzureProvider(BaseProvider): if self._azure_zones is None: self.log.debug('azure_zones: loading') zones = {} - for zone in self._dns_client.zones.list(): - zones[zone['name']] = zone['id'] + for zone in self._dns_client.zones.list_by_resource_group(self._resource_group): + zones[zone.name] = zone.id self._azure_zones = zones return self._azure_zones @@ -109,8 +119,9 @@ class AzureProvider(BaseProvider): self.log.debug('_get_zone_id: id=%s', id) return id if create: + self.log.debug('_get_zone_id: no matching zone; creating %s', name) raise Exception - #TODO + #TODO, write code return None # Create a dictionary of record objects by zone and octodns record names @@ -128,7 +139,7 @@ class AzureProvider(BaseProvider): data = getattr(self, '_data_for_{}'.format(type))(type, azrecord) record = Record.new(zone, record_name, data, source=self) zone.add_record(record) - self._azure_records[record] = _AzureRecord(self._resource_group, record, record.data) + self._azure_records[record] = _AzureRecord(self._resource_group, record, data) self.log.info('populate: found %s records', len(zone.records)-before) @@ -137,24 +148,48 @@ class AzureProvider(BaseProvider): azrecord['type'].split('/')[-1] def _data_for_A(self, type, azrecord): + print('xxxxx\n',file=sys.stderr) + for ar in azrecords.arecords: + print(ar,file=sys.stderr) + + + v = [ARecord(ar.ipv4_address) for ar in azrecord.arecords] + # print(v, file=sys.stderr) + # print('99999999999999999999999999999999999999999999999999\n',file=sys.stderr) return { 'type': type, 'ttl': azrecord['ttl'], - 'values': [ar.ipv4_address for ar in azrecord.arecords] + 'value': v } - def _get_azure_record(record): - try: - return self._azure_records[record] - except: - raise + # def _get_azure_record(self, record): + # try: + # return self._azure_records[record] + # except: + # self._azure_records[record] = _AzureRecord(self._resource_group, record, record.data) + # return self._azure_records[record] + # except: + # raise def _apply_Create(self, change): new = change.new - ar = self._get_azure_record(new) + + #validate that the zone exists. + #self._get_zone_id(new.name, create=True) + + #ar = self._get_azure_record(new) + ar = _AzureRecord(self._resource_group, new, new.data) create = self._dns_client.record_sets.create_or_update - create(ar.resource_group_name, ar.zone_name, ar.relative_record_set_name, ar.record_type, ar.params) + print('find:{} {} {} {} {}\n'.format(ar.resource_group,ar.zone_name,ar.relative_record_set_name,ar.record_type,ar.params), file=sys.stderr) + for arec in ar.params['arecords']: + print(str(arec.ipv4_address) + ', ', file=sys.stderr) + print('\n',file=sys.stderr) + create(resource_group_name=ar.resource_group, + zone_name=ar.zone_name, + relative_record_set_name=ar.relative_record_set_name, + record_type=ar.record_type, + parameters=ar.params) # type plan: Plan class from .base def _apply(self, plan): From 852c1013889e6c6c4504692e7e91948b364c22a7 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Wed, 21 Jun 2017 17:08:16 -0700 Subject: [PATCH 11/64] Switch to an explicit SUPPORTS setup --- octodns/provider/cloudflare.py | 7 ++----- octodns/provider/dnsimple.py | 2 ++ octodns/provider/dyn.py | 2 ++ octodns/provider/ns1.py | 6 +++--- octodns/provider/powerdns.py | 2 ++ octodns/provider/route53.py | 5 ++--- octodns/provider/yaml.py | 2 ++ octodns/source/base.py | 7 ++++--- octodns/source/tinydns.py | 1 + tests/helpers.py | 1 + tests/test_octodns_provider_base.py | 13 +++++++++++-- 11 files changed, 32 insertions(+), 16 deletions(-) diff --git a/octodns/provider/cloudflare.py b/octodns/provider/cloudflare.py index eb44d30..a45bd44 100644 --- a/octodns/provider/cloudflare.py +++ b/octodns/provider/cloudflare.py @@ -36,7 +36,7 @@ class CloudflareProvider(BaseProvider): ''' SUPPORTS_GEO = False # TODO: support SRV - UNSUPPORTED_TYPES = ('ALIAS', 'NAPTR', 'PTR', 'SOA', 'SRV', 'SSHFP') + SUPPORTS = set(('A', 'AAAA', 'CNAME', 'MX', 'NS', 'SPF', 'TXT')) MIN_TTL = 120 TIMEOUT = 15 @@ -56,9 +56,6 @@ class CloudflareProvider(BaseProvider): self._zones = None self._zone_records = {} - def supports(self, record): - return record._type not in self.UNSUPPORTED_TYPES - def _request(self, method, path, params=None, data=None): self.log.debug('_request: method=%s, path=%s', method, path) @@ -167,7 +164,7 @@ class CloudflareProvider(BaseProvider): for record in records: name = zone.hostname_from_fqdn(record['name']) _type = record['type'] - if _type not in self.UNSUPPORTED_TYPES: + if _type in self.SUPPORTS: values[name][record['type']].append(record) for name, types in values.items(): diff --git a/octodns/provider/dnsimple.py b/octodns/provider/dnsimple.py index 763e446..cb0f2d7 100644 --- a/octodns/provider/dnsimple.py +++ b/octodns/provider/dnsimple.py @@ -91,6 +91,8 @@ class DnsimpleProvider(BaseProvider): account: 42 ''' SUPPORTS_GEO = False + SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CNAME', 'MX', 'NAPTR', 'NS', 'PTR', + 'SPF', 'SRV', 'SSHFP', 'TXT')) def __init__(self, id, token, account, *args, **kwargs): self.log = logging.getLogger('DnsimpleProvider[{}]'.format(id)) diff --git a/octodns/provider/dyn.py b/octodns/provider/dyn.py index ac0e21b..673e8d0 100644 --- a/octodns/provider/dyn.py +++ b/octodns/provider/dyn.py @@ -106,6 +106,7 @@ class DynProvider(BaseProvider): than one account active at a time. See DynProvider._check_dyn_sess for some related bits. ''' + RECORDS_TO_TYPE = { 'a_records': 'A', 'aaaa_records': 'AAAA', @@ -121,6 +122,7 @@ class DynProvider(BaseProvider): 'txt_records': 'TXT', } TYPE_TO_RECORDS = {v: k for k, v in RECORDS_TO_TYPE.items()} + SUPPORTS = set(TYPE_TO_RECORDS.keys()) # https://help.dyn.com/predefined-geotm-regions-groups/ REGION_CODES = { diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 5a51780..c50341d 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -22,6 +22,9 @@ class Ns1Provider(BaseProvider): api_key: env/NS1_API_KEY ''' SUPPORTS_GEO = False + SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CNAME', 'MX', 'NAPTR', 'NS', 'PTR', + 'SPF', 'SRV', 'TXT')) + ZONE_NOT_FOUND_MESSAGE = 'server error: zone not found' def __init__(self, id, api_key, *args, **kwargs): @@ -30,9 +33,6 @@ class Ns1Provider(BaseProvider): super(Ns1Provider, self).__init__(id, *args, **kwargs) self._client = NSONE(apiKey=api_key) - def supports(self, record): - return record._type != 'SSHFP' - def _data_for_A(self, _type, record): return { 'ttl': record['ttl'], diff --git a/octodns/provider/powerdns.py b/octodns/provider/powerdns.py index 4ff2568..21e4d44 100644 --- a/octodns/provider/powerdns.py +++ b/octodns/provider/powerdns.py @@ -14,6 +14,8 @@ from .base import BaseProvider class PowerDnsBaseProvider(BaseProvider): SUPPORTS_GEO = False + SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CNAME', 'MX', 'NAPTR', 'NS', 'PTR', + 'SPF', 'SSHFP', 'SRV', 'TXT')) TIMEOUT = 5 def __init__(self, id, host, api_key, port=8081, *args, **kwargs): diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index bc8bc34..12c1aff 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -220,6 +220,8 @@ class Route53Provider(BaseProvider): In general the account used will need full permissions on Route53. ''' SUPPORTS_GEO = True + SUPPORTS = set(('A', 'AAAA', 'CNAME', 'MX', 'NAPTR', 'NS', 'PTR', 'SPF', + 'SRV', 'TXT')) # This should be bumped when there are underlying changes made to the # health check config. @@ -239,9 +241,6 @@ class Route53Provider(BaseProvider): self._r53_rrsets = {} self._health_checks = None - def supports(self, record): - return record._type not in ('ALIAS', 'SSHFP') - @property def r53_zones(self): if self._r53_zones is None: diff --git a/octodns/provider/yaml.py b/octodns/provider/yaml.py index 7b2d209..c728caf 100644 --- a/octodns/provider/yaml.py +++ b/octodns/provider/yaml.py @@ -31,6 +31,8 @@ class YamlProvider(BaseProvider): enforce_order: True ''' SUPPORTS_GEO = True + SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CNAME', 'MX', 'NAPTR', 'NS', 'PTR', + 'SSHFP', 'SPF', 'SRV', 'TXT')) def __init__(self, id, directory, default_ttl=3600, enforce_order=True, *args, **kwargs): diff --git a/octodns/source/base.py b/octodns/source/base.py index 72ebaab..42d214b 100644 --- a/octodns/source/base.py +++ b/octodns/source/base.py @@ -16,6 +16,9 @@ class BaseSource(object): if not hasattr(self, 'SUPPORTS_GEO'): raise NotImplementedError('Abstract base class, SUPPORTS_GEO ' 'property missing') + if not hasattr(self, 'SUPPORTS'): + raise NotImplementedError('Abstract base class, SUPPORTS ' + 'property missing') def populate(self, zone, target=False): ''' @@ -25,9 +28,7 @@ class BaseSource(object): 'missing') def supports(self, record): - # Unless overriden and handled appropriaitely we'll assume that all - # record types are supported - return True + return record._type in self.SUPPORTS def __repr__(self): return self.__class__.__name__ diff --git a/octodns/source/tinydns.py b/octodns/source/tinydns.py index 70f0145..6805378 100644 --- a/octodns/source/tinydns.py +++ b/octodns/source/tinydns.py @@ -19,6 +19,7 @@ from .base import BaseSource class TinyDnsBaseSource(BaseSource): SUPPORTS_GEO = False + SUPPORTS = set(('A', 'CNAME', 'MX', 'NS')) split_re = re.compile(r':+') diff --git a/tests/helpers.py b/tests/helpers.py index df74e84..a8aafa3 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -17,6 +17,7 @@ class SimpleSource(object): class SimpleProvider(object): SUPPORTS_GEO = False + SUPPORTS = set(('A',)) def __init__(self, id='test'): pass diff --git a/tests/test_octodns_provider_base.py b/tests/test_octodns_provider_base.py index c7836c8..766bf65 100644 --- a/tests/test_octodns_provider_base.py +++ b/tests/test_octodns_provider_base.py @@ -16,6 +16,8 @@ from octodns.zone import Zone class HelperProvider(BaseProvider): log = getLogger('HelperProvider') + SUPPORTS = set(('A',)) + def __init__(self, extra_changes, apply_disabled=False, include_change_callback=None): self.__extra_changes = extra_changes @@ -58,10 +60,17 @@ class TestBaseProvider(TestCase): zone = Zone('unit.tests.', []) with self.assertRaises(NotImplementedError) as ctx: HasSupportsGeo('hassupportesgeo').populate(zone) + self.assertEquals('Abstract base class, SUPPORTS property missing', + ctx.exception.message) + + class HasSupports(HasSupportsGeo): + SUPPORTS = set(('A',)) + with self.assertRaises(NotImplementedError) as ctx: + HasSupports('hassupportes').populate(zone) self.assertEquals('Abstract base class, populate method missing', ctx.exception.message) - class HasPopulate(HasSupportsGeo): + class HasPopulate(HasSupports): def populate(self, zone, target=False): zone.add_record(Record.new(zone, '', { @@ -81,7 +90,7 @@ class TestBaseProvider(TestCase): 'value': '1.2.3.4' })) - self.assertTrue(HasSupportsGeo('hassupportesgeo') + self.assertTrue(HasSupports('hassupportesgeo') .supports(list(zone.records)[0])) plan = HasPopulate('haspopulate').plan(zone) From 3c1e409e6fea2347c4cd5d18c6fccc84387ae797 Mon Sep 17 00:00:00 2001 From: Heesu Hwang Date: Thu, 22 Jun 2017 14:22:40 -0700 Subject: [PATCH 12/64] Added support for CNAME, AAAA, MX, SRV, NS, PTR. TODO: add TXT. add zone creation. create tests --- octodns/provider/azuredns.py | 164 ++++++++++++++++++++++------------- 1 file changed, 102 insertions(+), 62 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 3a442e8..66ca8e3 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -22,7 +22,8 @@ from .base import BaseProvider # making Makefile. # Only made for A records. will have to adjust for more generic params types class _AzureRecord(object): - def __init__(self, resource_group, record, values=None): + def __init__(self, resource_group, record, values=None, ttl=1800): + # print('Here4',file=sys.stderr) self.resource_group = resource_group self.zone_name = record.zone.name[0:len(record.zone.name)-1] # strips last period self.relative_record_set_name = record.name or '@' @@ -31,22 +32,21 @@ class _AzureRecord(object): type_name = '{}records'.format(self.record_type).lower() class_name = '{}'.format(self.record_type).capitalize() + \ 'Record'.format(self.record_type) + if values == None: + return + # TODO: clean up this bit. data = values or record.data #This should fail if it gets to record.data? It only returns ttl. TODO - print('findme: ',file=sys.stderr) - print(data, file=sys.stderr) - print('\n',file=sys.stderr) #depending on mult values or not + #TODO: import explicitly. eval() uses for example ARecord from azure.mgmt.dns.models self.params = {} try: - self.params = {'ttl':record.ttl or 1800, \ + self.params = {'ttl':record.ttl or ttl, \ type_name:[eval(class_name)(ip) for ip in data['values']] or []} except KeyError: # means that doesn't have multiple values but single value - self.params = {'ttl':record.ttl or 1800, \ + self.params = {'ttl':record.ttl or ttl, \ type_name:[eval(class_name)(data['value'])] or []} - - # ar = _AzureRecord(self._resource_group, new, new.data) @@ -88,17 +88,17 @@ class AzureProvider(BaseProvider): self._resource_group = resource_group self._azure_zones = None # will be a dictionary. key: name. val: id. - self._azure_records = None # will be dict by octodns record, az record + self._azure_records = {} # will be dict by octodns record, az record - self._supported_types = ['A'] + self._supported_types = ['CNAME', 'A', 'AAAA', 'MX', 'SRV', 'NS', 'PTR'] + # TODO: add TXT - # TODO: health checks a la route53. + # TODO: health checks a la route53. # TODO: add support for all types. First skeleton: add A. def supports(self, record): - # TODO: possibly refactor return record._type in self._supported_types @property @@ -114,82 +114,121 @@ class AzureProvider(BaseProvider): # Given a zone name, returns the zone id. If DNE, creates it. def _get_zone_id(self, name, create=False): self.log.debug('_get_zone_id: name=%s', name) - if name in self.azure_zones: - id = self.azure_zones[name] + try: + id = self._dns_client.zones.get(self._resource_group, name) self.log.debug('_get_zone_id: id=%s', id) return id - if create: - self.log.debug('_get_zone_id: no matching zone; creating %s', name) - raise Exception - #TODO, write code - return None - + except: + if create: + self.log.debug('_get_zone_id: no matching zone; creating %s', name) + #TODO: write + return None #placeholder + return None + # Create a dictionary of record objects by zone and octodns record names # TODO: add geo parsing def populate(self, zone, target): - self.log.debug('populate: name=%s', zone.name) + zone_name = zone.name[0:len(zone.name)-1]#Azure zone names do not include suffix . + self.log.debug('populate: name=%s', zone_name) before = len(zone.records) - - zone_id = self._get_zone_id(zone.name) + zone_id = self._get_zone_id(zone_name) if zone_id: - records = defaultdict(list) + #records = defaultdict(list) for type in self._supported_types: - for azrecord in self.dns_client.record_sets.list_by_type(self._resource_group, zone.name, type): - record_name = azrecord.name - data = getattr(self, '_data_for_{}'.format(type))(type, azrecord) + # print('populate. type: {}'.format(type),file=sys.stderr) + for azrecord in self._dns_client.record_sets.list_by_type(self._resource_group, zone_name, type): + # print(azrecord, file=sys.stderr) + record_name = azrecord.name if azrecord.name != '@' else '' + data = self._type_and_ttl(type, azrecord, + getattr(self, '_data_for_{}'.format(type))(azrecord)) # TODO: azure online interface allows None values. must validate. record = Record.new(zone, record_name, data, source=self) + # print('HERE0',file=sys.stderr) zone.add_record(record) self._azure_records[record] = _AzureRecord(self._resource_group, record, data) - + # print('HERE1',file=sys.stderr) self.log.info('populate: found %s records', len(zone.records)-before) - # might not need + # might not need def _get_type(azrecord): azrecord['type'].split('/')[-1] - def _data_for_A(self, type, azrecord): - print('xxxxx\n',file=sys.stderr) - for ar in azrecords.arecords: - print(ar,file=sys.stderr) - - - v = [ARecord(ar.ipv4_address) for ar in azrecord.arecords] - # print(v, file=sys.stderr) - # print('99999999999999999999999999999999999999999999999999\n',file=sys.stderr) - return { - 'type': type, - 'ttl': azrecord['ttl'], - 'value': v + def _type_and_ttl(self, type, azrecord, data): + data['type'] = type + data['ttl'] = azrecord.ttl + return data + + def _data_for_A(self, azrecord): + return {'values': [ar.ipv4_address for ar in azrecord.arecords]} + + def _data_for_AAAA(self, azrecord): + return {'values': [ar.ipv6_address for ar in azrecord.aaaa_records]} + + def _data_for_TXT(self, azrecord): + print('azure',file=sys.stderr) + print([ar.value for ar in azrecord.txt_records], file=sys.stderr) + print('',file=sys.stderr) + return {'values': [ar.value for ar in azrecord.txt_records]} + + def _data_for_CNAME(self, azrecord): + try: + val = azrecord.cname_record.cname + if not val.endswith('.'): + val += '.' + return {'value': val} + except: + return {'value': '.'} #TODO: this is a bad fix. but octo checks that cnames have trailing '.' while azure allows creating cnames on the online interface with no value. + + def _data_for_PTR(self, azrecord): + try: + val = azrecord.ptr_records[0].ptdrname + if not val.endswith('.'): + val += '.' + return {'value': val} + except: + return {'value': '.' } #TODO: this is a bad fix. but octo checks that cnames have trailing '.' while azure allows creating cnames on the online interface with no value. + + def _data_for_MX(self, azrecord): + return {'values': [{'priority':ar.preference, + 'value':ar.exchange} for ar in azrecord.mx_records]} + + def _data_for_SRV(self, azrecord): + return {'values': [{'priority': ar.priority, + 'weight': ar.weight, + 'port': ar.port, + 'target': ar.target} for ar in azrecord.srv_records] } - - # def _get_azure_record(self, record): - # try: - # return self._azure_records[record] - # except: - # self._azure_records[record] = _AzureRecord(self._resource_group, record, record.data) - # return self._azure_records[record] - # except: - # raise - + + def _data_for_NS(self, azrecord): + def period_validate(string): + return string if string.endswith('.') else string + '.' + vals = [ar.nsdname for ar in azrecord.ns_records] + return {'values': [period_validate(val) for val in vals]} + def _apply_Create(self, change): new = change.new - + #validate that the zone exists. #self._get_zone_id(new.name, create=True) - #ar = self._get_azure_record(new) ar = _AzureRecord(self._resource_group, new, new.data) create = self._dns_client.record_sets.create_or_update - print('find:{} {} {} {} {}\n'.format(ar.resource_group,ar.zone_name,ar.relative_record_set_name,ar.record_type,ar.params), file=sys.stderr) - for arec in ar.params['arecords']: - print(str(arec.ipv4_address) + ', ', file=sys.stderr) - print('\n',file=sys.stderr) create(resource_group_name=ar.resource_group, zone_name=ar.zone_name, relative_record_set_name=ar.relative_record_set_name, record_type=ar.record_type, parameters=ar.params) + + def _apply_Delete(self, change): + existing = change.existing + ar = _AzureRecord(self._resource_group, existing) + delete = self._dns_client.record_sets.delete + delete(self._resource_group, ar.zone_name, ar.relative_record_set_name, + ar.record_type) + + def _apply_Update(self, change): + self._apply_Delete(change) + self._apply_Create(change) # type plan: Plan class from .base def _apply(self, plan): @@ -198,12 +237,9 @@ class AzureProvider(BaseProvider): self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name, len(changes)) - # validate that the zone exists. function creates zone if DNE. self._get_zone_id(desired.name) - - # Some parsing bits to call _mod_Create or _mod_Delete. # changes is a list of Delete and Create objects. for change in changes: @@ -232,4 +268,8 @@ class AzureProvider(BaseProvider): # existing.type == desired.type == Zone(desired.name, desired.sub_zones) # Zone(name, sub_zones) (str and set of strs) # changes.type = [Delete/Create] - \ No newline at end of file + + + # Starts with sync in main() of sync. + # {u'values': ['3.3.3.3', '4.4.4.4'], u'type': 'A', u'ttl': 3600} + # {u'type': u'A', u'value': [u'3.3.3.3', u'4.4.4.4'], u'ttl': 3600L} \ No newline at end of file From 8323b4c0ea134c45cddcf00e1667125524a7dbe6 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 23 Jun 2017 07:14:01 -0700 Subject: [PATCH 13/64] Complete refactor & rework of how validation is set up This is with an eye toward expanding it in the future both in terms of what it checks and to add the ability to ignore things. This commit does not intend to change any validation. It only reworks the flow and improves the error messaging. --- octodns/record.py | 350 ++++++++++----- tests/test_octodns_record.py | 807 ++++++++++++++++++++++++++++------- 2 files changed, 895 insertions(+), 262 deletions(-) diff --git a/octodns/record.py b/octodns/record.py index cacb147..827ad0a 100644 --- a/octodns/record.py +++ b/octodns/record.py @@ -54,7 +54,14 @@ class Delete(Change): return 'Delete {}'.format(self.existing) -_unescaped_semicolon_re = re.compile(r'\w;') +class ValidationError(Exception): + + def __init__(self, fqdn, reasons): + message = 'Invalid record {}\n - {}' \ + .format(fqdn, '\n - '.join(reasons)) + super(Exception, self).__init__(message) + self.fqdn = fqdn + self.reasons = reasons class Record(object): @@ -62,13 +69,13 @@ class Record(object): @classmethod def new(cls, zone, name, data, source=None): + fqdn = '{}.{}'.format(name, zone.name) if name else zone.name try: _type = data['type'] except KeyError: - fqdn = '{}.{}'.format(name, zone.name) if name else zone.name raise Exception('Invalid record {}, missing type'.format(fqdn)) try: - _type = { + _class = { 'A': ARecord, 'AAAA': AaaaRecord, 'ALIAS': AliasRecord, @@ -98,7 +105,21 @@ class Record(object): }[_type] except KeyError: raise Exception('Unknown record type: "{}"'.format(_type)) - return _type(zone, name, data, source=source) + reasons = _class.validate(name, data) + if reasons: + raise ValidationError(fqdn, reasons) + return _class(zone, name, data, source=source) + + @classmethod + def validate(cls, name, data): + reasons = [] + try: + ttl = int(data['ttl']) + if ttl < 0: + reasons.append('invalid ttl') + except KeyError: + reasons.append('missing ttl') + return reasons def __init__(self, zone, name, data, source=None): self.log.debug('__init__: zone.name=%s, type=%11s, name=%s', zone.name, @@ -106,11 +127,8 @@ class Record(object): self.zone = zone # force everything lower-case just to be safe self.name = str(name).lower() if name else name - try: - self.ttl = int(data['ttl']) - except KeyError: - raise Exception('Invalid record {}, missing ttl'.format(self.fqdn)) self.source = source + self.ttl = int(data['ttl']) octodns = data.get('octodns', {}) self.ignored = octodns.get('ignored', False) @@ -154,11 +172,17 @@ class GeoValue(object): geo_re = re.compile(r'^(?P\w\w)(-(?P\w\w)' r'(-(?P\w\w))?)?$') - def __init__(self, geo, values): - match = self.geo_re.match(geo) + @classmethod + def _validate_geo(cls, code): + reasons = [] + match = cls.geo_re.match(code) if not match: - raise Exception('Invalid geo "{}"'.format(geo)) + reasons.append('invalid geo "{}"'.format(code)) + return reasons + + def __init__(self, geo, values): self.code = geo + match = self.geo_re.match(geo) self.continent_code = match.group('continent_code') self.country_code = match.group('country_code') self.subdivision_code = match.group('subdivision_code') @@ -185,16 +209,29 @@ class GeoValue(object): class _ValuesMixin(object): - def __init__(self, zone, name, data, source=None): - super(_ValuesMixin, self).__init__(zone, name, data, source=source) + @classmethod + def validate(cls, name, data): + reasons = super(_ValuesMixin, cls).validate(name, data) + values = [] try: values = data['values'] except KeyError: try: values = [data['value']] except KeyError: - raise Exception('Invalid record {}, missing value(s)' - .format(self.fqdn)) + reasons.append('missing value(s)') + + for value in values: + reasons.extend(cls._validate_value(value)) + + return reasons + + def __init__(self, zone, name, data, source=None): + super(_ValuesMixin, self).__init__(zone, name, data, source=source) + try: + values = data['values'] + except KeyError: + values = [data['value']] self.values = sorted(self._process_values(values)) def changes(self, other, target): @@ -224,6 +261,21 @@ class _GeoMixin(_ValuesMixin): Must be included before `Record`. ''' + @classmethod + def validate(cls, name, data): + reasons = super(_GeoMixin, cls).validate(name, data) + try: + geo = dict(data['geo']) + # TODO: validate legal codes + for code, values in geo.items(): + reasons.extend(GeoValue._validate_geo(code)) + for value in values: + reasons.extend(cls._validate_value(value)) + except KeyError: + pass + return reasons + + # TODO: support 'value' as well # TODO: move away from "data" hash to strict params, it's kind of leaking # the yaml implementation into here and then forcing it back out into # non-yaml providers during input @@ -233,9 +285,8 @@ class _GeoMixin(_ValuesMixin): self.geo = dict(data['geo']) except KeyError: self.geo = {} - for k, vs in self.geo.items(): - vs = sorted(self._process_values(vs)) - self.geo[k] = GeoValue(k, vs) + for code, values in self.geo.items(): + self.geo[code] = GeoValue(code, values) def _data(self): ret = super(_GeoMixin, self)._data() @@ -264,41 +315,52 @@ class _GeoMixin(_ValuesMixin): class ARecord(_GeoMixin, Record): _type = 'A' + @classmethod + def _validate_value(self, value): + reasons = [] + try: + IPv4Address(unicode(value)) + except Exception: + reasons.append('invalid ip address "{}"'.format(value)) + return reasons + def _process_values(self, values): - for ip in values: - try: - IPv4Address(unicode(ip)) - except Exception: - raise Exception('Invalid record {}, value {} not a valid ip' - .format(self.fqdn, ip)) return values class AaaaRecord(_GeoMixin, Record): _type = 'AAAA' + @classmethod + def _validate_value(self, value): + reasons = [] + try: + IPv6Address(unicode(value)) + except Exception: + reasons.append('invalid ip address "{}"'.format(value)) + return reasons + def _process_values(self, values): - ret = [] - for ip in values: - try: - IPv6Address(unicode(ip)) - ret.append(ip.lower()) - except Exception: - raise Exception('Invalid record {}, value {} not a valid ip' - .format(self.fqdn, ip)) - return ret + return values class _ValueMixin(object): - def __init__(self, zone, name, data, source=None): - super(_ValueMixin, self).__init__(zone, name, data, source=source) + @classmethod + def validate(cls, name, data): + reasons = super(_ValueMixin, cls).validate(name, data) + value = None try: value = data['value'] except KeyError: - raise Exception('Invalid record {}, missing value' - .format(self.fqdn)) - self.value = self._process_value(value) + reasons.append('missing value') + if value: + reasons.extend(cls._validate_value(value)) + return reasons + + def __init__(self, zone, name, data, source=None): + super(_ValueMixin, self).__init__(zone, name, data, source=source) + self.value = self._process_value(data['value']) def changes(self, other, target): if self.value != other.value: @@ -319,25 +381,42 @@ class _ValueMixin(object): class AliasRecord(_ValueMixin, Record): _type = 'ALIAS' - def _process_value(self, value): + @classmethod + def _validate_value(self, value): + reasons = [] if not value.endswith('.'): - raise Exception('Invalid record {}, value ({}) missing trailing .' - .format(self.fqdn, value)) + reasons.append('missing trailing .') + return reasons + + def _process_value(self, value): return value class CnameRecord(_ValueMixin, Record): _type = 'CNAME' - def _process_value(self, value): + @classmethod + def _validate_value(cls, value): + reasons = [] if not value.endswith('.'): - raise Exception('Invalid record {}, value ({}) missing trailing .' - .format(self.fqdn, value)) - return value.lower() + reasons.append('missing trailing .') + return reasons + + def _process_value(self, value): + return value class MxValue(object): + @classmethod + def _validate_value(cls, value): + reasons = [] + if 'priority' not in value: + reasons.append('missing priority') + if 'value' not in value: + reasons.append('missing value') + return reasons + def __init__(self, value): # TODO: rename preference self.priority = int(value['priority']) @@ -363,19 +442,38 @@ class MxValue(object): class MxRecord(_ValuesMixin, Record): _type = 'MX' + @classmethod + def _validate_value(cls, value): + return MxValue._validate_value(value) + def _process_values(self, values): - ret = [] - for value in values: - try: - ret.append(MxValue(value)) - except KeyError as e: - raise Exception('Invalid value in record {}, missing {}' - .format(self.fqdn, e.args[0])) - return ret + return [MxValue(v) for v in values] class NaptrValue(object): + @classmethod + def _validate_value(cls, data): + reasons = [] + try: + int(data['order']) + except KeyError: + reasons.append('missing order') + except ValueError: + reasons.append('invalid order "{}"'.format(data['order'])) + try: + int(data['preference']) + except KeyError: + reasons.append('missing preference') + except ValueError: + reasons.append('invalid preference "{}"' + .format(data['preference'])) + # TODO: validate field data + for k in ('flags', 'service', 'regexp', 'replacement'): + if k not in data: + reasons.append('missing {}'.format(k)) + return reasons + def __init__(self, value): self.order = int(value['order']) self.preference = int(value['preference']) @@ -420,42 +518,65 @@ class NaptrValue(object): class NaptrRecord(_ValuesMixin, Record): _type = 'NAPTR' + @classmethod + def _validate_value(cls, value): + return NaptrValue._validate_value(value) + def _process_values(self, values): - ret = [] - for value in values: - try: - ret.append(NaptrValue(value)) - except KeyError as e: - raise Exception('Invalid value in record {}, missing {}' - .format(self.fqdn, e.args[0])) - return ret + return [NaptrValue(v) for v in values] class NsRecord(_ValuesMixin, Record): _type = 'NS' + @classmethod + def _validate_value(cls, value): + reasons = [] + if not value.endswith('.'): + reasons.append('missing trailing .') + return reasons + def _process_values(self, values): - ret = [] - for ns in values: - if not ns.endswith('.'): - raise Exception('Invalid record {}, value {} missing ' - 'trailing .'.format(self.fqdn, ns)) - ret.append(ns.lower()) - return ret + return values class PtrRecord(_ValueMixin, Record): _type = 'PTR' - def _process_value(self, value): + @classmethod + def _validate_value(cls, value): + reasons = [] if not value.endswith('.'): - raise Exception('Invalid record {}, value ({}) missing trailing .' - .format(self.fqdn, value)) - return value.lower() + reasons.append('missing trailing .') + return reasons + + def _process_value(self, value): + return value class SshfpValue(object): + @classmethod + def _validate_value(cls, value): + reasons = [] + # TODO: validate algorithm and fingerprint_type values + try: + int(value['algorithm']) + except KeyError: + reasons.append('missing algorithm') + except ValueError: + reasons.append('invalid algorithm "{}"'.format(value['algorithm'])) + try: + int(value['fingerprint_type']) + except KeyError: + reasons.append('missing fingerprint_type') + except ValueError: + reasons.append('invalid fingerprint_type "{}"' + .format(value['fingerprint_type'])) + if 'fingerprint' not in value: + reasons.append('missing fingerprint') + return reasons + def __init__(self, value): self.algorithm = int(value['algorithm']) self.fingerprint_type = int(value['fingerprint_type']) @@ -484,26 +605,61 @@ class SshfpValue(object): class SshfpRecord(_ValuesMixin, Record): _type = 'SSHFP' + @classmethod + def _validate_value(cls, value): + return SshfpValue._validate_value(value) + def _process_values(self, values): - ret = [] - for value in values: - try: - ret.append(SshfpValue(value)) - except KeyError as e: - raise Exception('Invalid value in record {}, missing {}' - .format(self.fqdn, e.args[0])) - return ret + return [SshfpValue(v) for v in values] + + +_unescaped_semicolon_re = re.compile(r'\w;') class SpfRecord(_ValuesMixin, Record): _type = 'SPF' + @classmethod + def _validate_value(cls, value): + if _unescaped_semicolon_re.search(value): + return ['unescaped ;'] + return [] + def _process_values(self, values): return values class SrvValue(object): + @classmethod + def _validate_value(self, value): + reasons = [] + # TODO: validate algorithm and fingerprint_type values + try: + int(value['priority']) + except KeyError: + reasons.append('missing priority') + except ValueError: + reasons.append('invalid priority "{}"'.format(value['priority'])) + try: + int(value['weight']) + except KeyError: + reasons.append('missing weight') + except ValueError: + reasons.append('invalid weight "{}"'.format(value['weight'])) + try: + int(value['port']) + except KeyError: + reasons.append('missing port') + except ValueError: + reasons.append('invalid port "{}"'.format(value['port'])) + try: + if not value['target'].endswith('.'): + reasons.append('missing trailing .') + except KeyError: + reasons.append('missing target') + return reasons + def __init__(self, value): self.priority = int(value['priority']) self.weight = int(value['weight']) @@ -537,28 +693,30 @@ class SrvRecord(_ValuesMixin, Record): _type = 'SRV' _name_re = re.compile(r'^_[^\.]+\.[^\.]+') - def __init__(self, zone, name, data, source=None): - if not self._name_re.match(name): - raise Exception('Invalid name {}.{}'.format(name, zone.name)) - super(SrvRecord, self).__init__(zone, name, data, source) + @classmethod + def validate(cls, name, data): + reasons = [] + if not cls._name_re.match(name): + reasons.append('invalid name') + reasons.extend(super(SrvRecord, cls).validate(name, data)) + return reasons + + @classmethod + def _validate_value(cls, value): + return SrvValue._validate_value(value) def _process_values(self, values): - ret = [] - for value in values: - try: - ret.append(SrvValue(value)) - except KeyError as e: - raise Exception('Invalid value in record {}, missing {}' - .format(self.fqdn, e.args[0])) - return ret + return [SrvValue(v) for v in values] class TxtRecord(_ValuesMixin, Record): _type = 'TXT' + @classmethod + def _validate_value(cls, value): + if _unescaped_semicolon_re.search(value): + return ['unescaped ;'] + return [] + def _process_values(self, values): - for value in values: - if _unescaped_semicolon_re.search(value): - raise Exception('Invalid record {}, unescaped ;' - .format(self.fqdn)) return values diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index 52505cb..99a502e 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -9,7 +9,8 @@ from unittest import TestCase from octodns.record import ARecord, AaaaRecord, AliasRecord, CnameRecord, \ Create, Delete, GeoValue, MxRecord, NaptrRecord, NaptrValue, NsRecord, \ - PtrRecord, Record, SshfpRecord, SpfRecord, SrvRecord, TxtRecord, Update + Record, SshfpRecord, SpfRecord, SrvRecord, TxtRecord, Update, \ + ValidationError from octodns.zone import Zone from helpers import GeoProvider, SimpleProvider @@ -42,15 +43,6 @@ class TestRecord(TestCase): self.assertEquals([b_value], b.values) self.assertEquals(b_data, b.data) - # missing ttl - with self.assertRaises(Exception) as ctx: - ARecord(self.zone, None, {'value': '1.1.1.1'}) - self.assertTrue('missing ttl' in ctx.exception.message) - # missing values & value - with self.assertRaises(Exception) as ctx: - ARecord(self.zone, None, {'ttl': 42}) - self.assertTrue('missing value(s)' in ctx.exception.message) - # top-level data = {'ttl': 30, 'value': '4.2.3.4'} self.assertEquals(self.zone.name, ARecord(self.zone, '', data).fqdn) @@ -104,20 +96,6 @@ class TestRecord(TestCase): DummyRecord().__repr__() - def test_invalid_a(self): - with self.assertRaises(Exception) as ctx: - ARecord(self.zone, 'a', { - 'ttl': 30, - 'value': 'foo', - }) - self.assertTrue('Invalid record' in ctx.exception.message) - with self.assertRaises(Exception) as ctx: - ARecord(self.zone, 'a', { - 'ttl': 30, - 'values': ['1.2.3.4', 'bar'], - }) - self.assertTrue('Invalid record' in ctx.exception.message) - def test_geo(self): geo_data = {'ttl': 42, 'values': ['5.2.3.4', '6.2.3.4'], 'geo': {'AF': ['1.1.1.1'], @@ -157,19 +135,6 @@ class TestRecord(TestCase): # Geo provider does consider lack of geo diffs to be changes self.assertTrue(geo.changes(other, geo_target)) - # invalid geo code - with self.assertRaises(Exception) as ctx: - ARecord(self.zone, 'geo', {'ttl': 42, - 'values': ['5.2.3.4', '6.2.3.4'], - 'geo': {'abc': ['1.1.1.1']}}) - self.assertEquals('Invalid geo "abc"', ctx.exception.message) - - with self.assertRaises(Exception) as ctx: - ARecord(self.zone, 'geo', {'ttl': 42, - 'values': ['5.2.3.4', '6.2.3.4'], - 'geo': {'NA-US': ['1.1.1']}}) - self.assertTrue('not a valid ip' in ctx.exception.message) - # __repr__ doesn't blow up geo.__repr__() @@ -187,30 +152,12 @@ class TestRecord(TestCase): self.assertEquals([b_value], b.values) self.assertEquals(b_data, b.data) - # missing values & value - with self.assertRaises(Exception) as ctx: - _type(self.zone, None, {'ttl': 42}) - self.assertTrue('missing value(s)' in ctx.exception.message) - def test_aaaa(self): a_values = ['2001:0db8:3c4d:0015:0000:0000:1a2f:1a2b', '2001:0db8:3c4d:0015:0000:0000:1a2f:1a3b'] b_value = '2001:0db8:3c4d:0015:0000:0000:1a2f:1a4b' self.assertMultipleValues(AaaaRecord, a_values, b_value) - with self.assertRaises(Exception) as ctx: - AaaaRecord(self.zone, 'a', { - 'ttl': 30, - 'value': 'foo', - }) - self.assertTrue('Invalid record' in ctx.exception.message) - with self.assertRaises(Exception) as ctx: - AaaaRecord(self.zone, 'a', { - 'ttl': 30, - 'values': [b_value, 'bar'], - }) - self.assertTrue('Invalid record' in ctx.exception.message) - def assertSingleValue(self, _type, a_value, b_value): a_data = {'ttl': 30, 'value': a_value} a = _type(self.zone, 'a', a_data) @@ -225,11 +172,6 @@ class TestRecord(TestCase): self.assertEquals(b_value, b.value) self.assertEquals(b_data, b.data) - # missing value - with self.assertRaises(Exception) as ctx: - _type(self.zone, None, {'ttl': 42}) - self.assertTrue('missing value' in ctx.exception.message) - target = SimpleProvider() # No changes with self self.assertFalse(a.changes(a, target)) @@ -251,15 +193,6 @@ class TestRecord(TestCase): self.assertEquals(a_data['value'], a.value) self.assertEquals(a_data, a.data) - # missing value - with self.assertRaises(Exception) as ctx: - AliasRecord(self.zone, None, {'ttl': 0}) - self.assertTrue('missing value' in ctx.exception.message) - # bad name - with self.assertRaises(Exception) as ctx: - AliasRecord(self.zone, None, {'ttl': 0, 'value': 'www.unit.tests'}) - self.assertTrue('missing trailing .' in ctx.exception.message) - target = SimpleProvider() # No changes with self self.assertFalse(a.changes(a, target)) @@ -277,19 +210,6 @@ class TestRecord(TestCase): self.assertSingleValue(CnameRecord, 'target.foo.com.', 'other.foo.com.') - with self.assertRaises(Exception) as ctx: - CnameRecord(self.zone, 'a', { - 'ttl': 30, - 'value': 'foo', - }) - self.assertTrue('Invalid record' in ctx.exception.message) - with self.assertRaises(Exception) as ctx: - CnameRecord(self.zone, 'a', { - 'ttl': 30, - 'values': ['foo.com.', 'bar.com'], - }) - self.assertTrue('Invalid record' in ctx.exception.message) - def test_mx(self): a_values = [{ 'priority': 10, @@ -319,15 +239,6 @@ class TestRecord(TestCase): self.assertEquals(b_value['value'], b.values[0].value) self.assertEquals(b_data, b.data) - # missing value - with self.assertRaises(Exception) as ctx: - MxRecord(self.zone, None, {'ttl': 42}) - self.assertTrue('missing value(s)' in ctx.exception.message) - # invalid value - with self.assertRaises(Exception) as ctx: - MxRecord(self.zone, None, {'ttl': 42, 'value': {}}) - self.assertTrue('Invalid value' in ctx.exception.message) - target = SimpleProvider() # No changes with self self.assertFalse(a.changes(a, target)) @@ -387,15 +298,6 @@ class TestRecord(TestCase): self.assertEquals(b_value[k], getattr(b.values[0], k)) self.assertEquals(b_data, b.data) - # missing value - with self.assertRaises(Exception) as ctx: - NaptrRecord(self.zone, None, {'ttl': 42}) - self.assertTrue('missing value' in ctx.exception.message) - # invalid value - with self.assertRaises(Exception) as ctx: - NaptrRecord(self.zone, None, {'ttl': 42, 'value': {}}) - self.assertTrue('Invalid value' in ctx.exception.message) - target = SimpleProvider() # No changes with self self.assertFalse(a.changes(a, target)) @@ -538,33 +440,6 @@ class TestRecord(TestCase): self.assertEquals([b_value], b.values) self.assertEquals(b_data, b.data) - # missing values & value - with self.assertRaises(Exception) as ctx: - NsRecord(self.zone, None, {'ttl': 42}) - self.assertTrue('missing value(s)' in ctx.exception.message) - - with self.assertRaises(Exception) as ctx: - NsRecord(self.zone, 'a', { - 'ttl': 30, - 'value': 'foo', - }) - self.assertTrue('Invalid record' in ctx.exception.message) - with self.assertRaises(Exception) as ctx: - NsRecord(self.zone, 'a', { - 'ttl': 30, - 'values': ['foo.com.', 'bar.com'], - }) - self.assertTrue('Invalid record' in ctx.exception.message) - - def test_ptr(self): - self.assertSingleValue(PtrRecord, 'foo.bar.com.', 'other.bar.com.') - with self.assertRaises(Exception) as ctx: - PtrRecord(self.zone, 'a', { - 'ttl': 30, - 'value': 'foo', - }) - self.assertTrue('Invalid record' in ctx.exception.message) - def test_sshfp(self): a_values = [{ 'algorithm': 10, @@ -599,15 +474,6 @@ class TestRecord(TestCase): self.assertEquals(b_value['fingerprint'], b.values[0].fingerprint) self.assertEquals(b_data, b.data) - # missing value - with self.assertRaises(Exception) as ctx: - SshfpRecord(self.zone, None, {'ttl': 42}) - self.assertTrue('missing value(s)' in ctx.exception.message) - # invalid value - with self.assertRaises(Exception) as ctx: - SshfpRecord(self.zone, None, {'ttl': 42, 'value': {}}) - self.assertTrue('Invalid value' in ctx.exception.message) - target = SimpleProvider() # No changes with self self.assertFalse(a.changes(a, target)) @@ -677,21 +543,6 @@ class TestRecord(TestCase): self.assertEquals(b_value['target'], b.values[0].target) self.assertEquals(b_data, b.data) - # invalid name - with self.assertRaises(Exception) as ctx: - SrvRecord(self.zone, 'bad', {'ttl': 42}) - self.assertEquals('Invalid name bad.unit.tests.', - ctx.exception.message) - - # missing value - with self.assertRaises(Exception) as ctx: - SrvRecord(self.zone, '_missing._tcp', {'ttl': 42}) - self.assertTrue('missing value(s)' in ctx.exception.message) - # invalid value - with self.assertRaises(Exception) as ctx: - SrvRecord(self.zone, '_missing._udp', {'ttl': 42, 'value': {}}) - self.assertTrue('Invalid value' in ctx.exception.message) - target = SimpleProvider() # No changes with self self.assertFalse(a.changes(a, target)) @@ -729,21 +580,6 @@ class TestRecord(TestCase): b_value = 'b other' self.assertMultipleValues(TxtRecord, a_values, b_value) - Record.new(self.zone, 'txt', { - 'ttl': 44, - 'type': 'TXT', - 'value': 'escaped\; foo', - }) - - with self.assertRaises(Exception) as ctx: - Record.new(self.zone, 'txt', { - 'ttl': 44, - 'type': 'TXT', - 'value': 'un-escaped; foo', - }) - self.assertEquals('Invalid record txt.unit.tests., unescaped ;', - ctx.exception.message) - def test_record_new(self): txt = Record.new(self.zone, 'txt', { 'ttl': 44, @@ -794,3 +630,642 @@ class TestRecord(TestCase): self.assertEquals('CA', geo.subdivision_code) self.assertEquals(values, geo.values) self.assertEquals(['NA-US', 'NA'], list(geo.parents)) + + +class TestRecordValidation(TestCase): + zone = Zone('unit.tests.', []) + + def test_base(self): + # no ttl + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'A', + 'value': '1.2.3.4', + }) + self.assertEquals(['missing ttl'], ctx.exception.reasons) + # invalid ttl + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, 'www', { + 'type': 'A', + 'ttl': -1, + 'value': '1.2.3.4', + }) + self.assertEquals('www.unit.tests.', ctx.exception.fqdn) + self.assertEquals(['invalid ttl'], ctx.exception.reasons) + + def test_A_and_values_mixin(self): + # doesn't blow up + Record.new(self.zone, '', { + 'type': 'A', + 'ttl': 600, + 'value': '1.2.3.4', + }) + Record.new(self.zone, '', { + 'type': 'A', + 'ttl': 600, + 'values': [ + '1.2.3.4', + '1.2.3.5', + ] + }) + + # missing value(s) + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'A', + 'ttl': 600, + }) + self.assertEquals(['missing value(s)'], ctx.exception.reasons) + # missing value(s) & ttl + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'A', + }) + self.assertEquals(['missing ttl', 'missing value(s)'], + ctx.exception.reasons) + + # invalid ip address + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'A', + 'ttl': 600, + 'value': 'hello' + }) + self.assertEquals(['invalid ip address "hello"'], + ctx.exception.reasons) + + # invalid ip addresses + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'A', + 'ttl': 600, + 'values': ['hello', 'goodbye'] + }) + self.assertEquals([ + 'invalid ip address "hello"', + 'invalid ip address "goodbye"' + ], ctx.exception.reasons) + + # invalid & valid ip addresses, no ttl + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'A', + 'values': ['1.2.3.4', 'hello', '5.6.7.8'] + }) + self.assertEquals([ + 'missing ttl', + 'invalid ip address "hello"', + ], ctx.exception.reasons) + + def test_geo(self): + Record.new(self.zone, '', { + 'geo': { + 'NA': ['1.2.3.5'], + 'NA-US': ['1.2.3.5', '1.2.3.6'] + }, + 'type': 'A', + 'ttl': 600, + 'value': '1.2.3.4', + }) + + # invalid ip address + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'geo': { + 'NA': ['hello'], + 'NA-US': ['1.2.3.5', '1.2.3.6'] + }, + 'type': 'A', + 'ttl': 600, + 'value': '1.2.3.4', + }) + self.assertEquals(['invalid ip address "hello"'], + ctx.exception.reasons) + + # invalid geo code + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'geo': { + 'XYZ': ['1.2.3.4'], + }, + 'type': 'A', + 'ttl': 600, + 'value': '1.2.3.4', + }) + self.assertEquals(['invalid geo "XYZ"'], ctx.exception.reasons) + + # invalid ip address + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'geo': { + 'NA': ['hello'], + 'NA-US': ['1.2.3.5', 'goodbye'] + }, + 'type': 'A', + 'ttl': 600, + 'value': '1.2.3.4', + }) + self.assertEquals([ + 'invalid ip address "hello"', + 'invalid ip address "goodbye"' + ], ctx.exception.reasons) + + def test_AAAA(self): + # doesn't blow up + Record.new(self.zone, '', { + 'type': 'AAAA', + 'ttl': 600, + 'value': '2601:644:500:e210:62f8:1dff:feb8:947a', + }) + Record.new(self.zone, '', { + 'type': 'AAAA', + 'ttl': 600, + 'values': [ + '2601:644:500:e210:62f8:1dff:feb8:947a', + '2601:644:500:e210:62f8:1dff:feb8:947b', + ] + }) + + # invalid ip address + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'AAAA', + 'ttl': 600, + 'value': 'hello' + }) + self.assertEquals(['invalid ip address "hello"'], + ctx.exception.reasons) + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'AAAA', + 'ttl': 600, + 'value': '1.2.3.4' + }) + self.assertEquals(['invalid ip address "1.2.3.4"'], + ctx.exception.reasons) + + # invalid ip addresses + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'AAAA', + 'ttl': 600, + 'values': ['hello', 'goodbye'] + }) + self.assertEquals([ + 'invalid ip address "hello"', + 'invalid ip address "goodbye"' + ], ctx.exception.reasons) + + def test_ALIAS_and_value_mixin(self): + # doesn't blow up + Record.new(self.zone, '', { + 'type': 'ALIAS', + 'ttl': 600, + 'value': 'foo.bar.com.', + }) + + # missing value + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'ALIAS', + 'ttl': 600, + }) + self.assertEquals(['missing value'], ctx.exception.reasons) + + # missing trailing . + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'ALIAS', + 'ttl': 600, + 'value': 'foo.bar.com', + }) + self.assertEquals(['missing trailing .'], ctx.exception.reasons) + + def test_CNAME(self): + # doesn't blow up + Record.new(self.zone, '', { + 'type': 'CNAME', + 'ttl': 600, + 'value': 'foo.bar.com.', + }) + + # missing trailing . + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'CNAME', + 'ttl': 600, + 'value': 'foo.bar.com', + }) + self.assertEquals(['missing trailing .'], ctx.exception.reasons) + + def test_MX(self): + # doesn't blow up + Record.new(self.zone, '', { + 'type': 'MX', + 'ttl': 600, + 'value': { + 'priority': 10, + 'value': 'foo.bar.com.' + } + }) + + # missing priority + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'MX', + 'ttl': 600, + 'value': { + 'value': 'foo.bar.com.' + } + }) + self.assertEquals(['missing priority'], ctx.exception.reasons) + + # missing value + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'MX', + 'ttl': 600, + 'value': { + 'priority': 10, + } + }) + self.assertEquals(['missing value'], ctx.exception.reasons) + + def test_NXPTR(self): + # doesn't blow up + Record.new(self.zone, '', { + 'type': 'NAPTR', + 'ttl': 600, + 'value': { + 'order': 10, + 'preference': 20, + 'flags': 'f', + 'service': 'srv', + 'regexp': '.*', + 'replacement': '.' + } + }) + + # missing X priority + value = { + 'order': 10, + 'preference': 20, + 'flags': 'f', + 'service': 'srv', + 'regexp': '.*', + 'replacement': '.' + } + for k in ('order', 'preference', 'flags', 'service', 'regexp', + 'replacement'): + v = dict(value) + del v[k] + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'NAPTR', + 'ttl': 600, + 'value': v + }) + self.assertEquals(['missing {}'.format(k)], ctx.exception.reasons) + + # non-int order + v = dict(value) + v['order'] = 'boo' + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'NAPTR', + 'ttl': 600, + 'value': v + }) + self.assertEquals(['invalid order "boo"'], ctx.exception.reasons) + + # non-int preference + v = dict(value) + v['preference'] = 'who' + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'NAPTR', + 'ttl': 600, + 'value': v + }) + self.assertEquals(['invalid preference "who"'], ctx.exception.reasons) + + def test_NS(self): + # doesn't blow up + Record.new(self.zone, '', { + 'type': 'NS', + 'ttl': 600, + 'values': [ + 'foo.bar.com.', + '1.2.3.4.' + ] + }) + + # missing value + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'NS', + 'ttl': 600, + }) + self.assertEquals(['missing value(s)'], ctx.exception.reasons) + + # no trailing . + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'NS', + 'ttl': 600, + 'value': 'foo.bar', + }) + self.assertEquals(['missing trailing .'], ctx.exception.reasons) + + def test_PTR(self): + # doesn't blow up (name & zone here don't make any sense, but not + # important) + Record.new(self.zone, '', { + 'type': 'PTR', + 'ttl': 600, + 'value': 'foo.bar.com.', + }) + + # missing value + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'PTR', + 'ttl': 600, + }) + self.assertEquals(['missing value'], ctx.exception.reasons) + + # no trailing . + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'PTR', + 'ttl': 600, + 'value': 'foo.bar', + }) + self.assertEquals(['missing trailing .'], ctx.exception.reasons) + + def test_SSHFP(self): + # doesn't blow up + Record.new(self.zone, '', { + 'type': 'SSHFP', + 'ttl': 600, + 'value': { + 'algorithm': 1, + 'fingerprint_type': 1, + 'fingerprint': 'bf6b6825d2977c511a475bbefb88aad54a92ac73' + } + }) + + # missing algorithm + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'SSHFP', + 'ttl': 600, + 'value': { + 'fingerprint_type': 1, + 'fingerprint': 'bf6b6825d2977c511a475bbefb88aad54a92ac73' + } + }) + self.assertEquals(['missing algorithm'], ctx.exception.reasons) + + # invalid algorithm + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'SSHFP', + 'ttl': 600, + 'value': { + 'algorithm': 'nope', + 'fingerprint_type': 1, + 'fingerprint': 'bf6b6825d2977c511a475bbefb88aad54a92ac73' + } + }) + self.assertEquals(['invalid algorithm "nope"'], ctx.exception.reasons) + + # missing fingerprint_type + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'SSHFP', + 'ttl': 600, + 'value': { + 'algorithm': 1, + 'fingerprint': 'bf6b6825d2977c511a475bbefb88aad54a92ac73' + } + }) + self.assertEquals(['missing fingerprint_type'], ctx.exception.reasons) + + # invalid fingerprint_type + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'SSHFP', + 'ttl': 600, + 'value': { + 'algorithm': 1, + 'fingerprint_type': 'yeeah', + 'fingerprint': 'bf6b6825d2977c511a475bbefb88aad54a92ac73' + } + }) + self.assertEquals(['invalid fingerprint_type "yeeah"'], + ctx.exception.reasons) + + # missing fingerprint + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'SSHFP', + 'ttl': 600, + 'value': { + 'algorithm': 1, + 'fingerprint_type': 1, + } + }) + self.assertEquals(['missing fingerprint'], ctx.exception.reasons) + + def test_SPF(self): + # doesn't blow up (name & zone here don't make any sense, but not + # important) + Record.new(self.zone, '', { + 'type': 'SPF', + 'ttl': 600, + 'values': [ + 'v=spf1 ip4:192.168.0.1/16-all', + 'v=spf1 ip4:10.1.2.1/24-all', + 'this has some\; semi-colons\; in it', + ] + }) + + # missing value + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'SPF', + 'ttl': 600, + }) + self.assertEquals(['missing value(s)'], ctx.exception.reasons) + + # missing escapes + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'SPF', + 'ttl': 600, + 'value': 'this has some; semi-colons\; in it', + }) + self.assertEquals(['unescaped ;'], ctx.exception.reasons) + + def test_SRV(self): + # doesn't blow up + Record.new(self.zone, '_srv._tcp', { + 'type': 'SRV', + 'ttl': 600, + 'value': { + 'priority': 1, + 'weight': 2, + 'port': 3, + 'target': 'foo.bar.baz.' + } + }) + + # invalid name + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, 'neup', { + 'type': 'SRV', + 'ttl': 600, + 'value': { + 'priority': 1, + 'weight': 2, + 'port': 3, + 'target': 'foo.bar.baz.' + } + }) + self.assertEquals(['invalid name'], ctx.exception.reasons) + + # missing priority + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '_srv._tcp', { + 'type': 'SRV', + 'ttl': 600, + 'value': { + 'weight': 2, + 'port': 3, + 'target': 'foo.bar.baz.' + } + }) + self.assertEquals(['missing priority'], ctx.exception.reasons) + + # invalid priority + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '_srv._tcp', { + 'type': 'SRV', + 'ttl': 600, + 'value': { + 'priority': 'foo', + 'weight': 2, + 'port': 3, + 'target': 'foo.bar.baz.' + } + }) + self.assertEquals(['invalid priority "foo"'], ctx.exception.reasons) + + # missing weight + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '_srv._tcp', { + 'type': 'SRV', + 'ttl': 600, + 'value': { + 'priority': 1, + 'port': 3, + 'target': 'foo.bar.baz.' + } + }) + self.assertEquals(['missing weight'], ctx.exception.reasons) + # invalid weight + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '_srv._tcp', { + 'type': 'SRV', + 'ttl': 600, + 'value': { + 'priority': 1, + 'weight': 'foo', + 'port': 3, + 'target': 'foo.bar.baz.' + } + }) + self.assertEquals(['invalid weight "foo"'], ctx.exception.reasons) + + # missing port + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '_srv._tcp', { + 'type': 'SRV', + 'ttl': 600, + 'value': { + 'priority': 1, + 'weight': 2, + 'target': 'foo.bar.baz.' + } + }) + self.assertEquals(['missing port'], ctx.exception.reasons) + # invalid port + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '_srv._tcp', { + 'type': 'SRV', + 'ttl': 600, + 'value': { + 'priority': 1, + 'weight': 2, + 'port': 'foo', + 'target': 'foo.bar.baz.' + } + }) + self.assertEquals(['invalid port "foo"'], ctx.exception.reasons) + + # missing target + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '_srv._tcp', { + 'type': 'SRV', + 'ttl': 600, + 'value': { + 'priority': 1, + 'weight': 2, + 'port': 3, + } + }) + self.assertEquals(['missing target'], ctx.exception.reasons) + # invalid target + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '_srv._tcp', { + 'type': 'SRV', + 'ttl': 600, + 'value': { + 'priority': 1, + 'weight': 2, + 'port': 3, + 'target': 'foo.bar.baz' + } + }) + self.assertEquals(['missing trailing .'], + ctx.exception.reasons) + + def test_TXT(self): + # doesn't blow up (name & zone here don't make any sense, but not + # important) + Record.new(self.zone, '', { + 'type': 'TXT', + 'ttl': 600, + 'values': [ + 'hello world', + 'this has some\; semi-colons\; in it', + ] + }) + + # missing value + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'TXT', + 'ttl': 600, + }) + self.assertEquals(['missing value(s)'], ctx.exception.reasons) + + # missing escapes + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'TXT', + 'ttl': 600, + 'value': 'this has some; semi-colons\; in it', + }) + self.assertEquals(['unescaped ;'], ctx.exception.reasons) From a97818b6ec745a624eeca6e2ace71f93dfc65918 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 23 Jun 2017 09:01:25 -0700 Subject: [PATCH 14/64] populating existing provider state is lenient - adds lenient flag to Record.new, problems during validation are just warnings if it's true - target populate calls during the plan phase pass lenient=True - make all of the provider.populate call logging consistent including both target and lenient - add source=self to Record.new in a few places that were missing it --- octodns/provider/base.py | 2 +- octodns/provider/cloudflare.py | 8 +++++--- octodns/provider/dnsimple.py | 8 +++++--- octodns/provider/dyn.py | 9 ++++++--- octodns/provider/ns1.py | 8 +++++--- octodns/provider/powerdns.py | 7 ++++--- octodns/provider/route53.py | 9 ++++++--- octodns/provider/yaml.py | 9 ++++++--- octodns/record.py | 15 ++++++++++----- octodns/source/base.py | 8 ++++++++ octodns/source/tinydns.py | 19 +++++++++++-------- tests/helpers.py | 4 ++-- tests/test_octodns_provider_base.py | 4 ++-- tests/test_octodns_provider_route53.py | 6 +++--- tests/test_octodns_record.py | 16 ++++++++++++++++ 15 files changed, 90 insertions(+), 42 deletions(-) diff --git a/octodns/provider/base.py b/octodns/provider/base.py index 385fe36..2fd4349 100644 --- a/octodns/provider/base.py +++ b/octodns/provider/base.py @@ -104,7 +104,7 @@ class BaseProvider(BaseSource): self.log.info('plan: desired=%s', desired.name) existing = Zone(desired.name, desired.sub_zones) - self.populate(existing, target=True) + self.populate(existing, target=True, lenient=True) # compute the changes at the zone/record level changes = existing.changes(desired, self) diff --git a/octodns/provider/cloudflare.py b/octodns/provider/cloudflare.py index a45bd44..51b2171 100644 --- a/octodns/provider/cloudflare.py +++ b/octodns/provider/cloudflare.py @@ -154,8 +154,9 @@ class CloudflareProvider(BaseProvider): return self._zone_records[zone.name] - def populate(self, zone, target=False): - self.log.debug('populate: name=%s', zone.name) + def populate(self, zone, target=False, lenient=False): + self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, + target, lenient) before = len(zone.records) records = self.zone_records(zone) @@ -171,7 +172,8 @@ class CloudflareProvider(BaseProvider): for _type, records in types.items(): data_for = getattr(self, '_data_for_{}'.format(_type)) data = data_for(_type, records) - record = Record.new(zone, name, data, source=self) + record = Record.new(zone, name, data, source=self, + lenient=lenient) zone.add_record(record) self.log.info('populate: found %s records', diff --git a/octodns/provider/dnsimple.py b/octodns/provider/dnsimple.py index cb0f2d7..91bd638 100644 --- a/octodns/provider/dnsimple.py +++ b/octodns/provider/dnsimple.py @@ -234,8 +234,9 @@ class DnsimpleProvider(BaseProvider): return self._zone_records[zone.name] - def populate(self, zone, target=False): - self.log.debug('populate: name=%s', zone.name) + def populate(self, zone, target=False, lenient=False): + self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, + target, lenient) values = defaultdict(lambda: defaultdict(list)) for record in self.zone_records(zone): @@ -252,7 +253,8 @@ class DnsimpleProvider(BaseProvider): for name, types in values.items(): for _type, records in types.items(): data_for = getattr(self, '_data_for_{}'.format(_type)) - record = Record.new(zone, name, data_for(_type, records)) + record = Record.new(zone, name, data_for(_type, records), + source=self, lenient=lenient) zone.add_record(record) self.log.info('populate: found %s records', diff --git a/octodns/provider/dyn.py b/octodns/provider/dyn.py index 673e8d0..5e0e1f3 100644 --- a/octodns/provider/dyn.py +++ b/octodns/provider/dyn.py @@ -338,8 +338,10 @@ class DynProvider(BaseProvider): return td_records - def populate(self, zone, target=False): - self.log.info('populate: zone=%s', zone.name) + def populate(self, zone, target=False, lenient=False): + self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, + target, lenient) + before = len(zone.records) self._check_dyn_sess() @@ -364,7 +366,8 @@ class DynProvider(BaseProvider): for _type, records in types.items(): data_for = getattr(self, '_data_for_{}'.format(_type)) data = data_for(_type, records) - record = Record.new(zone, name, data, source=self) + record = Record.new(zone, name, data, source=self, + lenient=lenient) if record not in td_records: zone.add_record(record) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index c50341d..33fb19c 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -111,8 +111,9 @@ class Ns1Provider(BaseProvider): 'values': values, } - def populate(self, zone, target=False): - self.log.debug('populate: name=%s', zone.name) + def populate(self, zone, target=False, lenient=False): + self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, + target, lenient) try: nsone_zone = self._client.loadZone(zone.name[:-1]) @@ -127,7 +128,8 @@ class Ns1Provider(BaseProvider): _type = record['type'] data_for = getattr(self, '_data_for_{}'.format(_type)) name = zone.hostname_from_fqdn(record['domain']) - record = Record.new(zone, name, data_for(_type, record)) + record = Record.new(zone, name, data_for(_type, record), + source=self, lenient=lenient) zone.add_record(record) self.log.info('populate: found %s records', diff --git a/octodns/provider/powerdns.py b/octodns/provider/powerdns.py index 21e4d44..d8cccae 100644 --- a/octodns/provider/powerdns.py +++ b/octodns/provider/powerdns.py @@ -146,8 +146,9 @@ class PowerDnsBaseProvider(BaseProvider): 'ttl': rrset['ttl'] } - def populate(self, zone, target=False): - self.log.debug('populate: name=%s', zone.name) + def populate(self, zone, target=False, lenient=False): + self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, + target, lenient) resp = None try: @@ -177,7 +178,7 @@ class PowerDnsBaseProvider(BaseProvider): data_for = getattr(self, '_data_for_{}'.format(_type)) record_name = zone.hostname_from_fqdn(rrset['name']) record = Record.new(zone, record_name, data_for(rrset), - source=self) + source=self, lenient=lenient) zone.add_record(record) self.log.info('populate: found %s records', diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index 12c1aff..3875bd6 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -418,8 +418,10 @@ class Route53Provider(BaseProvider): return self._r53_rrsets[zone_id] - def populate(self, zone, target=False): - self.log.debug('populate: name=%s', zone.name) + def populate(self, zone, target=False, lenient=False): + self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, + target, lenient) + before = len(zone.records) zone_id = self._get_zone_id(zone.name) @@ -449,7 +451,8 @@ class Route53Provider(BaseProvider): data['geo'] = geo else: data = data[0] - record = Record.new(zone, name, data, source=self) + record = Record.new(zone, name, data, source=self, + lenient=lenient) zone.add_record(record) self.log.info('populate: found %s records', diff --git a/octodns/provider/yaml.py b/octodns/provider/yaml.py index c728caf..fe1a406 100644 --- a/octodns/provider/yaml.py +++ b/octodns/provider/yaml.py @@ -45,8 +45,10 @@ class YamlProvider(BaseProvider): self.default_ttl = default_ttl self.enforce_order = enforce_order - def populate(self, zone, target=False): - self.log.debug('populate: zone=%s, target=%s', zone.name, target) + def populate(self, zone, target=False, lenient=False): + self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, + target, lenient) + if target: # When acting as a target we ignore any existing records so that we # create a completely new copy @@ -63,7 +65,8 @@ class YamlProvider(BaseProvider): for d in data: if 'ttl' not in d: d['ttl'] = self.default_ttl - record = Record.new(zone, name, d, source=self) + record = Record.new(zone, name, d, source=self, + lenient=lenient) zone.add_record(record) self.log.info('populate: found %s records', diff --git a/octodns/record.py b/octodns/record.py index 827ad0a..3f07c39 100644 --- a/octodns/record.py +++ b/octodns/record.py @@ -56,10 +56,12 @@ class Delete(Change): class ValidationError(Exception): + @classmethod + def build_message(cls, fqdn, reasons): + return 'Invalid record {}\n - {}'.format(fqdn, '\n - '.join(reasons)) + def __init__(self, fqdn, reasons): - message = 'Invalid record {}\n - {}' \ - .format(fqdn, '\n - '.join(reasons)) - super(Exception, self).__init__(message) + super(Exception, self).__init__(self.build_message(fqdn, reasons)) self.fqdn = fqdn self.reasons = reasons @@ -68,7 +70,7 @@ class Record(object): log = getLogger('Record') @classmethod - def new(cls, zone, name, data, source=None): + def new(cls, zone, name, data, source=None, lenient=False): fqdn = '{}.{}'.format(name, zone.name) if name else zone.name try: _type = data['type'] @@ -107,7 +109,10 @@ class Record(object): raise Exception('Unknown record type: "{}"'.format(_type)) reasons = _class.validate(name, data) if reasons: - raise ValidationError(fqdn, reasons) + if lenient: + cls.log.warn(ValidationError.build_message(fqdn, reasons)) + else: + raise ValidationError(fqdn, reasons) return _class(zone, name, data, source=source) @classmethod diff --git a/octodns/source/base.py b/octodns/source/base.py index 42d214b..2e2c5c2 100644 --- a/octodns/source/base.py +++ b/octodns/source/base.py @@ -23,6 +23,14 @@ class BaseSource(object): def populate(self, zone, target=False): ''' Loads all zones the provider knows about + + When `target` is True the populate call is being made to load the + current state of the provider. + + When `lenient` is True the populate call may skip record validation and + do a "best effort" load of data. That will allow through some common, + but not best practices stuff that we otherwise would reject. E.g. no + trailing . or mising escapes for ;. ''' raise NotImplementedError('Abstract base class, populate method ' 'missing') diff --git a/octodns/source/tinydns.py b/octodns/source/tinydns.py index 6805378..1b98092 100644 --- a/octodns/source/tinydns.py +++ b/octodns/source/tinydns.py @@ -81,19 +81,21 @@ class TinyDnsBaseSource(BaseSource): 'values': ['{}.'.format(r[0]) for r in records] } - def populate(self, zone, target=False): - self.log.debug('populate: zone=%s', zone.name) + def populate(self, zone, target=False, lenient=False): + self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, + target, lenient) + before = len(zone.records) if zone.name.endswith('in-addr.arpa.'): - self._populate_in_addr_arpa(zone) + self._populate_in_addr_arpa(zone, lenient) else: - self._populate_normal(zone) + self._populate_normal(zone, lenient) self.log.info('populate: found %s records', len(zone.records) - before) - def _populate_normal(self, zone): + def _populate_normal(self, zone, lenient): type_map = { '=': 'A', '^': None, @@ -129,14 +131,15 @@ class TinyDnsBaseSource(BaseSource): data_for = getattr(self, '_data_for_{}'.format(_type)) data = data_for(_type, d) if data: - record = Record.new(zone, name, data, source=self) + record = Record.new(zone, name, data, source=self, + lenient=lenient) try: zone.add_record(record) except SubzoneRecordException: self.log.debug('_populate_normal: skipping subzone ' 'record=%s', record) - def _populate_in_addr_arpa(self, zone): + def _populate_in_addr_arpa(self, zone, lenient): name_re = re.compile('(?P.+)\.{}$'.format(zone.name[:-1])) for line in self._lines(): @@ -170,7 +173,7 @@ class TinyDnsBaseSource(BaseSource): 'ttl': ttl, 'type': 'PTR', 'value': value - }, source=self) + }, source=self, lenient=lenient) try: zone.add_record(record) except DuplicateRecordException: diff --git a/tests/helpers.py b/tests/helpers.py index a8aafa3..adac81d 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -22,7 +22,7 @@ class SimpleProvider(object): def __init__(self, id='test'): pass - def populate(self, zone, source=True): + def populate(self, zone, source=False, lenient=False): pass def supports(self, record): @@ -38,7 +38,7 @@ class GeoProvider(object): def __init__(self, id='test'): pass - def populate(self, zone, source=True): + def populate(self, zone, source=False, lenient=False): pass def supports(self, record): diff --git a/tests/test_octodns_provider_base.py b/tests/test_octodns_provider_base.py index 766bf65..bd134bc 100644 --- a/tests/test_octodns_provider_base.py +++ b/tests/test_octodns_provider_base.py @@ -24,7 +24,7 @@ class HelperProvider(BaseProvider): self.apply_disabled = apply_disabled self.include_change_callback = include_change_callback - def populate(self, zone, target=False): + def populate(self, zone, target=False, lenient=False): pass def _include_change(self, change): @@ -72,7 +72,7 @@ class TestBaseProvider(TestCase): class HasPopulate(HasSupports): - def populate(self, zone, target=False): + def populate(self, zone, target=False, lenient=False): zone.add_record(Record.new(zone, '', { 'ttl': 60, 'type': 'A', diff --git a/tests/test_octodns_provider_route53.py b/tests/test_octodns_provider_route53.py index 5982b74..0a769f9 100644 --- a/tests/test_octodns_provider_route53.py +++ b/tests/test_octodns_provider_route53.py @@ -370,7 +370,7 @@ class TestRoute53Provider(TestCase): stubber.assert_no_pending_responses() # Delete by monkey patching in a populate that includes an extra record - def add_extra_populate(existing, target): + def add_extra_populate(existing, target, lenient): for record in self.expected.records: existing.records.add(record) record = Record.new(existing, 'extra', @@ -406,7 +406,7 @@ class TestRoute53Provider(TestCase): # Update by monkey patching in a populate that modifies the A record # with geos - def mod_geo_populate(existing, target): + def mod_geo_populate(existing, target, lenient): for record in self.expected.records: if record._type != 'A' or not record.geo: existing.records.add(record) @@ -502,7 +502,7 @@ class TestRoute53Provider(TestCase): # Update converting to non-geo by monkey patching in a populate that # modifies the A record with geos - def mod_add_geo_populate(existing, target): + def mod_add_geo_populate(existing, target, lenient): for record in self.expected.records: if record._type != 'A' or record.geo: existing.records.add(record) diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index 99a502e..6e40e18 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -643,6 +643,7 @@ class TestRecordValidation(TestCase): 'value': '1.2.3.4', }) self.assertEquals(['missing ttl'], ctx.exception.reasons) + # invalid ttl with self.assertRaises(ValidationError) as ctx: Record.new(self.zone, 'www', { @@ -653,6 +654,21 @@ class TestRecordValidation(TestCase): self.assertEquals('www.unit.tests.', ctx.exception.fqdn) self.assertEquals(['invalid ttl'], ctx.exception.reasons) + # no exception if we're in lenient mode + Record.new(self.zone, 'www', { + 'type': 'A', + 'ttl': -1, + 'value': '1.2.3.4', + }, lenient=True) + + # __init__ may still blow up, even if validation is lenient + with self.assertRaises(KeyError) as ctx: + Record.new(self.zone, 'www', { + 'type': 'A', + 'ttl': -1, + }, lenient=True) + self.assertEquals(('value',), ctx.exception.args) + def test_A_and_values_mixin(self): # doesn't blow up Record.new(self.zone, '', { From cfc0d586a13450ccf60ddc8b8fab260da958442b Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 23 Jun 2017 09:06:21 -0700 Subject: [PATCH 15/64] Log max_workers, useful to know --- octodns/manager.py | 1 + 1 file changed, 1 insertion(+) diff --git a/octodns/manager.py b/octodns/manager.py index 0366685..5feed8e 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -69,6 +69,7 @@ class Manager(object): manager_config = self.config.get('manager', {}) max_workers = manager_config.get('max_workers', 1) \ if max_workers is None else max_workers + self.log.info('__init__: max_workers=%d', max_workers) if max_workers > 1: self._executor = ThreadPoolExecutor(max_workers=max_workers) else: From a69ff64ae1db0b19eed101e1c992fc965e06b24a Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 23 Jun 2017 09:24:25 -0700 Subject: [PATCH 16/64] Add --lenient flag to dump --- octodns/cmds/dump.py | 5 ++++- octodns/manager.py | 4 ++-- tests/test_octodns_manager.py | 6 +++--- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/octodns/cmds/dump.py b/octodns/cmds/dump.py index e4c4987..9f5e0aa 100755 --- a/octodns/cmds/dump.py +++ b/octodns/cmds/dump.py @@ -18,6 +18,9 @@ def main(): parser.add_argument('--output-dir', required=True, help='The directory into which the results will be ' 'written (Note: will overwrite existing files)') + parser.add_argument('--lenient', action='store_true', default=False, + help='Ignore record validations and do a best effort ' + 'dump') parser.add_argument('zone', help='Zone to dump') parser.add_argument('source', nargs='+', help='Source(s) to pull data from') @@ -25,7 +28,7 @@ def main(): args = parser.parse_args() manager = Manager(args.config_file) - manager.dump(args.zone, args.output_dir, *args.source) + manager.dump(args.zone, args.output_dir, args.lenient, *args.source) if __name__ == '__main__': diff --git a/octodns/manager.py b/octodns/manager.py index 5feed8e..07719dd 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -323,7 +323,7 @@ class Manager(object): return zb.changes(za, _AggregateTarget(a + b)) - def dump(self, zone, output_dir, source, *sources): + def dump(self, zone, output_dir, lenient, source, *sources): ''' Dump zone data from the specified source ''' @@ -342,7 +342,7 @@ class Manager(object): zone = Zone(zone, self.configured_sub_zones(zone)) for source in sources: - source.populate(zone) + source.populate(zone, lenient=lenient) plan = target.plan(zone) target.apply(plan) diff --git a/tests/test_octodns_manager.py b/tests/test_octodns_manager.py index fa8bdd1..641c1ff 100644 --- a/tests/test_octodns_manager.py +++ b/tests/test_octodns_manager.py @@ -195,15 +195,15 @@ class TestManager(TestCase): manager = Manager(get_config_filename('simple.yaml')) with self.assertRaises(Exception) as ctx: - manager.dump('unit.tests.', tmpdir.dirname, 'nope') + manager.dump('unit.tests.', tmpdir.dirname, False, 'nope') self.assertEquals('Unknown source: nope', ctx.exception.message) - manager.dump('unit.tests.', tmpdir.dirname, 'in') + manager.dump('unit.tests.', tmpdir.dirname, False, 'in') # make sure this fails with an IOError and not a KeyError when # tyring to find sub zones with self.assertRaises(IOError): - manager.dump('unknown.zone.', tmpdir.dirname, 'in') + manager.dump('unknown.zone.', tmpdir.dirname, False, 'in') def test_validate_configs(self): Manager(get_config_filename('simple-validate.yaml')).validate_configs() From d2af8efe5c9d7b5f19bbb5ed6dacb52caac4e172 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 23 Jun 2017 09:49:11 -0700 Subject: [PATCH 17/64] Root CNAMEs are not allowed --- octodns/record.py | 8 ++++++++ tests/test_octodns_provider_route53.py | 6 ++++-- tests/test_octodns_record.py | 13 +++++++++++-- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/octodns/record.py b/octodns/record.py index 3f07c39..11876f5 100644 --- a/octodns/record.py +++ b/octodns/record.py @@ -400,6 +400,14 @@ class AliasRecord(_ValueMixin, Record): class CnameRecord(_ValueMixin, Record): _type = 'CNAME' + @classmethod + def validate(cls, name, data): + reasons = [] + if name == '': + reasons.append('root CNAME not allowed') + reasons.extend(super(CnameRecord, cls).validate(name, data)) + return reasons + @classmethod def _validate_value(cls, value): reasons = [] diff --git a/tests/test_octodns_provider_route53.py b/tests/test_octodns_provider_route53.py index 0a769f9..8960088 100644 --- a/tests/test_octodns_provider_route53.py +++ b/tests/test_octodns_provider_route53.py @@ -1260,8 +1260,10 @@ class TestRoute53Records(TestCase): False) self.assertEquals(c, c) d = _Route53Record(None, Record.new(existing, '', - {'ttl': 42, 'type': 'CNAME', - 'value': 'foo.bar.'}), + {'ttl': 42, 'type': 'MX', + 'value': { + 'priority': 10, + 'value': 'foo.bar.'}}), False) self.assertEquals(d, d) diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index 6e40e18..96a83a0 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -859,15 +859,24 @@ class TestRecordValidation(TestCase): def test_CNAME(self): # doesn't blow up - Record.new(self.zone, '', { + Record.new(self.zone, 'www', { 'type': 'CNAME', 'ttl': 600, 'value': 'foo.bar.com.', }) - # missing trailing . + # root cname is a no-no with self.assertRaises(ValidationError) as ctx: Record.new(self.zone, '', { + 'type': 'CNAME', + 'ttl': 600, + 'value': 'foo.bar.com.', + }) + self.assertEquals(['root CNAME not allowed'], ctx.exception.reasons) + + # missing trailing . + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, 'www', { 'type': 'CNAME', 'ttl': 600, 'value': 'foo.bar.com', From 615bc95976ca0dba4ce8d77b06d19205d1fe7a08 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 23 Jun 2017 09:49:25 -0700 Subject: [PATCH 18/64] CNAME cannot coexist with other records on a node --- octodns/zone.py | 18 +++++++++++++++--- tests/test_octodns_zone.py | 27 ++++++++++++++++++++++++++- 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/octodns/zone.py b/octodns/zone.py index 1822fec..03bc41c 100644 --- a/octodns/zone.py +++ b/octodns/zone.py @@ -19,6 +19,10 @@ class DuplicateRecordException(Exception): pass +class InvalidNodeException(Exception): + pass + + def _is_eligible(record): # Should this record be considered when computing changes # We ignore all top-level NS records @@ -59,9 +63,17 @@ class Zone(object): raise SubzoneRecordException('Record {} a managed sub-zone ' 'and not of type NS' .format(record.fqdn)) - if record in self.records: - raise DuplicateRecordException('Duplicate record {}, type {}' - .format(record.fqdn, record._type)) + for existing in self.records: + if record == existing: + raise DuplicateRecordException('Duplicate record {}, type {}' + .format(record.fqdn, + record._type)) + elif name == existing.name and (record._type == 'CNAME' or + existing._type == 'CNAME'): + raise InvalidNodeException('Invalid state, CNAME at {} ' + 'cannot coexist with other records' + .format(record.fqdn)) + self.records.add(record) def changes(self, desired, target): diff --git a/tests/test_octodns_zone.py b/tests/test_octodns_zone.py index 88bbb68..a4d7300 100644 --- a/tests/test_octodns_zone.py +++ b/tests/test_octodns_zone.py @@ -8,7 +8,8 @@ from __future__ import absolute_import, division, print_function, \ from unittest import TestCase from octodns.record import ARecord, AaaaRecord, Create, Delete, Record, Update -from octodns.zone import DuplicateRecordException, SubzoneRecordException, Zone +from octodns.zone import DuplicateRecordException, InvalidNodeException, \ + SubzoneRecordException, Zone from helpers import SimpleProvider @@ -205,3 +206,27 @@ class TestZone(TestCase): self.assertTrue(zone_missing.changes(zone_normal, provider)) self.assertFalse(zone_missing.changes(zone_ignored, provider)) + + def test_cname_coexisting(self): + zone = Zone('unit.tests.', []) + a = Record.new(zone, 'www', { + 'ttl': 60, + 'type': 'A', + 'value': '9.9.9.9', + }) + cname = Record.new(zone, 'www', { + 'ttl': 60, + 'type': 'CNAME', + 'value': 'foo.bar.com.', + }) + + # add cname to a + zone.add_record(a) + with self.assertRaises(InvalidNodeException): + zone.add_record(cname) + + # add a to cname + zone = Zone('unit.tests.', []) + zone.add_record(cname) + with self.assertRaises(InvalidNodeException): + zone.add_record(a) From 1340aee8a934c690d74001b3780880b5eee8b2fa Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 23 Jun 2017 13:04:38 -0700 Subject: [PATCH 19/64] MX RFC1035 - priority -> preference & value -> exchange --- octodns/provider/cloudflare.py | 8 ++-- octodns/provider/dnsimple.py | 8 ++-- octodns/provider/dyn.py | 6 +-- octodns/provider/ns1.py | 8 ++-- octodns/provider/powerdns.py | 8 ++-- octodns/provider/route53.py | 9 ++-- octodns/record.py | 41 +++++++++++------- octodns/source/tinydns.py | 4 +- tests/config/unit.tests.yaml | 12 +++--- tests/test_octodns_provider_dyn.py | 8 ++-- tests/test_octodns_provider_ns1.py | 8 ++-- tests/test_octodns_provider_route53.py | 12 +++--- tests/test_octodns_record.py | 58 ++++++++++++++++---------- tests/test_octodns_source_tinydns.py | 16 +++---- 14 files changed, 118 insertions(+), 88 deletions(-) diff --git a/octodns/provider/cloudflare.py b/octodns/provider/cloudflare.py index 51b2171..2ee8f8b 100644 --- a/octodns/provider/cloudflare.py +++ b/octodns/provider/cloudflare.py @@ -116,8 +116,8 @@ class CloudflareProvider(BaseProvider): values = [] for r in records: values.append({ - 'priority': r['priority'], - 'value': '{}.'.format(r['content']), + 'preference': r['priority'], + 'exchange': '{}.'.format(r['content']), }) return { 'ttl': records[0]['ttl'], @@ -207,8 +207,8 @@ class CloudflareProvider(BaseProvider): def _contents_for_MX(self, record): for value in record.values: yield { - 'priority': value.priority, - 'content': value.value + 'priority': value.preference, + 'content': value.exchange } def _apply_Create(self, change): diff --git a/octodns/provider/dnsimple.py b/octodns/provider/dnsimple.py index 91bd638..dc44d1b 100644 --- a/octodns/provider/dnsimple.py +++ b/octodns/provider/dnsimple.py @@ -128,8 +128,8 @@ class DnsimpleProvider(BaseProvider): values = [] for record in records: values.append({ - 'priority': record['priority'], - 'value': '{}.'.format(record['content']) + 'preference': record['priority'], + 'exchange': '{}.'.format(record['content']) }) return { 'ttl': records[0]['ttl'], @@ -290,9 +290,9 @@ class DnsimpleProvider(BaseProvider): def _params_for_MX(self, record): for value in record.values: yield { - 'content': value.value, + 'content': value.exchange, 'name': record.name, - 'priority': value.priority, + 'priority': value.preference, 'ttl': record.ttl, 'type': record._type } diff --git a/octodns/provider/dyn.py b/octodns/provider/dyn.py index 5e0e1f3..e21b93e 100644 --- a/octodns/provider/dyn.py +++ b/octodns/provider/dyn.py @@ -206,7 +206,7 @@ class DynProvider(BaseProvider): return { 'type': _type, 'ttl': records[0].ttl, - 'values': [{'priority': r.preference, 'value': r.exchange} + 'values': [{'preference': r.preference, 'exchange': r.exchange} for r in records], } @@ -400,8 +400,8 @@ class DynProvider(BaseProvider): def _kwargs_for_MX(self, record): return [{ - 'preference': v.priority, - 'exchange': v.value, + 'preference': v.preference, + 'exchange': v.exchange, 'ttl': record.ttl, } for v in record.values] diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 33fb19c..2f0a024 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -57,10 +57,10 @@ class Ns1Provider(BaseProvider): def _data_for_MX(self, _type, record): values = [] for answer in record['short_answers']: - priority, value = answer.split(' ', 1) + preference, exchange = answer.split(' ', 1) values.append({ - 'priority': priority, - 'value': value, + 'preference': preference, + 'exchange': exchange, }) return { 'ttl': record['ttl'], @@ -150,7 +150,7 @@ class Ns1Provider(BaseProvider): _params_for_PTR = _params_for_CNAME def _params_for_MX(self, record): - values = [(v.priority, v.value) for v in record.values] + values = [(v.preference, v.exchange) for v in record.values] return {'answers': values, 'ttl': record.ttl} def _params_for_NAPTR(self, record): diff --git a/octodns/provider/powerdns.py b/octodns/provider/powerdns.py index d8cccae..c6d11b0 100644 --- a/octodns/provider/powerdns.py +++ b/octodns/provider/powerdns.py @@ -83,10 +83,10 @@ class PowerDnsBaseProvider(BaseProvider): def _data_for_MX(self, rrset): values = [] for record in rrset['records']: - priority, value = record['content'].split(' ', 1) + preference, exchange = record['content'].split(' ', 1) values.append({ - 'priority': priority, - 'value': value, + 'preference': preference, + 'exchange': exchange, }) return { 'type': rrset['type'], @@ -208,7 +208,7 @@ class PowerDnsBaseProvider(BaseProvider): def _records_for_MX(self, record): return [{ - 'content': '{} {}'.format(v.priority, v.value), + 'content': '{} {}'.format(v.preference, v.exchange), 'disabled': False } for v in record.values] diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index 3875bd6..b4fcab8 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -96,7 +96,8 @@ class _Route53Record(object): _values_for_PTR = _values_for_value def _values_for_MX(self, record): - return ['{} {}'.format(v.priority, v.value) for v in record.values] + return ['{} {}'.format(v.preference, v.exchange) + for v in record.values] def _values_for_NAPTR(self, record): return ['{} {} "{}" "{}" "{}" {}' @@ -335,10 +336,10 @@ class Route53Provider(BaseProvider): def _data_for_MX(self, rrset): values = [] for rr in rrset['ResourceRecords']: - priority, value = rr['Value'].split(' ') + preference, exchange = rr['Value'].split(' ') values.append({ - 'priority': priority, - 'value': value, + 'preference': preference, + 'exchange': exchange, }) return { 'type': rrset['Type'], diff --git a/octodns/record.py b/octodns/record.py index 11876f5..0388911 100644 --- a/octodns/record.py +++ b/octodns/record.py @@ -424,32 +424,45 @@ class MxValue(object): @classmethod def _validate_value(cls, value): reasons = [] - if 'priority' not in value: - reasons.append('missing priority') - if 'value' not in value: - reasons.append('missing value') + if 'preference' not in value and 'priority' not in value: + reasons.append('missing preference') + exchange = None + try: + exchange = value.get('exchange', None) or value['value'] + if not exchange.endswith('.'): + reasons.append('missing trailing .') + except KeyError: + reasons.append('missing exchange') return reasons def __init__(self, value): - # TODO: rename preference - self.priority = int(value['priority']) - # TODO: rename to exchange? - self.value = value['value'].lower() + # RFC1035 says preference, half the providers use priority + try: + preference = value['preference'] + except KeyError: + preference = value['priority'] + self.preference = int(preference) + # UNTIL 1.0 remove value fallback + try: + exchange = value['exchange'] + except KeyError: + exchange = value['value'] + self.exchange = exchange @property def data(self): return { - 'priority': self.priority, - 'value': self.value, + 'preference': self.preference, + 'exchange': self.exchange, } def __cmp__(self, other): - if self.priority == other.priority: - return cmp(self.value, other.value) - return cmp(self.priority, other.priority) + if self.preference == other.preference: + return cmp(self.exchange, other.exchange) + return cmp(self.preference, other.preference) def __repr__(self): - return "'{} {}'".format(self.priority, self.value) + return "'{} {}'".format(self.preference, self.exchange) class MxRecord(_ValuesMixin, Record): diff --git a/octodns/source/tinydns.py b/octodns/source/tinydns.py index 1b98092..63cafa2 100644 --- a/octodns/source/tinydns.py +++ b/octodns/source/tinydns.py @@ -65,8 +65,8 @@ class TinyDnsBaseSource(BaseSource): 'ttl': ttl, 'type': _type, 'values': [{ - 'priority': r[1], - 'value': '{}.'.format(r[0]) + 'preference': r[1], + 'exchange': '{}.'.format(r[0]) } for r in records] } diff --git a/tests/config/unit.tests.yaml b/tests/config/unit.tests.yaml index d18bf59..8be1614 100644 --- a/tests/config/unit.tests.yaml +++ b/tests/config/unit.tests.yaml @@ -60,12 +60,12 @@ mx: ttl: 300 type: MX values: - - priority: 40 - value: smtp-1.unit.tests. - - priority: 20 - value: smtp-2.unit.tests. - - priority: 30 - value: smtp-3.unit.tests. + - exchange: smtp-1.unit.tests. + preference: 40 + - exchange: smtp-2.unit.tests. + preference: 20 + - exchange: smtp-3.unit.tests. + preference: 30 - priority: 10 value: smtp-4.unit.tests. naptr: diff --git a/tests/test_octodns_provider_dyn.py b/tests/test_octodns_provider_dyn.py index 307e640..bebd3e3 100644 --- a/tests/test_octodns_provider_dyn.py +++ b/tests/test_octodns_provider_dyn.py @@ -46,11 +46,11 @@ class TestDynProvider(TestCase): 'type': 'MX', 'ttl': 302, 'values': [{ - 'priority': 10, - 'value': 'smtp-1.unit.tests.' + 'preference': 10, + 'exchange': 'smtp-1.unit.tests.' }, { - 'priority': 20, - 'value': 'smtp-2.unit.tests.' + 'preference': 20, + 'exchange': 'smtp-2.unit.tests.' }] }), ('naptr', { diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index acb0125..ecc107c 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -44,11 +44,11 @@ class TestNs1Provider(TestCase): 'ttl': 35, 'type': 'MX', 'values': [{ - 'priority': 10, - 'value': 'mx1.unit.tests.', + 'preference': 10, + 'exchange': 'mx1.unit.tests.', }, { - 'priority': 20, - 'value': 'mx2.unit.tests.', + 'preference': 20, + 'exchange': 'mx2.unit.tests.', }] })) expected.add(Record.new(zone, 'naptr', { diff --git a/tests/test_octodns_provider_route53.py b/tests/test_octodns_provider_route53.py index 8960088..cad58f8 100644 --- a/tests/test_octodns_provider_route53.py +++ b/tests/test_octodns_provider_route53.py @@ -52,11 +52,11 @@ class TestRoute53Provider(TestCase): 'Goodbye World?']}), ('', {'ttl': 64, 'type': 'MX', 'values': [{ - 'priority': 10, - 'value': 'smtp-1.unit.tests.', + 'preference': 10, + 'exchange': 'smtp-1.unit.tests.', }, { - 'priority': 20, - 'value': 'smtp-2.unit.tests.', + 'preference': 20, + 'exchange': 'smtp-2.unit.tests.', }]}), ('naptr', {'ttl': 65, 'type': 'NAPTR', 'value': { @@ -1262,8 +1262,8 @@ class TestRoute53Records(TestCase): d = _Route53Record(None, Record.new(existing, '', {'ttl': 42, 'type': 'MX', 'value': { - 'priority': 10, - 'value': 'foo.bar.'}}), + 'preference': 10, + 'exchange': 'foo.bar.'}}), False) self.assertEquals(d, d) diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index 96a83a0..cb87c70 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -212,45 +212,49 @@ class TestRecord(TestCase): def test_mx(self): a_values = [{ - 'priority': 10, - 'value': 'smtp1' + 'preference': 10, + 'exchange': 'smtp1.' }, { 'priority': 20, - 'value': 'smtp2' + 'value': 'smtp2.' }] a_data = {'ttl': 30, 'values': a_values} a = MxRecord(self.zone, 'a', a_data) self.assertEquals('a', a.name) self.assertEquals('a.unit.tests.', a.fqdn) self.assertEquals(30, a.ttl) - self.assertEquals(a_values[0]['priority'], a.values[0].priority) - self.assertEquals(a_values[0]['value'], a.values[0].value) - self.assertEquals(a_values[1]['priority'], a.values[1].priority) - self.assertEquals(a_values[1]['value'], a.values[1].value) + self.assertEquals(a_values[0]['preference'], a.values[0].preference) + self.assertEquals(a_values[0]['exchange'], a.values[0].exchange) + self.assertEquals(a_values[1]['priority'], a.values[1].preference) + self.assertEquals(a_values[1]['value'], a.values[1].exchange) + a_data['values'][1] = { + 'preference': 20, + 'exchange': 'smtp2.', + } self.assertEquals(a_data, a.data) b_value = { - 'priority': 12, - 'value': 'smtp3', + 'preference': 12, + 'exchange': 'smtp3.', } b_data = {'ttl': 30, 'value': b_value} b = MxRecord(self.zone, 'b', b_data) - self.assertEquals(b_value['priority'], b.values[0].priority) - self.assertEquals(b_value['value'], b.values[0].value) + self.assertEquals(b_value['preference'], b.values[0].preference) + self.assertEquals(b_value['exchange'], b.values[0].exchange) self.assertEquals(b_data, b.data) target = SimpleProvider() # No changes with self self.assertFalse(a.changes(a, target)) - # Diff in priority causes change + # Diff in preference causes change other = MxRecord(self.zone, 'a', {'ttl': 30, 'values': a_values}) - other.values[0].priority = 22 + other.values[0].preference = 22 change = a.changes(other, target) self.assertEqual(change.existing, a) self.assertEqual(change.new, other) # Diff in value causes change - other.values[0].priority = a.values[0].priority - other.values[0].value = 'smtpX' + other.values[0].preference = a.values[0].preference + other.values[0].exchange = 'smtpX' change = a.changes(other, target) self.assertEqual(change.existing, a) self.assertEqual(change.new, other) @@ -889,8 +893,8 @@ class TestRecordValidation(TestCase): 'type': 'MX', 'ttl': 600, 'value': { - 'priority': 10, - 'value': 'foo.bar.com.' + 'preference': 10, + 'exchange': 'foo.bar.com.' } }) @@ -900,10 +904,10 @@ class TestRecordValidation(TestCase): 'type': 'MX', 'ttl': 600, 'value': { - 'value': 'foo.bar.com.' + 'exchange': 'foo.bar.com.' } }) - self.assertEquals(['missing priority'], ctx.exception.reasons) + self.assertEquals(['missing preference'], ctx.exception.reasons) # missing value with self.assertRaises(ValidationError) as ctx: @@ -911,10 +915,22 @@ class TestRecordValidation(TestCase): 'type': 'MX', 'ttl': 600, 'value': { - 'priority': 10, + 'preference': 10, } }) - self.assertEquals(['missing value'], ctx.exception.reasons) + self.assertEquals(['missing exchange'], ctx.exception.reasons) + + # missing trailing . + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'MX', + 'ttl': 600, + 'value': { + 'preference': 10, + 'exchange': 'foo.bar.com' + } + }) + self.assertEquals(['missing trailing .'], ctx.exception.reasons) def test_NXPTR(self): # doesn't blow up diff --git a/tests/test_octodns_source_tinydns.py b/tests/test_octodns_source_tinydns.py index b4cea06..5792b25 100644 --- a/tests/test_octodns_source_tinydns.py +++ b/tests/test_octodns_source_tinydns.py @@ -68,22 +68,22 @@ class TestTinyDnsFileSource(TestCase): 'type': 'MX', 'ttl': 3600, 'values': [{ - 'priority': 10, - 'value': 'smtp-1-host.example.com.', + 'preference': 10, + 'exchange': 'smtp-1-host.example.com.', }, { - 'priority': 20, - 'value': 'smtp-2-host.example.com.', + 'preference': 20, + 'exchange': 'smtp-2-host.example.com.', }] }), ('smtp', { 'type': 'MX', 'ttl': 1800, 'values': [{ - 'priority': 30, - 'value': 'smtp-1-host.example.com.', + 'preference': 30, + 'exchange': 'smtp-1-host.example.com.', }, { - 'priority': 40, - 'value': 'smtp-2-host.example.com.', + 'preference': 40, + 'exchange': 'smtp-2-host.example.com.', }] }), ): From 6fc82fd279e32c9e1f35d98a8d32575ba49831a3 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 23 Jun 2017 13:17:32 -0700 Subject: [PATCH 20/64] Validate that MX preference parses as int --- octodns/record.py | 8 +++++++- tests/test_octodns_record.py | 16 ++++++++++++++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/octodns/record.py b/octodns/record.py index 0388911..03ce675 100644 --- a/octodns/record.py +++ b/octodns/record.py @@ -424,8 +424,14 @@ class MxValue(object): @classmethod def _validate_value(cls, value): reasons = [] - if 'preference' not in value and 'priority' not in value: + try: + # seperate lines to have preference set in the ValueError case + preference = value.get('preference', None) or value['priority'] + int(preference) + except KeyError: reasons.append('missing preference') + except ValueError: + reasons.append('invalid preference "{}"'.format(preference)) exchange = None try: exchange = value.get('exchange', None) or value['value'] diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index cb87c70..0230e2c 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -898,7 +898,7 @@ class TestRecordValidation(TestCase): } }) - # missing priority + # missing preference with self.assertRaises(ValidationError) as ctx: Record.new(self.zone, '', { 'type': 'MX', @@ -909,7 +909,19 @@ class TestRecordValidation(TestCase): }) self.assertEquals(['missing preference'], ctx.exception.reasons) - # missing value + # invalid preference + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'MX', + 'ttl': 600, + 'value': { + 'preference': 'nope', + 'exchange': 'foo.bar.com.' + } + }) + self.assertEquals(['invalid preference "nope"'], ctx.exception.reasons) + + # missing exchange with self.assertRaises(ValidationError) as ctx: Record.new(self.zone, '', { 'type': 'MX', From 3ce0d71e62f95aa86cf698435851176905f8a194 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 23 Jun 2017 13:28:22 -0700 Subject: [PATCH 21/64] NAPTR RFC2915 - validate flags (partial) - punting on service, regex & replacement validation for now - clean up MX a smidge --- octodns/record.py | 19 +++++++++++++------ tests/test_octodns_record.py | 15 +++++++++++++-- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/octodns/record.py b/octodns/record.py index 03ce675..1d2b349 100644 --- a/octodns/record.py +++ b/octodns/record.py @@ -425,13 +425,12 @@ class MxValue(object): def _validate_value(cls, value): reasons = [] try: - # seperate lines to have preference set in the ValueError case - preference = value.get('preference', None) or value['priority'] - int(preference) + int(value.get('preference', None) or value['priority']) except KeyError: reasons.append('missing preference') except ValueError: - reasons.append('invalid preference "{}"'.format(preference)) + reasons.append('invalid preference "{}"' + .format(value['preference'])) exchange = None try: exchange = value.get('exchange', None) or value['value'] @@ -483,6 +482,7 @@ class MxRecord(_ValuesMixin, Record): class NaptrValue(object): + LEGAL_FLAGS = ('S', 'A', 'U', 'P') @classmethod def _validate_value(cls, data): @@ -500,8 +500,15 @@ class NaptrValue(object): except ValueError: reasons.append('invalid preference "{}"' .format(data['preference'])) - # TODO: validate field data - for k in ('flags', 'service', 'regexp', 'replacement'): + try: + flags = data['flags'] + if flags not in cls.LEGAL_FLAGS: + reasons.append('invalid flags "{}"'.format(flags)) + except KeyError: + reasons.append('missing flags') + + # TODO: validate these... they're non-trivial + for k in ('service', 'regexp', 'replacement'): if k not in data: reasons.append('missing {}'.format(k)) return reasons diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index 0230e2c..08d3ad7 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -952,7 +952,7 @@ class TestRecordValidation(TestCase): 'value': { 'order': 10, 'preference': 20, - 'flags': 'f', + 'flags': 'S', 'service': 'srv', 'regexp': '.*', 'replacement': '.' @@ -963,7 +963,7 @@ class TestRecordValidation(TestCase): value = { 'order': 10, 'preference': 20, - 'flags': 'f', + 'flags': 'S', 'service': 'srv', 'regexp': '.*', 'replacement': '.' @@ -1002,6 +1002,17 @@ class TestRecordValidation(TestCase): }) self.assertEquals(['invalid preference "who"'], ctx.exception.reasons) + # unrecognized flags + v = dict(value) + v['flags'] = 'X' + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'NAPTR', + 'ttl': 600, + 'value': v + }) + self.assertEquals(['invalid flags "X"'], ctx.exception.reasons) + def test_NS(self): # doesn't blow up Record.new(self.zone, '', { From 4e3cc6b46ac7e324ff028e4056ee4e04f4f0c243 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 23 Jun 2017 13:35:04 -0700 Subject: [PATCH 22/64] SSHFP RFC4255 - validate algorithm & fingerprint_type - unrecognized wording for invalid values --- octodns/record.py | 18 ++++++++++++------ tests/test_octodns_record.py | 30 +++++++++++++++++++++++++++++- 2 files changed, 41 insertions(+), 7 deletions(-) diff --git a/octodns/record.py b/octodns/record.py index 1d2b349..23bfd45 100644 --- a/octodns/record.py +++ b/octodns/record.py @@ -482,7 +482,7 @@ class MxRecord(_ValuesMixin, Record): class NaptrValue(object): - LEGAL_FLAGS = ('S', 'A', 'U', 'P') + VALID_FLAGS = ('S', 'A', 'U', 'P') @classmethod def _validate_value(cls, data): @@ -502,8 +502,8 @@ class NaptrValue(object): .format(data['preference'])) try: flags = data['flags'] - if flags not in cls.LEGAL_FLAGS: - reasons.append('invalid flags "{}"'.format(flags)) + if flags not in cls.VALID_FLAGS: + reasons.append('unrecognized flags "{}"'.format(flags)) except KeyError: reasons.append('missing flags') @@ -594,19 +594,25 @@ class PtrRecord(_ValueMixin, Record): class SshfpValue(object): + VALID_ALGORITHMS = (1, 2) + VALID_FINGERPRINT_TYPES = (1,) @classmethod def _validate_value(cls, value): reasons = [] - # TODO: validate algorithm and fingerprint_type values try: - int(value['algorithm']) + algorithm = int(value['algorithm']) + if algorithm not in cls.VALID_ALGORITHMS: + reasons.append('unrecognized algorithm "{}"'.format(algorithm)) except KeyError: reasons.append('missing algorithm') except ValueError: reasons.append('invalid algorithm "{}"'.format(value['algorithm'])) try: - int(value['fingerprint_type']) + fingerprint_type = int(value['fingerprint_type']) + if fingerprint_type not in cls.VALID_FINGERPRINT_TYPES: + reasons.append('unrecognized fingerprint_type "{}"' + .format(fingerprint_type)) except KeyError: reasons.append('missing fingerprint_type') except ValueError: diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index 08d3ad7..1d64081 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -1011,7 +1011,7 @@ class TestRecordValidation(TestCase): 'ttl': 600, 'value': v }) - self.assertEquals(['invalid flags "X"'], ctx.exception.reasons) + self.assertEquals(['unrecognized flags "X"'], ctx.exception.reasons) def test_NS(self): # doesn't blow up @@ -1104,6 +1104,20 @@ class TestRecordValidation(TestCase): }) self.assertEquals(['invalid algorithm "nope"'], ctx.exception.reasons) + # unrecognized algorithm + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'SSHFP', + 'ttl': 600, + 'value': { + 'algorithm': 42, + 'fingerprint_type': 1, + 'fingerprint': 'bf6b6825d2977c511a475bbefb88aad54a92ac73' + } + }) + self.assertEquals(['unrecognized algorithm "42"'], + ctx.exception.reasons) + # missing fingerprint_type with self.assertRaises(ValidationError) as ctx: Record.new(self.zone, '', { @@ -1130,6 +1144,20 @@ class TestRecordValidation(TestCase): self.assertEquals(['invalid fingerprint_type "yeeah"'], ctx.exception.reasons) + # unrecognized fingerprint_type + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'SSHFP', + 'ttl': 600, + 'value': { + 'algorithm': 1, + 'fingerprint_type': 42, + 'fingerprint': 'bf6b6825d2977c511a475bbefb88aad54a92ac73' + } + }) + self.assertEquals(['unrecognized fingerprint_type "42"'], + ctx.exception.reasons) + # missing fingerprint with self.assertRaises(ValidationError) as ctx: Record.new(self.zone, '', { From 598acc943d48eca9ce0c461137d7dfeb242e9d7f Mon Sep 17 00:00:00 2001 From: Heesu Hwang Date: Fri, 23 Jun 2017 15:19:27 -0700 Subject: [PATCH 23/64] Added full support of Azure DNS. TODO: testing. --- .gitignore | 5 +- doit.txt | 4 + octodns/provider/azuredns.py | 297 ++++++++++++++++++----------------- 3 files changed, 164 insertions(+), 142 deletions(-) create mode 100755 doit.txt diff --git a/.gitignore b/.gitignore index eca95c9..5a3f3f9 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ nosetests.xml octodns.egg-info/ output/ tmp/ -Makefile build/ -config/ \ No newline at end of file +config/ +./rb.txt +./doit.txt diff --git a/doit.txt b/doit.txt new file mode 100755 index 0000000..1ff2da6 --- /dev/null +++ b/doit.txt @@ -0,0 +1,4 @@ +#!/bin/bash +#script to rebuild octodns quickly + +octodns-sync --config-file=./config/production.yaml --doit diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 66ca8e3..07b39fe 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -8,54 +8,66 @@ import sys from azure.common.credentials import ServicePrincipalCredentials from azure.mgmt.dns import DnsManagementClient -from azure.mgmt.dns.models import * +from azure.mgmt.dns.models import ARecord, AaaaRecord, CnameRecord, MxRecord, \ + SrvRecord, NsRecord, PtrRecord, TxtRecord, Zone + +from functools import reduce -from collections import defaultdict -# from incf.countryutils.transformations import cca_to_ctca2 TODO: add geo sup. import logging import re from ..record import Record, Update from .base import BaseProvider - -#TODO: changes made to master include adding /build, Makefile to .gitignore and -# making Makefile. -# Only made for A records. will have to adjust for more generic params types class _AzureRecord(object): - def __init__(self, resource_group, record, values=None, ttl=1800): - # print('Here4',file=sys.stderr) + ''' + Wrapper for OctoDNS record. + azuredns.py: + class: octodns.provider.azuredns._AzureRecord + An _AzureRecord is easily accessible to the Azure DNS Management library + functions and is used to wrap all relevant data to create a record in + Azure. + ''' + + def __init__(self, resource_group, record, values=None): + ''' + :param resource_group: The name of resource group in Azure + :type resource_group: str + :param record: An OctoDNS record + :type record: ..record.Record + :param values: Parameters for a record. eg IP address, port, domain + name, etc. Values usually read from record.data + :type values: {'values': [...]} or {'value': [...]} + + :type return: _AzureRecord + ''' self.resource_group = resource_group - self.zone_name = record.zone.name[0:len(record.zone.name)-1] # strips last period + self.zone_name = record.zone.name[0:len(record.zone.name)-1] self.relative_record_set_name = record.name or '@' self.record_type = record._type - type_name = '{}records'.format(self.record_type).lower() + data = values or record.data + format_u_s = '' if record._type == 'A' else '_' + key_name ='{}{}records'.format(self.record_type, format_u_s).lower() class_name = '{}'.format(self.record_type).capitalize() + \ 'Record'.format(self.record_type) - if values == None: - return - # TODO: clean up this bit. - data = values or record.data #This should fail if it gets to record.data? It only returns ttl. TODO - - #depending on mult values or not - #TODO: import explicitly. eval() uses for example ARecord from azure.mgmt.dns.models - self.params = {} - try: - self.params = {'ttl':record.ttl or ttl, \ - type_name:[eval(class_name)(ip) for ip in data['values']] or []} - except KeyError: # means that doesn't have multiple values but single value - self.params = {'ttl':record.ttl or ttl, \ - type_name:[eval(class_name)(data['value'])] or []} + self.params = None + if not self.record_type == 'CNAME': + self.params = self._params(data, key_name, eval(class_name)) + else: + self.params = {'cname_record': CnameRecord(data['value'])} + self.params['ttl'] = record.ttl + def _params(self, data, key_name, azure_class): + return {key_name: [azure_class(v) for v in data['values']]} \ + if 'values' in data else {key_name: [azure_class(data['value'])]} - class AzureProvider(BaseProvider): ''' Azure DNS Provider - - azure.py: - class: octodns.provider.azure.AzureProvider + + azuredns.py: + class: octodns.provider.azuredns.AzureProvider # Current support of authentication of access to Azure services only # includes using a Service Principal: # https://docs.microsoft.com/en-us/azure/azure-resource-manager/ @@ -71,90 +83,118 @@ class AzureProvider(BaseProvider): # Resource Group name req: resource_group: - testing: test authentication vars located in /home/t-hehwan/vars.txt + TODO: change the config file to use env variables instead of hard-coded keys? + + personal notes: testing: test authentication vars located in /home/t-hehwan/vars.txt ''' - SUPPORTS_GEO = False # TODO. Will add support as project progresses. + SUPPORTS_GEO = False + SUPPORTS = set(('A', 'AAAA', 'CNAME', 'MX', 'NS', 'PTR', 'SRV', 'TXT')) - def __init__(self, id, client_id, key, directory_id, sub_id, resource_group, *args, **kwargs): + def __init__(self, id, client_id, key, directory_id, sub_id, resource_group, + *args, **kwargs): self.log = logging.getLogger('AzureProvider[{}]'.format(id)) self.log.debug('__init__: id=%s, client_id=%s, ' 'key=***, directory_id:%s', id, client_id, directory_id) super(AzureProvider, self).__init__(id, *args, **kwargs) credentials = ServicePrincipalCredentials( - client_id = client_id, secret = key, tenant = directory_id + client_id, secret = key, tenant = directory_id ) self._dns_client = DnsManagementClient(credentials, sub_id) self._resource_group = resource_group - - self._azure_zones = None # will be a dictionary. key: name. val: id. - self._azure_records = {} # will be dict by octodns record, az record - - self._supported_types = ['CNAME', 'A', 'AAAA', 'MX', 'SRV', 'NS', 'PTR'] - # TODO: add TXT - - # TODO: health checks a la route53. - - + self._azure_zones = set() - # TODO: add support for all types. First skeleton: add A. - def supports(self, record): - return record._type in self._supported_types - - @property - def azure_zones(self): - if self._azure_zones is None: - self.log.debug('azure_zones: loading') - zones = {} - for zone in self._dns_client.zones.list_by_resource_group(self._resource_group): - zones[zone.name] = zone.id - self._azure_zones = zones - return self._azure_zones - - # Given a zone name, returns the zone id. If DNE, creates it. - def _get_zone_id(self, name, create=False): - self.log.debug('_get_zone_id: name=%s', name) + def _populate_zones(self): + self.log.debug('azure_zones: loading') + for zone in self._dns_client.zones.list_by_resource_group( + self._resource_group): + self._azure_zones.add(zone.name) + + def _check_zone(self, name, create=False): + ''' + Checks whether a zone specified in a source exist in Azure server. + Note that Azure zones omit end '.' eg: contoso.com vs contoso.com. + Returns the name if it exists. + + :param name: Name of a zone to checks + :type name: str + :param create: If True, creates the zone of that name. + :type create: bool + + :type return: str or None + ''' + self.log.debug('_check_zone: name=%s', name) try: - id = self._dns_client.zones.get(self._resource_group, name) - self.log.debug('_get_zone_id: id=%s', id) - return id + if name in self._azure_zones: + return name + if self._dns_client.zones.get(self._resource_group, name): + self._azure_zones.add(name) + return name except: if create: - self.log.debug('_get_zone_id: no matching zone; creating %s', name) - #TODO: write - return None #placeholder - return None + try: + self.log.debug('_check_zone: no matching zone; creating %s', + name) + if self._dns_client.zones.create_or_update( + self._resource_group, name, Zone('global')): #TODO: figure out what location should be + return name + except: + raise + return None - # Create a dictionary of record objects by zone and octodns record names - # TODO: add geo parsing - def populate(self, zone, target): - zone_name = zone.name[0:len(zone.name)-1]#Azure zone names do not include suffix . + def populate(self, zone, target=False): + ''' + Required function of manager.py. + + Special notes for Azure. Azure zone names omit final '.' + Azure record names for '' are represented by '@' + Azure records created through online interface may have null values + (eg, no IP address for A record). Specific quirks such as these are + responsible for any strange parsing. + + :param zone: A dns zone + :type zone: octodns.zone.Zone + :param target: Checks if Azure is source or target of config. + Currently only supports as a target. Does not use. + :type target: bool + + + TODO: azure interface allows null values. If this attempts to populate with them, will fail. add safety check (simply delete records with null values?) + + :type return: void + ''' + zone_name = zone.name[0:len(zone.name)-1] self.log.debug('populate: name=%s', zone_name) before = len(zone.records) - zone_id = self._get_zone_id(zone_name) - if zone_id: - #records = defaultdict(list) - for type in self._supported_types: - # print('populate. type: {}'.format(type),file=sys.stderr) - for azrecord in self._dns_client.record_sets.list_by_type(self._resource_group, zone_name, type): - # print(azrecord, file=sys.stderr) + + self._populate_zones() + if self._check_zone(zone_name): + for typ in self.SUPPORTS: + for azrecord in self._dns_client.record_sets.list_by_type( + self._resource_group, zone_name, typ): record_name = azrecord.name if azrecord.name != '@' else '' - data = self._type_and_ttl(type, azrecord, - getattr(self, '_data_for_{}'.format(type))(azrecord)) # TODO: azure online interface allows None values. must validate. + data = self._type_and_ttl(typ, azrecord.ttl, + getattr(self, '_data_for_{}'.format(typ))(azrecord)) + record = Record.new(zone, record_name, data, source=self) - # print('HERE0',file=sys.stderr) zone.add_record(record) - self._azure_records[record] = _AzureRecord(self._resource_group, record, data) - # print('HERE1',file=sys.stderr) + self.log.info('populate: found %s records', len(zone.records)-before) - - # might not need - def _get_type(azrecord): - azrecord['type'].split('/')[-1] - - def _type_and_ttl(self, type, azrecord, data): - data['type'] = type - data['ttl'] = azrecord.ttl + + def _type_and_ttl(self, typ, ttl, data): + ''' Adds type and ttl fields to return dictionary. + + :param typ: The type of a record + :type typ: str + :param ttl: The ttl of a record + :type ttl: int + :param data: Dictionary holding values of a record. eg, IP addresses + :type data: {'values': [...]} or {'value': [...]} + + :type return: {...} + ''' + data['type'] = typ + data['ttl'] = ttl return data def _data_for_A(self, azrecord): @@ -164,12 +204,10 @@ class AzureProvider(BaseProvider): return {'values': [ar.ipv6_address for ar in azrecord.aaaa_records]} def _data_for_TXT(self, azrecord): - print('azure',file=sys.stderr) - print([ar.value for ar in azrecord.txt_records], file=sys.stderr) - print('',file=sys.stderr) - return {'values': [ar.value for ar in azrecord.txt_records]} + return {'values': \ + [reduce((lambda a,b:a+b), ar.value) for ar in azrecord.txt_records]} - def _data_for_CNAME(self, azrecord): + def _data_for_CNAME(self, azrecord): #TODO: see TODO in population comment. try: val = azrecord.cname_record.cname if not val.endswith('.'): @@ -178,7 +216,7 @@ class AzureProvider(BaseProvider): except: return {'value': '.'} #TODO: this is a bad fix. but octo checks that cnames have trailing '.' while azure allows creating cnames on the online interface with no value. - def _data_for_PTR(self, azrecord): + def _data_for_PTR(self, azrecord): #TODO: see TODO in population comment. try: val = azrecord.ptr_records[0].ptdrname if not val.endswith('.'): @@ -198,21 +236,23 @@ class AzureProvider(BaseProvider): 'target': ar.target} for ar in azrecord.srv_records] } - def _data_for_NS(self, azrecord): + def _data_for_NS(self, azrecord): #TODO: see TODO in population comment. def period_validate(string): return string if string.endswith('.') else string + '.' vals = [ar.nsdname for ar in azrecord.ns_records] return {'values': [period_validate(val) for val in vals]} def _apply_Create(self, change): - new = change.new - - #validate that the zone exists. - #self._get_zone_id(new.name, create=True) - - ar = _AzureRecord(self._resource_group, new, new.data) - + ''' A record from change must be created. + + :param change: a change object + :type change: octodns.record.Change + + :type return: void + ''' + ar = _AzureRecord(self._resource_group, change.new) create = self._dns_client.record_sets.create_or_update + create(resource_group_name=ar.resource_group, zone_name=ar.zone_name, relative_record_set_name=ar.relative_record_set_name, @@ -220,56 +260,33 @@ class AzureProvider(BaseProvider): parameters=ar.params) def _apply_Delete(self, change): - existing = change.existing - ar = _AzureRecord(self._resource_group, existing) + ar = _AzureRecord(self._resource_group, change.existing) delete = self._dns_client.record_sets.delete + delete(self._resource_group, ar.zone_name, ar.relative_record_set_name, ar.record_type) def _apply_Update(self, change): - self._apply_Delete(change) self._apply_Create(change) - # type plan: Plan class from .base def _apply(self, plan): + ''' + Required function of manager.py + + :param plan: Contains the zones and changes to be made + :type plan: octodns.provider.base.Plan + + :type return: void + ''' desired = plan.desired changes = plan.changes self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name, len(changes)) - # validate that the zone exists. function creates zone if DNE. - self._get_zone_id(desired.name) + azure_zone_name = desired.name[0:len(desired.name)-1] + self._check_zone(azure_zone_name, create=True) - # Some parsing bits to call _mod_Create or _mod_Delete. - # changes is a list of Delete and Create objects. for change in changes: class_name = change.__class__.__name__ getattr(self, '_apply_{}'.format(class_name))(change) - - - # ********** - # Figuring out what object plan is. - - # self._executor = ThreadPoolExecutor(max_workers) - # futures.append(self._executor.submit(self._populate_and_plan, - # zone_name, sources, targets)) - # plans = [p for f in futures for p in f.results()] - # type of plans[0] == type of one output of _populate_and_plan - - # for target, plan in plans: - # apply(plan) - - - # type(target) == BaseProvider - # type(plan) == Plan() - - # Plan(existing, desired, changes) - # existing.type == desired.type == Zone(desired.name, desired.sub_zones) - # Zone(name, sub_zones) (str and set of strs) - # changes.type = [Delete/Create] - - - # Starts with sync in main() of sync. - # {u'values': ['3.3.3.3', '4.4.4.4'], u'type': 'A', u'ttl': 3600} - # {u'type': u'A', u'value': [u'3.3.3.3', u'4.4.4.4'], u'ttl': 3600L} \ No newline at end of file From 5e4d68094fe3e782540e623ab6fd303c37be5c19 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 24 Jun 2017 17:14:48 -0700 Subject: [PATCH 24/64] Add meta record support with provider id to zone Support replace=True in zone.add_record --- octodns/manager.py | 14 +++++++++++++- octodns/zone.py | 7 ++++++- tests/test_octodns_manager.py | 6 ++++++ tests/test_octodns_zone.py | 6 ++++++ 4 files changed, 31 insertions(+), 2 deletions(-) diff --git a/octodns/manager.py b/octodns/manager.py index 07719dd..e6fe253 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -13,6 +13,7 @@ import logging from .provider.base import BaseProvider from .provider.yaml import YamlProvider +from .record import Record from .yaml import safe_load from .zone import Zone @@ -59,7 +60,7 @@ class MainThreadExecutor(object): class Manager(object): log = logging.getLogger('Manager') - def __init__(self, config_file, max_workers=None): + def __init__(self, config_file, max_workers=None, include_meta=False): self.log.info('__init__: config_file=%s', config_file) # Read our config file @@ -75,6 +76,10 @@ class Manager(object): else: self._executor = MainThreadExecutor() + self.include_meta = include_meta or manager_config.get('include_meta', + False) + self.log.info('__init__: max_workers=%s', self.include_meta) + self.log.debug('__init__: configuring providers') self.providers = {} for provider_name, provider_config in self.config['providers'].items(): @@ -176,6 +181,13 @@ class Manager(object): plans = [] for target in targets: + if self.include_meta: + meta = Record.new(zone, 'octodns-meta', { + 'type': 'TXT', + 'ttl': 60, + 'value': 'provider={}'.format(target.id) + }) + zone.add_record(meta, replace=True) plan = target.plan(zone) if plan: plans.append((target, plan)) diff --git a/octodns/zone.py b/octodns/zone.py index 03bc41c..9d405bb 100644 --- a/octodns/zone.py +++ b/octodns/zone.py @@ -49,9 +49,13 @@ class Zone(object): def hostname_from_fqdn(self, fqdn): return self._name_re.sub('', fqdn) - def add_record(self, record): + def add_record(self, record, replace=False): name = record.name last = name.split('.')[-1] + + if replace and record in self.records: + self.records.remove(record) + if last in self.sub_zones: if name != last: # it's a record for something under a sub-zone @@ -63,6 +67,7 @@ class Zone(object): raise SubzoneRecordException('Record {} a managed sub-zone ' 'and not of type NS' .format(record.fqdn)) + # TODO: this is pretty inefficent for existing in self.records: if record == existing: raise DuplicateRecordException('Duplicate record {}, type {}' diff --git a/tests/test_octodns_manager.py b/tests/test_octodns_manager.py index 641c1ff..45a3b55 100644 --- a/tests/test_octodns_manager.py +++ b/tests/test_octodns_manager.py @@ -128,6 +128,12 @@ class TestManager(TestCase): .sync(dry_run=False, force=True) self.assertEquals(19, tc) + # Include meta + tc = Manager(get_config_filename('simple.yaml'), max_workers=1, + include_meta=True) \ + .sync(dry_run=False, force=True) + self.assertEquals(23, tc) + def test_eligible_targets(self): with TemporaryDirectory() as tmpdir: environ['YAML_TMP_DIR'] = tmpdir.dirname diff --git a/tests/test_octodns_zone.py b/tests/test_octodns_zone.py index a4d7300..f310397 100644 --- a/tests/test_octodns_zone.py +++ b/tests/test_octodns_zone.py @@ -39,6 +39,7 @@ class TestZone(TestCase): a = ARecord(zone, 'a', {'ttl': 42, 'value': '1.1.1.1'}) b = ARecord(zone, 'b', {'ttl': 42, 'value': '1.1.1.1'}) + c = ARecord(zone, 'a', {'ttl': 43, 'value': '2.2.2.2'}) zone.add_record(a) self.assertEquals(zone.records, set([a])) @@ -48,6 +49,11 @@ class TestZone(TestCase): self.assertEquals('Duplicate record a.unit.tests., type A', ctx.exception.message) self.assertEquals(zone.records, set([a])) + + # can add duplicate with replace=True + zone.add_record(c, replace=True) + self.assertEquals('2.2.2.2', list(zone.records)[0].values[0]) + # Can add dup name, with different type zone.add_record(b) self.assertEquals(zone.records, set([a, b])) From cc47bd70348ac711da8dc6d3ef2ce4b74a2642b4 Mon Sep 17 00:00:00 2001 From: Heesu Hwang Date: Tue, 27 Jun 2017 12:10:57 -0700 Subject: [PATCH 25/64] Fixed bug for MX and SRV. Added Azure test suite as well. --- octodns/provider/azuredns.py | 287 ++++++++++++++---------- rb.txt | 8 - tests/test_octodns_provider_azuredns.py | 171 ++++++++++++++ 3 files changed, 342 insertions(+), 124 deletions(-) delete mode 100755 rb.txt create mode 100644 tests/test_octodns_provider_azuredns.py diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 07b39fe..013613a 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -4,7 +4,6 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -import sys from azure.common.credentials import ServicePrincipalCredentials from azure.mgmt.dns import DnsManagementClient @@ -14,53 +13,120 @@ from azure.mgmt.dns.models import ARecord, AaaaRecord, CnameRecord, MxRecord, \ from functools import reduce import logging -import re -from ..record import Record, Update +from ..record import Record from .base import BaseProvider + class _AzureRecord(object): - ''' - Wrapper for OctoDNS record. - azuredns.py: + ''' Wrapper for OctoDNS record. + azuredns.py: class: octodns.provider.azuredns._AzureRecord - An _AzureRecord is easily accessible to the Azure DNS Management library - functions and is used to wrap all relevant data to create a record in + An _AzureRecord is easily accessible to Azure DNS Management library + functions and is used to wrap all relevant data to create a record in Azure. ''' - + def __init__(self, resource_group, record, values=None): - ''' + ''' :param resource_group: The name of resource group in Azure - :type resource_group: str - :param record: An OctoDNS record + :type resource_group: str + :param record: An OctoDNS record :type record: ..record.Record :param values: Parameters for a record. eg IP address, port, domain name, etc. Values usually read from record.data :type values: {'values': [...]} or {'value': [...]} - + :type return: _AzureRecord ''' self.resource_group = resource_group - self.zone_name = record.zone.name[0:len(record.zone.name)-1] + self.zone_name = record.zone.name[0:len(record.zone.name) - 1] self.relative_record_set_name = record.name or '@' self.record_type = record._type - + data = values or record.data format_u_s = '' if record._type == 'A' else '_' - key_name ='{}{}records'.format(self.record_type, format_u_s).lower() + key_name = '{}{}records'.format(self.record_type, format_u_s).lower() class_name = '{}'.format(self.record_type).capitalize() + \ 'Record'.format(self.record_type) - self.params = None - if not self.record_type == 'CNAME': - self.params = self._params(data, key_name, eval(class_name)) - else: - self.params = {'cname_record': CnameRecord(data['value'])} + + self.params = getattr(self, '_params_for_{}'.format(record._type)) + self.params = self.params(data, key_name, eval(class_name)) self.params['ttl'] = record.ttl def _params(self, data, key_name, azure_class): return {key_name: [azure_class(v) for v in data['values']]} \ if 'values' in data else {key_name: [azure_class(data['value'])]} + _params_for_A = _params + _params_for_AAAA = _params + _params_for_NS = _params + _params_for_PTR = _params + _params_for_TXT = _params + + def _params_for_SRV(self, data, key_name, azure_class): + params = [] + if 'values' in data: + for vals in data['values']: + params.append(azure_class(vals['priority'], + vals['weight'], + vals['port'], + vals['target'])) + else: + params.append(azure_class(data['value']['priority'], + data['value']['weight'], + data['value']['port'], + data['value']['target'])) + return {key_name: params} + + def _params_for_MX(self, data, key_name, azure_class): + params = [] + if 'values' in data: + for vals in data['values']: + params.append(azure_class(vals['priority'], + vals['value'])) + else: + params.append(azure_class(data['value']['priority'], + data['value']['value'])) + return {key_name: params} + + def _params_for_CNAME(self, data, key_name, azure_class): + return {'cname_record': CnameRecord(data['value'])} + + def _equals(self, b): + def parse_dict(params): + vals = [] + for char in params: + if char != 'ttl': + list_records = params[char] + try: + for record in list_records: + vals.append(record.__dict__) + except: + vals.append(list_records.__dict__) + vals.sort() + return vals + + return (self.resource_group == b.resource_group) & \ + (self.zone_name == b.zone_name) & \ + (self.record_type == b.record_type) & \ + (self.params['ttl'] == b.params['ttl']) & \ + (parse_dict(self.params) == parse_dict(b.params)) & \ + (self.relative_record_set_name == b.relative_record_set_name) + + def __str__(self): + string = 'Zone: {}; '.format(self.zone_name) + string += 'Name: {}; '.format(self.relative_record_set_name) + string += 'Type: {}; '.format(self.record_type) + string += 'Ttl: {}; '.format(self.params['ttl']) + for char in self.params: + if char != 'ttl': + try: + for rec in self.params[char]: + string += 'Record: {}; '.format(rec.__dict__) + except: + string += 'Record: {}; '.format(self.params[char].__dict__) + return string + class AzureProvider(BaseProvider): ''' @@ -68,11 +134,11 @@ class AzureProvider(BaseProvider): azuredns.py: class: octodns.provider.azuredns.AzureProvider - # Current support of authentication of access to Azure services only - # includes using a Service Principal: + # Current support of authentication of access to Azure services only + # includes using a Service Principal: # https://docs.microsoft.com/en-us/azure/azure-resource-manager/ # resource-group-create-service-principal-portal - # The Azure Active Directory Application ID (referred to client ID) req: + # The Azure Active Directory Application ID (aka client ID) req: client_id: # Authentication Key Value req: key: @@ -82,32 +148,33 @@ class AzureProvider(BaseProvider): sub_id: # Resource Group name req: resource_group: - - TODO: change the config file to use env variables instead of hard-coded keys? - - personal notes: testing: test authentication vars located in /home/t-hehwan/vars.txt + + TODO: change config file to use env vars instead of hard-coded keys + + personal notes: testing: test authentication vars located in + /home/t-hehwan/vars.txt ''' SUPPORTS_GEO = False SUPPORTS = set(('A', 'AAAA', 'CNAME', 'MX', 'NS', 'PTR', 'SRV', 'TXT')) - - def __init__(self, id, client_id, key, directory_id, sub_id, resource_group, - *args, **kwargs): + + def __init__(self, id, client_id, key, directory_id, sub_id, + resource_group, *args, **kwargs): self.log = logging.getLogger('AzureProvider[{}]'.format(id)) self.log.debug('__init__: id=%s, client_id=%s, ' - 'key=***, directory_id:%s', id, client_id, directory_id) + 'key=***, directory_id:%s', id, client_id, directory_id) super(AzureProvider, self).__init__(id, *args, **kwargs) credentials = ServicePrincipalCredentials( - client_id, secret = key, tenant = directory_id + client_id, secret=key, tenant=directory_id ) self._dns_client = DnsManagementClient(credentials, sub_id) self._resource_group = resource_group self._azure_zones = set() - + def _populate_zones(self): self.log.debug('azure_zones: loading') - for zone in self._dns_client.zones.list_by_resource_group( - self._resource_group): + list_zones = self._dns_client.zones.list_by_resource_group + for zone in list_zones(self._resource_group): self._azure_zones.add(zone.name) def _check_zone(self, name, create=False): @@ -115,12 +182,12 @@ class AzureProvider(BaseProvider): Checks whether a zone specified in a source exist in Azure server. Note that Azure zones omit end '.' eg: contoso.com vs contoso.com. Returns the name if it exists. - + :param name: Name of a zone to checks :type name: str :param create: If True, creates the zone of that name. :type create: bool - + :type return: str or None ''' self.log.debug('_check_zone: name=%s', name) @@ -130,163 +197,151 @@ class AzureProvider(BaseProvider): if self._dns_client.zones.get(self._resource_group, name): self._azure_zones.add(name) return name - except: + except: # TODO: figure out what location should be if create: try: - self.log.debug('_check_zone: no matching zone; creating %s', - name) - if self._dns_client.zones.create_or_update( - self._resource_group, name, Zone('global')): #TODO: figure out what location should be + self.log.debug('_check_zone:no matching zone; creating %s', + name) + create_zone = self._dns_client.zones.create_or_update + if create_zone(self._resource_group, name, Zone('global')): return name except: raise return None - - def populate(self, zone, target=False): + + def populate(self, zone, target=False, lenient=False): ''' Required function of manager.py. - + Special notes for Azure. Azure zone names omit final '.' Azure record names for '' are represented by '@' Azure records created through online interface may have null values (eg, no IP address for A record). Specific quirks such as these are responsible for any strange parsing. - + :param zone: A dns zone :type zone: octodns.zone.Zone - :param target: Checks if Azure is source or target of config. + :param target: Checks if Azure is source or target of config. Currently only supports as a target. Does not use. :type target: bool - - - TODO: azure interface allows null values. If this attempts to populate with them, will fail. add safety check (simply delete records with null values?) - + + + TODO: azure interface allows null values. If this attempts to + populate with them, will fail. add safety check (simply delete + records with null values?) + :type return: void ''' - zone_name = zone.name[0:len(zone.name)-1] + zone_name = zone.name[0:len(zone.name) - 1] self.log.debug('populate: name=%s', zone_name) before = len(zone.records) self._populate_zones() if self._check_zone(zone_name): for typ in self.SUPPORTS: - for azrecord in self._dns_client.record_sets.list_by_type( - self._resource_group, zone_name, typ): + records = self._dns_client.record_sets.list_by_type + for azrecord in records(self._resource_group, zone_name, typ): record_name = azrecord.name if azrecord.name != '@' else '' - data = self._type_and_ttl(typ, azrecord.ttl, - getattr(self, '_data_for_{}'.format(typ))(azrecord)) - + data = getattr(self, '_data_for_{}'.format(typ))(azrecord) + data['type'] = typ + data['ttl'] = azrecord.ttl + record = Record.new(zone, record_name, data, source=self) zone.add_record(record) - self.log.info('populate: found %s records', len(zone.records)-before) - - def _type_and_ttl(self, typ, ttl, data): - ''' Adds type and ttl fields to return dictionary. - - :param typ: The type of a record - :type typ: str - :param ttl: The ttl of a record - :type ttl: int - :param data: Dictionary holding values of a record. eg, IP addresses - :type data: {'values': [...]} or {'value': [...]} - - :type return: {...} - ''' - data['type'] = typ - data['ttl'] = ttl - return data - + self.log.info('populate: found %s records', len(zone.records) - before) + def _data_for_A(self, azrecord): return {'values': [ar.ipv4_address for ar in azrecord.arecords]} - + def _data_for_AAAA(self, azrecord): return {'values': [ar.ipv6_address for ar in azrecord.aaaa_records]} - + def _data_for_TXT(self, azrecord): - return {'values': \ - [reduce((lambda a,b:a+b), ar.value) for ar in azrecord.txt_records]} + return {'values': [reduce((lambda a, b: a + b), ar.value) + for ar in azrecord.txt_records]} - def _data_for_CNAME(self, azrecord): #TODO: see TODO in population comment. + def _data_for_CNAME(self, azrecord): # TODO: see TODO in pop comment. try: val = azrecord.cname_record.cname if not val.endswith('.'): val += '.' return {'value': val} except: - return {'value': '.'} #TODO: this is a bad fix. but octo checks that cnames have trailing '.' while azure allows creating cnames on the online interface with no value. - - def _data_for_PTR(self, azrecord): #TODO: see TODO in population comment. + return {'value': '.'} # TODO: this is a bad fix. but octo checks + # that cnames have trailing '.' while azure allows creating cnames + # on the online interface with no value. + + def _data_for_PTR(self, azrecord): # TODO: see TODO in population comment. try: val = azrecord.ptr_records[0].ptdrname if not val.endswith('.'): val += '.' return {'value': val} except: - return {'value': '.' } #TODO: this is a bad fix. but octo checks that cnames have trailing '.' while azure allows creating cnames on the online interface with no value. - + return {'value': '.'} + def _data_for_MX(self, azrecord): - return {'values': [{'priority':ar.preference, - 'value':ar.exchange} for ar in azrecord.mx_records]} - + return {'values': [{'priority': ar.preference, 'value': ar.exchange} + for ar in azrecord.mx_records] + } + def _data_for_SRV(self, azrecord): - return {'values': [{'priority': ar.priority, - 'weight': ar.weight, - 'port': ar.port, - 'target': ar.target} for ar in azrecord.srv_records] - } - - def _data_for_NS(self, azrecord): #TODO: see TODO in population comment. + return {'values': [{'priority': ar.priority, 'weight': ar.weight, + 'port': ar.port, 'target': ar.target} + for ar in azrecord.srv_records] + } + + def _data_for_NS(self, azrecord): # TODO: see TODO in population comment. def period_validate(string): return string if string.endswith('.') else string + '.' vals = [ar.nsdname for ar in azrecord.ns_records] return {'values': [period_validate(val) for val in vals]} def _apply_Create(self, change): - ''' A record from change must be created. - + '''A record from change must be created. + :param change: a change object :type change: octodns.record.Change - + :type return: void ''' ar = _AzureRecord(self._resource_group, change.new) create = self._dns_client.record_sets.create_or_update - - create(resource_group_name=ar.resource_group, - zone_name=ar.zone_name, - relative_record_set_name=ar.relative_record_set_name, - record_type=ar.record_type, + + create(resource_group_name=ar.resource_group, + zone_name=ar.zone_name, + relative_record_set_name=ar.relative_record_set_name, + record_type=ar.record_type, parameters=ar.params) - + def _apply_Delete(self, change): ar = _AzureRecord(self._resource_group, change.existing) delete = self._dns_client.record_sets.delete - - delete(self._resource_group, ar.zone_name, ar.relative_record_set_name, + + delete(self._resource_group, ar.zone_name, ar.relative_record_set_name, ar.record_type) - + def _apply_Update(self, change): self._apply_Create(change) - + def _apply(self, plan): ''' Required function of manager.py - + :param plan: Contains the zones and changes to be made :type plan: octodns.provider.base.Plan - + :type return: void ''' desired = plan.desired changes = plan.changes - self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name, + self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name, len(changes)) - - azure_zone_name = desired.name[0:len(desired.name)-1] + + azure_zone_name = desired.name[0:len(desired.name) - 1] self._check_zone(azure_zone_name, create=True) - + for change in changes: class_name = change.__class__.__name__ getattr(self, '_apply_{}'.format(class_name))(change) - diff --git a/rb.txt b/rb.txt deleted file mode 100755 index d6a7949..0000000 --- a/rb.txt +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -#script to rebuild octodns quickly - -sudo rm -r /home/t-hehwan/GitHub/octodns/build -sudo rm -r /home/t-hehwan/GitHub/octodns/octodns.egg-info -sudo python /home/t-hehwan/GitHub/octodns/setup.py -q build -sudo python /home/t-hehwan/GitHub/octodns/setup.py -q install -octodns-sync --config-file=./config/production.yaml \ No newline at end of file diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py new file mode 100644 index 0000000..db4d293 --- /dev/null +++ b/tests/test_octodns_provider_azuredns.py @@ -0,0 +1,171 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from octodns.record import Create, Delete, Record, Update +from octodns.provider.azuredns import _AzureRecord, AzureProvider +from octodns.zone import Zone + +from azure.mgmt.dns.models import ARecord, AaaaRecord, CnameRecord, MxRecord, \ + SrvRecord, NsRecord, PtrRecord, TxtRecord, Zone as AzureZone + +from octodns.zone import Zone + +from unittest import TestCase +import sys + + +class Test_AzureRecord(TestCase): + zone = Zone(name='unit.tests.', sub_zones=[]) + octo_records = [] + octo_records.append(Record.new(zone, '', { + 'ttl': 0, + 'type': 'A', + 'values': ['1.2.3.4', '10.10.10.10'] + })) + octo_records.append(Record.new(zone, 'a', { + 'ttl': 1, + 'type': 'A', + 'values': ['1.2.3.4', '1.1.1.1'], + })) + octo_records.append(Record.new(zone, 'aa', { + 'ttl': 9001, + 'type': 'A', + 'values': ['1.2.4.3'] + })) + octo_records.append(Record.new(zone, 'aaa', { + 'ttl': 2, + 'type': 'A', + 'values': ['1.1.1.3'] + })) + octo_records.append(Record.new(zone, 'cname', { + 'ttl': 3, + 'type': 'CNAME', + 'value': 'a.unit.tests.', + })) + octo_records.append(Record.new(zone, '', { + 'ttl': 3, + 'type': 'MX', + 'values': [{ + 'priority': 10, + 'value': 'mx1.unit.tests.', + }, { + 'priority': 20, + 'value': 'mx2.unit.tests.', + }] + })) + octo_records.append(Record.new(zone, '', { + 'ttl': 4, + 'type': 'NS', + 'values': ['ns1.unit.tests.', 'ns2.unit.tests.'], + })) + octo_records.append(Record.new(zone, '', { + 'ttl': 5, + 'type': 'NS', + 'value': 'ns1.unit.tests.', + })) + octo_records.append(Record.new(zone, '_srv._tcp', { + 'ttl': 6, + 'type': 'SRV', + 'values': [{ + 'priority': 10, + 'weight': 20, + 'port': 30, + 'target': 'foo-1.unit.tests.', + }, { + 'priority': 12, + 'weight': 30, + 'port': 30, + 'target': 'foo-2.unit.tests.', + }] + })) + + azure_records = [] + _base0 = _AzureRecord('TestAzure', octo_records[0]) + _base0.zone_name = 'unit.tests' + _base0.relative_record_set_name = '@' + _base0.record_type = 'A' + _base0.params['ttl'] = 0 + _base0.params['arecords'] = [ARecord('1.2.3.4'), ARecord('10.10.10.10')] + azure_records.append(_base0) + + _base1 = _AzureRecord('TestAzure', octo_records[1]) + _base1.zone_name = 'unit.tests' + _base1.relative_record_set_name = 'a' + _base1.record_type = 'A' + _base1.params['ttl'] = 1 + _base1.params['arecords'] = [ARecord('1.2.3.4'), ARecord('1.1.1.1')] + azure_records.append(_base1) + + _base2 = _AzureRecord('TestAzure', octo_records[2]) + _base2.zone_name = 'unit.tests' + _base2.relative_record_set_name = 'aa' + _base2.record_type = 'A' + _base2.params['ttl'] = 9001 + _base2.params['arecords'] = ARecord('1.2.4.3') + azure_records.append(_base2) + + _base3 = _AzureRecord('TestAzure', octo_records[3]) + _base3.zone_name = 'unit.tests' + _base3.relative_record_set_name = 'aaa' + _base3.record_type = 'A' + _base3.params['ttl'] = 2 + _base3.params['arecords'] = ARecord('1.1.1.3') + azure_records.append(_base3) + + _base4 = _AzureRecord('TestAzure', octo_records[4]) + _base4.zone_name = 'unit.tests' + _base4.relative_record_set_name = 'cname' + _base4.record_type = 'CNAME' + _base4.params['ttl'] = 3 + _base4.params['cname_record'] = CnameRecord('a.unit.tests.') + azure_records.append(_base4) + + _base5 = _AzureRecord('TestAzure', octo_records[5]) + _base5.zone_name = 'unit.tests' + _base5.relative_record_set_name = '@' + _base5.record_type = 'MX' + _base5.params['ttl'] = 3 + _base5.params['mx_records'] = [MxRecord(10, 'mx1.unit.tests.'), + MxRecord(20, 'mx2.unit.tests.')] + azure_records.append(_base5) + + _base6 = _AzureRecord('TestAzure', octo_records[6]) + _base6.zone_name = 'unit.tests' + _base6.relative_record_set_name = '@' + _base6.record_type = 'NS' + _base6.params['ttl'] = 4 + _base6.params['ns_records'] = [NsRecord('ns1.unit.tests.'), + NsRecord('ns2.unit.tests.')] + azure_records.append(_base6) + + _base7 = _AzureRecord('TestAzure', octo_records[7]) + _base7.zone_name = 'unit.tests' + _base7.relative_record_set_name = '@' + _base7.record_type = 'NS' + _base7.params['ttl'] = 5 + _base7.params['ns_records'] = [NsRecord('ns1.unit.tests.')] + azure_records.append(_base7) + + _base8 = _AzureRecord('TestAzure', octo_records[8]) + _base8.zone_name = 'unit.tests' + _base8.relative_record_set_name = '_srv._tcp' + _base8.record_type = 'SRV' + _base8.params['ttl'] = 6 + _base8.params['srv_records'] = [SrvRecord(10, 20, 30, 'foo-1.unit.tests.'), + SrvRecord(12, 30, 30, 'foo-2.unit.tests.')] + azure_records.append(_base8) + + def test_azure_record(self): + assert(len(self.azure_records) == len(self.octo_records)) + for i in range(len(self.azure_records)): + octo = _AzureRecord('TestAzure', self.octo_records[i]) + assert(self.azure_records[i]._equals(octo)) + + +class TestAzureDnsProvider(TestCase): + def test_populate(self): + pass # placeholder From 3a2ccdcac09497540ce8d681a3ae33c96fd3f389 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Wed, 28 Jun 2017 03:09:41 -0700 Subject: [PATCH 26/64] Manually join self.values to avoid double escapes, e.g. \\; --- octodns/record.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/octodns/record.py b/octodns/record.py index 23bfd45..6ee9dff 100644 --- a/octodns/record.py +++ b/octodns/record.py @@ -254,9 +254,10 @@ class _ValuesMixin(object): return ret def __repr__(self): + values = "['{}']".format("', '".join([str(v) for v in self.values])) return '<{} {} {}, {}, {}>'.format(self.__class__.__name__, self._type, self.ttl, - self.fqdn, self.values) + self.fqdn, values) class _GeoMixin(_ValuesMixin): From 0fb88a959a156ef2a4be6cf6ef00b70ccabfe3b6 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Wed, 28 Jun 2017 03:26:23 -0700 Subject: [PATCH 27/64] Add retry to ns1 provider --- octodns/provider/ns1.py | 29 +++++++++++++++++++++++++---- requirements.txt | 2 +- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 2f0a024..3b8ad00 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -7,7 +7,8 @@ from __future__ import absolute_import, division, print_function, \ from logging import getLogger from nsone import NSONE -from nsone.rest.errors import ResourceException +from nsone.rest.errors import RateLimitException, ResourceException +from time import sleep from ..record import Record from .base import BaseProvider @@ -25,6 +26,7 @@ class Ns1Provider(BaseProvider): SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CNAME', 'MX', 'NAPTR', 'NS', 'PTR', 'SPF', 'SRV', 'TXT')) + RATE_LIMIT_DELAY = 1 ZONE_NOT_FOUND_MESSAGE = 'server error: zone not found' def __init__(self, id, api_key, *args, **kwargs): @@ -171,7 +173,14 @@ class Ns1Provider(BaseProvider): name = self._get_name(new) _type = new._type params = getattr(self, '_params_for_{}'.format(_type))(new) - getattr(nsone_zone, 'add_{}'.format(_type))(name, **params) + meth = getattr(nsone_zone, 'add_{}'.format(_type)) + try: + meth(name, **params) + except RateLimitException: + self.log.warn('_apply_Create: rate limit encountered, pausing ' + 'and trying again') + sleep(self.RATE_LIMIT_DELAY) + meth(name, **params) def _apply_Update(self, nsone_zone, change): existing = change.existing @@ -180,14 +189,26 @@ class Ns1Provider(BaseProvider): record = nsone_zone.loadRecord(name, _type) new = change.new params = getattr(self, '_params_for_{}'.format(_type))(new) - record.update(**params) + try: + record.update(**params) + except RateLimitException: + self.log.warn('_apply_Update: rate limit encountered, pausing ' + 'and trying again') + sleep(self.RATE_LIMIT_DELAY) + record.update(**params) def _apply_Delete(self, nsone_zone, change): existing = change.existing name = self._get_name(existing) _type = existing._type record = nsone_zone.loadRecord(name, _type) - record.delete() + try: + record.delete() + except RateLimitException: + self.log.warn('_apply_Delete: rate limit encountered, pausing ' + 'and trying again') + sleep(self.RATE_LIMIT_DELAY) + record.delete() def _apply(self, plan): desired = plan.desired diff --git a/requirements.txt b/requirements.txt index b10ca4c..93a8521 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,7 @@ incf.countryutils==1.0 ipaddress==1.0.18 jmespath==0.9.0 natsort==5.0.3 -nsone==0.9.10 +nsone==0.9.14 python-dateutil==2.6.0 requests==2.13.0 s3transfer==0.1.10 From a44b82c2c79e6025c605f77fd797c2254c2939dd Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Wed, 28 Jun 2017 04:11:46 -0700 Subject: [PATCH 28/64] NS1 rate_limit_delay param, unit tests for rate limit handling --- octodns/provider/ns1.py | 13 ++++++----- tests/test_octodns_provider_ns1.py | 35 +++++++++++++++++++++++++----- 2 files changed, 36 insertions(+), 12 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 3b8ad00..0f3db1f 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -26,14 +26,15 @@ class Ns1Provider(BaseProvider): SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CNAME', 'MX', 'NAPTR', 'NS', 'PTR', 'SPF', 'SRV', 'TXT')) - RATE_LIMIT_DELAY = 1 ZONE_NOT_FOUND_MESSAGE = 'server error: zone not found' - def __init__(self, id, api_key, *args, **kwargs): + def __init__(self, id, api_key, rate_limit_delay=1, *args, **kwargs): self.log = getLogger('Ns1Provider[{}]'.format(id)) - self.log.debug('__init__: id=%s, api_key=***', id) + self.log.debug('__init__: id=%s, api_key=***, rate_limit_delay=%d', id, + rate_limit_delay) super(Ns1Provider, self).__init__(id, *args, **kwargs) self._client = NSONE(apiKey=api_key) + self.rate_limit_delay = rate_limit_delay def _data_for_A(self, _type, record): return { @@ -179,7 +180,7 @@ class Ns1Provider(BaseProvider): except RateLimitException: self.log.warn('_apply_Create: rate limit encountered, pausing ' 'and trying again') - sleep(self.RATE_LIMIT_DELAY) + sleep(self.rate_limit_delay) meth(name, **params) def _apply_Update(self, nsone_zone, change): @@ -194,7 +195,7 @@ class Ns1Provider(BaseProvider): except RateLimitException: self.log.warn('_apply_Update: rate limit encountered, pausing ' 'and trying again') - sleep(self.RATE_LIMIT_DELAY) + sleep(self.rate_limit_delay) record.update(**params) def _apply_Delete(self, nsone_zone, change): @@ -207,7 +208,7 @@ class Ns1Provider(BaseProvider): except RateLimitException: self.log.warn('_apply_Delete: rate limit encountered, pausing ' 'and trying again') - sleep(self.RATE_LIMIT_DELAY) + sleep(self.rate_limit_delay) record.delete() def _apply(self, plan): diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index ecc107c..5e53cfd 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -6,7 +6,8 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals from mock import Mock, call, patch -from nsone.rest.errors import AuthException, ResourceException +from nsone.rest.errors import AuthException, RateLimitException, \ + ResourceException from unittest import TestCase from octodns.record import Delete, Record, Update @@ -192,7 +193,7 @@ class TestNs1Provider(TestCase): @patch('nsone.NSONE.createZone') @patch('nsone.NSONE.loadZone') def test_sync(self, load_mock, create_mock): - provider = Ns1Provider('test', 'api-key') + provider = Ns1Provider('test', 'api-key', rate_limit_delay=0) desired = Zone('unit.tests.', []) desired.records.update(self.expected) @@ -225,7 +226,15 @@ class TestNs1Provider(TestCase): create_mock.reset_mock() load_mock.side_effect = \ ResourceException('server error: zone not found') - create_mock.side_effect = None + # ugh, need a mock zone with a mock prop since we're using getattr, we + # can actually control side effects on `meth` with that. + mock_zone = Mock() + mock_zone.add_SRV = Mock() + mock_zone.add_SRV.side_effect = [ + RateLimitException('boo'), + None, + ] + create_mock.side_effect = [mock_zone] got_n = provider.apply(plan) self.assertEquals(expected_n, got_n) @@ -245,12 +254,26 @@ class TestNs1Provider(TestCase): self.assertEquals(2, len(plan.changes)) self.assertIsInstance(plan.changes[0], Update) self.assertIsInstance(plan.changes[1], Delete) - + # ugh, we need a mock record that can be returned from loadRecord for + # the update and delete targets, we can add our side effects to that to + # trigger rate limit handling + mock_record = Mock() + mock_record.update.side_effect = [ + RateLimitException('one'), + None, + ] + mock_record.delete.side_effect = [ + RateLimitException('two'), + None, + ] + nsone_zone.loadRecord.side_effect = [mock_record, mock_record] got_n = provider.apply(plan) self.assertEquals(2, got_n) nsone_zone.loadRecord.assert_has_calls([ call('unit.tests', u'A'), - call().update(answers=[u'1.2.3.4'], ttl=32), call('delete-me', u'A'), - call().delete() + ]) + mock_record.assert_has_calls([ + call.update(answers=[u'1.2.3.4'], ttl=32), + call.delete() ]) From 67c2f9767ba7e74c66f21fbe938050ef86a7007c Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Wed, 28 Jun 2017 04:46:59 -0700 Subject: [PATCH 29/64] CHANGELOG, version bump, pass at release script --- CHANGELOG.md | 44 ++++++++++++++++++++++++++++++++++++++++++++ octodns/__init__.py | 2 +- script/release | 12 ++++++++++++ 3 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 CHANGELOG.md create mode 100755 script/release diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..1cb1204 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,44 @@ + +## v0.8.4 - 2017-03-14 - It's been too long + +Lots of updates based on our internal use, needs, and feedback & suggestions +from our OSS users. There's too much to list out since the previous release was +cut, but I'll try to cover the highlights/important bits and promise to do +better in the future :fingers_crossed: + +#### Major: + +* Complete rework of record validation with lenient mode support added to + octodns-dump so that data with validation problems can be dumped to config + files as a starting point. octoDNS now also ignores validation errors when + pulling the current state from a provider before planning changes. In both + cases this is best effort. +* Naming of record keys are based on RFC-1035 and friends, previous names have + been kept for backwards compatibility until the 1.0 release. +* Provider record type support is now explicit, i.e. opt-in, rather than + opt-out. This prevents bugs/oversights in record handling where providers + don't support (new) record types and didn't correctly ignore them. +* ALIAS support for DNSimple, Dyn, NS1, PowerDNS +* Ignored record support added, `octodns:\n ignored: True` +* Ns1Provider added + +#### Miscellaneous + +* Use a 3rd party lib for nautrual sorting of keys, rather than my old + implementation. Sorting can be disabled in the YamlProvider with + `enforce_order: False`. +* Semi-colon/escaping fixes and improvements. +* Meta record support, `TXT octodns-meta.`. For now just + `provider=`. Optionally turned on with `include_meta` manager + config val. +* Validations check for CNAMEs co-existing with other records and error out if + found. Was a common mistaken/unknown issue and this surfaces the problem + early. +* Sizeable refactor in the way Route53 record translation works to make it + cleaner/less hacky +* Lots of docs type-o fixes +* Fixed some pretty major bugs in DnsimpleProvider +* Relax UnsafePlan checks a bit, more to come here +* Set User-Agent header on Dyn health checks + +## v0.8.0 - 2017-03-14 - First public release diff --git a/octodns/__init__.py b/octodns/__init__.py index 4806766..b6287e5 100644 --- a/octodns/__init__.py +++ b/octodns/__init__.py @@ -5,4 +5,4 @@ OctoDNS: DNS as code - Tools for managing DNS across multiple providers from __future__ import absolute_import, division, print_function, \ unicode_literals -__VERSION__ = '0.8.0' +__VERSION__ = '0.8.4' diff --git a/script/release b/script/release new file mode 100755 index 0000000..16e7641 --- /dev/null +++ b/script/release @@ -0,0 +1,12 @@ +#!/bin/bash + +set -e + +cd "$(dirname $0)"/.. +ROOT=$(pwd) + +VERSION=$(grep __VERSION__ $ROOT/octodns/__init__.py | sed -e "s/.* = '//" -e "s/'$//") + +git tag -s v$VERSION -m "Release $VERSION" +python setup.py sdist upload +echo "Updloaded $VERSION" From 08d3fda99ef17e4ae0ca529989c14a356b8f925c Mon Sep 17 00:00:00 2001 From: Heesu Hwang Date: Wed, 28 Jun 2017 14:00:59 -0700 Subject: [PATCH 30/64] safety check for azure null values --- octodns/provider/azuredns.py | 104 ++++++++++++++++++++--------------- 1 file changed, 61 insertions(+), 43 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 013613a..6a2db41 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -11,7 +11,7 @@ from azure.mgmt.dns.models import ARecord, AaaaRecord, CnameRecord, MxRecord, \ SrvRecord, NsRecord, PtrRecord, TxtRecord, Zone from functools import reduce - +import sys import logging from ..record import Record from .base import BaseProvider @@ -26,7 +26,7 @@ class _AzureRecord(object): Azure. ''' - def __init__(self, resource_group, record, values=None): + def __init__(self, resource_group, record, delete=False): ''' :param resource_group: The name of resource group in Azure :type resource_group: str @@ -43,14 +43,16 @@ class _AzureRecord(object): self.relative_record_set_name = record.name or '@' self.record_type = record._type - data = values or record.data + if delete: + return + format_u_s = '' if record._type == 'A' else '_' key_name = '{}{}records'.format(self.record_type, format_u_s).lower() class_name = '{}'.format(self.record_type).capitalize() + \ 'Record'.format(self.record_type) self.params = getattr(self, '_params_for_{}'.format(record._type)) - self.params = self.params(data, key_name, eval(class_name)) + self.params = self.params(record.data, key_name, eval(class_name)) self.params['ttl'] = record.ttl def _params(self, data, key_name, azure_class): @@ -71,7 +73,7 @@ class _AzureRecord(object): vals['weight'], vals['port'], vals['target'])) - else: + else: # single value at key 'value' params.append(azure_class(data['value']['priority'], data['value']['weight'], data['value']['port'], @@ -82,17 +84,23 @@ class _AzureRecord(object): params = [] if 'values' in data: for vals in data['values']: - params.append(azure_class(vals['priority'], - vals['value'])) - else: - params.append(azure_class(data['value']['priority'], - data['value']['value'])) + params.append(azure_class(vals['preference'], + vals['exchange'])) + else: # single value at key 'value' + params.append(azure_class(data['value']['preference'], + data['value']['exchange'])) return {key_name: params} def _params_for_CNAME(self, data, key_name, azure_class): return {'cname_record': CnameRecord(data['value'])} def _equals(self, b): + '''Checks whether two records are equal by comparing all fields. + :param b: Another _AzureRecord object + :type b: _AzureRecord + + :type return: bool + ''' def parse_dict(params): vals = [] for char in params: @@ -114,6 +122,9 @@ class _AzureRecord(object): (self.relative_record_set_name == b.relative_record_set_name) def __str__(self): + '''String representation of an _AzureRecord. + :type return: str + ''' string = 'Zone: {}; '.format(self.zone_name) string += 'Name: {}; '.format(self.relative_record_set_name) string += 'Type: {}; '.format(self.record_type) @@ -128,6 +139,10 @@ class _AzureRecord(object): return string +def period_validate(string): + return string if string.endswith('.') else string + '.' + + class AzureProvider(BaseProvider): ''' Azure DNS Provider @@ -213,22 +228,24 @@ class AzureProvider(BaseProvider): ''' Required function of manager.py. - Special notes for Azure. Azure zone names omit final '.' - Azure record names for '' are represented by '@' + Special notes for Azure. + Azure zone names omit final '.' + Azure root records names are represented by '@'. OctoDNS uses '' Azure records created through online interface may have null values - (eg, no IP address for A record). Specific quirks such as these are - responsible for any strange parsing. + (eg, no IP address for A record). + Azure online interface allows constructing records with null values + which are destroyed by _apply. + + Specific quirks such as these are responsible for any non-obvious + parsing in this function and the functions '_params_for_*'. :param zone: A dns zone :type zone: octodns.zone.Zone :param target: Checks if Azure is source or target of config. Currently only supports as a target. Does not use. :type target: bool - - - TODO: azure interface allows null values. If this attempts to - populate with them, will fail. add safety check (simply delete - records with null values?) + :param lenient: Unused. Check octodns.manager for usage. + :type lenient: bool :type return: void ''' @@ -261,40 +278,38 @@ class AzureProvider(BaseProvider): return {'values': [reduce((lambda a, b: a + b), ar.value) for ar in azrecord.txt_records]} - def _data_for_CNAME(self, azrecord): # TODO: see TODO in pop comment. + def _data_for_CNAME(self, azrecord): + '''Parsing data from Azure DNS Client record call + :param azrecord: a return of a call to list azure records + :type azrecord: azure.mgmt.dns.models.RecordSet + + :type return: dict + + CNAME and PTR both use the catch block to catch possible empty + records. Refer to population comment. + ''' try: - val = azrecord.cname_record.cname - if not val.endswith('.'): - val += '.' - return {'value': val} + return {'value': period_validate(azrecord.cname_record.cname)} except: - return {'value': '.'} # TODO: this is a bad fix. but octo checks - # that cnames have trailing '.' while azure allows creating cnames - # on the online interface with no value. + return {'value': '.'} - def _data_for_PTR(self, azrecord): # TODO: see TODO in population comment. + def _data_for_PTR(self, azrecord): try: - val = azrecord.ptr_records[0].ptdrname - if not val.endswith('.'): - val += '.' - return {'value': val} + return {'value': period_validate(azrecord.ptr_records[0].ptdrname)} except: return {'value': '.'} def _data_for_MX(self, azrecord): - return {'values': [{'priority': ar.preference, 'value': ar.exchange} - for ar in azrecord.mx_records] - } + return {'values': [{'preference': ar.preference, + 'exchange': ar.exchange} + for ar in azrecord.mx_records]} def _data_for_SRV(self, azrecord): return {'values': [{'priority': ar.priority, 'weight': ar.weight, 'port': ar.port, 'target': ar.target} - for ar in azrecord.srv_records] - } + for ar in azrecord.srv_records]} - def _data_for_NS(self, azrecord): # TODO: see TODO in population comment. - def period_validate(string): - return string if string.endswith('.') else string + '.' + def _data_for_NS(self, azrecord): vals = [ar.nsdname for ar in azrecord.ns_records] return {'values': [period_validate(val) for val in vals]} @@ -315,15 +330,18 @@ class AzureProvider(BaseProvider): record_type=ar.record_type, parameters=ar.params) + print('* Success Create/Update: {}'.format(ar), file=sys.stderr) + + _apply_Update = _apply_Create + def _apply_Delete(self, change): - ar = _AzureRecord(self._resource_group, change.existing) + ar = _AzureRecord(self._resource_group, change.existing, delete=True) delete = self._dns_client.record_sets.delete delete(self._resource_group, ar.zone_name, ar.relative_record_set_name, ar.record_type) - def _apply_Update(self, change): - self._apply_Create(change) + print('* Success Delete: {}'.format(ar), file=sys.stderr) def _apply(self, plan): ''' From b0de5de445f5cffdba2760939c9d9eb96c97b2bb Mon Sep 17 00:00:00 2001 From: anthonyvia Date: Thu, 29 Jun 2017 09:55:52 -0700 Subject: [PATCH 31/64] Supply 'Marker' to Route53 client when paging in order to correctly retrieve the next page of results. --- octodns/provider/route53.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index b4fcab8..490d630 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -250,7 +250,7 @@ class Route53Provider(BaseProvider): more = True start = {} while more: - resp = self._conn.list_hosted_zones() + resp = self._conn.list_hosted_zones(**start) for z in resp['HostedZones']: zones[z['Name']] = z['Id'] more = resp['IsTruncated'] From 0b2275c4e6388d25a28e501152fae4a88f2cf0a1 Mon Sep 17 00:00:00 2001 From: Heesu Hwang Date: Fri, 30 Jun 2017 11:06:42 -0700 Subject: [PATCH 32/64] Added complete error testing suite for azuredns --- octodns/provider/azuredns.py | 64 ++-- tests/test_octodns_provider_azuredns.py | 456 ++++++++++++++++-------- 2 files changed, 339 insertions(+), 181 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 6a2db41..cf0e2d5 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -128,6 +128,8 @@ class _AzureRecord(object): string = 'Zone: {}; '.format(self.zone_name) string += 'Name: {}; '.format(self.relative_record_set_name) string += 'Type: {}; '.format(self.record_type) + if not hasattr(self, 'params'): + return string string += 'Ttl: {}; '.format(self.params['ttl']) for char in self.params: if char != 'ttl': @@ -139,7 +141,7 @@ class _AzureRecord(object): return string -def period_validate(string): +def _validate_per(string): return string if string.endswith('.') else string + '.' @@ -209,20 +211,18 @@ class AzureProvider(BaseProvider): try: if name in self._azure_zones: return name - if self._dns_client.zones.get(self._resource_group, name): - self._azure_zones.add(name) - return name - except: # TODO: figure out what location should be + self._dns_client.zones.get(self._resource_group, name) + self._azure_zones.add(name) + return name + except: if create: - try: - self.log.debug('_check_zone:no matching zone; creating %s', - name) - create_zone = self._dns_client.zones.create_or_update - if create_zone(self._resource_group, name, Zone('global')): - return name - except: - raise - return None + self.log.debug('_check_zone:no matching zone; creating %s', + name) + create_zone = self._dns_client.zones.create_or_update + create_zone(self._resource_group, name, Zone('global')) + return name + else: + raise def populate(self, zone, target=False, lenient=False): ''' @@ -254,17 +254,21 @@ class AzureProvider(BaseProvider): before = len(zone.records) self._populate_zones() - if self._check_zone(zone_name): - for typ in self.SUPPORTS: - records = self._dns_client.record_sets.list_by_type - for azrecord in records(self._resource_group, zone_name, typ): - record_name = azrecord.name if azrecord.name != '@' else '' - data = getattr(self, '_data_for_{}'.format(typ))(azrecord) - data['type'] = typ - data['ttl'] = azrecord.ttl - - record = Record.new(zone, record_name, data, source=self) - zone.add_record(record) + self._check_zone(zone_name) + + _records = set() + records = self._dns_client.record_sets.list_by_dns_zone + for azrecord in records(self._resource_group, zone_name): + if azrecord.type in self.SUPPORTS: + _records.add(azrecord) + for azrecord in _records: + record_name = azrecord.name if azrecord.name != '@' else '' + data = getattr(self, '_data_for_{}'.format(azrecord.type)) + data = data(azrecord) + data['type'] = azrecord.type + data['ttl'] = azrecord.ttl + record = Record.new(zone, record_name, data, source=self) + zone.add_record(record) self.log.info('populate: found %s records', len(zone.records) - before) @@ -289,13 +293,13 @@ class AzureProvider(BaseProvider): records. Refer to population comment. ''' try: - return {'value': period_validate(azrecord.cname_record.cname)} + return {'value': _validate_per(azrecord.cname_record.cname)} except: return {'value': '.'} def _data_for_PTR(self, azrecord): try: - return {'value': period_validate(azrecord.ptr_records[0].ptdrname)} + return {'value': _validate_per(azrecord.ptr_records[0].ptdrname)} except: return {'value': '.'} @@ -311,7 +315,7 @@ class AzureProvider(BaseProvider): def _data_for_NS(self, azrecord): vals = [ar.nsdname for ar in azrecord.ns_records] - return {'values': [period_validate(val) for val in vals]} + return {'values': [_validate_per(val) for val in vals]} def _apply_Create(self, change): '''A record from change must be created. @@ -330,7 +334,7 @@ class AzureProvider(BaseProvider): record_type=ar.record_type, parameters=ar.params) - print('* Success Create/Update: {}'.format(ar), file=sys.stderr) + self.log.debug('* Success Create/Update: {}'.format(ar)) _apply_Update = _apply_Create @@ -341,7 +345,7 @@ class AzureProvider(BaseProvider): delete(self._resource_group, ar.zone_name, ar.relative_record_set_name, ar.record_type) - print('* Success Delete: {}'.format(ar), file=sys.stderr) + self.log.debug('* Success Delete: {}'.format(ar)) def _apply(self, plan): ''' diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index db4d293..edb2db2 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -5,167 +5,321 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -from octodns.record import Create, Delete, Record, Update -from octodns.provider.azuredns import _AzureRecord, AzureProvider +from octodns.record import Create, Delete, Record +from octodns.provider.azuredns import _AzureRecord, AzureProvider, \ + _validate_per from octodns.zone import Zone +from octodns.provider.base import Plan from azure.mgmt.dns.models import ARecord, AaaaRecord, CnameRecord, MxRecord, \ - SrvRecord, NsRecord, PtrRecord, TxtRecord, Zone as AzureZone + SrvRecord, NsRecord, PtrRecord, TxtRecord, RecordSet, SoaRecord, \ + Zone as AzureZone +from msrestazure.azure_exceptions import CloudError -from octodns.zone import Zone from unittest import TestCase -import sys +from mock import Mock, patch -class Test_AzureRecord(TestCase): - zone = Zone(name='unit.tests.', sub_zones=[]) - octo_records = [] - octo_records.append(Record.new(zone, '', { - 'ttl': 0, - 'type': 'A', - 'values': ['1.2.3.4', '10.10.10.10'] - })) - octo_records.append(Record.new(zone, 'a', { - 'ttl': 1, - 'type': 'A', - 'values': ['1.2.3.4', '1.1.1.1'], - })) - octo_records.append(Record.new(zone, 'aa', { - 'ttl': 9001, - 'type': 'A', - 'values': ['1.2.4.3'] - })) - octo_records.append(Record.new(zone, 'aaa', { - 'ttl': 2, - 'type': 'A', - 'values': ['1.1.1.3'] - })) - octo_records.append(Record.new(zone, 'cname', { - 'ttl': 3, - 'type': 'CNAME', - 'value': 'a.unit.tests.', - })) - octo_records.append(Record.new(zone, '', { - 'ttl': 3, - 'type': 'MX', - 'values': [{ - 'priority': 10, - 'value': 'mx1.unit.tests.', - }, { - 'priority': 20, - 'value': 'mx2.unit.tests.', - }] - })) - octo_records.append(Record.new(zone, '', { - 'ttl': 4, - 'type': 'NS', - 'values': ['ns1.unit.tests.', 'ns2.unit.tests.'], - })) - octo_records.append(Record.new(zone, '', { - 'ttl': 5, - 'type': 'NS', - 'value': 'ns1.unit.tests.', - })) - octo_records.append(Record.new(zone, '_srv._tcp', { - 'ttl': 6, - 'type': 'SRV', - 'values': [{ - 'priority': 10, - 'weight': 20, - 'port': 30, - 'target': 'foo-1.unit.tests.', - }, { - 'priority': 12, - 'weight': 30, - 'port': 30, - 'target': 'foo-2.unit.tests.', - }] - })) - - azure_records = [] - _base0 = _AzureRecord('TestAzure', octo_records[0]) - _base0.zone_name = 'unit.tests' - _base0.relative_record_set_name = '@' - _base0.record_type = 'A' - _base0.params['ttl'] = 0 - _base0.params['arecords'] = [ARecord('1.2.3.4'), ARecord('10.10.10.10')] - azure_records.append(_base0) - - _base1 = _AzureRecord('TestAzure', octo_records[1]) - _base1.zone_name = 'unit.tests' - _base1.relative_record_set_name = 'a' - _base1.record_type = 'A' - _base1.params['ttl'] = 1 - _base1.params['arecords'] = [ARecord('1.2.3.4'), ARecord('1.1.1.1')] - azure_records.append(_base1) - - _base2 = _AzureRecord('TestAzure', octo_records[2]) - _base2.zone_name = 'unit.tests' - _base2.relative_record_set_name = 'aa' - _base2.record_type = 'A' - _base2.params['ttl'] = 9001 - _base2.params['arecords'] = ARecord('1.2.4.3') - azure_records.append(_base2) - - _base3 = _AzureRecord('TestAzure', octo_records[3]) - _base3.zone_name = 'unit.tests' - _base3.relative_record_set_name = 'aaa' - _base3.record_type = 'A' - _base3.params['ttl'] = 2 - _base3.params['arecords'] = ARecord('1.1.1.3') - azure_records.append(_base3) - - _base4 = _AzureRecord('TestAzure', octo_records[4]) - _base4.zone_name = 'unit.tests' - _base4.relative_record_set_name = 'cname' - _base4.record_type = 'CNAME' - _base4.params['ttl'] = 3 - _base4.params['cname_record'] = CnameRecord('a.unit.tests.') - azure_records.append(_base4) - - _base5 = _AzureRecord('TestAzure', octo_records[5]) - _base5.zone_name = 'unit.tests' - _base5.relative_record_set_name = '@' - _base5.record_type = 'MX' - _base5.params['ttl'] = 3 - _base5.params['mx_records'] = [MxRecord(10, 'mx1.unit.tests.'), - MxRecord(20, 'mx2.unit.tests.')] - azure_records.append(_base5) - - _base6 = _AzureRecord('TestAzure', octo_records[6]) - _base6.zone_name = 'unit.tests' - _base6.relative_record_set_name = '@' - _base6.record_type = 'NS' - _base6.params['ttl'] = 4 - _base6.params['ns_records'] = [NsRecord('ns1.unit.tests.'), - NsRecord('ns2.unit.tests.')] - azure_records.append(_base6) - - _base7 = _AzureRecord('TestAzure', octo_records[7]) - _base7.zone_name = 'unit.tests' - _base7.relative_record_set_name = '@' - _base7.record_type = 'NS' - _base7.params['ttl'] = 5 - _base7.params['ns_records'] = [NsRecord('ns1.unit.tests.')] - azure_records.append(_base7) - - _base8 = _AzureRecord('TestAzure', octo_records[8]) - _base8.zone_name = 'unit.tests' - _base8.relative_record_set_name = '_srv._tcp' - _base8.record_type = 'SRV' - _base8.params['ttl'] = 6 - _base8.params['srv_records'] = [SrvRecord(10, 20, 30, 'foo-1.unit.tests.'), - SrvRecord(12, 30, 30, 'foo-2.unit.tests.')] - azure_records.append(_base8) +zone = Zone(name='unit.tests.', sub_zones=[]) +octo_records = [] +octo_records.append(Record.new(zone, '', { + 'ttl': 0, + 'type': 'A', + 'values': ['1.2.3.4', '10.10.10.10']})) +octo_records.append(Record.new(zone, 'a', { + 'ttl': 1, + 'type': 'A', + 'values': ['1.2.3.4', '1.1.1.1']})) +octo_records.append(Record.new(zone, 'aa', { + 'ttl': 9001, + 'type': 'A', + 'values': ['1.2.4.3']})) +octo_records.append(Record.new(zone, 'aaa', { + 'ttl': 2, + 'type': 'A', + 'values': ['1.1.1.3']})) +octo_records.append(Record.new(zone, 'cname', { + 'ttl': 3, + 'type': 'CNAME', + 'value': 'a.unit.tests.'})) +octo_records.append(Record.new(zone, 'mx1', { + 'ttl': 3, + 'type': 'MX', + 'values': [{ + 'priority': 10, + 'value': 'mx1.unit.tests.', + }, { + 'priority': 20, + 'value': 'mx2.unit.tests.', + }]})) +octo_records.append(Record.new(zone, 'mx2', { + 'ttl': 3, + 'type': 'MX', + 'values': [{ + 'priority': 10, + 'value': 'mx1.unit.tests.', + }]})) +octo_records.append(Record.new(zone, '', { + 'ttl': 4, + 'type': 'NS', + 'values': ['ns1.unit.tests.', 'ns2.unit.tests.']})) +octo_records.append(Record.new(zone, 'foo', { + 'ttl': 5, + 'type': 'NS', + 'value': 'ns1.unit.tests.'})) +octo_records.append(Record.new(zone, '_srv._tcp', { + 'ttl': 6, + 'type': 'SRV', + 'values': [{ + 'priority': 10, + 'weight': 20, + 'port': 30, + 'target': 'foo-1.unit.tests.', + }, { + 'priority': 12, + 'weight': 30, + 'port': 30, + 'target': 'foo-2.unit.tests.', + }]})) +octo_records.append(Record.new(zone, '_srv2._tcp', { + 'ttl': 7, + 'type': 'SRV', + 'values': [{ + 'priority': 12, + 'weight': 17, + 'port': 1, + 'target': 'srvfoo.unit.tests.', + }]})) + +azure_records = [] +_base0 = _AzureRecord('TestAzure', octo_records[0]) +_base0.zone_name = 'unit.tests' +_base0.relative_record_set_name = '@' +_base0.record_type = 'A' +_base0.params['ttl'] = 0 +_base0.params['arecords'] = [ARecord('1.2.3.4'), ARecord('10.10.10.10')] +azure_records.append(_base0) + +_base1 = _AzureRecord('TestAzure', octo_records[1]) +_base1.zone_name = 'unit.tests' +_base1.relative_record_set_name = 'a' +_base1.record_type = 'A' +_base1.params['ttl'] = 1 +_base1.params['arecords'] = [ARecord('1.2.3.4'), ARecord('1.1.1.1')] +azure_records.append(_base1) + +_base2 = _AzureRecord('TestAzure', octo_records[2]) +_base2.zone_name = 'unit.tests' +_base2.relative_record_set_name = 'aa' +_base2.record_type = 'A' +_base2.params['ttl'] = 9001 +_base2.params['arecords'] = ARecord('1.2.4.3') +azure_records.append(_base2) + +_base3 = _AzureRecord('TestAzure', octo_records[3]) +_base3.zone_name = 'unit.tests' +_base3.relative_record_set_name = 'aaa' +_base3.record_type = 'A' +_base3.params['ttl'] = 2 +_base3.params['arecords'] = ARecord('1.1.1.3') +azure_records.append(_base3) + +_base4 = _AzureRecord('TestAzure', octo_records[4]) +_base4.zone_name = 'unit.tests' +_base4.relative_record_set_name = 'cname' +_base4.record_type = 'CNAME' +_base4.params['ttl'] = 3 +_base4.params['cname_record'] = CnameRecord('a.unit.tests.') +azure_records.append(_base4) + +_base5 = _AzureRecord('TestAzure', octo_records[5]) +_base5.zone_name = 'unit.tests' +_base5.relative_record_set_name = 'mx1' +_base5.record_type = 'MX' +_base5.params['ttl'] = 3 +_base5.params['mx_records'] = [MxRecord(10, 'mx1.unit.tests.'), + MxRecord(20, 'mx2.unit.tests.')] +azure_records.append(_base5) + +_base6 = _AzureRecord('TestAzure', octo_records[6]) +_base6.zone_name = 'unit.tests' +_base6.relative_record_set_name = 'mx2' +_base6.record_type = 'MX' +_base6.params['ttl'] = 3 +_base6.params['mx_records'] = [MxRecord(10, 'mx1.unit.tests.')] +azure_records.append(_base6) + +_base7 = _AzureRecord('TestAzure', octo_records[7]) +_base7.zone_name = 'unit.tests' +_base7.relative_record_set_name = '@' +_base7.record_type = 'NS' +_base7.params['ttl'] = 4 +_base7.params['ns_records'] = [NsRecord('ns1.unit.tests.'), + NsRecord('ns2.unit.tests.')] +azure_records.append(_base7) +_base8 = _AzureRecord('TestAzure', octo_records[8]) +_base8.zone_name = 'unit.tests' +_base8.relative_record_set_name = 'foo' +_base8.record_type = 'NS' +_base8.params['ttl'] = 5 +_base8.params['ns_records'] = [NsRecord('ns1.unit.tests.')] +azure_records.append(_base8) + +_base9 = _AzureRecord('TestAzure', octo_records[9]) +_base9.zone_name = 'unit.tests' +_base9.relative_record_set_name = '_srv._tcp' +_base9.record_type = 'SRV' +_base9.params['ttl'] = 6 +_base9.params['srv_records'] = [SrvRecord(10, 20, 30, 'foo-1.unit.tests.'), + SrvRecord(12, 30, 30, 'foo-2.unit.tests.')] +azure_records.append(_base9) + +_base10 = _AzureRecord('TestAzure', octo_records[10]) +_base10.zone_name = 'unit.tests' +_base10.relative_record_set_name = '_srv2._tcp' +_base10.record_type = 'SRV' +_base10.params['ttl'] = 7 +_base10.params['srv_records'] = [SrvRecord(12, 17, 1, 'srvfoo.unit.tests.')] +azure_records.append(_base10) + + +class Test_AzureRecord(TestCase): def test_azure_record(self): - assert(len(self.azure_records) == len(self.octo_records)) - for i in range(len(self.azure_records)): - octo = _AzureRecord('TestAzure', self.octo_records[i]) - assert(self.azure_records[i]._equals(octo)) + assert(len(azure_records) == len(octo_records)) + for i in range(len(azure_records)): + octo = _AzureRecord('TestAzure', octo_records[i]) + assert(azure_records[i]._equals(octo)) + string = str(azure_records[i]) + assert(('Ttl: ' in string)) + + +class TestValidatePeriod(TestCase): + def test_validate_per(self): + for expected, test in [['a.', 'a'], + ['a.', 'a.'], + ['foo.bar.', 'foo.bar.'], + ['foo.bar.', 'foo.bar']]: + self.assertEquals(expected, _validate_per(test)) class TestAzureDnsProvider(TestCase): - def test_populate(self): - pass # placeholder + def _provider(self): + return self._get_provider('mock_spc', 'mock_dns_client') + + @patch('octodns.provider.azuredns.DnsManagementClient') + @patch('octodns.provider.azuredns.ServicePrincipalCredentials') + def _get_provider(self, mock_spc, mock_dns_client): + '''Returns a mock AzureProvider object to use in testing. + + :param mock_spc: placeholder + :type mock_spc: str + :param mock_dns_client: placeholder + :type mock_dns_client: str + + :type return: AzureProvider + ''' + return AzureProvider('mock_id', 'mock_client', 'mock_key', + 'mock_directory', 'mock_sub', 'mock_rg') + + def test_populate_records(self): + provider = self._get_provider() + + rs = [] + rs.append(RecordSet(name='a1', ttl=0, type='A', + arecords=[ARecord('1.1.1.1')])) + rs.append(RecordSet(name='a2', ttl=1, type='A', + arecords=[ARecord('1.1.1.1'), + ARecord('2.2.2.2')])) + rs.append(RecordSet(name='aaaa1', ttl=2, type='AAAA', + aaaa_records=[AaaaRecord('1:1ec:1::1')])) + rs.append(RecordSet(name='aaaa2', ttl=3, type='AAAA', + aaaa_records=[AaaaRecord('1:1ec:1::1'), + AaaaRecord('1:1ec:1::2')])) + rs.append(RecordSet(name='cname1', ttl=4, type='CNAME', + cname_record=CnameRecord('cname.unit.test.'))) + rs.append(RecordSet(name='cname2', ttl=5, type='CNAME', + cname_record=None)) + rs.append(RecordSet(name='mx1', ttl=6, type='MX', + mx_records=[MxRecord(10, 'mx1.unit.test.')])) + rs.append(RecordSet(name='mx2', ttl=7, type='MX', + mx_records=[MxRecord(10, 'mx1.unit.test.'), + MxRecord(11, 'mx2.unit.test.')])) + rs.append(RecordSet(name='ns1', ttl=8, type='NS', + ns_records=[NsRecord('ns1.unit.test.')])) + rs.append(RecordSet(name='ns2', ttl=9, type='NS', + ns_records=[NsRecord('ns1.unit.test.'), + NsRecord('ns2.unit.test.')])) + rs.append(RecordSet(name='ptr1', ttl=10, type='PTR', + ptr_records=[PtrRecord('ptr1.unit.test.')])) + rs.append(RecordSet(name='ptr2', ttl=11, type='PTR', + ptr_records=[PtrRecord('ptr1.unit.test.'), + PtrRecord('ptr2.unit.test.')])) + rs.append(RecordSet(name='_srv1._tcp', ttl=12, type='SRV', + srv_records=[SrvRecord(1, 2, 3, '1unit.tests.')])) + rs.append(RecordSet(name='_srv2._tcp', ttl=13, type='SRV', + srv_records=[SrvRecord(1, 2, 3, '1unit.tests.'), + SrvRecord(4, 5, 6, '2unit.tests.')])) + rs.append(RecordSet(name='txt1', ttl=14, type='TXT', + txt_records=[TxtRecord('sample text1')])) + rs.append(RecordSet(name='txt2', ttl=15, type='TXT', + txt_records=[TxtRecord('sample text1'), + TxtRecord('sample text2')])) + rs.append(RecordSet(name='', ttl=16, type='SOA', + soa_record=[SoaRecord()])) + + record_list = provider._dns_client.record_sets.list_by_dns_zone + record_list.return_value = rs + + provider.populate(zone) + + self.assertEquals(len(zone.records), 16) + + def test_populate_zone(self): + provider = self._get_provider() + + zone_list = provider._dns_client.zones.list_by_resource_group + zone_list.return_value = [AzureZone(location='global'), + AzureZone(location='global')] + + provider._populate_zones() + + self.assertEquals(len(provider._azure_zones), 1) + + def test_bad_zone_response(self): + provider = self._get_provider() + + _get = provider._dns_client.zones.get + _get.side_effect = CloudError(Mock(status=404), 'Azure Error') + trip = False + try: + provider._check_zone('unit.test', create=False) + except CloudError: + trip = True + self.assertEquals(trip, True) + + def test_apply(self): + provider = self._get_provider() + + changes = [] + deletes = [] + for i in octo_records: + changes.append(Create(i)) + deletes.append(Delete(i)) + + self.assertEquals(11, provider.apply(Plan(None, zone, changes))) + self.assertEquals(11, provider.apply(Plan(zone, zone, deletes))) + + def test_create_zone(self): + provider = self._get_provider() + + changes = [] + for i in octo_records: + changes.append(Create(i)) + desired = Zone('unit2.test.', []) + + _get = provider._dns_client.zones.get + _get.side_effect = CloudError(Mock(status=404), 'Azure Error') + + self.assertEquals(11, provider.apply(Plan(None, desired, changes))) From 824cf4e98c3cd9e334fe9def9402dfefdb4abc16 Mon Sep 17 00:00:00 2001 From: Heesu Hwang Date: Fri, 30 Jun 2017 17:12:18 -0700 Subject: [PATCH 33/64] Changed code as per PR review. Only major change is refactoring _check_zones. Many more comments --- .gitignore | 2 - doit.txt | 4 - octodns/provider/azuredns.py | 150 ++++++++++++++++-------- tests/test_octodns_provider_azuredns.py | 43 ++++++- 4 files changed, 140 insertions(+), 59 deletions(-) delete mode 100755 doit.txt diff --git a/.gitignore b/.gitignore index 5a3f3f9..c45a684 100644 --- a/.gitignore +++ b/.gitignore @@ -11,5 +11,3 @@ output/ tmp/ build/ config/ -./rb.txt -./doit.txt diff --git a/doit.txt b/doit.txt deleted file mode 100755 index 1ff2da6..0000000 --- a/doit.txt +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -#script to rebuild octodns quickly - -octodns-sync --config-file=./config/production.yaml --doit diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index cf0e2d5..91e298c 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -9,6 +9,7 @@ from azure.common.credentials import ServicePrincipalCredentials from azure.mgmt.dns import DnsManagementClient from azure.mgmt.dns.models import ARecord, AaaaRecord, CnameRecord, MxRecord, \ SrvRecord, NsRecord, PtrRecord, TxtRecord, Zone +from msrestazure.azure_exceptions import CloudError from functools import reduce import sys @@ -18,7 +19,8 @@ from .base import BaseProvider class _AzureRecord(object): - ''' Wrapper for OctoDNS record. + '''Wrapper for OctoDNS record for AzureProvider to make dns_client calls. + azuredns.py: class: octodns.provider.azuredns._AzureRecord An _AzureRecord is easily accessible to Azure DNS Management library @@ -27,7 +29,19 @@ class _AzureRecord(object): ''' def __init__(self, resource_group, record, delete=False): - ''' + '''Contructor for _AzureRecord. + + Notes on Azure records: An Azure record set has the form + RecordSet(name=<...>, type=<...>, arecords=[...], aaaa_records, ..) + When constructing an azure record as done in self._apply_Create, + the argument parameters for an A record would be + parameters={'ttl': , 'arecords': [ARecord(),]}. + As another example for CNAME record: + parameters={'ttl': , 'cname_record': CnameRecord()}. + + Below, key_name and class_name are the dictionary key and Azure + Record class respectively. + :param resource_group: The name of resource group in Azure :type resource_group: str :param record: An OctoDNS record @@ -39,15 +53,18 @@ class _AzureRecord(object): :type return: _AzureRecord ''' self.resource_group = resource_group - self.zone_name = record.zone.name[0:len(record.zone.name) - 1] + self.zone_name = record.zone.name[:len(record.zone.name) - 1] self.relative_record_set_name = record.name or '@' self.record_type = record._type if delete: return + # Refer to function docstring for key_name and class_name. format_u_s = '' if record._type == 'A' else '_' key_name = '{}{}records'.format(self.record_type, format_u_s).lower() + if record._type == 'CNAME': + key_name = key_name[:len(key_name) - 1] class_name = '{}'.format(self.record_type).capitalize() + \ 'Record'.format(self.record_type) @@ -56,8 +73,10 @@ class _AzureRecord(object): self.params['ttl'] = record.ttl def _params(self, data, key_name, azure_class): - return {key_name: [azure_class(v) for v in data['values']]} \ - if 'values' in data else {key_name: [azure_class(data['value'])]} + if 'values' in data: + return {key_name: [azure_class(v) for v in data['values']]} + else: # Else there is a singular data point keyed by 'value'. + return {key_name: [azure_class(data['value'])]} _params_for_A = _params _params_for_AAAA = _params @@ -73,7 +92,7 @@ class _AzureRecord(object): vals['weight'], vals['port'], vals['target'])) - else: # single value at key 'value' + else: # Else there is a singular data point keyed by 'value'. params.append(azure_class(data['value']['priority'], data['value']['weight'], data['value']['port'], @@ -86,7 +105,7 @@ class _AzureRecord(object): for vals in data['values']: params.append(azure_class(vals['preference'], vals['exchange'])) - else: # single value at key 'value' + else: # Else there is a singular data point keyed by 'value'. params.append(azure_class(data['value']['preference'], data['value']['exchange'])) return {key_name: params} @@ -141,10 +160,14 @@ class _AzureRecord(object): return string -def _validate_per(string): +def _check_endswith_dot(string): return string if string.endswith('.') else string + '.' +def _parse_azure_type(string): + return string.split('/')[len(string.split('/')) - 1] + + class AzureProvider(BaseProvider): ''' Azure DNS Provider @@ -155,21 +178,44 @@ class AzureProvider(BaseProvider): # includes using a Service Principal: # https://docs.microsoft.com/en-us/azure/azure-resource-manager/ # resource-group-create-service-principal-portal - # The Azure Active Directory Application ID (aka client ID) req: + # The Azure Active Directory Application ID (aka client ID): client_id: - # Authentication Key Value req: + # Authentication Key Value: (note this should be secret) key: - # Directory ID (referred to tenant ID) req: + # Directory ID (aka tenant ID): directory_id: - # Subscription ID req: + # Subscription ID: sub_id: - # Resource Group name req: + # Resource Group name: resource_group: - - TODO: change config file to use env vars instead of hard-coded keys - - personal notes: testing: test authentication vars located in - /home/t-hehwan/vars.txt + # All are required to authenticate. + + Example config file with variables: + " + --- + providers: + config: + class: octodns.provider.yaml.YamlProvider + directory: ./config (example path to directory of zone files) + azuredns: + class: octodns.provider.azuredns.AzureProvider + client_id: env/AZURE_APPLICATION_ID + key: env/AZURE_AUTHENICATION_KEY + directory_id: env/AZURE_DIRECTORY_ID + sub_id: env/AZURE_SUBSCRIPTION_ID + resource_group: 'TestResource1' + + zones: + example.com.: + sources: + - config + targets: + - azuredns + " + The first four variables above can be hidden in environment variables + and octoDNS will automatically search for them in the shell. It is + possible to also hard-code into the config file. resource_group can + also be an environment variable but might likely change. ''' SUPPORTS_GEO = False SUPPORTS = set(('A', 'AAAA', 'CNAME', 'MX', 'NS', 'PTR', 'SRV', 'TXT')) @@ -214,19 +260,24 @@ class AzureProvider(BaseProvider): self._dns_client.zones.get(self._resource_group, name) self._azure_zones.add(name) return name - except: - if create: - self.log.debug('_check_zone:no matching zone; creating %s', - name) - create_zone = self._dns_client.zones.create_or_update - create_zone(self._resource_group, name, Zone('global')) - return name - else: - raise + except CloudError as err: + msg = 'The Resource \'Microsoft.Network/dnszones/{}\''.format(name) + msg += ' under resource group \'{}\''.format(self._resource_group) + msg += ' was not found.' + if msg == err.message: + # Then the only error is that the zone doesn't currently exist + if create: + self.log.debug('_check_zone:no matching zone; creating %s', + name) + create_zone = self._dns_client.zones.create_or_update + create_zone(self._resource_group, name, Zone('global')) + return name + else: + return + raise def populate(self, zone, target=False, lenient=False): - ''' - Required function of manager.py. + '''Required function of manager.py to collect records from zone. Special notes for Azure. Azure zone names omit final '.' @@ -249,26 +300,28 @@ class AzureProvider(BaseProvider): :type return: void ''' - zone_name = zone.name[0:len(zone.name) - 1] - self.log.debug('populate: name=%s', zone_name) + self.log.debug('populate: name=%s', zone.name) before = len(zone.records) + zone_name = zone.name[:len(zone.name) - 1] self._populate_zones() self._check_zone(zone_name) _records = set() records = self._dns_client.record_sets.list_by_dns_zone - for azrecord in records(self._resource_group, zone_name): - if azrecord.type in self.SUPPORTS: - _records.add(azrecord) - for azrecord in _records: - record_name = azrecord.name if azrecord.name != '@' else '' - data = getattr(self, '_data_for_{}'.format(azrecord.type)) - data = data(azrecord) - data['type'] = azrecord.type - data['ttl'] = azrecord.ttl - record = Record.new(zone, record_name, data, source=self) - zone.add_record(record) + if self._check_zone(zone_name): + for azrecord in records(self._resource_group, zone_name): + if _parse_azure_type(azrecord.type) in self.SUPPORTS: + _records.add(azrecord) + for azrecord in _records: + record_name = azrecord.name if azrecord.name != '@' else '' + typ = _parse_azure_type(azrecord.type) + data = getattr(self, '_data_for_{}'.format(typ)) + data = data(azrecord) + data['type'] = typ + data['ttl'] = azrecord.ttl + record = Record.new(zone, record_name, data, source=self) + zone.add_record(record) self.log.info('populate: found %s records', len(zone.records) - before) @@ -293,13 +346,14 @@ class AzureProvider(BaseProvider): records. Refer to population comment. ''' try: - return {'value': _validate_per(azrecord.cname_record.cname)} + return {'value': _check_endswith_dot(azrecord.cname_record.cname)} except: return {'value': '.'} def _data_for_PTR(self, azrecord): try: - return {'value': _validate_per(azrecord.ptr_records[0].ptdrname)} + ptdrname = azrecord.ptr_records[0].ptdrname + return {'value': _check_endswith_dot(ptdrname)} except: return {'value': '.'} @@ -315,7 +369,7 @@ class AzureProvider(BaseProvider): def _data_for_NS(self, azrecord): vals = [ar.nsdname for ar in azrecord.ns_records] - return {'values': [_validate_per(val) for val in vals]} + return {'values': [_check_endswith_dot(val) for val in vals]} def _apply_Create(self, change): '''A record from change must be created. @@ -334,7 +388,7 @@ class AzureProvider(BaseProvider): record_type=ar.record_type, parameters=ar.params) - self.log.debug('* Success Create/Update: {}'.format(ar)) + print('* Success Create/Update: {}'.format(ar), file=sys.stderr) _apply_Update = _apply_Create @@ -345,7 +399,7 @@ class AzureProvider(BaseProvider): delete(self._resource_group, ar.zone_name, ar.relative_record_set_name, ar.record_type) - self.log.debug('* Success Delete: {}'.format(ar)) + print('* Success Delete: {}'.format(ar), file=sys.stderr) def _apply(self, plan): ''' @@ -361,7 +415,7 @@ class AzureProvider(BaseProvider): self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name, len(changes)) - azure_zone_name = desired.name[0:len(desired.name) - 1] + azure_zone_name = desired.name[:len(desired.name) - 1] self._check_zone(azure_zone_name, create=True) for change in changes: diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index edb2db2..abce5aa 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -7,7 +7,7 @@ from __future__ import absolute_import, division, print_function, \ from octodns.record import Create, Delete, Record from octodns.provider.azuredns import _AzureRecord, AzureProvider, \ - _validate_per + _check_endswith_dot, _parse_azure_type from octodns.zone import Zone from octodns.provider.base import Plan @@ -195,13 +195,22 @@ class Test_AzureRecord(TestCase): assert(('Ttl: ' in string)) -class TestValidatePeriod(TestCase): - def test_validate_per(self): +class Test_ParseAzureType(TestCase): + def test_parse_azure_type(self): + for expected, test in [['A', 'Microsoft.Network/dnszones/A'], + ['AAAA', 'Microsoft.Network/dnszones/AAAA'], + ['NS', 'Microsoft.Network/dnszones/NS'], + ['MX', 'Microsoft.Network/dnszones/MX']]: + self.assertEquals(expected, _parse_azure_type(test)) + + +class Test_CheckEndswithDot(TestCase): + def test_check_endswith_dot(self): for expected, test in [['a.', 'a'], ['a.', 'a.'], ['foo.bar.', 'foo.bar.'], ['foo.bar.', 'foo.bar']]: - self.assertEquals(expected, _validate_per(test)) + self.assertEquals(expected, _check_endswith_dot(test)) class TestAzureDnsProvider(TestCase): @@ -319,7 +328,31 @@ class TestAzureDnsProvider(TestCase): changes.append(Create(i)) desired = Zone('unit2.test.', []) + err_msg = 'The Resource \'Microsoft.Network/dnszones/unit2.test\' ' + err_msg += 'under resource group \'mock_rg\' was not found.' _get = provider._dns_client.zones.get - _get.side_effect = CloudError(Mock(status=404), 'Azure Error') + _get.side_effect = CloudError(Mock(status=404), err_msg) self.assertEquals(11, provider.apply(Plan(None, desired, changes))) + + def test_check_zone_no_create(self): + provider = self._get_provider() + + rs = [] + rs.append(RecordSet(name='a1', ttl=0, type='A', + arecords=[ARecord('1.1.1.1')])) + rs.append(RecordSet(name='a2', ttl=1, type='A', + arecords=[ARecord('1.1.1.1'), + ARecord('2.2.2.2')])) + + record_list = provider._dns_client.record_sets.list_by_dns_zone + record_list.return_value = rs + + err_msg = 'The Resource \'Microsoft.Network/dnszones/unit3.test\' ' + err_msg += 'under resource group \'mock_rg\' was not found.' + _get = provider._dns_client.zones.get + _get.side_effect = CloudError(Mock(status=404), err_msg) + + provider.populate(Zone('unit3.test.', [])) + + self.assertEquals(len(zone.records), 0) From ec4261e7da97bf8711c830965f18f7da3e725439 Mon Sep 17 00:00:00 2001 From: Heesu Hwang Date: Fri, 30 Jun 2017 17:29:16 -0700 Subject: [PATCH 34/64] Fixed typed in _data_for_PTR and amended test case to check for it --- octodns/provider/azuredns.py | 4 ++-- tests/test_octodns_provider_azuredns.py | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 91e298c..4fe5184 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -352,8 +352,8 @@ class AzureProvider(BaseProvider): def _data_for_PTR(self, azrecord): try: - ptdrname = azrecord.ptr_records[0].ptdrname - return {'value': _check_endswith_dot(ptdrname)} + ptrdname = azrecord.ptr_records[0].ptrdname + return {'value': _check_endswith_dot(ptrdname)} except: return {'value': '.'} diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index abce5aa..5ffa055 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -263,8 +263,7 @@ class TestAzureDnsProvider(TestCase): rs.append(RecordSet(name='ptr1', ttl=10, type='PTR', ptr_records=[PtrRecord('ptr1.unit.test.')])) rs.append(RecordSet(name='ptr2', ttl=11, type='PTR', - ptr_records=[PtrRecord('ptr1.unit.test.'), - PtrRecord('ptr2.unit.test.')])) + ptr_records=[PtrRecord(None)])) rs.append(RecordSet(name='_srv1._tcp', ttl=12, type='SRV', srv_records=[SrvRecord(1, 2, 3, '1unit.tests.')])) rs.append(RecordSet(name='_srv2._tcp', ttl=13, type='SRV', From d9806e851f2ef7a2a8717cda1fa49cd8687e67e7 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sun, 2 Jul 2017 10:45:58 -0700 Subject: [PATCH 35/64] NS1 RateLimitException, just sleep for e.period --- octodns/provider/ns1.py | 18 ++++++++---------- tests/test_octodns_provider_ns1.py | 8 ++++---- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 0f3db1f..bca6118 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -28,13 +28,11 @@ class Ns1Provider(BaseProvider): ZONE_NOT_FOUND_MESSAGE = 'server error: zone not found' - def __init__(self, id, api_key, rate_limit_delay=1, *args, **kwargs): + def __init__(self, id, api_key, *args, **kwargs): self.log = getLogger('Ns1Provider[{}]'.format(id)) - self.log.debug('__init__: id=%s, api_key=***, rate_limit_delay=%d', id, - rate_limit_delay) + self.log.debug('__init__: id=%s, api_key=***', id) super(Ns1Provider, self).__init__(id, *args, **kwargs) self._client = NSONE(apiKey=api_key) - self.rate_limit_delay = rate_limit_delay def _data_for_A(self, _type, record): return { @@ -177,10 +175,10 @@ class Ns1Provider(BaseProvider): meth = getattr(nsone_zone, 'add_{}'.format(_type)) try: meth(name, **params) - except RateLimitException: + except RateLimitException as e: self.log.warn('_apply_Create: rate limit encountered, pausing ' 'and trying again') - sleep(self.rate_limit_delay) + sleep(e.period) meth(name, **params) def _apply_Update(self, nsone_zone, change): @@ -192,10 +190,10 @@ class Ns1Provider(BaseProvider): params = getattr(self, '_params_for_{}'.format(_type))(new) try: record.update(**params) - except RateLimitException: + except RateLimitException as e: self.log.warn('_apply_Update: rate limit encountered, pausing ' 'and trying again') - sleep(self.rate_limit_delay) + sleep(e.period) record.update(**params) def _apply_Delete(self, nsone_zone, change): @@ -205,10 +203,10 @@ class Ns1Provider(BaseProvider): record = nsone_zone.loadRecord(name, _type) try: record.delete() - except RateLimitException: + except RateLimitException as e: self.log.warn('_apply_Delete: rate limit encountered, pausing ' 'and trying again') - sleep(self.rate_limit_delay) + sleep(e.period) record.delete() def _apply(self, plan): diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index 5e53cfd..0398459 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -193,7 +193,7 @@ class TestNs1Provider(TestCase): @patch('nsone.NSONE.createZone') @patch('nsone.NSONE.loadZone') def test_sync(self, load_mock, create_mock): - provider = Ns1Provider('test', 'api-key', rate_limit_delay=0) + provider = Ns1Provider('test', 'api-key') desired = Zone('unit.tests.', []) desired.records.update(self.expected) @@ -231,7 +231,7 @@ class TestNs1Provider(TestCase): mock_zone = Mock() mock_zone.add_SRV = Mock() mock_zone.add_SRV.side_effect = [ - RateLimitException('boo'), + RateLimitException('boo', period=0), None, ] create_mock.side_effect = [mock_zone] @@ -259,11 +259,11 @@ class TestNs1Provider(TestCase): # trigger rate limit handling mock_record = Mock() mock_record.update.side_effect = [ - RateLimitException('one'), + RateLimitException('one', period=0), None, ] mock_record.delete.side_effect = [ - RateLimitException('two'), + RateLimitException('two', period=0), None, ] nsone_zone.loadRecord.side_effect = [mock_record, mock_record] From 06fb57855015a372b60aeea5342023247d9fd8a2 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sun, 2 Jul 2017 10:47:13 -0700 Subject: [PATCH 36/64] Include sleep duration in ns1 RateLimitException log --- octodns/provider/ns1.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index bca6118..65db64c 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -177,7 +177,7 @@ class Ns1Provider(BaseProvider): meth(name, **params) except RateLimitException as e: self.log.warn('_apply_Create: rate limit encountered, pausing ' - 'and trying again') + 'for %ds and trying again', e.period) sleep(e.period) meth(name, **params) @@ -192,7 +192,7 @@ class Ns1Provider(BaseProvider): record.update(**params) except RateLimitException as e: self.log.warn('_apply_Update: rate limit encountered, pausing ' - 'and trying again') + 'for %ds and trying again', e.period) sleep(e.period) record.update(**params) @@ -205,7 +205,7 @@ class Ns1Provider(BaseProvider): record.delete() except RateLimitException as e: self.log.warn('_apply_Delete: rate limit encountered, pausing ' - 'and trying again') + 'for %ds and trying again', e.period) sleep(e.period) record.delete() From 908698da492bd0e71c111bae4d59669ea4e83827 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sun, 2 Jul 2017 18:23:45 -0700 Subject: [PATCH 37/64] Fix major performance issue with add_record O(N^2) Before, 1-2k record took ~10s and more than that was just painful, 5k took forever. This records things to keep a dict of nodes with a set of records so that we can quickly "jump" to the point we're looking for without having to search. 10k records now takes ~5s. --- octodns/__init__.py | 4 +- octodns/zone.py | 49 +++++++++++++++-------- tests/test_octodns_provider_cloudflare.py | 2 +- tests/test_octodns_provider_dnsimple.py | 2 +- tests/test_octodns_provider_ns1.py | 3 +- tests/test_octodns_provider_powerdns.py | 2 +- tests/test_octodns_provider_route53.py | 12 +++--- tests/test_octodns_zone.py | 2 +- 8 files changed, 45 insertions(+), 31 deletions(-) diff --git a/octodns/__init__.py b/octodns/__init__.py index b6287e5..94ef299 100644 --- a/octodns/__init__.py +++ b/octodns/__init__.py @@ -1,6 +1,4 @@ -''' -OctoDNS: DNS as code - Tools for managing DNS across multiple providers -''' +'OctoDNS: DNS as code - Tools for managing DNS across multiple providers' from __future__ import absolute_import, division, print_function, \ unicode_literals diff --git a/octodns/zone.py b/octodns/zone.py index 9d405bb..74e5d9e 100644 --- a/octodns/zone.py +++ b/octodns/zone.py @@ -5,6 +5,7 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals +from collections import defaultdict from logging import getLogger import re @@ -39,13 +40,19 @@ class Zone(object): # Force everyting to lowercase just to be safe self.name = str(name).lower() if name else name self.sub_zones = sub_zones - self.records = set() + # We're grouping by node, it allows us to efficently search for + # duplicates and detect when CNAMEs co-exist with other records + self._records = defaultdict(set) # optional leading . to match empty hostname # optional trailing . b/c some sources don't have it on their fqdn self._name_re = re.compile('\.?{}?$'.format(name)) self.log.debug('__init__: zone=%s, sub_zones=%s', self, sub_zones) + @property + def records(self): + return set([r for _, node in self._records.items() for r in node]) + def hostname_from_fqdn(self, fqdn): return self._name_re.sub('', fqdn) @@ -53,9 +60,6 @@ class Zone(object): name = record.name last = name.split('.')[-1] - if replace and record in self.records: - self.records.remove(record) - if last in self.sub_zones: if name != last: # it's a record for something under a sub-zone @@ -67,19 +71,30 @@ class Zone(object): raise SubzoneRecordException('Record {} a managed sub-zone ' 'and not of type NS' .format(record.fqdn)) - # TODO: this is pretty inefficent - for existing in self.records: - if record == existing: - raise DuplicateRecordException('Duplicate record {}, type {}' - .format(record.fqdn, - record._type)) - elif name == existing.name and (record._type == 'CNAME' or - existing._type == 'CNAME'): - raise InvalidNodeException('Invalid state, CNAME at {} ' - 'cannot coexist with other records' - .format(record.fqdn)) - - self.records.add(record) + + if replace: + # will remove it if it exists + self._records[name].discard(record) + + node = self._records[name] + if record in node: + # We already have a record at this node of this type + raise DuplicateRecordException('Duplicate record {}, type {}' + .format(record.fqdn, + record._type)) + elif ((record._type == 'CNAME' and len(node) > 0) or + ('CNAME' in map(lambda r: r._type, node))): + # We're adding a CNAME to existing records or adding to an existing + # CNAME + raise InvalidNodeException('Invalid state, CNAME at {} cannot ' + 'coexist with other records' + .format(record.fqdn)) + + node.add(record) + + def _remove_record(self, record): + 'Only for use in tests' + self._records[record.name].discard(record) def changes(self, desired, target): self.log.debug('changes: zone=%s, target=%s', self, target) diff --git a/tests/test_octodns_provider_cloudflare.py b/tests/test_octodns_provider_cloudflare.py index 3f652a1..5dcae30 100644 --- a/tests/test_octodns_provider_cloudflare.py +++ b/tests/test_octodns_provider_cloudflare.py @@ -33,7 +33,7 @@ class TestCloudflareProvider(TestCase): })) for record in list(expected.records): if record.name == 'sub' and record._type == 'NS': - expected.records.remove(record) + expected._remove_record(record) break empty = {'result': [], 'result_info': {'count': 0, 'per_page': 0}} diff --git a/tests/test_octodns_provider_dnsimple.py b/tests/test_octodns_provider_dnsimple.py index 1f62bfd..aed1e8b 100644 --- a/tests/test_octodns_provider_dnsimple.py +++ b/tests/test_octodns_provider_dnsimple.py @@ -33,7 +33,7 @@ class TestDnsimpleProvider(TestCase): })) for record in list(expected.records): if record.name == 'sub' and record._type == 'NS': - expected.records.remove(record) + expected._remove_record(record) break def test_populate(self): diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index 0398459..ce1353b 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -196,7 +196,8 @@ class TestNs1Provider(TestCase): provider = Ns1Provider('test', 'api-key') desired = Zone('unit.tests.', []) - desired.records.update(self.expected) + for r in self.expected: + desired.add_record(r) plan = provider.plan(desired) # everything except the root NS diff --git a/tests/test_octodns_provider_powerdns.py b/tests/test_octodns_provider_powerdns.py index 01e7d83..5fcd80a 100644 --- a/tests/test_octodns_provider_powerdns.py +++ b/tests/test_octodns_provider_powerdns.py @@ -253,7 +253,7 @@ class TestPowerDnsProvider(TestCase): plan = provider.plan(expected) self.assertFalse(plan) # remove it now that we don't need the unrelated change any longer - expected.records.remove(unrelated_record) + expected._remove_record(unrelated_record) # ttl diff with requests_mock() as mock: diff --git a/tests/test_octodns_provider_route53.py b/tests/test_octodns_provider_route53.py index cad58f8..be624ff 100644 --- a/tests/test_octodns_provider_route53.py +++ b/tests/test_octodns_provider_route53.py @@ -372,11 +372,11 @@ class TestRoute53Provider(TestCase): # Delete by monkey patching in a populate that includes an extra record def add_extra_populate(existing, target, lenient): for record in self.expected.records: - existing.records.add(record) + existing.add_record(record) record = Record.new(existing, 'extra', {'ttl': 99, 'type': 'A', 'values': ['9.9.9.9']}) - existing.records.add(record) + existing.add_record(record) provider.populate = add_extra_populate change_resource_record_sets_params = { @@ -409,7 +409,7 @@ class TestRoute53Provider(TestCase): def mod_geo_populate(existing, target, lenient): for record in self.expected.records: if record._type != 'A' or not record.geo: - existing.records.add(record) + existing.add_record(record) record = Record.new(existing, '', { 'ttl': 61, 'type': 'A', @@ -420,7 +420,7 @@ class TestRoute53Provider(TestCase): 'NA-US-KY': ['7.2.3.4'] } }) - existing.records.add(record) + existing.add_record(record) provider.populate = mod_geo_populate change_resource_record_sets_params = { @@ -505,7 +505,7 @@ class TestRoute53Provider(TestCase): def mod_add_geo_populate(existing, target, lenient): for record in self.expected.records: if record._type != 'A' or record.geo: - existing.records.add(record) + existing.add_record(record) record = Record.new(existing, 'simple', { 'ttl': 61, 'type': 'A', @@ -514,7 +514,7 @@ class TestRoute53Provider(TestCase): 'OC': ['3.2.3.4', '4.2.3.4'], } }) - existing.records.add(record) + existing.add_record(record) provider.populate = mod_add_geo_populate change_resource_record_sets_params = { diff --git a/tests/test_octodns_zone.py b/tests/test_octodns_zone.py index f310397..8d75100 100644 --- a/tests/test_octodns_zone.py +++ b/tests/test_octodns_zone.py @@ -77,7 +77,7 @@ class TestZone(TestCase): # add a record, delete a record -> [Delete, Create] c = ARecord(before, 'c', {'ttl': 42, 'value': '1.1.1.1'}) after.add_record(c) - after.records.remove(b) + after._remove_record(b) self.assertEquals(after.records, set([a, c])) changes = before.changes(after, target) self.assertEquals(2, len(changes)) From bdceac42beb485ed30417a1ae85db1f6e45673b2 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sun, 2 Jul 2017 18:40:58 -0700 Subject: [PATCH 38/64] Fix stacktraces on MainThreadExecutor --- octodns/manager.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/octodns/manager.py b/octodns/manager.py index e6fe253..8439eb6 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -6,7 +6,7 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals from StringIO import StringIO -from concurrent.futures import Future, ThreadPoolExecutor +from concurrent.futures import ThreadPoolExecutor from importlib import import_module from os import environ import logging @@ -38,6 +38,17 @@ class _AggregateTarget(object): return True +class MakeThreadFuture(object): + + def __init__(self, func, args, kwargs): + self.func = func + self.args = args + self.kwargs = kwargs + + def result(self): + return self.func(*self.args, **self.kwargs) + + class MainThreadExecutor(object): ''' Dummy executor that runs things on the main thread during the involcation @@ -48,13 +59,7 @@ class MainThreadExecutor(object): ''' def submit(self, func, *args, **kwargs): - future = Future() - try: - future.set_result(func(*args, **kwargs)) - except Exception as e: - # TODO: get right stacktrace here - future.set_exception(e) - return future + return MakeThreadFuture(func, args, kwargs) class Manager(object): From 9637218c8b544c397bcd5d433de47cafbfad973d Mon Sep 17 00:00:00 2001 From: Petter Hassberg Date: Mon, 3 Jul 2017 16:10:48 +0200 Subject: [PATCH 39/64] Add lenient to abstract BaseSource signature --- octodns/source/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/source/base.py b/octodns/source/base.py index 2e2c5c2..4ace09f 100644 --- a/octodns/source/base.py +++ b/octodns/source/base.py @@ -20,7 +20,7 @@ class BaseSource(object): raise NotImplementedError('Abstract base class, SUPPORTS ' 'property missing') - def populate(self, zone, target=False): + def populate(self, zone, target=False, lenient=False): ''' Loads all zones the provider knows about From cd9d7254f06b1f5d24524bae0e236367642ff899 Mon Sep 17 00:00:00 2001 From: Heesu Hwang Date: Mon, 3 Jul 2017 11:03:23 -0700 Subject: [PATCH 40/64] Fixed stray prints and assert errors. Added versions of required azure libraries --- octodns/provider/azuredns.py | 30 +++++++++++++++++++++--------- requirements.txt | 3 +++ 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 4fe5184..0abddb0 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -7,17 +7,30 @@ from __future__ import absolute_import, division, print_function, \ from azure.common.credentials import ServicePrincipalCredentials from azure.mgmt.dns import DnsManagementClient -from azure.mgmt.dns.models import ARecord, AaaaRecord, CnameRecord, MxRecord, \ - SrvRecord, NsRecord, PtrRecord, TxtRecord, Zone from msrestazure.azure_exceptions import CloudError -from functools import reduce -import sys +# Imports are used: 'self.params(record.data, key_name, eval(class_name))' +# To pass pyflakes import statement tests. +from azure.mgmt.dns.models import ARecord, AaaaRecord, CnameRecord, MxRecord, \ + SrvRecord, NsRecord, PtrRecord, TxtRecord +from azure.mgmt.dns.models import Zone + import logging +from functools import reduce from ..record import Record from .base import BaseProvider +ARecord +AaaaRecord +CnameRecord +MxRecord +SrvRecord +NsRecord +PtrRecord +TxtRecord + + class _AzureRecord(object): '''Wrapper for OctoDNS record for AzureProvider to make dns_client calls. @@ -65,8 +78,7 @@ class _AzureRecord(object): key_name = '{}{}records'.format(self.record_type, format_u_s).lower() if record._type == 'CNAME': key_name = key_name[:len(key_name) - 1] - class_name = '{}'.format(self.record_type).capitalize() + \ - 'Record'.format(self.record_type) + class_name = '{}'.format(self.record_type).capitalize() + 'Record' self.params = getattr(self, '_params_for_{}'.format(record._type)) self.params = self.params(record.data, key_name, eval(class_name)) @@ -111,7 +123,7 @@ class _AzureRecord(object): return {key_name: params} def _params_for_CNAME(self, data, key_name, azure_class): - return {'cname_record': CnameRecord(data['value'])} + return {'cname_record': azure_class(data['value'])} def _equals(self, b): '''Checks whether two records are equal by comparing all fields. @@ -388,7 +400,7 @@ class AzureProvider(BaseProvider): record_type=ar.record_type, parameters=ar.params) - print('* Success Create/Update: {}'.format(ar), file=sys.stderr) + self.log.debug('* Success Create/Update: {}'.format(ar)) _apply_Update = _apply_Create @@ -399,7 +411,7 @@ class AzureProvider(BaseProvider): delete(self._resource_group, ar.zone_name, ar.relative_record_set_name, ar.record_type) - print('* Success Delete: {}'.format(ar), file=sys.stderr) + self.log.debug('* Success Delete: {}'.format(ar)) def _apply(self, plan): ''' diff --git a/requirements.txt b/requirements.txt index b10ca4c..62c485a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,3 +16,6 @@ python-dateutil==2.6.0 requests==2.13.0 s3transfer==0.1.10 six==1.10.0 +azure-mgmt-dns==1.0.1 +azure-common==1.1.6 +msrestazure==0.4.10 \ No newline at end of file From 348a6ca783af214632831961abd0a5aec8a2225a Mon Sep 17 00:00:00 2001 From: Heesu Hwang Date: Mon, 3 Jul 2017 16:03:20 -0700 Subject: [PATCH 41/64] Changed to map types to Azure Records isntead of implicitly using eval --- octodns/provider/azuredns.py | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 0abddb0..4ed783c 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -9,11 +9,8 @@ from azure.common.credentials import ServicePrincipalCredentials from azure.mgmt.dns import DnsManagementClient from msrestazure.azure_exceptions import CloudError -# Imports are used: 'self.params(record.data, key_name, eval(class_name))' -# To pass pyflakes import statement tests. from azure.mgmt.dns.models import ARecord, AaaaRecord, CnameRecord, MxRecord, \ - SrvRecord, NsRecord, PtrRecord, TxtRecord -from azure.mgmt.dns.models import Zone + SrvRecord, NsRecord, PtrRecord, TxtRecord, Zone import logging from functools import reduce @@ -21,16 +18,6 @@ from ..record import Record from .base import BaseProvider -ARecord -AaaaRecord -CnameRecord -MxRecord -SrvRecord -NsRecord -PtrRecord -TxtRecord - - class _AzureRecord(object): '''Wrapper for OctoDNS record for AzureProvider to make dns_client calls. @@ -40,6 +27,16 @@ class _AzureRecord(object): functions and is used to wrap all relevant data to create a record in Azure. ''' + TYPE_MAP = { + 'A': ARecord, + 'AAAA': AaaaRecord, + 'CNAME': CnameRecord, + 'MX': MxRecord, + 'SRV': SrvRecord, + 'NS': NsRecord, + 'PTR': PtrRecord, + 'TXT': TxtRecord + } def __init__(self, resource_group, record, delete=False): '''Contructor for _AzureRecord. @@ -78,10 +75,10 @@ class _AzureRecord(object): key_name = '{}{}records'.format(self.record_type, format_u_s).lower() if record._type == 'CNAME': key_name = key_name[:len(key_name) - 1] - class_name = '{}'.format(self.record_type).capitalize() + 'Record' + azure_class = self.TYPE_MAP[self.record_type] self.params = getattr(self, '_params_for_{}'.format(record._type)) - self.params = self.params(record.data, key_name, eval(class_name)) + self.params = self.params(record.data, key_name, azure_class) self.params['ttl'] = record.ttl def _params(self, data, key_name, azure_class): From 99578f328c18fc72f874da090952d7cd5e040e2f Mon Sep 17 00:00:00 2001 From: Heesu Hwang Date: Wed, 5 Jul 2017 09:45:38 -0700 Subject: [PATCH 42/64] add azure to README. order reqs, change comments slightly, alphabetize functions --- README.md | 1 + octodns/provider/azuredns.py | 76 +++++++++++++------------ requirements.txt | 8 +-- tests/test_octodns_provider_azuredns.py | 1 - 4 files changed, 45 insertions(+), 41 deletions(-) diff --git a/README.md b/README.md index 0e63e51..05cf979 100644 --- a/README.md +++ b/README.md @@ -149,6 +149,7 @@ The above command pulled the existing data out of Route53 and placed the results | Provider | Record Support | GeoDNS Support | Notes | |--|--|--|--| +| [AzureProvider](/octodns/provider/azuredns.py) | A, AAAA, CNAME, MX, NS, PTR, SRV, TXT | No | | | [CloudflareProvider](/octodns/provider/cloudflare.py) | A, AAAA, CNAME, MX, NS, SPF, TXT | No | | | [DnsimpleProvider](/octodns/provider/dnsimple.py) | All | No | | | [DynProvider](/octodns/provider/dyn.py) | All | Yes | | diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 4ed783c..4433b1e 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -56,9 +56,8 @@ class _AzureRecord(object): :type resource_group: str :param record: An OctoDNS record :type record: ..record.Record - :param values: Parameters for a record. eg IP address, port, domain - name, etc. Values usually read from record.data - :type values: {'values': [...]} or {'value': [...]} + :param delete: If true, omit data parsing; not needed to delete + :type delete: bool :type return: _AzureRecord ''' @@ -93,6 +92,20 @@ class _AzureRecord(object): _params_for_PTR = _params _params_for_TXT = _params + def _params_for_CNAME(self, data, key_name, azure_class): + return {key_name: azure_class(data['value'])} + + def _params_for_MX(self, data, key_name, azure_class): + params = [] + if 'values' in data: + for vals in data['values']: + params.append(azure_class(vals['preference'], + vals['exchange'])) + else: # Else there is a singular data point keyed by 'value'. + params.append(azure_class(data['value']['preference'], + data['value']['exchange'])) + return {key_name: params} + def _params_for_SRV(self, data, key_name, azure_class): params = [] if 'values' in data: @@ -108,20 +121,6 @@ class _AzureRecord(object): data['value']['target'])) return {key_name: params} - def _params_for_MX(self, data, key_name, azure_class): - params = [] - if 'values' in data: - for vals in data['values']: - params.append(azure_class(vals['preference'], - vals['exchange'])) - else: # Else there is a singular data point keyed by 'value'. - params.append(azure_class(data['value']['preference'], - data['value']['exchange'])) - return {key_name: params} - - def _params_for_CNAME(self, data, key_name, azure_class): - return {'cname_record': azure_class(data['value'])} - def _equals(self, b): '''Checks whether two records are equal by comparing all fields. :param b: Another _AzureRecord object @@ -174,6 +173,13 @@ def _check_endswith_dot(string): def _parse_azure_type(string): + '''Converts string representing an Azure RecordSet type to usual type. + + :param string: the Azure type. eg: + :type string: str + + :type return: str + ''' return string.split('/')[len(string.split('/')) - 1] @@ -223,8 +229,7 @@ class AzureProvider(BaseProvider): " The first four variables above can be hidden in environment variables and octoDNS will automatically search for them in the shell. It is - possible to also hard-code into the config file. resource_group can - also be an environment variable but might likely change. + possible to also hard-code into the config file: eg, resource_group. ''' SUPPORTS_GEO = False SUPPORTS = set(('A', 'AAAA', 'CNAME', 'MX', 'NS', 'PTR', 'SRV', 'TXT')) @@ -250,8 +255,8 @@ class AzureProvider(BaseProvider): self._azure_zones.add(zone.name) def _check_zone(self, name, create=False): - ''' - Checks whether a zone specified in a source exist in Azure server. + '''Checks whether a zone specified in a source exist in Azure server. + Note that Azure zones omit end '.' eg: contoso.com vs contoso.com. Returns the name if it exists. @@ -302,7 +307,7 @@ class AzureProvider(BaseProvider): :param zone: A dns zone :type zone: octodns.zone.Zone :param target: Checks if Azure is source or target of config. - Currently only supports as a target. Does not use. + Currently only supports as a target. Unused. :type target: bool :param lenient: Unused. Check octodns.manager for usage. :type lenient: bool @@ -340,10 +345,6 @@ class AzureProvider(BaseProvider): def _data_for_AAAA(self, azrecord): return {'values': [ar.ipv6_address for ar in azrecord.aaaa_records]} - def _data_for_TXT(self, azrecord): - return {'values': [reduce((lambda a, b: a + b), ar.value) - for ar in azrecord.txt_records]} - def _data_for_CNAME(self, azrecord): '''Parsing data from Azure DNS Client record call :param azrecord: a return of a call to list azure records @@ -359,6 +360,15 @@ class AzureProvider(BaseProvider): except: return {'value': '.'} + def _data_for_MX(self, azrecord): + return {'values': [{'preference': ar.preference, + 'exchange': ar.exchange} + for ar in azrecord.mx_records]} + + def _data_for_NS(self, azrecord): + vals = [ar.nsdname for ar in azrecord.ns_records] + return {'values': [_check_endswith_dot(val) for val in vals]} + def _data_for_PTR(self, azrecord): try: ptrdname = azrecord.ptr_records[0].ptrdname @@ -366,19 +376,14 @@ class AzureProvider(BaseProvider): except: return {'value': '.'} - def _data_for_MX(self, azrecord): - return {'values': [{'preference': ar.preference, - 'exchange': ar.exchange} - for ar in azrecord.mx_records]} - def _data_for_SRV(self, azrecord): return {'values': [{'priority': ar.priority, 'weight': ar.weight, 'port': ar.port, 'target': ar.target} for ar in azrecord.srv_records]} - def _data_for_NS(self, azrecord): - vals = [ar.nsdname for ar in azrecord.ns_records] - return {'values': [_check_endswith_dot(val) for val in vals]} + def _data_for_TXT(self, azrecord): + return {'values': [reduce((lambda a, b: a + b), ar.value) + for ar in azrecord.txt_records]} def _apply_Create(self, change): '''A record from change must be created. @@ -411,8 +416,7 @@ class AzureProvider(BaseProvider): self.log.debug('* Success Delete: {}'.format(ar)) def _apply(self, plan): - ''' - Required function of manager.py + '''Required function of manager.py to actually apply a record change. :param plan: Contains the zones and changes to be made :type plan: octodns.provider.base.Plan diff --git a/requirements.txt b/requirements.txt index 9eb8284..5d8089a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,8 @@ # These are known good versions. You're free to use others and things will # likely work, but no promises are made, especilly if you go older. PyYaml==3.12 +azure-mgmt-dns==1.0.1 +azure-common==1.1.6 boto3==1.4.4 botocore==1.5.4 dnspython==1.15.0 @@ -10,12 +12,10 @@ futures==3.0.5 incf.countryutils==1.0 ipaddress==1.0.18 jmespath==0.9.0 +msrestazure==0.4.10 natsort==5.0.3 nsone==0.9.14 python-dateutil==2.6.0 requests==2.13.0 s3transfer==0.1.10 -six==1.10.0 -azure-mgmt-dns==1.0.1 -azure-common==1.1.6 -msrestazure==0.4.10 \ No newline at end of file +six==1.10.0 \ No newline at end of file diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index 5ffa055..59cf551 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -16,7 +16,6 @@ from azure.mgmt.dns.models import ARecord, AaaaRecord, CnameRecord, MxRecord, \ Zone as AzureZone from msrestazure.azure_exceptions import CloudError - from unittest import TestCase from mock import Mock, patch From 8d7b3fb101959898a576e735dea606e616d26614 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Wed, 5 Jul 2017 14:09:05 -0700 Subject: [PATCH 43/64] Remove ; escapes before sending to ns1 and when pulling from --- octodns/provider/ns1.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 65db64c..e7b8ffb 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -42,8 +42,16 @@ class Ns1Provider(BaseProvider): } _data_for_AAAA = _data_for_A - _data_for_SPF = _data_for_A - _data_for_TXT = _data_for_A + + def _data_for_SPF(self, _type, record): + values = [v.replace(';', '\;') for v in record['short_answers']] + return { + 'ttl': record['ttl'], + 'type': _type, + 'values': values + } + + _data_for_TXT = _data_for_SPF def _data_for_CNAME(self, _type, record): return { @@ -141,8 +149,15 @@ class Ns1Provider(BaseProvider): _params_for_AAAA = _params_for_A _params_for_NS = _params_for_A - _params_for_SPF = _params_for_A - _params_for_TXT = _params_for_A + + def _params_for_SPF(self, record): + # NS1 seems to be the only provider that doesn't want things escaped in + # values so we have to strip them here and add them when going the + # other way + values = [v.replace('\\', '') for v in record.values] + return {'answers': values, 'ttl': record.ttl} + + _params_for_TXT = _params_for_SPF def _params_for_CNAME(self, record): return {'answers': [record.value], 'ttl': record.ttl} From 818c1e9cc6ba6d155d43a4f3152615d48dabe091 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Wed, 5 Jul 2017 14:28:01 -0700 Subject: [PATCH 44/64] Unit tests for ns1 escape handling and fix --- octodns/provider/ns1.py | 2 +- tests/test_octodns_provider_ns1.py | 34 ++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index e7b8ffb..7757812 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -154,7 +154,7 @@ class Ns1Provider(BaseProvider): # NS1 seems to be the only provider that doesn't want things escaped in # values so we have to strip them here and add them when going the # other way - values = [v.replace('\\', '') for v in record.values] + values = [v.replace('\;', ';') for v in record.values] return {'answers': values, 'ttl': record.ttl} _params_for_TXT = _params_for_SPF diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index 0398459..afcdd41 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -277,3 +277,37 @@ class TestNs1Provider(TestCase): call.update(answers=[u'1.2.3.4'], ttl=32), call.delete() ]) + + def test_escaping(self): + provider = Ns1Provider('test', 'api-key') + + record = { + 'ttl': 31, + 'short_answers': ['foo; bar baz; blip'] + } + self.assertEquals(['foo\; bar baz\; blip'], + provider._data_for_SPF('SPF', record)['values']) + + record = { + 'ttl': 31, + 'short_answers': ['no', 'foo; bar baz; blip', 'yes'] + } + self.assertEquals(['no', 'foo\; bar baz\; blip', 'yes'], + provider._data_for_TXT('TXT', record)['values']) + + zone = Zone('unit.tests.', []) + record = Record.new(zone, 'spf', { + 'ttl': 34, + 'type': 'SPF', + 'value': 'foo\; bar baz\; blip' + }) + self.assertEquals(['foo; bar baz; blip'], + provider._params_for_SPF(record)['answers']) + + record = Record.new(zone, 'txt', { + 'ttl': 35, + 'type': 'TXT', + 'value': 'foo\; bar baz\; blip' + }) + self.assertEquals(['foo; bar baz; blip'], + provider._params_for_TXT(record)['answers']) From 22a05639160bf7c0417d82af78e0ac4df3d43f15 Mon Sep 17 00:00:00 2001 From: Joe Williams Date: Mon, 10 Jul 2017 16:00:50 -0700 Subject: [PATCH 45/64] fix unsafe plan text interpolation --- octodns/provider/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/octodns/provider/base.py b/octodns/provider/base.py index 2fd4349..bcf566a 100644 --- a/octodns/provider/base.py +++ b/octodns/provider/base.py @@ -56,8 +56,8 @@ class Plan(object): delete_pcent = self.change_counts['Delete'] / existing_record_count if update_pcent > self.MAX_SAFE_UPDATE_PCENT: - raise UnsafePlan('Too many updates, %s is over %s percent' - '(%s/%s)', + raise UnsafePlan('Too many updates, {} is over {} percent' + '({}/{})'.format, update_pcent, self.MAX_SAFE_UPDATE_PCENT * 100, self.change_counts['Update'], From 2b58e065e8569e336aaec2cd3f759f03ed69867a Mon Sep 17 00:00:00 2001 From: Joe Williams Date: Tue, 11 Jul 2017 07:09:20 -0700 Subject: [PATCH 46/64] fix format --- octodns/provider/base.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/octodns/provider/base.py b/octodns/provider/base.py index bcf566a..8c97edb 100644 --- a/octodns/provider/base.py +++ b/octodns/provider/base.py @@ -57,18 +57,18 @@ class Plan(object): if update_pcent > self.MAX_SAFE_UPDATE_PCENT: raise UnsafePlan('Too many updates, {} is over {} percent' - '({}/{})'.format, + '({}/{})'.format( update_pcent, self.MAX_SAFE_UPDATE_PCENT * 100, self.change_counts['Update'], - existing_record_count) + existing_record_count)) if delete_pcent > self.MAX_SAFE_DELETE_PCENT: - raise UnsafePlan('Too many deletes, %s is over %s percent' - '(%s/%s)', + raise UnsafePlan('Too many deletes, {} is over {} percent' + '({}/{})'.format( delete_pcent, self.MAX_SAFE_DELETE_PCENT * 100, self.change_counts['Delete'], - existing_record_count) + existing_record_count)) def __repr__(self): return 'Creates={}, Updates={}, Deletes={}, Existing Records={}' \ From 11a246da81928640387a8af36f0b3da8a93cfbe2 Mon Sep 17 00:00:00 2001 From: Joe Williams Date: Tue, 11 Jul 2017 07:20:18 -0700 Subject: [PATCH 47/64] whitespace --- octodns/provider/base.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/octodns/provider/base.py b/octodns/provider/base.py index 8c97edb..d2561ba 100644 --- a/octodns/provider/base.py +++ b/octodns/provider/base.py @@ -58,17 +58,17 @@ class Plan(object): if update_pcent > self.MAX_SAFE_UPDATE_PCENT: raise UnsafePlan('Too many updates, {} is over {} percent' '({}/{})'.format( - update_pcent, - self.MAX_SAFE_UPDATE_PCENT * 100, - self.change_counts['Update'], - existing_record_count)) + update_pcent, + self.MAX_SAFE_UPDATE_PCENT * 100, + self.change_counts['Update'], + existing_record_count)) if delete_pcent > self.MAX_SAFE_DELETE_PCENT: raise UnsafePlan('Too many deletes, {} is over {} percent' '({}/{})'.format( - delete_pcent, - self.MAX_SAFE_DELETE_PCENT * 100, - self.change_counts['Delete'], - existing_record_count)) + delete_pcent, + self.MAX_SAFE_DELETE_PCENT * 100, + self.change_counts['Delete'], + existing_record_count)) def __repr__(self): return 'Creates={}, Updates={}, Deletes={}, Existing Records={}' \ From 5b746845ed567169edb523a2149d385743c84c0c Mon Sep 17 00:00:00 2001 From: Joe Williams Date: Tue, 11 Jul 2017 07:36:24 -0700 Subject: [PATCH 48/64] add tests --- tests/test_octodns_provider_base.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_octodns_provider_base.py b/tests/test_octodns_provider_base.py index bd134bc..e44adc0 100644 --- a/tests/test_octodns_provider_base.py +++ b/tests/test_octodns_provider_base.py @@ -214,9 +214,11 @@ class TestBaseProvider(TestCase): for i in range(int(Plan.MIN_EXISTING_RECORDS * Plan.MAX_SAFE_UPDATE_PCENT) + 1)] - with self.assertRaises(UnsafePlan): + with self.assertRaises(UnsafePlan) as ctx: Plan(zone, zone, changes).raise_if_unsafe() + self.assertTrue('Too many updates' in ctx.exception.message) + def test_safe_updates_min_existing_pcent(self): # MAX_SAFE_UPDATE_PCENT is safe when more # than MIN_EXISTING_RECORDS exist @@ -260,9 +262,11 @@ class TestBaseProvider(TestCase): for i in range(int(Plan.MIN_EXISTING_RECORDS * Plan.MAX_SAFE_DELETE_PCENT) + 1)] - with self.assertRaises(UnsafePlan): + with self.assertRaises(UnsafePlan) as ctx: Plan(zone, zone, changes).raise_if_unsafe() + self.assertTrue('Too many deletes' in ctx.exception.message) + def test_safe_deletes_min_existing_pcent(self): # MAX_SAFE_DELETE_PCENT is safe when more # than MIN_EXISTING_RECORDS exist From f0258d4b2a1bfcc8b3b1b8236f15ccfcfa4a3c61 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 21 Jul 2017 08:53:21 -0700 Subject: [PATCH 49/64] Release v0.8.5 --- CHANGELOG.md | 15 +++++++++++++-- octodns/__init__.py | 2 +- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1cb1204..d58d4de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ +## v0.8.5 - 2017-07-21 - Azure, NS1 escaping, & large zones -## v0.8.4 - 2017-03-14 - It's been too long +Relatively small delta this go around. No major themes or anything, just steady +progress. + +* AzureProvider added thanks to work by + [Heesu Hwang](https://github.com/h-hwang). +* Fixed some escaping issues with NS1 TXT and SPF records that were tracked down + with the help of [Blake Stoddard](https://github.com/blakestoddard). +* Some tweaks were made to Zone.records to vastly improve handling of zones with + very large numbers of records, no more O(N^2). + +## v0.8.4 - 2017-06-28 - It's been too long Lots of updates based on our internal use, needs, and feedback & suggestions from our OSS users. There's too much to list out since the previous release was @@ -28,7 +39,7 @@ better in the future :fingers_crossed: implementation. Sorting can be disabled in the YamlProvider with `enforce_order: False`. * Semi-colon/escaping fixes and improvements. -* Meta record support, `TXT octodns-meta.`. For now just +* Meta record support, `TXT octodns-meta.`. For now just `provider=`. Optionally turned on with `include_meta` manager config val. * Validations check for CNAMEs co-existing with other records and error out if diff --git a/octodns/__init__.py b/octodns/__init__.py index 94ef299..601734c 100644 --- a/octodns/__init__.py +++ b/octodns/__init__.py @@ -3,4 +3,4 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -__VERSION__ = '0.8.4' +__VERSION__ = '0.8.5' From 7f8a01a81da107fb0babffeaed49ba67372e3bfc Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Tue, 25 Jul 2017 09:15:30 -0700 Subject: [PATCH 50/64] Improved/actionable keys out of order error message --- octodns/yaml.py | 10 +++++++--- tests/test_octodns_yaml.py | 4 ++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/octodns/yaml.py b/octodns/yaml.py index d4ab541..98bafdb 100644 --- a/octodns/yaml.py +++ b/octodns/yaml.py @@ -21,9 +21,13 @@ class SortEnforcingLoader(SafeLoader): self.flatten_mapping(node) ret = self.construct_pairs(node) keys = [d[0] for d in ret] - if keys != sorted(keys, key=_natsort_key): - raise ConstructorError(None, None, "keys out of order: {}" - .format(', '.join(keys)), node.start_mark) + keys_sorted = sorted(keys, key=_natsort_key) + for key in keys: + expected = keys_sorted.pop(0) + if key != expected: + raise ConstructorError(None, None, 'keys out of order: ' + 'expected {} got {} at {}' + .format(expected, key, node.start_mark)) return dict(ret) diff --git a/tests/test_octodns_yaml.py b/tests/test_octodns_yaml.py index 0f454b3..effe231 100644 --- a/tests/test_octodns_yaml.py +++ b/tests/test_octodns_yaml.py @@ -48,8 +48,8 @@ class TestYaml(TestCase): '*.11.2': 'd' '*.10.1': 'c' ''') - self.assertEquals('keys out of order: *.2.2, *.1.2, *.11.2, *.10.1', - ctx.exception.problem) + self.assertTrue('keys out of order: expected *.1.2 got *.2.2 at' in + ctx.exception.problem) buf = StringIO() safe_dump({ From 268620c9398713252298e0685fa149dfbb635538 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 12 Aug 2017 12:54:53 -0700 Subject: [PATCH 51/64] Add support for increasing Route53 retries --- octodns/provider/route53.py | 13 +++++++++++-- requirements.txt | 14 +++++++------- tests/test_octodns_provider_route53.py | 8 ++++++++ 3 files changed, 26 insertions(+), 9 deletions(-) diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index 490d630..6f9adc2 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -6,6 +6,7 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals from boto3 import client +from botocore.config import Config from collections import defaultdict from incf.countryutils.transformations import cca_to_ctca2 from uuid import uuid4 @@ -229,14 +230,22 @@ class Route53Provider(BaseProvider): HEALTH_CHECK_VERSION = '0000' def __init__(self, id, access_key_id, secret_access_key, max_changes=1000, - *args, **kwargs): + client_max_attempts=None, *args, **kwargs): self.max_changes = max_changes self.log = logging.getLogger('Route53Provider[{}]'.format(id)) self.log.debug('__init__: id=%s, access_key_id=%s, ' 'secret_access_key=***', id, access_key_id) super(Route53Provider, self).__init__(id, *args, **kwargs) + + config = None + if client_max_attempts is not None: + self.log.info('__init__: setting max_attempts to %d', + client_max_attempts) + config = Config(retries={'max_attempts': client_max_attempts}) + self._conn = client('route53', aws_access_key_id=access_key_id, - aws_secret_access_key=secret_access_key) + aws_secret_access_key=secret_access_key, + config=config) self._r53_zones = None self._r53_rrsets = {} diff --git a/requirements.txt b/requirements.txt index 5d8089a..2aec6d0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,19 +3,19 @@ PyYaml==3.12 azure-mgmt-dns==1.0.1 azure-common==1.1.6 -boto3==1.4.4 -botocore==1.5.4 +boto3==1.4.6 +botocore==1.6.0 dnspython==1.15.0 -docutils==0.13.1 +docutils==0.14 dyn==1.7.10 -futures==3.0.5 +futures==3.1.1 incf.countryutils==1.0 ipaddress==1.0.18 -jmespath==0.9.0 +jmespath==0.9.3 msrestazure==0.4.10 natsort==5.0.3 nsone==0.9.14 -python-dateutil==2.6.0 +python-dateutil==2.6.1 requests==2.13.0 s3transfer==0.1.10 -six==1.10.0 \ No newline at end of file +six==1.10.0 diff --git a/tests/test_octodns_provider_route53.py b/tests/test_octodns_provider_route53.py index be624ff..97dae4f 100644 --- a/tests/test_octodns_provider_route53.py +++ b/tests/test_octodns_provider_route53.py @@ -1232,6 +1232,14 @@ class TestRoute53Provider(TestCase): 'Type': 'TXT', })) + def test_client_max_attempts(self): + provider = Route53Provider('test', 'abc', '123', + client_max_attempts=42) + # NOTE: this will break if boto ever changes the impl details... + self.assertEquals(43, provider._conn.meta.events + ._unique_id_handlers['retry-config-route53'] + ['handler']._checker.__dict__['_max_attempts']) + class TestRoute53Records(TestCase): From 75ca21a6cd6cba22ad773a73ce9c028f8e07890a Mon Sep 17 00:00:00 2001 From: Patrick O'Brien Date: Tue, 15 Aug 2017 16:09:05 -0700 Subject: [PATCH 52/64] Allow scheme to be specified for powerdns This allows a scheme to be set for the PowerDNS API. It defaults to http to retain backwards compatibility. --- octodns/provider/powerdns.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/octodns/provider/powerdns.py b/octodns/provider/powerdns.py index c6d11b0..62b6fd8 100644 --- a/octodns/provider/powerdns.py +++ b/octodns/provider/powerdns.py @@ -18,11 +18,13 @@ class PowerDnsBaseProvider(BaseProvider): 'SPF', 'SSHFP', 'SRV', 'TXT')) TIMEOUT = 5 - def __init__(self, id, host, api_key, port=8081, *args, **kwargs): + def __init__(self, id, host, api_key, port=8081, scheme="http", *args, + **kwargs): super(PowerDnsBaseProvider, self).__init__(id, *args, **kwargs) self.host = host self.port = port + self.scheme = scheme sess = Session() sess.headers.update({'X-API-Key': api_key}) @@ -31,8 +33,8 @@ class PowerDnsBaseProvider(BaseProvider): def _request(self, method, path, data=None): self.log.debug('_request: method=%s, path=%s', method, path) - url = 'http://{}:{}/api/v1/servers/localhost/{}' \ - .format(self.host, self.port, path) + url = '{}://{}:{}/api/v1/servers/localhost/{}' \ + .format(self.scheme, self.host, self.port, path) resp = self._sess.request(method, url, json=data, timeout=self.TIMEOUT) self.log.debug('_request: status=%d', resp.status_code) resp.raise_for_status() From a2c9950d28ad40d548a0ad5f4ef27c8cdd9af97d Mon Sep 17 00:00:00 2001 From: Heesu Hwang Date: Mon, 21 Aug 2017 10:28:43 -0700 Subject: [PATCH 53/64] Fixed inconsistency bug with adding TXT records with Azure. --- octodns/provider/azuredns.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 4433b1e..ee3a6ed 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -90,7 +90,6 @@ class _AzureRecord(object): _params_for_AAAA = _params _params_for_NS = _params _params_for_PTR = _params - _params_for_TXT = _params def _params_for_CNAME(self, data, key_name, azure_class): return {key_name: azure_class(data['value'])} @@ -121,6 +120,12 @@ class _AzureRecord(object): data['value']['target'])) return {key_name: params} + def _params_for_TXT(self, data, key_name, azure_class): + if 'values' in data: + return {key_name: [azure_class([v]) for v in data['values']]} + else: # API for TxtRecord has list of str, even for singleton + return {key_name: [azure_class([data['value']])]} + def _equals(self, b): '''Checks whether two records are equal by comparing all fields. :param b: Another _AzureRecord object From 9623f4e7833dc546527e8031597acd08b7052a1d Mon Sep 17 00:00:00 2001 From: Heesu Hwang Date: Mon, 21 Aug 2017 11:02:28 -0700 Subject: [PATCH 54/64] updated testfile to include test cases for new TXT data parsing --- tests/test_octodns_provider_azuredns.py | 33 +++++++++++++++++++++---- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index 59cf551..598fe48 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -90,6 +90,14 @@ octo_records.append(Record.new(zone, '_srv2._tcp', { 'port': 1, 'target': 'srvfoo.unit.tests.', }]})) +octo_records.append(Record.new(zone, 'txt1', { + 'ttl': 8, + 'type': 'TXT', + 'value': 'txt singleton test'})) +octo_records.append(Record.new(zone, 'txt2', { + 'ttl': 9, + 'type': 'TXT', + 'values': ['txt multiple test', 'txt multiple test 2']})) azure_records = [] _base0 = _AzureRecord('TestAzure', octo_records[0]) @@ -183,6 +191,23 @@ _base10.params['ttl'] = 7 _base10.params['srv_records'] = [SrvRecord(12, 17, 1, 'srvfoo.unit.tests.')] azure_records.append(_base10) +_base11 = _AzureRecord('TestAzure', octo_records[11]) +_base11.zone_name = 'unit.tests' +_base11.relative_record_set_name = 'txt1' +_base11.record_type = 'TXT' +_base11.params['ttl'] = 8 +_base11.params['txt_records'] = [TxtRecord(['txt singleton test'])] +azure_records.append(_base11) + +_base12 = _AzureRecord('TestAzure', octo_records[12]) +_base12.zone_name = 'unit.tests' +_base12.relative_record_set_name = 'txt2' +_base12.record_type = 'TXT' +_base12.params['ttl'] = 9 +_base12.params['txt_records'] = [TxtRecord(['txt multiple test']), + TxtRecord(['txt multiple test 2'])] +azure_records.append(_base12) + class Test_AzureRecord(TestCase): def test_azure_record(self): @@ -190,8 +215,6 @@ class Test_AzureRecord(TestCase): for i in range(len(azure_records)): octo = _AzureRecord('TestAzure', octo_records[i]) assert(azure_records[i]._equals(octo)) - string = str(azure_records[i]) - assert(('Ttl: ' in string)) class Test_ParseAzureType(TestCase): @@ -315,8 +338,8 @@ class TestAzureDnsProvider(TestCase): changes.append(Create(i)) deletes.append(Delete(i)) - self.assertEquals(11, provider.apply(Plan(None, zone, changes))) - self.assertEquals(11, provider.apply(Plan(zone, zone, deletes))) + self.assertEquals(13, provider.apply(Plan(None, zone, changes))) + self.assertEquals(13, provider.apply(Plan(zone, zone, deletes))) def test_create_zone(self): provider = self._get_provider() @@ -331,7 +354,7 @@ class TestAzureDnsProvider(TestCase): _get = provider._dns_client.zones.get _get.side_effect = CloudError(Mock(status=404), err_msg) - self.assertEquals(11, provider.apply(Plan(None, desired, changes))) + self.assertEquals(13, provider.apply(Plan(None, desired, changes))) def test_check_zone_no_create(self): provider = self._get_provider() From a46ee23cc5742c368037d98b8b9b88c65471f139 Mon Sep 17 00:00:00 2001 From: Heesu Hwang Date: Mon, 21 Aug 2017 11:59:18 -0700 Subject: [PATCH 55/64] Slight refactor to make parsing value vs values separate from return --- octodns/provider/azuredns.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index ee3a6ed..1757274 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -81,10 +81,11 @@ class _AzureRecord(object): self.params['ttl'] = record.ttl def _params(self, data, key_name, azure_class): - if 'values' in data: - return {key_name: [azure_class(v) for v in data['values']]} - else: # Else there is a singular data point keyed by 'value'. - return {key_name: [azure_class(data['value'])]} + try: + values = data['values'] + except KeyError: + values = [data['value']] + return {key_name: [azure_class(v) for v in values]} _params_for_A = _params _params_for_AAAA = _params @@ -121,10 +122,11 @@ class _AzureRecord(object): return {key_name: params} def _params_for_TXT(self, data, key_name, azure_class): - if 'values' in data: - return {key_name: [azure_class([v]) for v in data['values']]} - else: # API for TxtRecord has list of str, even for singleton - return {key_name: [azure_class([data['value']])]} + try: # API for TxtRecord has list of str, even for singleton + values = data['values'] + except KeyError: + values = [data['value']] + return {key_name: [azure_class([v]) for v in values]} def _equals(self, b): '''Checks whether two records are equal by comparing all fields. From 4cae1e2bdb71b9e4d4298c727a51932ec3ef8b67 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 26 Aug 2017 08:18:17 -0700 Subject: [PATCH 56/64] Add CAA Record class and tests --- octodns/record.py | 75 +++++++++++++++---- tests/test_octodns_record.py | 138 ++++++++++++++++++++++++++++++++++- 2 files changed, 195 insertions(+), 18 deletions(-) diff --git a/octodns/record.py b/octodns/record.py index 6ee9dff..cc9949f 100644 --- a/octodns/record.py +++ b/octodns/record.py @@ -81,29 +81,16 @@ class Record(object): 'A': ARecord, 'AAAA': AaaaRecord, 'ALIAS': AliasRecord, - # cert + 'CAA': CaaRecord, 'CNAME': CnameRecord, - # dhcid - # dname - # dnskey - # ds - # ipseckey - # key - # kx - # loc 'MX': MxRecord, 'NAPTR': NaptrRecord, 'NS': NsRecord, - # nsap 'PTR': PtrRecord, - # px - # rp - # soa - would it even make sense? 'SPF': SpfRecord, 'SRV': SrvRecord, 'SSHFP': SshfpRecord, 'TXT': TxtRecord, - # url }[_type] except KeyError: raise Exception('Unknown record type: "{}"'.format(_type)) @@ -398,6 +385,66 @@ class AliasRecord(_ValueMixin, Record): return value +class CaaValue(object): + # https://tools.ietf.org/html/rfc6844#page-5 + + @classmethod + def _validate_value(cls, value): + reasons = [] + try: + flags = int(value.get('flags', 0)) + if flags not in (0, 1): + reasons.append('invalid flags "{}"'.format(flags)) + except ValueError: + reasons.append('invalid flags "{}"'.format(value['flags'])) + + try: + tag = value['tag'] + if tag not in ('issue', 'issuewild', 'iodef'): + reasons.append('invalid tag "{}"'.format(tag)) + except KeyError: + reasons.append('missing tag') + + if 'value' not in value: + reasons.append('missing value') + + return reasons + + def __init__(self, value): + self.flags = int(value.get('flags', 0)) + self.tag = value['tag'] + self.value = value['value'] + + @property + def data(self): + return { + 'flags': self.flags, + 'tag': self.tag, + 'value': self.value, + } + + def __cmp__(self, other): + if self.flags == other.flags: + if self.tag == other.tag: + return cmp(self.value, other.value) + return cmp(self.tag, other.tag) + return cmp(self.flags, other.flags) + + def __repr__(self): + return "'{} {} {}'".format(self.flags, self.tag, self.value) + + +class CaaRecord(_ValuesMixin, Record): + _type = 'CAA' + + @classmethod + def _validate_value(cls, value): + return CaaValue._validate_value(value) + + def _process_values(self, values): + return [CaaValue(v) for v in values] + + class CnameRecord(_ValueMixin, Record): _type = 'CNAME' diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index 1d64081..10e3869 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -7,10 +7,10 @@ from __future__ import absolute_import, division, print_function, \ from unittest import TestCase -from octodns.record import ARecord, AaaaRecord, AliasRecord, CnameRecord, \ - Create, Delete, GeoValue, MxRecord, NaptrRecord, NaptrValue, NsRecord, \ - Record, SshfpRecord, SpfRecord, SrvRecord, TxtRecord, Update, \ - ValidationError +from octodns.record import ARecord, AaaaRecord, AliasRecord, CaaRecord, \ + CnameRecord, Create, Delete, GeoValue, MxRecord, NaptrRecord, \ + NaptrValue, NsRecord, Record, SshfpRecord, SpfRecord, SrvRecord, \ + TxtRecord, Update, ValidationError from octodns.zone import Zone from helpers import GeoProvider, SimpleProvider @@ -206,6 +206,66 @@ class TestRecord(TestCase): # __repr__ doesn't blow up a.__repr__() + def test_caa(self): + a_values = [{ + 'flags': 0, + 'tag': 'issue', + 'value': 'ca.example.net', + }, { + 'flags': 1, + 'tag': 'iodef', + 'value': 'mailto:security@example.com', + }] + a_data = {'ttl': 30, 'values': a_values} + a = CaaRecord(self.zone, 'a', a_data) + self.assertEquals('a', a.name) + self.assertEquals('a.unit.tests.', a.fqdn) + self.assertEquals(30, a.ttl) + self.assertEquals(a_values[0]['flags'], a.values[0].flags) + self.assertEquals(a_values[0]['tag'], a.values[0].tag) + self.assertEquals(a_values[0]['value'], a.values[0].value) + self.assertEquals(a_values[1]['flags'], a.values[1].flags) + self.assertEquals(a_values[1]['tag'], a.values[1].tag) + self.assertEquals(a_values[1]['value'], a.values[1].value) + self.assertEquals(a_data, a.data) + + b_value = { + 'tag': 'iodef', + 'value': 'http://iodef.example.com/', + } + b_data = {'ttl': 30, 'value': b_value} + b = CaaRecord(self.zone, 'b', b_data) + self.assertEquals(0, b.values[0].flags) + self.assertEquals(b_value['tag'], b.values[0].tag) + self.assertEquals(b_value['value'], b.values[0].value) + b_data['value']['flags'] = 0 + self.assertEquals(b_data, b.data) + + target = SimpleProvider() + # No changes with self + self.assertFalse(a.changes(a, target)) + # Diff in flags causes change + other = CaaRecord(self.zone, 'a', {'ttl': 30, 'values': a_values}) + other.values[0].flags = 1 + change = a.changes(other, target) + self.assertEqual(change.existing, a) + self.assertEqual(change.new, other) + # Diff in tag causes change + other.values[0].flags = a.values[0].flags + other.values[0].tag = 'foo' + change = a.changes(other, target) + self.assertEqual(change.existing, a) + self.assertEqual(change.new, other) + # Diff in value causes change + other.values[0].tag = a.values[0].tag + other.values[0].value = 'bar' + change = a.changes(other, target) + self.assertEqual(change.existing, a) + self.assertEqual(change.new, other) + + # __repr__ doesn't blow up + a.__repr__() + def test_cname(self): self.assertSingleValue(CnameRecord, 'target.foo.com.', 'other.foo.com.') @@ -861,6 +921,76 @@ class TestRecordValidation(TestCase): }) self.assertEquals(['missing trailing .'], ctx.exception.reasons) + def test_CAA(self): + # doesn't blow up + Record.new(self.zone, '', { + 'type': 'CAA', + 'ttl': 600, + 'value': { + 'flags': 1, + 'tag': 'iodef', + 'value': 'http://foo.bar.com/' + } + }) + + # invalid flags + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'CAA', + 'ttl': 600, + 'value': { + 'flags': 42, + 'tag': 'iodef', + 'value': 'http://foo.bar.com/', + } + }) + self.assertEquals(['invalid flags "42"'], ctx.exception.reasons) + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'CAA', + 'ttl': 600, + 'value': { + 'flags': 'nope', + 'tag': 'iodef', + 'value': 'http://foo.bar.com/', + } + }) + self.assertEquals(['invalid flags "nope"'], ctx.exception.reasons) + + # missing tag + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'CAA', + 'ttl': 600, + 'value': { + 'value': 'http://foo.bar.com/', + } + }) + self.assertEquals(['missing tag'], ctx.exception.reasons) + + # invalid tag + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'CAA', + 'ttl': 600, + 'value': { + 'tag': 'xyz', + 'value': 'http://foo.bar.com/', + } + }) + self.assertEquals(['invalid tag "xyz"'], ctx.exception.reasons) + + # missing value + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'CAA', + 'ttl': 600, + 'value': { + 'tag': 'iodef', + } + }) + self.assertEquals(['missing value'], ctx.exception.reasons) + def test_CNAME(self): # doesn't blow up Record.new(self.zone, 'www', { From 1e68cd6ae98a8d6891010174bd73992e68fbae58 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 26 Aug 2017 09:03:59 -0700 Subject: [PATCH 57/64] Add CAA support to Dyn, PowerDNS, and Route53 --- octodns/provider/dyn.py | 16 ++++++++++++++++ octodns/provider/powerdns.py | 25 +++++++++++++++++++++++-- octodns/provider/route53.py | 23 +++++++++++++++++++++-- octodns/record.py | 11 +++-------- requirements.txt | 4 ++-- tests/test_octodns_record.py | 18 +++--------------- 6 files changed, 68 insertions(+), 29 deletions(-) diff --git a/octodns/provider/dyn.py b/octodns/provider/dyn.py index e21b93e..3b7b9ea 100644 --- a/octodns/provider/dyn.py +++ b/octodns/provider/dyn.py @@ -111,6 +111,7 @@ class DynProvider(BaseProvider): 'a_records': 'A', 'aaaa_records': 'AAAA', 'alias_records': 'ALIAS', + 'caa_records': 'CAA', 'cname_records': 'CNAME', 'mx_records': 'MX', 'naptr_records': 'NAPTR', @@ -194,6 +195,14 @@ class DynProvider(BaseProvider): 'value': record.alias } + def _data_for_CAA(self, _type, records): + return { + 'type': _type, + 'ttl': records[0].ttl, + 'values': [{'flags': r.flags, 'tag': r.tag, 'value': r.value} + for r in records], + } + def _data_for_CNAME(self, _type, records): record = records[0] return { @@ -382,6 +391,13 @@ class DynProvider(BaseProvider): _kwargs_for_AAAA = _kwargs_for_A + def _kwargs_for_CAA(self, record): + return [{ + 'flags': v.flags, + 'tag': v.tag, + 'value': v.value, + } for v in record.values] + def _kwargs_for_CNAME(self, record): return [{ 'cname': record.value, diff --git a/octodns/provider/powerdns.py b/octodns/provider/powerdns.py index 62b6fd8..55ca0b1 100644 --- a/octodns/provider/powerdns.py +++ b/octodns/provider/powerdns.py @@ -14,8 +14,8 @@ from .base import BaseProvider class PowerDnsBaseProvider(BaseProvider): SUPPORTS_GEO = False - SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CNAME', 'MX', 'NAPTR', 'NS', 'PTR', - 'SPF', 'SSHFP', 'SRV', 'TXT')) + SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CAA', 'CNAME', 'MX', 'NAPTR', 'NS', + 'PTR', 'SPF', 'SSHFP', 'SRV', 'TXT')) TIMEOUT = 5 def __init__(self, id, host, api_key, port=8081, scheme="http", *args, @@ -61,6 +61,21 @@ class PowerDnsBaseProvider(BaseProvider): _data_for_AAAA = _data_for_multiple _data_for_NS = _data_for_multiple + def _data_for_CAA(self, rrset): + values = [] + for record in rrset['records']: + flags, tag, value = record['content'].split(' ', 2) + values.append({ + 'flags': flags, + 'tag': tag, + 'value': value, + }) + return { + 'type': rrset['type'], + 'values': values, + 'ttl': rrset['ttl'] + } + def _data_for_single(self, rrset): return { 'type': rrset['type'], @@ -194,6 +209,12 @@ class PowerDnsBaseProvider(BaseProvider): _records_for_AAAA = _records_for_multiple _records_for_NS = _records_for_multiple + def _records_for_CAA(self, record): + return [{ + 'content': '{} {} "{}"'.format(v.flags, v.tag, v.value), + 'disabled': False + } for v in record.values] + def _records_for_single(self, record): return [{'content': record.value, 'disabled': False}] diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index 6f9adc2..0600511 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -90,6 +90,10 @@ class _Route53Record(object): _values_for_AAAA = _values_for_values _values_for_NS = _values_for_values + def _values_for_CAA(self, record): + return ['{} {} "{}"'.format(v.flags, v.tag, v.value) + for v in record.values] + def _values_for_value(self, record): return [record.value] @@ -222,8 +226,8 @@ class Route53Provider(BaseProvider): In general the account used will need full permissions on Route53. ''' SUPPORTS_GEO = True - SUPPORTS = set(('A', 'AAAA', 'CNAME', 'MX', 'NAPTR', 'NS', 'PTR', 'SPF', - 'SRV', 'TXT')) + SUPPORTS = set(('A', 'AAAA', 'CAA', 'CNAME', 'MX', 'NAPTR', 'NS', 'PTR', + 'SPF', 'SRV', 'TXT')) # This should be bumped when there are underlying changes made to the # health check config. @@ -319,6 +323,21 @@ class Route53Provider(BaseProvider): _data_for_A = _data_for_geo _data_for_AAAA = _data_for_geo + def _data_for_CAA(self, rrset): + values = [] + for rr in rrset['ResourceRecords']: + flags, tag, value = rr['Value'].split(' ') + values.append({ + 'flags': flags, + 'tag': tag, + 'value': value[1:-1], + }) + return { + 'type': rrset['Type'], + 'values': values, + 'ttl': int(rrset['TTL']) + } + def _data_for_single(self, rrset): return { 'type': rrset['Type'], diff --git a/octodns/record.py b/octodns/record.py index cc9949f..aa41606 100644 --- a/octodns/record.py +++ b/octodns/record.py @@ -393,18 +393,13 @@ class CaaValue(object): reasons = [] try: flags = int(value.get('flags', 0)) - if flags not in (0, 1): + if flags not in (0, 128): reasons.append('invalid flags "{}"'.format(flags)) except ValueError: reasons.append('invalid flags "{}"'.format(value['flags'])) - try: - tag = value['tag'] - if tag not in ('issue', 'issuewild', 'iodef'): - reasons.append('invalid tag "{}"'.format(tag)) - except KeyError: + if 'tag' not in value: reasons.append('missing tag') - if 'value' not in value: reasons.append('missing value') @@ -431,7 +426,7 @@ class CaaValue(object): return cmp(self.flags, other.flags) def __repr__(self): - return "'{} {} {}'".format(self.flags, self.tag, self.value) + return '{} {} "{}"'.format(self.flags, self.tag, self.value) class CaaRecord(_ValuesMixin, Record): diff --git a/requirements.txt b/requirements.txt index 2aec6d0..d2be70f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,10 +4,10 @@ PyYaml==3.12 azure-mgmt-dns==1.0.1 azure-common==1.1.6 boto3==1.4.6 -botocore==1.6.0 +botocore==1.6.8 dnspython==1.15.0 docutils==0.14 -dyn==1.7.10 +dyn==1.8.0 futures==3.1.1 incf.countryutils==1.0 ipaddress==1.0.18 diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index 10e3869..215abe1 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -212,7 +212,7 @@ class TestRecord(TestCase): 'tag': 'issue', 'value': 'ca.example.net', }, { - 'flags': 1, + 'flags': 128, 'tag': 'iodef', 'value': 'mailto:security@example.com', }] @@ -246,7 +246,7 @@ class TestRecord(TestCase): self.assertFalse(a.changes(a, target)) # Diff in flags causes change other = CaaRecord(self.zone, 'a', {'ttl': 30, 'values': a_values}) - other.values[0].flags = 1 + other.values[0].flags = 128 change = a.changes(other, target) self.assertEqual(change.existing, a) self.assertEqual(change.new, other) @@ -927,7 +927,7 @@ class TestRecordValidation(TestCase): 'type': 'CAA', 'ttl': 600, 'value': { - 'flags': 1, + 'flags': 128, 'tag': 'iodef', 'value': 'http://foo.bar.com/' } @@ -968,18 +968,6 @@ class TestRecordValidation(TestCase): }) self.assertEquals(['missing tag'], ctx.exception.reasons) - # invalid tag - with self.assertRaises(ValidationError) as ctx: - Record.new(self.zone, '', { - 'type': 'CAA', - 'ttl': 600, - 'value': { - 'tag': 'xyz', - 'value': 'http://foo.bar.com/', - } - }) - self.assertEquals(['invalid tag "xyz"'], ctx.exception.reasons) - # missing value with self.assertRaises(ValidationError) as ctx: Record.new(self.zone, '', { From 22591ae84b80812b0f01bc3c7c7fc12b18310869 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 26 Aug 2017 09:38:21 -0700 Subject: [PATCH 58/64] Add CAA support for NS1 --- octodns/provider/ns1.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 7757812..f7cbef1 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -23,8 +23,8 @@ class Ns1Provider(BaseProvider): api_key: env/NS1_API_KEY ''' SUPPORTS_GEO = False - SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CNAME', 'MX', 'NAPTR', 'NS', 'PTR', - 'SPF', 'SRV', 'TXT')) + SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CAA', 'CNAME', 'MX', 'NAPTR', 'NS', + 'PTR', 'SPF', 'SRV', 'TXT')) ZONE_NOT_FOUND_MESSAGE = 'server error: zone not found' @@ -53,6 +53,21 @@ class Ns1Provider(BaseProvider): _data_for_TXT = _data_for_SPF + def _data_for_CAA(self, _type, record): + values = [] + for answer in record['short_answers']: + flags, tag, value = answer.split(' ', 2) + values.append({ + 'flags': flags, + 'tag': tag, + 'value': value, + }) + return { + 'ttl': record['ttl'], + 'type': _type, + 'values': values, + } + def _data_for_CNAME(self, _type, record): return { 'ttl': record['ttl'], @@ -159,6 +174,10 @@ class Ns1Provider(BaseProvider): _params_for_TXT = _params_for_SPF + def _params_for_CAA(self, record): + values = [(v.flags, v.tag, v.value) for v in record.values] + return {'answers': values, 'ttl': record.ttl} + def _params_for_CNAME(self, record): return {'answers': [record.value], 'ttl': record.ttl} From c24c793bcbe59acdac97b819e649c8dceca103fe Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 26 Aug 2017 15:28:09 -0700 Subject: [PATCH 59/64] CAA unit tests for provider support --- octodns/provider/powerdns.py | 2 +- tests/config/unit.tests.yaml | 5 +++++ tests/fixtures/powerdns-full-data.json | 12 ++++++++++ tests/test_octodns_provider_dyn.py | 22 +++++++++++++++++-- tests/test_octodns_provider_ns1.py | 14 ++++++++++++ tests/test_octodns_provider_powerdns.py | 6 ++--- tests/test_octodns_provider_route53.py | 29 ++++++++++++++++++------- tests/test_octodns_provider_yaml.py | 2 +- 8 files changed, 77 insertions(+), 15 deletions(-) diff --git a/octodns/provider/powerdns.py b/octodns/provider/powerdns.py index 55ca0b1..20cfe8b 100644 --- a/octodns/provider/powerdns.py +++ b/octodns/provider/powerdns.py @@ -68,7 +68,7 @@ class PowerDnsBaseProvider(BaseProvider): values.append({ 'flags': flags, 'tag': tag, - 'value': value, + 'value': value[1:-1], }) return { 'type': rrset['type'], diff --git a/tests/config/unit.tests.yaml b/tests/config/unit.tests.yaml index 8be1614..5241406 100644 --- a/tests/config/unit.tests.yaml +++ b/tests/config/unit.tests.yaml @@ -31,6 +31,11 @@ values: - 6.2.3.4. - 7.2.3.4. + - type: CAA + values: + - flags: 0 + tag: issue + value: ca.unit.tests _srv._tcp: ttl: 600 type: SRV diff --git a/tests/fixtures/powerdns-full-data.json b/tests/fixtures/powerdns-full-data.json index 72ce016..b8f8bf3 100644 --- a/tests/fixtures/powerdns-full-data.json +++ b/tests/fixtures/powerdns-full-data.json @@ -230,6 +230,18 @@ ], "ttl": 300, "type": "A" + }, + { + "comments": [], + "name": "unit.tests.", + "records": [ + { + "content": "0 issue \"ca.unit.tests\"", + "disabled": false + } + ], + "ttl": 3600, + "type": "CAA" } ], "serial": 2017012803, diff --git a/tests/test_octodns_provider_dyn.py b/tests/test_octodns_provider_dyn.py index bebd3e3..9be253d 100644 --- a/tests/test_octodns_provider_dyn.py +++ b/tests/test_octodns_provider_dyn.py @@ -109,6 +109,14 @@ class TestDynProvider(TestCase): 'weight': 22, 'port': 20, 'target': 'foo-2.unit.tests.' + }]}), + ('', { + 'type': 'CAA', + 'ttl': 308, + 'values': [{ + 'flags': 0, + 'tag': 'issue', + 'value': 'ca.unit.tests' }]})): expected.add_record(Record.new(expected, name, data)) @@ -321,6 +329,16 @@ class TestDynProvider(TestCase): 'ttl': 307, 'zone': 'unit.tests', }], + 'caa_records': [{ + 'fqdn': 'unit.tests', + 'rdata': {'flags': 0, + 'tag': 'issue', + 'value': 'ca.unit.tests'}, + 'record_id': 12, + 'record_type': 'cAA', + 'ttl': 308, + 'zone': 'unit.tests', + }], }} ] got = Zone('unit.tests.', []) @@ -414,10 +432,10 @@ class TestDynProvider(TestCase): update_mock.assert_called() add_mock.assert_called() # Once for each dyn record (8 Records, 2 of which have dual values) - self.assertEquals(14, len(add_mock.call_args_list)) + self.assertEquals(15, len(add_mock.call_args_list)) execute_mock.assert_has_calls([call('/Zone/unit.tests/', 'GET', {}), call('/Zone/unit.tests/', 'GET', {})]) - self.assertEquals(9, len(plan.changes)) + self.assertEquals(10, len(plan.changes)) execute_mock.reset_mock() diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index 4df56b3..cde23b0 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -96,6 +96,15 @@ class TestNs1Provider(TestCase): 'type': 'NS', 'values': ['ns3.unit.tests.', 'ns4.unit.tests.'], })) + expected.add(Record.new(zone, '', { + 'ttl': 40, + 'type': 'CAA', + 'value': { + 'flags': 0, + 'tag': 'issue', + 'value': 'ca.unit.tests', + }, + })) nsone_records = [{ 'type': 'A', @@ -141,6 +150,11 @@ class TestNs1Provider(TestCase): 'ttl': 39, 'short_answers': ['ns3.unit.tests.', 'ns4.unit.tests.'], 'domain': 'sub.unit.tests.', + }, { + 'type': 'CAA', + 'ttl': 40, + 'short_answers': ['0 issue ca.unit.tests'], + 'domain': 'unit.tests.', }] @patch('nsone.NSONE.loadZone') diff --git a/tests/test_octodns_provider_powerdns.py b/tests/test_octodns_provider_powerdns.py index 5fcd80a..b6e02ff 100644 --- a/tests/test_octodns_provider_powerdns.py +++ b/tests/test_octodns_provider_powerdns.py @@ -79,7 +79,7 @@ class TestPowerDnsProvider(TestCase): source = YamlProvider('test', join(dirname(__file__), 'config')) source.populate(expected) expected_n = len(expected.records) - 1 - self.assertEquals(14, expected_n) + self.assertEquals(15, expected_n) # No diffs == no changes with requests_mock() as mock: @@ -87,7 +87,7 @@ class TestPowerDnsProvider(TestCase): zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals(14, len(zone.records)) + self.assertEquals(15, len(zone.records)) changes = expected.changes(zone, provider) self.assertEquals(0, len(changes)) @@ -167,7 +167,7 @@ class TestPowerDnsProvider(TestCase): expected = Zone('unit.tests.', []) source = YamlProvider('test', join(dirname(__file__), 'config')) source.populate(expected) - self.assertEquals(15, len(expected.records)) + self.assertEquals(16, len(expected.records)) # A small change to a single record with requests_mock() as mock: diff --git a/tests/test_octodns_provider_route53.py b/tests/test_octodns_provider_route53.py index 97dae4f..1cd4548 100644 --- a/tests/test_octodns_provider_route53.py +++ b/tests/test_octodns_provider_route53.py @@ -77,6 +77,12 @@ class TestRoute53Provider(TestCase): {'ttl': 67, 'type': 'NS', 'values': ['8.2.3.4.', '9.2.3.4.']}), ('sub', {'ttl': 68, 'type': 'NS', 'values': ['5.2.3.4.', '6.2.3.4.']}), + ('', + {'ttl': 69, 'type': 'CAA', 'value': { + 'flags': 0, + 'tag': 'issue', + 'value': 'ca.unit.tests' + }}), ): record = Record.new(expected, name, data) expected.add_record(record) @@ -300,6 +306,13 @@ class TestRoute53Provider(TestCase): 'Value': 'ns1.unit.tests.', }], 'TTL': 69, + }, { + 'Name': 'unit.tests.', + 'Type': 'CAA', + 'ResourceRecords': [{ + 'Value': '0 issue "ca.unit.tests"', + }], + 'TTL': 69, }], 'IsTruncated': False, 'MaxItems': '100', @@ -347,7 +360,7 @@ class TestRoute53Provider(TestCase): {'HostedZoneId': 'z42'}) plan = provider.plan(self.expected) - self.assertEquals(8, len(plan.changes)) + self.assertEquals(9, len(plan.changes)) for change in plan.changes: self.assertIsInstance(change, Create) stubber.assert_no_pending_responses() @@ -366,7 +379,7 @@ class TestRoute53Provider(TestCase): 'SubmittedAt': '2017-01-29T01:02:03Z', }}, {'HostedZoneId': 'z42', 'ChangeBatch': ANY}) - self.assertEquals(8, provider.apply(plan)) + self.assertEquals(9, provider.apply(plan)) stubber.assert_no_pending_responses() # Delete by monkey patching in a populate that includes an extra record @@ -579,7 +592,7 @@ class TestRoute53Provider(TestCase): {}) plan = provider.plan(self.expected) - self.assertEquals(8, len(plan.changes)) + self.assertEquals(9, len(plan.changes)) for change in plan.changes: self.assertIsInstance(change, Create) stubber.assert_no_pending_responses() @@ -626,7 +639,7 @@ class TestRoute53Provider(TestCase): 'SubmittedAt': '2017-01-29T01:02:03Z', }}, {'HostedZoneId': 'z42', 'ChangeBatch': ANY}) - self.assertEquals(8, provider.apply(plan)) + self.assertEquals(9, provider.apply(plan)) stubber.assert_no_pending_responses() def test_health_checks_pagination(self): @@ -1174,16 +1187,16 @@ class TestRoute53Provider(TestCase): @patch('octodns.provider.route53.Route53Provider._really_apply') def test_apply_1(self, really_apply_mock): - # 17 RRs with max of 18 should only get applied in one call - provider, plan = self._get_test_plan(18) + # 18 RRs with max of 19 should only get applied in one call + provider, plan = self._get_test_plan(19) provider.apply(plan) really_apply_mock.assert_called_once() @patch('octodns.provider.route53.Route53Provider._really_apply') def test_apply_2(self, really_apply_mock): - # 17 RRs with max of 17 should only get applied in two calls - provider, plan = self._get_test_plan(17) + # 18 RRs with max of 17 should only get applied in two calls + provider, plan = self._get_test_plan(18) provider.apply(plan) self.assertEquals(2, really_apply_mock.call_count) diff --git a/tests/test_octodns_provider_yaml.py b/tests/test_octodns_provider_yaml.py index 9438f01..36cd8d6 100644 --- a/tests/test_octodns_provider_yaml.py +++ b/tests/test_octodns_provider_yaml.py @@ -30,7 +30,7 @@ class TestYamlProvider(TestCase): # without it we see everything source.populate(zone) - self.assertEquals(15, len(zone.records)) + self.assertEquals(16, len(zone.records)) # Assumption here is that a clean round-trip means that everything # worked as expected, data that went in came back out and could be From f5ad26e1f989501fb415ca6ddbbadc1e12d13836 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 26 Aug 2017 15:31:57 -0700 Subject: [PATCH 60/64] Fixes for dnsimple CAA support --- tests/test_octodns_provider_dnsimple.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_octodns_provider_dnsimple.py b/tests/test_octodns_provider_dnsimple.py index aed1e8b..950d460 100644 --- a/tests/test_octodns_provider_dnsimple.py +++ b/tests/test_octodns_provider_dnsimple.py @@ -78,14 +78,14 @@ class TestDnsimpleProvider(TestCase): zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals(14, len(zone.records)) + self.assertEquals(15, len(zone.records)) changes = self.expected.changes(zone, provider) self.assertEquals(0, len(changes)) # 2nd populate makes no network calls/all from cache again = Zone('unit.tests.', []) provider.populate(again) - self.assertEquals(14, len(again.records)) + self.assertEquals(15, len(again.records)) # bust the cache del provider._zone_records[zone.name] @@ -147,7 +147,7 @@ class TestDnsimpleProvider(TestCase): }), ]) # expected number of total calls - self.assertEquals(26, provider._client._request.call_count) + self.assertEquals(27, provider._client._request.call_count) provider._client._request.reset_mock() From e43da949a3663d04f40d66633f29c85507cafaa7 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 26 Aug 2017 15:39:54 -0700 Subject: [PATCH 61/64] Add CAA for CF, DNSimple, and README --- README.md | 6 ++-- octodns/provider/cloudflare.py | 24 ++++++++++++++- octodns/provider/dnsimple.py | 29 +++++++++++++++++-- .../cloudflare-dns_records-page-2.json | 23 +++++++++++++-- tests/fixtures/dnsimple-page-2.json | 18 +++++++++++- tests/test_octodns_provider_cloudflare.py | 12 ++++---- 6 files changed, 97 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 05cf979..1f103f1 100644 --- a/README.md +++ b/README.md @@ -150,12 +150,12 @@ The above command pulled the existing data out of Route53 and placed the results | Provider | Record Support | GeoDNS Support | Notes | |--|--|--|--| | [AzureProvider](/octodns/provider/azuredns.py) | A, AAAA, CNAME, MX, NS, PTR, SRV, TXT | No | | -| [CloudflareProvider](/octodns/provider/cloudflare.py) | A, AAAA, CNAME, MX, NS, SPF, TXT | No | | -| [DnsimpleProvider](/octodns/provider/dnsimple.py) | All | No | | +| [CloudflareProvider](/octodns/provider/cloudflare.py) | A, AAAA, CAA, CNAME, MX, NS, SPF, TXT | No | CAA tags restricted | +| [DnsimpleProvider](/octodns/provider/dnsimple.py) | All | No | CAA tags restricted | | [DynProvider](/octodns/provider/dyn.py) | All | Yes | | | [Ns1Provider](/octodns/provider/ns1.py) | All | No | | | [PowerDnsProvider](/octodns/provider/powerdns.py) | All | No | | -| [Route53](/octodns/provider/route53.py) | A, AAAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | Yes | | +| [Route53](/octodns/provider/route53.py) | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | Yes | | | [TinyDNSSource](/octodns/source/tinydns.py) | A, CNAME, MX, NS, PTR | No | read-only | | [YamlProvider](/octodns/provider/yaml.py) | All | Yes | config | diff --git a/octodns/provider/cloudflare.py b/octodns/provider/cloudflare.py index 2ee8f8b..a4fce9b 100644 --- a/octodns/provider/cloudflare.py +++ b/octodns/provider/cloudflare.py @@ -36,7 +36,7 @@ class CloudflareProvider(BaseProvider): ''' SUPPORTS_GEO = False # TODO: support SRV - SUPPORTS = set(('A', 'AAAA', 'CNAME', 'MX', 'NS', 'SPF', 'TXT')) + SUPPORTS = set(('A', 'AAAA', 'CAA', 'CNAME', 'MX', 'NS', 'SPF', 'TXT')) MIN_TTL = 120 TIMEOUT = 15 @@ -104,6 +104,20 @@ class CloudflareProvider(BaseProvider): 'values': [r['content'].replace(';', '\;') for r in records], } + def _data_for_CAA(self, _type, records): + values = [] + for r in records: + values.append({ + 'flags': r['flags'], + 'tag': r['tag'], + 'value': r['content'], + }) + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': values, + } + def _data_for_CNAME(self, _type, records): only = records[0] return { @@ -197,6 +211,14 @@ class CloudflareProvider(BaseProvider): _contents_for_NS = _contents_for_multiple _contents_for_SPF = _contents_for_multiple + def _contents_for_CAA(self, record): + for value in record.values: + yield { + 'flags': value.flags, + 'tag': value.tag, + 'value': value.value, + } + def _contents_for_TXT(self, record): for value in record.values: yield {'content': value.replace('\;', ';')} diff --git a/octodns/provider/dnsimple.py b/octodns/provider/dnsimple.py index dc44d1b..43b5b9b 100644 --- a/octodns/provider/dnsimple.py +++ b/octodns/provider/dnsimple.py @@ -91,8 +91,8 @@ class DnsimpleProvider(BaseProvider): account: 42 ''' SUPPORTS_GEO = False - SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CNAME', 'MX', 'NAPTR', 'NS', 'PTR', - 'SPF', 'SRV', 'SSHFP', 'TXT')) + SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CAA', 'CNAME', 'MX', 'NAPTR', 'NS', + 'PTR', 'SPF', 'SRV', 'SSHFP', 'TXT')) def __init__(self, id, token, account, *args, **kwargs): self.log = logging.getLogger('DnsimpleProvider[{}]'.format(id)) @@ -114,6 +114,21 @@ class DnsimpleProvider(BaseProvider): _data_for_SPF = _data_for_multiple _data_for_TXT = _data_for_multiple + def _data_for_CAA(self, _type, records): + values = [] + for record in records: + flags, tag, value = record['content'].split(' ') + values.append({ + 'flags': flags, + 'tag': tag, + 'value': value[1:-1], + }) + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': values + } + def _data_for_CNAME(self, _type, records): record = records[0] return { @@ -275,6 +290,16 @@ class DnsimpleProvider(BaseProvider): _params_for_SPF = _params_for_multiple _params_for_TXT = _params_for_multiple + def _params_for_CAA(self, record): + for value in record.values: + yield { + 'content': '{} {} "{}"'.format(value.flags, value.tag, + value.value), + 'name': record.name, + 'ttl': record.ttl, + 'type': record._type + } + def _params_for_single(self, record): yield { 'content': record.value, diff --git a/tests/fixtures/cloudflare-dns_records-page-2.json b/tests/fixtures/cloudflare-dns_records-page-2.json index 24c49d5..9800155 100644 --- a/tests/fixtures/cloudflare-dns_records-page-2.json +++ b/tests/fixtures/cloudflare-dns_records-page-2.json @@ -118,14 +118,33 @@ "meta": { "auto_added": false } + }, + { + "id": "fc223b34cd5611334422ab3322997667", + "type": "CAA", + "name": "unit.tests", + "content": "ca.unit.tests", + "flags": 0, + "tag": "issue", + "proxiable": false, + "proxied": false, + "ttl": 3600, + "locked": false, + "zone_id": "ff12ab34cd5611334422ab3322997650", + "zone_name": "unit.tests", + "modified_on": "2017-03-11T18:01:42.961566Z", + "created_on": "2017-03-11T18:01:42.961566Z", + "meta": { + "auto_added": false + } } ], "result_info": { "page": 2, "per_page": 10, "total_pages": 2, - "count": 7, - "total_count": 17 + "count": 8, + "total_count": 19 }, "success": true, "errors": [], diff --git a/tests/fixtures/dnsimple-page-2.json b/tests/fixtures/dnsimple-page-2.json index f50704b..40aaa48 100644 --- a/tests/fixtures/dnsimple-page-2.json +++ b/tests/fixtures/dnsimple-page-2.json @@ -159,12 +159,28 @@ "system_record": false, "created_at": "2017-03-09T15:55:09Z", "updated_at": "2017-03-09T15:55:09Z" + }, + { + "id": 12188803, + "zone_id": "unit.tests", + "parent_id": null, + "name": "", + "content": "0 issue \"ca.unit.tests\"", + "ttl": 3600, + "priority": null, + "type": "CAA", + "regions": [ + "global" + ], + "system_record": false, + "created_at": "2017-03-09T15:55:09Z", + "updated_at": "2017-03-09T15:55:09Z" } ], "pagination": { "current_page": 2, "per_page": 20, - "total_entries": 29, + "total_entries": 30, "total_pages": 2 } } diff --git a/tests/test_octodns_provider_cloudflare.py b/tests/test_octodns_provider_cloudflare.py index 5dcae30..04a46e0 100644 --- a/tests/test_octodns_provider_cloudflare.py +++ b/tests/test_octodns_provider_cloudflare.py @@ -118,7 +118,7 @@ class TestCloudflareProvider(TestCase): zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals(9, len(zone.records)) + self.assertEquals(10, len(zone.records)) changes = self.expected.changes(zone, provider) self.assertEquals(0, len(changes)) @@ -126,7 +126,7 @@ class TestCloudflareProvider(TestCase): # re-populating the same zone/records comes out of cache, no calls again = Zone('unit.tests.', []) provider.populate(again) - self.assertEquals(9, len(again.records)) + self.assertEquals(10, len(again.records)) def test_apply(self): provider = CloudflareProvider('test', 'email', 'token') @@ -140,12 +140,12 @@ class TestCloudflareProvider(TestCase): 'id': 42, } }, # zone create - ] + [None] * 16 # individual record creates + ] + [None] * 17 # individual record creates # non-existant zone, create everything plan = provider.plan(self.expected) - self.assertEquals(9, len(plan.changes)) - self.assertEquals(9, provider.apply(plan)) + self.assertEquals(10, len(plan.changes)) + self.assertEquals(10, provider.apply(plan)) provider._request.assert_has_calls([ # created the domain @@ -170,7 +170,7 @@ class TestCloudflareProvider(TestCase): }), ], True) # expected number of total calls - self.assertEquals(18, provider._request.call_count) + self.assertEquals(19, provider._request.call_count) provider._request.reset_mock() From ba6dc9858e6e572d46f5b9d0d7778210b53c0410 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 28 Aug 2017 13:40:25 -0700 Subject: [PATCH 62/64] Get out of the business of validating CAA records Seem to be pretty inconsistently implemented/validated across providers so just shrug and move on. --- octodns/record.py | 2 +- tests/test_octodns_record.py | 15 +++++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/octodns/record.py b/octodns/record.py index aa41606..8ef80be 100644 --- a/octodns/record.py +++ b/octodns/record.py @@ -393,7 +393,7 @@ class CaaValue(object): reasons = [] try: flags = int(value.get('flags', 0)) - if flags not in (0, 128): + if flags < 0 or flags > 255: reasons.append('invalid flags "{}"'.format(flags)) except ValueError: reasons.append('invalid flags "{}"'.format(value['flags'])) diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index 215abe1..51676a3 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -939,12 +939,23 @@ class TestRecordValidation(TestCase): 'type': 'CAA', 'ttl': 600, 'value': { - 'flags': 42, + 'flags': -42, 'tag': 'iodef', 'value': 'http://foo.bar.com/', } }) - self.assertEquals(['invalid flags "42"'], ctx.exception.reasons) + self.assertEquals(['invalid flags "-42"'], ctx.exception.reasons) + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'CAA', + 'ttl': 600, + 'value': { + 'flags': 442, + 'tag': 'iodef', + 'value': 'http://foo.bar.com/', + } + }) + self.assertEquals(['invalid flags "442"'], ctx.exception.reasons) with self.assertRaises(ValidationError) as ctx: Record.new(self.zone, '', { 'type': 'CAA', From a558fde6df1c2c0763fcfd22642d72441f9cb698 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Wed, 6 Sep 2017 12:08:08 -0700 Subject: [PATCH 63/64] Fixes for cloudflare CAA support --- octodns/provider/cloudflare.py | 15 +++++++-------- tests/fixtures/cloudflare-dns_records-page-2.json | 8 +++++--- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/octodns/provider/cloudflare.py b/octodns/provider/cloudflare.py index a4fce9b..dd53b3a 100644 --- a/octodns/provider/cloudflare.py +++ b/octodns/provider/cloudflare.py @@ -107,11 +107,8 @@ class CloudflareProvider(BaseProvider): def _data_for_CAA(self, _type, records): values = [] for r in records: - values.append({ - 'flags': r['flags'], - 'tag': r['tag'], - 'value': r['content'], - }) + data = r['data'] + values.append(data) return { 'ttl': records[0]['ttl'], 'type': _type, @@ -214,9 +211,11 @@ class CloudflareProvider(BaseProvider): def _contents_for_CAA(self, record): for value in record.values: yield { - 'flags': value.flags, - 'tag': value.tag, - 'value': value.value, + 'data': { + 'flags': value.flags, + 'tag': value.tag, + 'value': value.value, + } } def _contents_for_TXT(self, record): diff --git a/tests/fixtures/cloudflare-dns_records-page-2.json b/tests/fixtures/cloudflare-dns_records-page-2.json index 9800155..195d6de 100644 --- a/tests/fixtures/cloudflare-dns_records-page-2.json +++ b/tests/fixtures/cloudflare-dns_records-page-2.json @@ -123,9 +123,11 @@ "id": "fc223b34cd5611334422ab3322997667", "type": "CAA", "name": "unit.tests", - "content": "ca.unit.tests", - "flags": 0, - "tag": "issue", + "data": { + "flags": 0, + "tag": "issue", + "value": "ca.unit.tests" + }, "proxiable": false, "proxied": false, "ttl": 3600, From ce7b2ef181adeda05353a776c5b52f50495e7a3a Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Wed, 6 Sep 2017 13:23:26 -0700 Subject: [PATCH 64/64] Cut v0.8.6 --- octodns/__init__.py | 2 +- script/release | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/octodns/__init__.py b/octodns/__init__.py index 601734c..bfb1905 100644 --- a/octodns/__init__.py +++ b/octodns/__init__.py @@ -3,4 +3,4 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -__VERSION__ = '0.8.5' +__VERSION__ = '0.8.6' diff --git a/script/release b/script/release index 16e7641..d8fabf2 100755 --- a/script/release +++ b/script/release @@ -8,5 +8,7 @@ ROOT=$(pwd) VERSION=$(grep __VERSION__ $ROOT/octodns/__init__.py | sed -e "s/.* = '//" -e "s/'$//") git tag -s v$VERSION -m "Release $VERSION" +git push origin v$VERSION +echo "Tagged and pushed v$VERSION" python setup.py sdist upload echo "Updloaded $VERSION"