diff --git a/.git_hooks_pre-commit b/.git_hooks_pre-commit index 9cb854a..c17906b 100755 --- a/.git_hooks_pre-commit +++ b/.git_hooks_pre-commit @@ -6,6 +6,6 @@ HOOKS=`dirname $0` GIT=`dirname $HOOKS` ROOT=`dirname $GIT` -source $ROOT/env/bin/activate +. $ROOT/env/bin/activate $ROOT/script/lint $ROOT/script/test diff --git a/.gitignore b/.gitignore index 842a688..c45a684 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,5 @@ nosetests.xml octodns.egg-info/ output/ tmp/ +build/ +config/ diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..9c97cdc --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,68 @@ +## v0.8.7 - 2017-09-29 - OVH support + +Adds an OVH provider. + +## v0.8.6 - 2017-09-06 - CAA record type, + +Misc fixes and improvments. + +* Azure TXT record fix +* PowerDNS api support for https +* Configurable Route53 max retries and max-attempts +* Improved key ordering error message + +## v0.8.5 - 2017-07-21 - Azure, NS1 escaping, & large zones + +Relatively small delta this go around. No major themes or anything, just steady +progress. + +* AzureProvider added thanks to work by + [Heesu Hwang](https://github.com/h-hwang). +* Fixed some escaping issues with NS1 TXT and SPF records that were tracked down + with the help of [Blake Stoddard](https://github.com/blakestoddard). +* Some tweaks were made to Zone.records to vastly improve handling of zones with + very large numbers of records, no more O(N^2). + +## v0.8.4 - 2017-06-28 - It's been too long + +Lots of updates based on our internal use, needs, and feedback & suggestions +from our OSS users. There's too much to list out since the previous release was +cut, but I'll try to cover the highlights/important bits and promise to do +better in the future :fingers_crossed: + +#### Major: + +* Complete rework of record validation with lenient mode support added to + octodns-dump so that data with validation problems can be dumped to config + files as a starting point. octoDNS now also ignores validation errors when + pulling the current state from a provider before planning changes. In both + cases this is best effort. +* Naming of record keys are based on RFC-1035 and friends, previous names have + been kept for backwards compatibility until the 1.0 release. +* Provider record type support is now explicit, i.e. opt-in, rather than + opt-out. This prevents bugs/oversights in record handling where providers + don't support (new) record types and didn't correctly ignore them. +* ALIAS support for DNSimple, Dyn, NS1, PowerDNS +* Ignored record support added, `octodns:\n ignored: True` +* Ns1Provider added + +#### Miscellaneous + +* Use a 3rd party lib for nautrual sorting of keys, rather than my old + implementation. Sorting can be disabled in the YamlProvider with + `enforce_order: False`. +* Semi-colon/escaping fixes and improvements. +* Meta record support, `TXT octodns-meta.`. For now just + `provider=`. Optionally turned on with `include_meta` manager + config val. +* Validations check for CNAMEs co-existing with other records and error out if + found. Was a common mistaken/unknown issue and this surfaces the problem + early. +* Sizeable refactor in the way Route53 record translation works to make it + cleaner/less hacky +* Lots of docs type-o fixes +* Fixed some pretty major bugs in DnsimpleProvider +* Relax UnsafePlan checks a bit, more to come here +* Set User-Agent header on Dyn health checks + +## v0.8.0 - 2017-03-14 - First public release diff --git a/README.md b/README.md index 0e63e51..a910b5b 100644 --- a/README.md +++ b/README.md @@ -149,12 +149,15 @@ The above command pulled the existing data out of Route53 and placed the results | Provider | Record Support | GeoDNS Support | Notes | |--|--|--|--| -| [CloudflareProvider](/octodns/provider/cloudflare.py) | A, AAAA, CNAME, MX, NS, SPF, TXT | No | | -| [DnsimpleProvider](/octodns/provider/dnsimple.py) | All | No | | +| [AzureProvider](/octodns/provider/azuredns.py) | A, AAAA, CNAME, MX, NS, PTR, SRV, TXT | No | | +| [CloudflareProvider](/octodns/provider/cloudflare.py) | A, AAAA, CAA, CNAME, MX, NS, SPF, TXT | No | CAA tags restricted | +| [DnsimpleProvider](/octodns/provider/dnsimple.py) | All | No | CAA tags restricted | | [DynProvider](/octodns/provider/dyn.py) | All | Yes | | +| [GoogleCloudProvider](/octodns/provider/googlecloud.py) | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | No | | | [Ns1Provider](/octodns/provider/ns1.py) | All | No | | +| [OVH](/octodns/provider/ovh.py) | A, AAAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, SSHFP, TXT | No | | | [PowerDnsProvider](/octodns/provider/powerdns.py) | All | No | | -| [Route53](/octodns/provider/route53.py) | A, AAAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | Yes | | +| [Route53](/octodns/provider/route53.py) | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | Yes | | | [TinyDNSSource](/octodns/source/tinydns.py) | A, CNAME, MX, NS, PTR | No | read-only | | [YamlProvider](/octodns/provider/yaml.py) | All | Yes | config | diff --git a/octodns/__init__.py b/octodns/__init__.py index 4806766..3740dec 100644 --- a/octodns/__init__.py +++ b/octodns/__init__.py @@ -1,8 +1,6 @@ -''' -OctoDNS: DNS as code - Tools for managing DNS across multiple providers -''' +'OctoDNS: DNS as code - Tools for managing DNS across multiple providers' from __future__ import absolute_import, division, print_function, \ unicode_literals -__VERSION__ = '0.8.0' +__VERSION__ = '0.8.7' diff --git a/octodns/cmds/dump.py b/octodns/cmds/dump.py index e4c4987..9f5e0aa 100755 --- a/octodns/cmds/dump.py +++ b/octodns/cmds/dump.py @@ -18,6 +18,9 @@ def main(): parser.add_argument('--output-dir', required=True, help='The directory into which the results will be ' 'written (Note: will overwrite existing files)') + parser.add_argument('--lenient', action='store_true', default=False, + help='Ignore record validations and do a best effort ' + 'dump') parser.add_argument('zone', help='Zone to dump') parser.add_argument('source', nargs='+', help='Source(s) to pull data from') @@ -25,7 +28,7 @@ def main(): args = parser.parse_args() manager = Manager(args.config_file) - manager.dump(args.zone, args.output_dir, *args.source) + manager.dump(args.zone, args.output_dir, args.lenient, *args.source) if __name__ == '__main__': diff --git a/octodns/manager.py b/octodns/manager.py index 0366685..36a3592 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -6,13 +6,14 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals from StringIO import StringIO -from concurrent.futures import Future, ThreadPoolExecutor +from concurrent.futures import ThreadPoolExecutor from importlib import import_module from os import environ import logging -from .provider.base import BaseProvider +from .provider.base import BaseProvider, Plan from .provider.yaml import YamlProvider +from .record import Record from .yaml import safe_load from .zone import Zone @@ -37,6 +38,17 @@ class _AggregateTarget(object): return True +class MakeThreadFuture(object): + + def __init__(self, func, args, kwargs): + self.func = func + self.args = args + self.kwargs = kwargs + + def result(self): + return self.func(*self.args, **self.kwargs) + + class MainThreadExecutor(object): ''' Dummy executor that runs things on the main thread during the involcation @@ -47,19 +59,13 @@ class MainThreadExecutor(object): ''' def submit(self, func, *args, **kwargs): - future = Future() - try: - future.set_result(func(*args, **kwargs)) - except Exception as e: - # TODO: get right stacktrace here - future.set_exception(e) - return future + return MakeThreadFuture(func, args, kwargs) class Manager(object): log = logging.getLogger('Manager') - def __init__(self, config_file, max_workers=None): + def __init__(self, config_file, max_workers=None, include_meta=False): self.log.info('__init__: config_file=%s', config_file) # Read our config file @@ -69,11 +75,16 @@ class Manager(object): manager_config = self.config.get('manager', {}) max_workers = manager_config.get('max_workers', 1) \ if max_workers is None else max_workers + self.log.info('__init__: max_workers=%d', max_workers) if max_workers > 1: self._executor = ThreadPoolExecutor(max_workers=max_workers) else: self._executor = MainThreadExecutor() + self.include_meta = include_meta or manager_config.get('include_meta', + False) + self.log.info('__init__: max_workers=%s', self.include_meta) + self.log.debug('__init__: configuring providers') self.providers = {} for provider_name, provider_config in self.config['providers'].items(): @@ -175,6 +186,13 @@ class Manager(object): plans = [] for target in targets: + if self.include_meta: + meta = Record.new(zone, 'octodns-meta', { + 'type': 'TXT', + 'ttl': 60, + 'value': 'provider={}'.format(target.id) + }) + zone.add_record(meta, replace=True) plan = target.plan(zone) if plan: plans.append((target, plan)) @@ -322,7 +340,7 @@ class Manager(object): return zb.changes(za, _AggregateTarget(a + b)) - def dump(self, zone, output_dir, source, *sources): + def dump(self, zone, output_dir, lenient, source, *sources): ''' Dump zone data from the specified source ''' @@ -341,9 +359,11 @@ class Manager(object): zone = Zone(zone, self.configured_sub_zones(zone)) for source in sources: - source.populate(zone) + source.populate(zone, lenient=lenient) plan = target.plan(zone) + if plan is None: + plan = Plan(zone, zone, []) target.apply(plan) def validate_configs(self): diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py new file mode 100644 index 0000000..1757274 --- /dev/null +++ b/octodns/provider/azuredns.py @@ -0,0 +1,443 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from azure.common.credentials import ServicePrincipalCredentials +from azure.mgmt.dns import DnsManagementClient +from msrestazure.azure_exceptions import CloudError + +from azure.mgmt.dns.models import ARecord, AaaaRecord, CnameRecord, MxRecord, \ + SrvRecord, NsRecord, PtrRecord, TxtRecord, Zone + +import logging +from functools import reduce +from ..record import Record +from .base import BaseProvider + + +class _AzureRecord(object): + '''Wrapper for OctoDNS record for AzureProvider to make dns_client calls. + + azuredns.py: + class: octodns.provider.azuredns._AzureRecord + An _AzureRecord is easily accessible to Azure DNS Management library + functions and is used to wrap all relevant data to create a record in + Azure. + ''' + TYPE_MAP = { + 'A': ARecord, + 'AAAA': AaaaRecord, + 'CNAME': CnameRecord, + 'MX': MxRecord, + 'SRV': SrvRecord, + 'NS': NsRecord, + 'PTR': PtrRecord, + 'TXT': TxtRecord + } + + def __init__(self, resource_group, record, delete=False): + '''Contructor for _AzureRecord. + + Notes on Azure records: An Azure record set has the form + RecordSet(name=<...>, type=<...>, arecords=[...], aaaa_records, ..) + When constructing an azure record as done in self._apply_Create, + the argument parameters for an A record would be + parameters={'ttl': , 'arecords': [ARecord(),]}. + As another example for CNAME record: + parameters={'ttl': , 'cname_record': CnameRecord()}. + + Below, key_name and class_name are the dictionary key and Azure + Record class respectively. + + :param resource_group: The name of resource group in Azure + :type resource_group: str + :param record: An OctoDNS record + :type record: ..record.Record + :param delete: If true, omit data parsing; not needed to delete + :type delete: bool + + :type return: _AzureRecord + ''' + self.resource_group = resource_group + self.zone_name = record.zone.name[:len(record.zone.name) - 1] + self.relative_record_set_name = record.name or '@' + self.record_type = record._type + + if delete: + return + + # Refer to function docstring for key_name and class_name. + format_u_s = '' if record._type == 'A' else '_' + key_name = '{}{}records'.format(self.record_type, format_u_s).lower() + if record._type == 'CNAME': + key_name = key_name[:len(key_name) - 1] + azure_class = self.TYPE_MAP[self.record_type] + + self.params = getattr(self, '_params_for_{}'.format(record._type)) + self.params = self.params(record.data, key_name, azure_class) + self.params['ttl'] = record.ttl + + def _params(self, data, key_name, azure_class): + try: + values = data['values'] + except KeyError: + values = [data['value']] + return {key_name: [azure_class(v) for v in values]} + + _params_for_A = _params + _params_for_AAAA = _params + _params_for_NS = _params + _params_for_PTR = _params + + def _params_for_CNAME(self, data, key_name, azure_class): + return {key_name: azure_class(data['value'])} + + def _params_for_MX(self, data, key_name, azure_class): + params = [] + if 'values' in data: + for vals in data['values']: + params.append(azure_class(vals['preference'], + vals['exchange'])) + else: # Else there is a singular data point keyed by 'value'. + params.append(azure_class(data['value']['preference'], + data['value']['exchange'])) + return {key_name: params} + + def _params_for_SRV(self, data, key_name, azure_class): + params = [] + if 'values' in data: + for vals in data['values']: + params.append(azure_class(vals['priority'], + vals['weight'], + vals['port'], + vals['target'])) + else: # Else there is a singular data point keyed by 'value'. + params.append(azure_class(data['value']['priority'], + data['value']['weight'], + data['value']['port'], + data['value']['target'])) + return {key_name: params} + + def _params_for_TXT(self, data, key_name, azure_class): + try: # API for TxtRecord has list of str, even for singleton + values = data['values'] + except KeyError: + values = [data['value']] + return {key_name: [azure_class([v]) for v in values]} + + def _equals(self, b): + '''Checks whether two records are equal by comparing all fields. + :param b: Another _AzureRecord object + :type b: _AzureRecord + + :type return: bool + ''' + def parse_dict(params): + vals = [] + for char in params: + if char != 'ttl': + list_records = params[char] + try: + for record in list_records: + vals.append(record.__dict__) + except: + vals.append(list_records.__dict__) + vals.sort() + return vals + + return (self.resource_group == b.resource_group) & \ + (self.zone_name == b.zone_name) & \ + (self.record_type == b.record_type) & \ + (self.params['ttl'] == b.params['ttl']) & \ + (parse_dict(self.params) == parse_dict(b.params)) & \ + (self.relative_record_set_name == b.relative_record_set_name) + + def __str__(self): + '''String representation of an _AzureRecord. + :type return: str + ''' + string = 'Zone: {}; '.format(self.zone_name) + string += 'Name: {}; '.format(self.relative_record_set_name) + string += 'Type: {}; '.format(self.record_type) + if not hasattr(self, 'params'): + return string + string += 'Ttl: {}; '.format(self.params['ttl']) + for char in self.params: + if char != 'ttl': + try: + for rec in self.params[char]: + string += 'Record: {}; '.format(rec.__dict__) + except: + string += 'Record: {}; '.format(self.params[char].__dict__) + return string + + +def _check_endswith_dot(string): + return string if string.endswith('.') else string + '.' + + +def _parse_azure_type(string): + '''Converts string representing an Azure RecordSet type to usual type. + + :param string: the Azure type. eg: + :type string: str + + :type return: str + ''' + return string.split('/')[len(string.split('/')) - 1] + + +class AzureProvider(BaseProvider): + ''' + Azure DNS Provider + + azuredns.py: + class: octodns.provider.azuredns.AzureProvider + # Current support of authentication of access to Azure services only + # includes using a Service Principal: + # https://docs.microsoft.com/en-us/azure/azure-resource-manager/ + # resource-group-create-service-principal-portal + # The Azure Active Directory Application ID (aka client ID): + client_id: + # Authentication Key Value: (note this should be secret) + key: + # Directory ID (aka tenant ID): + directory_id: + # Subscription ID: + sub_id: + # Resource Group name: + resource_group: + # All are required to authenticate. + + Example config file with variables: + " + --- + providers: + config: + class: octodns.provider.yaml.YamlProvider + directory: ./config (example path to directory of zone files) + azuredns: + class: octodns.provider.azuredns.AzureProvider + client_id: env/AZURE_APPLICATION_ID + key: env/AZURE_AUTHENICATION_KEY + directory_id: env/AZURE_DIRECTORY_ID + sub_id: env/AZURE_SUBSCRIPTION_ID + resource_group: 'TestResource1' + + zones: + example.com.: + sources: + - config + targets: + - azuredns + " + The first four variables above can be hidden in environment variables + and octoDNS will automatically search for them in the shell. It is + possible to also hard-code into the config file: eg, resource_group. + ''' + SUPPORTS_GEO = False + SUPPORTS = set(('A', 'AAAA', 'CNAME', 'MX', 'NS', 'PTR', 'SRV', 'TXT')) + + def __init__(self, id, client_id, key, directory_id, sub_id, + resource_group, *args, **kwargs): + self.log = logging.getLogger('AzureProvider[{}]'.format(id)) + self.log.debug('__init__: id=%s, client_id=%s, ' + 'key=***, directory_id:%s', id, client_id, directory_id) + super(AzureProvider, self).__init__(id, *args, **kwargs) + + credentials = ServicePrincipalCredentials( + client_id, secret=key, tenant=directory_id + ) + self._dns_client = DnsManagementClient(credentials, sub_id) + self._resource_group = resource_group + self._azure_zones = set() + + def _populate_zones(self): + self.log.debug('azure_zones: loading') + list_zones = self._dns_client.zones.list_by_resource_group + for zone in list_zones(self._resource_group): + self._azure_zones.add(zone.name) + + def _check_zone(self, name, create=False): + '''Checks whether a zone specified in a source exist in Azure server. + + Note that Azure zones omit end '.' eg: contoso.com vs contoso.com. + Returns the name if it exists. + + :param name: Name of a zone to checks + :type name: str + :param create: If True, creates the zone of that name. + :type create: bool + + :type return: str or None + ''' + self.log.debug('_check_zone: name=%s', name) + try: + if name in self._azure_zones: + return name + self._dns_client.zones.get(self._resource_group, name) + self._azure_zones.add(name) + return name + except CloudError as err: + msg = 'The Resource \'Microsoft.Network/dnszones/{}\''.format(name) + msg += ' under resource group \'{}\''.format(self._resource_group) + msg += ' was not found.' + if msg == err.message: + # Then the only error is that the zone doesn't currently exist + if create: + self.log.debug('_check_zone:no matching zone; creating %s', + name) + create_zone = self._dns_client.zones.create_or_update + create_zone(self._resource_group, name, Zone('global')) + return name + else: + return + raise + + def populate(self, zone, target=False, lenient=False): + '''Required function of manager.py to collect records from zone. + + Special notes for Azure. + Azure zone names omit final '.' + Azure root records names are represented by '@'. OctoDNS uses '' + Azure records created through online interface may have null values + (eg, no IP address for A record). + Azure online interface allows constructing records with null values + which are destroyed by _apply. + + Specific quirks such as these are responsible for any non-obvious + parsing in this function and the functions '_params_for_*'. + + :param zone: A dns zone + :type zone: octodns.zone.Zone + :param target: Checks if Azure is source or target of config. + Currently only supports as a target. Unused. + :type target: bool + :param lenient: Unused. Check octodns.manager for usage. + :type lenient: bool + + :type return: void + ''' + self.log.debug('populate: name=%s', zone.name) + before = len(zone.records) + + zone_name = zone.name[:len(zone.name) - 1] + self._populate_zones() + self._check_zone(zone_name) + + _records = set() + records = self._dns_client.record_sets.list_by_dns_zone + if self._check_zone(zone_name): + for azrecord in records(self._resource_group, zone_name): + if _parse_azure_type(azrecord.type) in self.SUPPORTS: + _records.add(azrecord) + for azrecord in _records: + record_name = azrecord.name if azrecord.name != '@' else '' + typ = _parse_azure_type(azrecord.type) + data = getattr(self, '_data_for_{}'.format(typ)) + data = data(azrecord) + data['type'] = typ + data['ttl'] = azrecord.ttl + record = Record.new(zone, record_name, data, source=self) + zone.add_record(record) + + self.log.info('populate: found %s records', len(zone.records) - before) + + def _data_for_A(self, azrecord): + return {'values': [ar.ipv4_address for ar in azrecord.arecords]} + + def _data_for_AAAA(self, azrecord): + return {'values': [ar.ipv6_address for ar in azrecord.aaaa_records]} + + def _data_for_CNAME(self, azrecord): + '''Parsing data from Azure DNS Client record call + :param azrecord: a return of a call to list azure records + :type azrecord: azure.mgmt.dns.models.RecordSet + + :type return: dict + + CNAME and PTR both use the catch block to catch possible empty + records. Refer to population comment. + ''' + try: + return {'value': _check_endswith_dot(azrecord.cname_record.cname)} + except: + return {'value': '.'} + + def _data_for_MX(self, azrecord): + return {'values': [{'preference': ar.preference, + 'exchange': ar.exchange} + for ar in azrecord.mx_records]} + + def _data_for_NS(self, azrecord): + vals = [ar.nsdname for ar in azrecord.ns_records] + return {'values': [_check_endswith_dot(val) for val in vals]} + + def _data_for_PTR(self, azrecord): + try: + ptrdname = azrecord.ptr_records[0].ptrdname + return {'value': _check_endswith_dot(ptrdname)} + except: + return {'value': '.'} + + def _data_for_SRV(self, azrecord): + return {'values': [{'priority': ar.priority, 'weight': ar.weight, + 'port': ar.port, 'target': ar.target} + for ar in azrecord.srv_records]} + + def _data_for_TXT(self, azrecord): + return {'values': [reduce((lambda a, b: a + b), ar.value) + for ar in azrecord.txt_records]} + + def _apply_Create(self, change): + '''A record from change must be created. + + :param change: a change object + :type change: octodns.record.Change + + :type return: void + ''' + ar = _AzureRecord(self._resource_group, change.new) + create = self._dns_client.record_sets.create_or_update + + create(resource_group_name=ar.resource_group, + zone_name=ar.zone_name, + relative_record_set_name=ar.relative_record_set_name, + record_type=ar.record_type, + parameters=ar.params) + + self.log.debug('* Success Create/Update: {}'.format(ar)) + + _apply_Update = _apply_Create + + def _apply_Delete(self, change): + ar = _AzureRecord(self._resource_group, change.existing, delete=True) + delete = self._dns_client.record_sets.delete + + delete(self._resource_group, ar.zone_name, ar.relative_record_set_name, + ar.record_type) + + self.log.debug('* Success Delete: {}'.format(ar)) + + def _apply(self, plan): + '''Required function of manager.py to actually apply a record change. + + :param plan: Contains the zones and changes to be made + :type plan: octodns.provider.base.Plan + + :type return: void + ''' + desired = plan.desired + changes = plan.changes + self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name, + len(changes)) + + azure_zone_name = desired.name[:len(desired.name) - 1] + self._check_zone(azure_zone_name, create=True) + + for change in changes: + class_name = change.__class__.__name__ + getattr(self, '_apply_{}'.format(class_name))(change) diff --git a/octodns/provider/base.py b/octodns/provider/base.py index e73c01d..921b32a 100644 --- a/octodns/provider/base.py +++ b/octodns/provider/base.py @@ -21,10 +21,14 @@ class Plan(object): MAX_SAFE_DELETE_PCENT = .3 MIN_EXISTING_RECORDS = 10 - def __init__(self, existing, desired, changes): + def __init__(self, existing, desired, changes, + update_pcent_threshold=MAX_SAFE_UPDATE_PCENT, + delete_pcent_threshold=MAX_SAFE_DELETE_PCENT): self.existing = existing self.desired = desired self.changes = changes + self.update_pcent_threshold = update_pcent_threshold + self.delete_pcent_threshold = delete_pcent_threshold change_counts = { 'Create': 0, @@ -55,20 +59,20 @@ class Plan(object): update_pcent = self.change_counts['Update'] / existing_record_count delete_pcent = self.change_counts['Delete'] / existing_record_count - if update_pcent > self.MAX_SAFE_UPDATE_PCENT: - raise UnsafePlan('Too many updates, %s is over %s percent' - '(%s/%s)', - update_pcent, - self.MAX_SAFE_UPDATE_PCENT * 100, - self.change_counts['Update'], - existing_record_count) - if delete_pcent > self.MAX_SAFE_DELETE_PCENT: - raise UnsafePlan('Too many deletes, %s is over %s percent' - '(%s/%s)', - delete_pcent, - self.MAX_SAFE_DELETE_PCENT * 100, - self.change_counts['Delete'], - existing_record_count) + if update_pcent > self.update_pcent_threshold: + raise UnsafePlan('Too many updates, {} is over {} percent' + '({}/{})'.format( + update_pcent, + self.MAX_SAFE_UPDATE_PCENT * 100, + self.change_counts['Update'], + existing_record_count)) + if delete_pcent > self.delete_pcent_threshold: + raise UnsafePlan('Too many deletes, {} is over {} percent' + '({}/{})'.format( + delete_pcent, + self.MAX_SAFE_DELETE_PCENT * 100, + self.change_counts['Delete'], + existing_record_count)) def __repr__(self): return 'Creates={}, Updates={}, Deletes={}, Existing Records={}' \ @@ -79,11 +83,19 @@ class Plan(object): class BaseProvider(BaseSource): - def __init__(self, id, apply_disabled=False): + def __init__(self, id, apply_disabled=False, + update_pcent_threshold=Plan.MAX_SAFE_UPDATE_PCENT, + delete_pcent_threshold=Plan.MAX_SAFE_DELETE_PCENT): super(BaseProvider, self).__init__(id) - self.log.debug('__init__: id=%s, apply_disabled=%s', id, - apply_disabled) + self.log.debug('__init__: id=%s, apply_disabled=%s, ' + 'update_pcent_threshold=%d, delete_pcent_threshold=%d', + id, + apply_disabled, + update_pcent_threshold, + delete_pcent_threshold) self.apply_disabled = apply_disabled + self.update_pcent_threshold = update_pcent_threshold + self.delete_pcent_threshold = delete_pcent_threshold def _include_change(self, change): ''' @@ -104,7 +116,7 @@ class BaseProvider(BaseSource): self.log.info('plan: desired=%s', desired.name) existing = Zone(desired.name, desired.sub_zones) - self.populate(existing, target=True) + self.populate(existing, target=True, lenient=True) # compute the changes at the zone/record level changes = existing.changes(desired, self) @@ -124,7 +136,9 @@ class BaseProvider(BaseSource): changes += extra if changes: - plan = Plan(existing, desired, changes) + plan = Plan(existing, desired, changes, + self.update_pcent_threshold, + self.delete_pcent_threshold) self.log.info('plan: %s', plan) return plan self.log.info('plan: No changes') diff --git a/octodns/provider/cloudflare.py b/octodns/provider/cloudflare.py index eb44d30..dd53b3a 100644 --- a/octodns/provider/cloudflare.py +++ b/octodns/provider/cloudflare.py @@ -36,7 +36,7 @@ class CloudflareProvider(BaseProvider): ''' SUPPORTS_GEO = False # TODO: support SRV - UNSUPPORTED_TYPES = ('ALIAS', 'NAPTR', 'PTR', 'SOA', 'SRV', 'SSHFP') + SUPPORTS = set(('A', 'AAAA', 'CAA', 'CNAME', 'MX', 'NS', 'SPF', 'TXT')) MIN_TTL = 120 TIMEOUT = 15 @@ -56,9 +56,6 @@ class CloudflareProvider(BaseProvider): self._zones = None self._zone_records = {} - def supports(self, record): - return record._type not in self.UNSUPPORTED_TYPES - def _request(self, method, path, params=None, data=None): self.log.debug('_request: method=%s, path=%s', method, path) @@ -107,6 +104,17 @@ class CloudflareProvider(BaseProvider): 'values': [r['content'].replace(';', '\;') for r in records], } + def _data_for_CAA(self, _type, records): + values = [] + for r in records: + data = r['data'] + values.append(data) + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': values, + } + def _data_for_CNAME(self, _type, records): only = records[0] return { @@ -119,8 +127,8 @@ class CloudflareProvider(BaseProvider): values = [] for r in records: values.append({ - 'priority': r['priority'], - 'value': '{}.'.format(r['content']), + 'preference': r['priority'], + 'exchange': '{}.'.format(r['content']), }) return { 'ttl': records[0]['ttl'], @@ -157,8 +165,9 @@ class CloudflareProvider(BaseProvider): return self._zone_records[zone.name] - def populate(self, zone, target=False): - self.log.debug('populate: name=%s', zone.name) + def populate(self, zone, target=False, lenient=False): + self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, + target, lenient) before = len(zone.records) records = self.zone_records(zone) @@ -167,14 +176,15 @@ class CloudflareProvider(BaseProvider): for record in records: name = zone.hostname_from_fqdn(record['name']) _type = record['type'] - if _type not in self.UNSUPPORTED_TYPES: + if _type in self.SUPPORTS: values[name][record['type']].append(record) for name, types in values.items(): for _type, records in types.items(): data_for = getattr(self, '_data_for_{}'.format(_type)) data = data_for(_type, records) - record = Record.new(zone, name, data, source=self) + record = Record.new(zone, name, data, source=self, + lenient=lenient) zone.add_record(record) self.log.info('populate: found %s records', @@ -198,6 +208,16 @@ class CloudflareProvider(BaseProvider): _contents_for_NS = _contents_for_multiple _contents_for_SPF = _contents_for_multiple + def _contents_for_CAA(self, record): + for value in record.values: + yield { + 'data': { + 'flags': value.flags, + 'tag': value.tag, + 'value': value.value, + } + } + def _contents_for_TXT(self, record): for value in record.values: yield {'content': value.replace('\;', ';')} @@ -208,8 +228,8 @@ class CloudflareProvider(BaseProvider): def _contents_for_MX(self, record): for value in record.values: yield { - 'priority': value.priority, - 'content': value.value + 'priority': value.preference, + 'content': value.exchange } def _apply_Create(self, change): diff --git a/octodns/provider/dnsimple.py b/octodns/provider/dnsimple.py index 763e446..43b5b9b 100644 --- a/octodns/provider/dnsimple.py +++ b/octodns/provider/dnsimple.py @@ -91,6 +91,8 @@ class DnsimpleProvider(BaseProvider): account: 42 ''' SUPPORTS_GEO = False + SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CAA', 'CNAME', 'MX', 'NAPTR', 'NS', + 'PTR', 'SPF', 'SRV', 'SSHFP', 'TXT')) def __init__(self, id, token, account, *args, **kwargs): self.log = logging.getLogger('DnsimpleProvider[{}]'.format(id)) @@ -112,6 +114,21 @@ class DnsimpleProvider(BaseProvider): _data_for_SPF = _data_for_multiple _data_for_TXT = _data_for_multiple + def _data_for_CAA(self, _type, records): + values = [] + for record in records: + flags, tag, value = record['content'].split(' ') + values.append({ + 'flags': flags, + 'tag': tag, + 'value': value[1:-1], + }) + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': values + } + def _data_for_CNAME(self, _type, records): record = records[0] return { @@ -126,8 +143,8 @@ class DnsimpleProvider(BaseProvider): values = [] for record in records: values.append({ - 'priority': record['priority'], - 'value': '{}.'.format(record['content']) + 'preference': record['priority'], + 'exchange': '{}.'.format(record['content']) }) return { 'ttl': records[0]['ttl'], @@ -232,8 +249,9 @@ class DnsimpleProvider(BaseProvider): return self._zone_records[zone.name] - def populate(self, zone, target=False): - self.log.debug('populate: name=%s', zone.name) + def populate(self, zone, target=False, lenient=False): + self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, + target, lenient) values = defaultdict(lambda: defaultdict(list)) for record in self.zone_records(zone): @@ -250,7 +268,8 @@ class DnsimpleProvider(BaseProvider): for name, types in values.items(): for _type, records in types.items(): data_for = getattr(self, '_data_for_{}'.format(_type)) - record = Record.new(zone, name, data_for(_type, records)) + record = Record.new(zone, name, data_for(_type, records), + source=self, lenient=lenient) zone.add_record(record) self.log.info('populate: found %s records', @@ -271,6 +290,16 @@ class DnsimpleProvider(BaseProvider): _params_for_SPF = _params_for_multiple _params_for_TXT = _params_for_multiple + def _params_for_CAA(self, record): + for value in record.values: + yield { + 'content': '{} {} "{}"'.format(value.flags, value.tag, + value.value), + 'name': record.name, + 'ttl': record.ttl, + 'type': record._type + } + def _params_for_single(self, record): yield { 'content': record.value, @@ -286,9 +315,9 @@ class DnsimpleProvider(BaseProvider): def _params_for_MX(self, record): for value in record.values: yield { - 'content': value.value, + 'content': value.exchange, 'name': record.name, - 'priority': value.priority, + 'priority': value.preference, 'ttl': record.ttl, 'type': record._type } diff --git a/octodns/provider/dyn.py b/octodns/provider/dyn.py index 714f2eb..7423981 100644 --- a/octodns/provider/dyn.py +++ b/octodns/provider/dyn.py @@ -157,10 +157,12 @@ class DynProvider(BaseProvider): than one account active at a time. See DynProvider._check_dyn_sess for some related bits. ''' + RECORDS_TO_TYPE = { 'a_records': 'A', 'aaaa_records': 'AAAA', 'alias_records': 'ALIAS', + 'caa_records': 'CAA', 'cname_records': 'CNAME', 'mx_records': 'MX', 'naptr_records': 'NAPTR', @@ -172,6 +174,7 @@ class DynProvider(BaseProvider): 'txt_records': 'TXT', } TYPE_TO_RECORDS = {v: k for k, v in RECORDS_TO_TYPE.items()} + SUPPORTS = set(TYPE_TO_RECORDS.keys()) # https://help.dyn.com/predefined-geotm-regions-groups/ REGION_CODES = { @@ -247,6 +250,14 @@ class DynProvider(BaseProvider): 'value': record.alias } + def _data_for_CAA(self, _type, records): + return { + 'type': _type, + 'ttl': records[0].ttl, + 'values': [{'flags': r.flags, 'tag': r.tag, 'value': r.value} + for r in records], + } + def _data_for_CNAME(self, _type, records): record = records[0] return { @@ -259,7 +270,7 @@ class DynProvider(BaseProvider): return { 'type': _type, 'ttl': records[0].ttl, - 'values': [{'priority': r.preference, 'value': r.exchange} + 'values': [{'preference': r.preference, 'exchange': r.exchange} for r in records], } @@ -334,7 +345,9 @@ class DynProvider(BaseProvider): for td in get_all_dsf_services(): try: fqdn, _type = td.label.split(':', 1) - except ValueError: + except ValueError as e: + self.log.warn("Failed to load TraficDirector '%s': %s", + td.label, e.message) continue tds[fqdn][_type] = td self._traffic_directors = dict(tds) @@ -391,8 +404,10 @@ class DynProvider(BaseProvider): return td_records - def populate(self, zone, target=False): - self.log.info('populate: zone=%s', zone.name) + def populate(self, zone, target=False, lenient=False): + self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, + target, lenient) + before = len(zone.records) self._check_dyn_sess() @@ -417,7 +432,8 @@ class DynProvider(BaseProvider): for _type, records in types.items(): data_for = getattr(self, '_data_for_{}'.format(_type)) data = data_for(_type, records) - record = Record.new(zone, name, data, source=self) + record = Record.new(zone, name, data, source=self, + lenient=lenient) if record not in td_records: zone.add_record(record) @@ -460,6 +476,13 @@ class DynProvider(BaseProvider): _kwargs_for_AAAA = _kwargs_for_A + def _kwargs_for_CAA(self, record): + return [{ + 'flags': v.flags, + 'tag': v.tag, + 'value': v.value, + } for v in record.values] + def _kwargs_for_CNAME(self, record): return [{ 'cname': record.value, @@ -478,8 +501,8 @@ class DynProvider(BaseProvider): def _kwargs_for_MX(self, record): return [{ - 'preference': v.priority, - 'exchange': v.value, + 'preference': v.preference, + 'exchange': v.exchange, 'ttl': record.ttl, } for v in record.values] @@ -517,7 +540,7 @@ class DynProvider(BaseProvider): return [{ 'txtdata': v, 'ttl': record.ttl, - } for v in record.values] + } for v in record.chunked_values] def _kwargs_for_SRV(self, record): return [{ diff --git a/octodns/provider/googlecloud.py b/octodns/provider/googlecloud.py new file mode 100644 index 0000000..6ca0794 --- /dev/null +++ b/octodns/provider/googlecloud.py @@ -0,0 +1,333 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +import shlex +import time +from logging import getLogger +from uuid import uuid4 + +from google.cloud import dns + +from .base import BaseProvider +from ..record import Record + + +class GoogleCloudProvider(BaseProvider): + """ + Google Cloud DNS provider + + google_cloud: + class: octodns.provider.googlecloud.GoogleCloudProvider + # Credentials file for a service_account or other account can be + # specified with the GOOGLE_APPLICATION_CREDENTIALS environment + # variable. (https://console.cloud.google.com/apis/credentials) + # + # The project to work on (not required) + # project: foobar + # + # The File with the google credentials (not required). If used, the + # "project" parameter needs to be set, else it will fall back to the + # "default credentials" + # credentials_file: ~/google_cloud_credentials_file.json + # + """ + + SUPPORTS = set(('A', 'AAAA', 'CAA', 'CNAME', 'MX', 'NAPTR', + 'NS', 'PTR', 'SPF', 'SRV', 'TXT')) + SUPPORTS_GEO = False + + CHANGE_LOOP_WAIT = 5 + + def __init__(self, id, project=None, credentials_file=None, + *args, **kwargs): + + if credentials_file: + self.gcloud_client = dns.Client.from_service_account_json( + credentials_file, project=project) + else: + self.gcloud_client = dns.Client(project=project) + + # Logger + self.log = getLogger('GoogleCloudProvider[{}]'.format(id)) + self.id = id + + self._gcloud_zones = {} + + super(GoogleCloudProvider, self).__init__(id, *args, **kwargs) + + def _apply(self, plan): + """Required function of manager.py to actually apply a record change. + + :param plan: Contains the zones and changes to be made + :type plan: octodns.provider.base.Plan + + :type return: void + """ + desired = plan.desired + changes = plan.changes + + self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name, + len(changes)) + + # Get gcloud zone, or create one if none existed before. + if desired.name not in self.gcloud_zones: + gcloud_zone = self._create_gcloud_zone(desired.name) + else: + gcloud_zone = self.gcloud_zones.get(desired.name) + + gcloud_changes = gcloud_zone.changes() + + for change in changes: + class_name = change.__class__.__name__ + _rrset_func = getattr( + self, '_rrset_for_{}'.format(change.record._type)) + + if class_name == 'Create': + gcloud_changes.add_record_set( + _rrset_func(gcloud_zone, change.record)) + elif class_name == 'Delete': + gcloud_changes.delete_record_set( + _rrset_func(gcloud_zone, change.record)) + elif class_name == 'Update': + gcloud_changes.delete_record_set( + _rrset_func(gcloud_zone, change.existing)) + gcloud_changes.add_record_set( + _rrset_func(gcloud_zone, change.new)) + else: + raise RuntimeError('Change type "{}" for change "{!s}" ' + 'is none of "Create", "Delete" or "Update' + .format(class_name, change)) + + gcloud_changes.create() + + for i in range(120): + gcloud_changes.reload() + # https://cloud.google.com/dns/api/v1/changes#resource + # status can be one of either "pending" or "done" + if gcloud_changes.status != 'pending': + break + self.log.debug("Waiting for changes to complete") + time.sleep(self.CHANGE_LOOP_WAIT) + + if gcloud_changes.status != 'done': + raise RuntimeError("Timeout reached after {} seconds".format( + i * self.CHANGE_LOOP_WAIT)) + + def _create_gcloud_zone(self, dns_name): + """Creates a google cloud ManagedZone with dns_name, and zone named + derived from it. calls .create() method and returns it. + + :param dns_name: fqdn of zone to create + :type dns_name: str + + :type return: new google.cloud.dns.ManagedZone + """ + # Zone name must begin with a letter, end with a letter or digit, + # and only contain lowercase letters, digits or dashes + zone_name = '{}-{}'.format( + dns_name[:-1].replace('.', '-'), uuid4().hex) + + gcloud_zone = self.gcloud_client.zone( + name=zone_name, + dns_name=dns_name + ) + gcloud_zone.create(client=self.gcloud_client) + + # add this new zone to the list of zones. + self._gcloud_zones[gcloud_zone.dns_name] = gcloud_zone + + self.log.info("Created zone {}. Fqdn {}.".format(zone_name, dns_name)) + + return gcloud_zone + + def _get_gcloud_records(self, gcloud_zone, page_token=None): + """ Generator function which yields ResourceRecordSet for the managed + gcloud zone, until there are no more records to pull. + + :param gcloud_zone: zone to pull records from + :type gcloud_zone: google.cloud.dns.ManagedZone + :param page_token: page token for the page to get + + :return: a resource record set + :type return: google.cloud.dns.ResourceRecordSet + """ + gcloud_iterator = gcloud_zone.list_resource_record_sets( + page_token=page_token) + for gcloud_record in gcloud_iterator: + yield gcloud_record + # This is to get results which may be on a "paged" page. + # (if more than max_results) entries. + if gcloud_iterator.next_page_token: + for gcloud_record in self._get_gcloud_records( + gcloud_zone, gcloud_iterator.next_page_token): + # yield from is in python 3 only. + yield gcloud_record + + def _get_cloud_zones(self, page_token=None): + """Load all ManagedZones into the self._gcloud_zones dict which is + mapped with the dns_name as key. + + :return: void + """ + + gcloud_zones = self.gcloud_client.list_zones(page_token=page_token) + for gcloud_zone in gcloud_zones: + self._gcloud_zones[gcloud_zone.dns_name] = gcloud_zone + + if gcloud_zones.next_page_token: + self._get_cloud_zones(gcloud_zones.next_page_token) + + @property + def gcloud_zones(self): + if not self._gcloud_zones: + self._get_cloud_zones() + return self._gcloud_zones + + def populate(self, zone, target=False, lenient=False): + """Required function of manager.py to collect records from zone. + + :param zone: A dns zone + :type zone: octodns.zone.Zone + :param target: Unused. + :type target: bool + :param lenient: Unused. Check octodns.manager for usage. + :type lenient: bool + + :type return: void + """ + + self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, + target, lenient) + before = len(zone.records) + + gcloud_zone = self.gcloud_zones.get(zone.name) + + if gcloud_zone: + for gcloud_record in self._get_gcloud_records(gcloud_zone): + if gcloud_record.record_type.upper() not in self.SUPPORTS: + continue + + record_name = gcloud_record.name + if record_name.endswith(zone.name): + # google cloud always return fqdn. Make relative record + # here. "root" records will then get the '' record_name, + # which is also the way octodns likes it. + record_name = record_name[:-(len(zone.name) + 1)] + typ = gcloud_record.record_type.upper() + data = getattr(self, '_data_for_{}'.format(typ)) + data = data(gcloud_record) + data['type'] = typ + data['ttl'] = gcloud_record.ttl + self.log.debug('populate: adding record {} records: {!s}' + .format(record_name, data)) + record = Record.new(zone, record_name, data, source=self) + zone.add_record(record) + + self.log.info('populate: found %s records', len(zone.records) - before) + + def _data_for_A(self, gcloud_record): + return { + 'values': gcloud_record.rrdatas + } + + _data_for_AAAA = _data_for_A + + def _data_for_CAA(self, gcloud_record): + return { + 'values': [{ + 'flags': v[0], + 'tag': v[1], + 'value': v[2]} + for v in [shlex.split(g) for g in gcloud_record.rrdatas]]} + + def _data_for_CNAME(self, gcloud_record): + return { + 'value': gcloud_record.rrdatas[0] + } + + def _data_for_MX(self, gcloud_record): + return {'values': [{ + "preference": v[0], + "exchange": v[1]} + for v in [shlex.split(g) for g in gcloud_record.rrdatas]]} + + def _data_for_NAPTR(self, gcloud_record): + return {'values': [{ + 'order': v[0], + 'preference': v[1], + 'flags': v[2], + 'service': v[3], + 'regexp': v[4], + 'replacement': v[5]} + for v in [shlex.split(g) for g in gcloud_record.rrdatas]]} + + _data_for_NS = _data_for_A + + _data_for_PTR = _data_for_CNAME + + def _data_for_SPF(self, gcloud_record): + if len(gcloud_record.rrdatas) > 1: + return { + 'values': gcloud_record.rrdatas} + return { + 'value': gcloud_record.rrdatas[0]} + + def _data_for_SRV(self, gcloud_record): + return {'values': [{ + 'priority': v[0], + 'weight': v[1], + 'port': v[2], + 'target': v[3]} + for v in [shlex.split(g) for g in gcloud_record.rrdatas]]} + + _data_for_TXT = _data_for_SPF + + def _rrset_for_A(self, gcloud_zone, record): + return gcloud_zone.resource_record_set( + record.fqdn, record._type, record.ttl, record.values) + + _rrset_for_AAAA = _rrset_for_A + + def _rrset_for_CAA(self, gcloud_zone, record): + return gcloud_zone.resource_record_set( + record.fqdn, record._type, record.ttl, [ + '{} {} {}'.format(v.flags, v.tag, v.value) + for v in record.values]) + + def _rrset_for_CNAME(self, gcloud_zone, record): + return gcloud_zone.resource_record_set( + record.fqdn, record._type, record.ttl, [record.value]) + + def _rrset_for_MX(self, gcloud_zone, record): + return gcloud_zone.resource_record_set( + record.fqdn, record._type, record.ttl, [ + '{} {}'.format(v.preference, v.exchange) + for v in record.values]) + + def _rrset_for_NAPTR(self, gcloud_zone, record): + return gcloud_zone.resource_record_set( + record.fqdn, record._type, record.ttl, [ + '{} {} "{}" "{}" "{}" {}'.format( + v.order, v.preference, v.flags, v.service, + v.regexp, v.replacement) for v in record.values]) + + _rrset_for_NS = _rrset_for_A + + _rrset_for_PTR = _rrset_for_CNAME + + def _rrset_for_SPF(self, gcloud_zone, record): + return gcloud_zone.resource_record_set( + record.fqdn, record._type, record.ttl, record.chunked_values) + + def _rrset_for_SRV(self, gcloud_zone, record): + return gcloud_zone.resource_record_set( + record.fqdn, record._type, record.ttl, [ + '{} {} {} {}' + .format(v.priority, v.weight, v.port, v.target) + for v in record.values]) + + _rrset_for_TXT = _rrset_for_SPF diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 5a51780..f7cbef1 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -7,7 +7,8 @@ from __future__ import absolute_import, division, print_function, \ from logging import getLogger from nsone import NSONE -from nsone.rest.errors import ResourceException +from nsone.rest.errors import RateLimitException, ResourceException +from time import sleep from ..record import Record from .base import BaseProvider @@ -22,6 +23,9 @@ class Ns1Provider(BaseProvider): api_key: env/NS1_API_KEY ''' SUPPORTS_GEO = False + SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CAA', 'CNAME', 'MX', 'NAPTR', 'NS', + 'PTR', 'SPF', 'SRV', 'TXT')) + ZONE_NOT_FOUND_MESSAGE = 'server error: zone not found' def __init__(self, id, api_key, *args, **kwargs): @@ -30,9 +34,6 @@ class Ns1Provider(BaseProvider): super(Ns1Provider, self).__init__(id, *args, **kwargs) self._client = NSONE(apiKey=api_key) - def supports(self, record): - return record._type != 'SSHFP' - def _data_for_A(self, _type, record): return { 'ttl': record['ttl'], @@ -41,8 +42,31 @@ class Ns1Provider(BaseProvider): } _data_for_AAAA = _data_for_A - _data_for_SPF = _data_for_A - _data_for_TXT = _data_for_A + + def _data_for_SPF(self, _type, record): + values = [v.replace(';', '\;') for v in record['short_answers']] + return { + 'ttl': record['ttl'], + 'type': _type, + 'values': values + } + + _data_for_TXT = _data_for_SPF + + def _data_for_CAA(self, _type, record): + values = [] + for answer in record['short_answers']: + flags, tag, value = answer.split(' ', 2) + values.append({ + 'flags': flags, + 'tag': tag, + 'value': value, + }) + return { + 'ttl': record['ttl'], + 'type': _type, + 'values': values, + } def _data_for_CNAME(self, _type, record): return { @@ -57,10 +81,10 @@ class Ns1Provider(BaseProvider): def _data_for_MX(self, _type, record): values = [] for answer in record['short_answers']: - priority, value = answer.split(' ', 1) + preference, exchange = answer.split(' ', 1) values.append({ - 'priority': priority, - 'value': value, + 'preference': preference, + 'exchange': exchange, }) return { 'ttl': record['ttl'], @@ -111,8 +135,9 @@ class Ns1Provider(BaseProvider): 'values': values, } - def populate(self, zone, target=False): - self.log.debug('populate: name=%s', zone.name) + def populate(self, zone, target=False, lenient=False): + self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, + target, lenient) try: nsone_zone = self._client.loadZone(zone.name[:-1]) @@ -127,7 +152,8 @@ class Ns1Provider(BaseProvider): _type = record['type'] data_for = getattr(self, '_data_for_{}'.format(_type)) name = zone.hostname_from_fqdn(record['domain']) - record = Record.new(zone, name, data_for(_type, record)) + record = Record.new(zone, name, data_for(_type, record), + source=self, lenient=lenient) zone.add_record(record) self.log.info('populate: found %s records', @@ -138,8 +164,19 @@ class Ns1Provider(BaseProvider): _params_for_AAAA = _params_for_A _params_for_NS = _params_for_A - _params_for_SPF = _params_for_A - _params_for_TXT = _params_for_A + + def _params_for_SPF(self, record): + # NS1 seems to be the only provider that doesn't want things escaped in + # values so we have to strip them here and add them when going the + # other way + values = [v.replace('\;', ';') for v in record.values] + return {'answers': values, 'ttl': record.ttl} + + _params_for_TXT = _params_for_SPF + + def _params_for_CAA(self, record): + values = [(v.flags, v.tag, v.value) for v in record.values] + return {'answers': values, 'ttl': record.ttl} def _params_for_CNAME(self, record): return {'answers': [record.value], 'ttl': record.ttl} @@ -148,7 +185,7 @@ class Ns1Provider(BaseProvider): _params_for_PTR = _params_for_CNAME def _params_for_MX(self, record): - values = [(v.priority, v.value) for v in record.values] + values = [(v.preference, v.exchange) for v in record.values] return {'answers': values, 'ttl': record.ttl} def _params_for_NAPTR(self, record): @@ -169,7 +206,14 @@ class Ns1Provider(BaseProvider): name = self._get_name(new) _type = new._type params = getattr(self, '_params_for_{}'.format(_type))(new) - getattr(nsone_zone, 'add_{}'.format(_type))(name, **params) + meth = getattr(nsone_zone, 'add_{}'.format(_type)) + try: + meth(name, **params) + except RateLimitException as e: + self.log.warn('_apply_Create: rate limit encountered, pausing ' + 'for %ds and trying again', e.period) + sleep(e.period) + meth(name, **params) def _apply_Update(self, nsone_zone, change): existing = change.existing @@ -178,14 +222,26 @@ class Ns1Provider(BaseProvider): record = nsone_zone.loadRecord(name, _type) new = change.new params = getattr(self, '_params_for_{}'.format(_type))(new) - record.update(**params) + try: + record.update(**params) + except RateLimitException as e: + self.log.warn('_apply_Update: rate limit encountered, pausing ' + 'for %ds and trying again', e.period) + sleep(e.period) + record.update(**params) def _apply_Delete(self, nsone_zone, change): existing = change.existing name = self._get_name(existing) _type = existing._type record = nsone_zone.loadRecord(name, _type) - record.delete() + try: + record.delete() + except RateLimitException as e: + self.log.warn('_apply_Delete: rate limit encountered, pausing ' + 'for %ds and trying again', e.period) + sleep(e.period) + record.delete() def _apply(self, plan): desired = plan.desired diff --git a/octodns/provider/ovh.py b/octodns/provider/ovh.py new file mode 100644 index 0000000..b890862 --- /dev/null +++ b/octodns/provider/ovh.py @@ -0,0 +1,322 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +import logging +from collections import defaultdict + +import ovh + +from octodns.record import Record +from .base import BaseProvider + + +class OvhProvider(BaseProvider): + """ + OVH provider using API v6 + + ovh: + class: octodns.provider.ovh.OvhProvider + # OVH api v6 endpoint + endpoint: ovh-eu + # API application key + application_key: 1234 + # API application secret + application_secret: 1234 + # API consumer key + consumer_key: 1234 + """ + + SUPPORTS_GEO = False + + SUPPORTS = set(('A', 'AAAA', 'CNAME', 'MX', 'NAPTR', 'NS', 'PTR', 'SPF', + 'SRV', 'SSHFP', 'TXT')) + + def __init__(self, id, endpoint, application_key, application_secret, + consumer_key, *args, **kwargs): + self.log = logging.getLogger('OvhProvider[{}]'.format(id)) + self.log.debug('__init__: id=%s, endpoint=%s, application_key=%s, ' + 'application_secret=***, consumer_key=%s', id, endpoint, + application_key, consumer_key) + super(OvhProvider, self).__init__(id, *args, **kwargs) + self._client = ovh.Client( + endpoint=endpoint, + application_key=application_key, + application_secret=application_secret, + consumer_key=consumer_key, + ) + + def populate(self, zone, target=False, lenient=False): + self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, + target, lenient) + zone_name = zone.name[:-1] + records = self.get_records(zone_name=zone_name) + + values = defaultdict(lambda: defaultdict(list)) + for record in records: + values[record['subDomain']][record['fieldType']].append(record) + + before = len(zone.records) + for name, types in values.items(): + for _type, records in types.items(): + data_for = getattr(self, '_data_for_{}'.format(_type)) + record = Record.new(zone, name, data_for(_type, records), + source=self, lenient=lenient) + zone.add_record(record) + + self.log.info('populate: found %s records', + len(zone.records) - before) + + def _apply(self, plan): + desired = plan.desired + changes = plan.changes + zone_name = desired.name[:-1] + self.log.info('_apply: zone=%s, len(changes)=%d', desired.name, + len(changes)) + for change in changes: + class_name = change.__class__.__name__ + getattr(self, '_apply_{}'.format(class_name).lower())(zone_name, + change) + + # We need to refresh the zone to really apply the changes + self._client.post('/domain/zone/{}/refresh'.format(zone_name)) + + def _apply_create(self, zone_name, change): + new = change.new + params_for = getattr(self, '_params_for_{}'.format(new._type)) + for params in params_for(new): + self.create_record(zone_name, params) + + def _apply_update(self, zone_name, change): + self._apply_delete(zone_name, change) + self._apply_create(zone_name, change) + + def _apply_delete(self, zone_name, change): + existing = change.existing + self.delete_records(zone_name, existing._type, existing.name) + + @staticmethod + def _data_for_multiple(_type, records): + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': [record['target'] for record in records] + } + + @staticmethod + def _data_for_single(_type, records): + record = records[0] + return { + 'ttl': record['ttl'], + 'type': _type, + 'value': record['target'] + } + + @staticmethod + def _data_for_MX(_type, records): + values = [] + for record in records: + preference, exchange = record['target'].split(' ', 1) + values.append({ + 'preference': preference, + 'exchange': exchange, + }) + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': values, + } + + @staticmethod + def _data_for_NAPTR(_type, records): + values = [] + for record in records: + order, preference, flags, service, regexp, replacement = record[ + 'target'].split(' ', 5) + values.append({ + 'flags': flags[1:-1], + 'order': order, + 'preference': preference, + 'regexp': regexp[1:-1], + 'replacement': replacement, + 'service': service[1:-1], + }) + return { + 'type': _type, + 'ttl': records[0]['ttl'], + 'values': values + } + + @staticmethod + def _data_for_SRV(_type, records): + values = [] + for record in records: + priority, weight, port, target = record['target'].split(' ', 3) + values.append({ + 'port': port, + 'priority': priority, + 'target': '{}.'.format(target), + 'weight': weight + }) + return { + 'type': _type, + 'ttl': records[0]['ttl'], + 'values': values + } + + @staticmethod + def _data_for_SSHFP(_type, records): + values = [] + for record in records: + algorithm, fingerprint_type, fingerprint = record['target'].split( + ' ', 2) + values.append({ + 'algorithm': algorithm, + 'fingerprint': fingerprint, + 'fingerprint_type': fingerprint_type + }) + return { + 'type': _type, + 'ttl': records[0]['ttl'], + 'values': values + } + + _data_for_A = _data_for_multiple + _data_for_AAAA = _data_for_multiple + _data_for_NS = _data_for_multiple + _data_for_TXT = _data_for_multiple + _data_for_SPF = _data_for_multiple + _data_for_PTR = _data_for_single + _data_for_CNAME = _data_for_single + + @staticmethod + def _params_for_multiple(record): + for value in record.values: + yield { + 'target': value, + 'subDomain': record.name, + 'ttl': record.ttl, + 'fieldType': record._type, + } + + @staticmethod + def _params_for_single(record): + yield { + 'target': record.value, + 'subDomain': record.name, + 'ttl': record.ttl, + 'fieldType': record._type + } + + @staticmethod + def _params_for_MX(record): + for value in record.values: + yield { + 'target': '%d %s' % (value.preference, value.exchange), + 'subDomain': record.name, + 'ttl': record.ttl, + 'fieldType': record._type + } + + @staticmethod + def _params_for_NAPTR(record): + for value in record.values: + content = '{} {} "{}" "{}" "{}" {}' \ + .format(value.order, value.preference, value.flags, + value.service, value.regexp, value.replacement) + yield { + 'target': content, + 'subDomain': record.name, + 'ttl': record.ttl, + 'fieldType': record._type + } + + @staticmethod + def _params_for_SRV(record): + for value in record.values: + yield { + 'subDomain': '{} {} {} {}'.format(value.priority, + value.weight, value.port, + value.target), + 'target': record.name, + 'ttl': record.ttl, + 'fieldType': record._type + } + + @staticmethod + def _params_for_SSHFP(record): + for value in record.values: + yield { + 'subDomain': '{} {} {}'.format(value.algorithm, + value.fingerprint_type, + value.fingerprint), + 'target': record.name, + 'ttl': record.ttl, + 'fieldType': record._type + } + + _params_for_A = _params_for_multiple + _params_for_AAAA = _params_for_multiple + _params_for_NS = _params_for_multiple + _params_for_SPF = _params_for_multiple + _params_for_TXT = _params_for_multiple + + _params_for_CNAME = _params_for_single + _params_for_PTR = _params_for_single + + def get_records(self, zone_name): + """ + List all records of a DNS zone + :param zone_name: Name of zone + :return: list of id's records + """ + records = self._client.get('/domain/zone/{}/record'.format(zone_name)) + return [self.get_record(zone_name, record_id) for record_id in records] + + def get_record(self, zone_name, record_id): + """ + Get record with given id + :param zone_name: Name of the zone + :param record_id: Id of the record + :return: Value of the record + """ + return self._client.get( + '/domain/zone/{}/record/{}'.format(zone_name, record_id)) + + def delete_records(self, zone_name, record_type, subdomain): + """ + Delete record from have fieldType=type and subDomain=subdomain + :param zone_name: Name of the zone + :param record_type: fieldType + :param subdomain: subDomain + """ + records = self._client.get('/domain/zone/{}/record'.format(zone_name), + fieldType=record_type, subDomain=subdomain) + for record in records: + self.delete_record(zone_name, record) + + def delete_record(self, zone_name, record_id): + """ + Delete record with a given id + :param zone_name: Name of the zone + :param record_id: Id of the record + """ + self.log.debug('Delete record: zone: %s, id %s', zone_name, + record_id) + self._client.delete( + '/domain/zone/{}/record/{}'.format(zone_name, record_id)) + + def create_record(self, zone_name, params): + """ + Create a record + :param zone_name: Name of the zone + :param params: {'fieldType': 'A', 'ttl': 60, 'subDomain': 'www', + 'target': '1.2.3.4' + """ + self.log.debug('Create record: zone: %s, id %s', zone_name, + params) + return self._client.post('/domain/zone/{}/record'.format(zone_name), + **params) diff --git a/octodns/provider/powerdns.py b/octodns/provider/powerdns.py index 8208833..26d219c 100644 --- a/octodns/provider/powerdns.py +++ b/octodns/provider/powerdns.py @@ -14,13 +14,17 @@ from .base import BaseProvider class PowerDnsBaseProvider(BaseProvider): SUPPORTS_GEO = False + SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CAA', 'CNAME', 'MX', 'NAPTR', 'NS', + 'PTR', 'SPF', 'SSHFP', 'SRV', 'TXT')) TIMEOUT = 5 - def __init__(self, id, host, api_key, port=8081, *args, **kwargs): + def __init__(self, id, host, api_key, port=8081, scheme="http", *args, + **kwargs): super(PowerDnsBaseProvider, self).__init__(id, *args, **kwargs) self.host = host self.port = port + self.scheme = scheme sess = Session() sess.headers.update({'X-API-Key': api_key}) @@ -29,8 +33,8 @@ class PowerDnsBaseProvider(BaseProvider): def _request(self, method, path, data=None): self.log.debug('_request: method=%s, path=%s', method, path) - url = 'http://{}:{}/api/v1/servers/localhost/{}' \ - .format(self.host, self.port, path) + url = '{}://{}:{}/api/v1/servers/localhost/{}' \ + .format(self.scheme, self.host, self.port, path) resp = self._sess.request(method, url, json=data, timeout=self.TIMEOUT) self.log.debug('_request: status=%d', resp.status_code) resp.raise_for_status() @@ -57,6 +61,21 @@ class PowerDnsBaseProvider(BaseProvider): _data_for_AAAA = _data_for_multiple _data_for_NS = _data_for_multiple + def _data_for_CAA(self, rrset): + values = [] + for record in rrset['records']: + flags, tag, value = record['content'].split(' ', 2) + values.append({ + 'flags': flags, + 'tag': tag, + 'value': value[1:-1], + }) + return { + 'type': rrset['type'], + 'values': values, + 'ttl': rrset['ttl'] + } + def _data_for_single(self, rrset): return { 'type': rrset['type'], @@ -81,10 +100,10 @@ class PowerDnsBaseProvider(BaseProvider): def _data_for_MX(self, rrset): values = [] for record in rrset['records']: - priority, value = record['content'].split(' ', 1) + preference, exchange = record['content'].split(' ', 1) values.append({ - 'priority': priority, - 'value': value, + 'preference': preference, + 'exchange': exchange, }) return { 'type': rrset['type'], @@ -144,8 +163,9 @@ class PowerDnsBaseProvider(BaseProvider): 'ttl': rrset['ttl'] } - def populate(self, zone, target=False): - self.log.debug('populate: name=%s', zone.name) + def populate(self, zone, target=False, lenient=False): + self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, + target, lenient) resp = None try: @@ -175,7 +195,7 @@ class PowerDnsBaseProvider(BaseProvider): data_for = getattr(self, '_data_for_{}'.format(_type)) record_name = zone.hostname_from_fqdn(rrset['name']) record = Record.new(zone, record_name, data_for(rrset), - source=self) + source=self, lenient=lenient) zone.add_record(record) self.log.info('populate: found %s records', @@ -189,6 +209,12 @@ class PowerDnsBaseProvider(BaseProvider): _records_for_AAAA = _records_for_multiple _records_for_NS = _records_for_multiple + def _records_for_CAA(self, record): + return [{ + 'content': '{} {} "{}"'.format(v.flags, v.tag, v.value), + 'disabled': False + } for v in record.values] + def _records_for_single(self, record): return [{'content': record.value, 'disabled': False}] @@ -205,7 +231,7 @@ class PowerDnsBaseProvider(BaseProvider): def _records_for_MX(self, record): return [{ - 'content': '{} {}'.format(v.priority, v.value), + 'content': '{} {}'.format(v.preference, v.exchange), 'disabled': False } for v in record.values] diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index f16fba6..d2af00f 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -6,6 +6,7 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals from boto3 import client +from botocore.config import Config from collections import defaultdict from incf.countryutils.transformations import cca_to_ctca2 from uuid import uuid4 @@ -89,6 +90,10 @@ class _Route53Record(object): _values_for_AAAA = _values_for_values _values_for_NS = _values_for_values + def _values_for_CAA(self, record): + return ['{} {} "{}"'.format(v.flags, v.tag, v.value) + for v in record.values] + def _values_for_value(self, record): return [record.value] @@ -96,7 +101,8 @@ class _Route53Record(object): _values_for_PTR = _values_for_value def _values_for_MX(self, record): - return ['{} {}'.format(v.priority, v.value) for v in record.values] + return ['{} {}'.format(v.preference, v.exchange) + for v in record.values] def _values_for_NAPTR(self, record): return ['{} {} "{}" "{}" "{}" {}' @@ -108,8 +114,7 @@ class _Route53Record(object): for v in record.values] def _values_for_quoted(self, record): - return ['"{}"'.format(v.replace('"', '\\"')) - for v in record.values] + return record.chunked_values _values_for_SPF = _values_for_quoted _values_for_TXT = _values_for_quoted @@ -220,28 +225,35 @@ class Route53Provider(BaseProvider): In general the account used will need full permissions on Route53. ''' SUPPORTS_GEO = True + SUPPORTS = set(('A', 'AAAA', 'CAA', 'CNAME', 'MX', 'NAPTR', 'NS', 'PTR', + 'SPF', 'SRV', 'TXT')) # This should be bumped when there are underlying changes made to the # health check config. HEALTH_CHECK_VERSION = '0001' def __init__(self, id, access_key_id, secret_access_key, max_changes=1000, - *args, **kwargs): + client_max_attempts=None, *args, **kwargs): self.max_changes = max_changes self.log = logging.getLogger('Route53Provider[{}]'.format(id)) self.log.debug('__init__: id=%s, access_key_id=%s, ' 'secret_access_key=***', id, access_key_id) super(Route53Provider, self).__init__(id, *args, **kwargs) + + config = None + if client_max_attempts is not None: + self.log.info('__init__: setting max_attempts to %d', + client_max_attempts) + config = Config(retries={'max_attempts': client_max_attempts}) + self._conn = client('route53', aws_access_key_id=access_key_id, - aws_secret_access_key=secret_access_key) + aws_secret_access_key=secret_access_key, + config=config) self._r53_zones = None self._r53_rrsets = {} self._health_checks = None - def supports(self, record): - return record._type not in ('ALIAS', 'SSHFP') - @property def r53_zones(self): if self._r53_zones is None: @@ -250,7 +262,7 @@ class Route53Provider(BaseProvider): more = True start = {} while more: - resp = self._conn.list_hosted_zones() + resp = self._conn.list_hosted_zones(**start) for z in resp['HostedZones']: zones[z['Name']] = z['Id'] more = resp['IsTruncated'] @@ -310,6 +322,21 @@ class Route53Provider(BaseProvider): _data_for_A = _data_for_geo _data_for_AAAA = _data_for_geo + def _data_for_CAA(self, rrset): + values = [] + for rr in rrset['ResourceRecords']: + flags, tag, value = rr['Value'].split(' ') + values.append({ + 'flags': flags, + 'tag': tag, + 'value': value[1:-1], + }) + return { + 'type': rrset['Type'], + 'values': values, + 'ttl': int(rrset['TTL']) + } + def _data_for_single(self, rrset): return { 'type': rrset['Type'], @@ -336,10 +363,10 @@ class Route53Provider(BaseProvider): def _data_for_MX(self, rrset): values = [] for rr in rrset['ResourceRecords']: - priority, value = rr['Value'].split(' ') + preference, exchange = rr['Value'].split(' ') values.append({ - 'priority': priority, - 'value': value, + 'preference': preference, + 'exchange': exchange, }) return { 'type': rrset['Type'], @@ -419,8 +446,10 @@ class Route53Provider(BaseProvider): return self._r53_rrsets[zone_id] - def populate(self, zone, target=False): - self.log.debug('populate: name=%s', zone.name) + def populate(self, zone, target=False, lenient=False): + self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, + target, lenient) + before = len(zone.records) zone_id = self._get_zone_id(zone.name) @@ -450,7 +479,8 @@ class Route53Provider(BaseProvider): data['geo'] = geo else: data = data[0] - record = Record.new(zone, name, data, source=self) + record = Record.new(zone, name, data, source=self, + lenient=lenient) zone.add_record(record) self.log.info('populate: found %s records', diff --git a/octodns/provider/yaml.py b/octodns/provider/yaml.py index 7b2d209..fe1a406 100644 --- a/octodns/provider/yaml.py +++ b/octodns/provider/yaml.py @@ -31,6 +31,8 @@ class YamlProvider(BaseProvider): enforce_order: True ''' SUPPORTS_GEO = True + SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CNAME', 'MX', 'NAPTR', 'NS', 'PTR', + 'SSHFP', 'SPF', 'SRV', 'TXT')) def __init__(self, id, directory, default_ttl=3600, enforce_order=True, *args, **kwargs): @@ -43,8 +45,10 @@ class YamlProvider(BaseProvider): self.default_ttl = default_ttl self.enforce_order = enforce_order - def populate(self, zone, target=False): - self.log.debug('populate: zone=%s, target=%s', zone.name, target) + def populate(self, zone, target=False, lenient=False): + self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, + target, lenient) + if target: # When acting as a target we ignore any existing records so that we # create a completely new copy @@ -61,7 +65,8 @@ class YamlProvider(BaseProvider): for d in data: if 'ttl' not in d: d['ttl'] = self.default_ttl - record = Record.new(zone, name, d, source=self) + record = Record.new(zone, name, d, source=self, + lenient=lenient) zone.add_record(record) self.log.info('populate: found %s records', diff --git a/octodns/record.py b/octodns/record.py index e86de2d..5430f1c 100644 --- a/octodns/record.py +++ b/octodns/record.py @@ -54,51 +54,64 @@ class Delete(Change): return 'Delete {}'.format(self.existing) -_unescaped_semicolon_re = re.compile(r'\w;') +class ValidationError(Exception): + + @classmethod + def build_message(cls, fqdn, reasons): + return 'Invalid record {}\n - {}'.format(fqdn, '\n - '.join(reasons)) + + def __init__(self, fqdn, reasons): + super(Exception, self).__init__(self.build_message(fqdn, reasons)) + self.fqdn = fqdn + self.reasons = reasons class Record(object): log = getLogger('Record') @classmethod - def new(cls, zone, name, data, source=None): + def new(cls, zone, name, data, source=None, lenient=False): + fqdn = '{}.{}'.format(name, zone.name) if name else zone.name try: _type = data['type'] except KeyError: - fqdn = '{}.{}'.format(name, zone.name) if name else zone.name raise Exception('Invalid record {}, missing type'.format(fqdn)) try: - _type = { + _class = { 'A': ARecord, 'AAAA': AaaaRecord, 'ALIAS': AliasRecord, - # cert + 'CAA': CaaRecord, 'CNAME': CnameRecord, - # dhcid - # dname - # dnskey - # ds - # ipseckey - # key - # kx - # loc 'MX': MxRecord, 'NAPTR': NaptrRecord, 'NS': NsRecord, - # nsap 'PTR': PtrRecord, - # px - # rp - # soa - would it even make sense? 'SPF': SpfRecord, 'SRV': SrvRecord, 'SSHFP': SshfpRecord, 'TXT': TxtRecord, - # url }[_type] except KeyError: raise Exception('Unknown record type: "{}"'.format(_type)) - return _type(zone, name, data, source=source) + reasons = _class.validate(name, data) + if reasons: + if lenient: + cls.log.warn(ValidationError.build_message(fqdn, reasons)) + else: + raise ValidationError(fqdn, reasons) + return _class(zone, name, data, source=source) + + @classmethod + def validate(cls, name, data): + reasons = [] + try: + ttl = int(data['ttl']) + if ttl < 0: + reasons.append('invalid ttl') + except KeyError: + reasons.append('missing ttl') + return reasons def __init__(self, zone, name, data, source=None): self.log.debug('__init__: zone.name=%s, type=%11s, name=%s', zone.name, @@ -106,11 +119,8 @@ class Record(object): self.zone = zone # force everything lower-case just to be safe self.name = str(name).lower() if name else name - try: - self.ttl = int(data['ttl']) - except KeyError: - raise Exception('Invalid record {}, missing ttl'.format(self.fqdn)) self.source = source + self.ttl = int(data['ttl']) self._octodns = data.get('octodns', {}) @@ -171,11 +181,17 @@ class GeoValue(object): geo_re = re.compile(r'^(?P\w\w)(-(?P\w\w)' r'(-(?P\w\w))?)?$') - def __init__(self, geo, values): - match = self.geo_re.match(geo) + @classmethod + def _validate_geo(cls, code): + reasons = [] + match = cls.geo_re.match(code) if not match: - raise Exception('Invalid geo "{}"'.format(geo)) + reasons.append('invalid geo "{}"'.format(code)) + return reasons + + def __init__(self, geo, values): self.code = geo + match = self.geo_re.match(geo) self.continent_code = match.group('continent_code') self.country_code = match.group('country_code') self.subdivision_code = match.group('subdivision_code') @@ -202,16 +218,29 @@ class GeoValue(object): class _ValuesMixin(object): - def __init__(self, zone, name, data, source=None): - super(_ValuesMixin, self).__init__(zone, name, data, source=source) + @classmethod + def validate(cls, name, data): + reasons = super(_ValuesMixin, cls).validate(name, data) + values = [] try: values = data['values'] except KeyError: try: values = [data['value']] except KeyError: - raise Exception('Invalid record {}, missing value(s)' - .format(self.fqdn)) + reasons.append('missing value(s)') + + for value in values: + reasons.extend(cls._validate_value(value)) + + return reasons + + def __init__(self, zone, name, data, source=None): + super(_ValuesMixin, self).__init__(zone, name, data, source=source) + try: + values = data['values'] + except KeyError: + values = [data['value']] self.values = sorted(self._process_values(values)) def changes(self, other, target): @@ -229,9 +258,10 @@ class _ValuesMixin(object): return ret def __repr__(self): + values = "['{}']".format("', '".join([str(v) for v in self.values])) return '<{} {} {}, {}, {}>'.format(self.__class__.__name__, self._type, self.ttl, - self.fqdn, self.values) + self.fqdn, values) class _GeoMixin(_ValuesMixin): @@ -241,6 +271,21 @@ class _GeoMixin(_ValuesMixin): Must be included before `Record`. ''' + @classmethod + def validate(cls, name, data): + reasons = super(_GeoMixin, cls).validate(name, data) + try: + geo = dict(data['geo']) + # TODO: validate legal codes + for code, values in geo.items(): + reasons.extend(GeoValue._validate_geo(code)) + for value in values: + reasons.extend(cls._validate_value(value)) + except KeyError: + pass + return reasons + + # TODO: support 'value' as well # TODO: move away from "data" hash to strict params, it's kind of leaking # the yaml implementation into here and then forcing it back out into # non-yaml providers during input @@ -250,9 +295,8 @@ class _GeoMixin(_ValuesMixin): self.geo = dict(data['geo']) except KeyError: self.geo = {} - for k, vs in self.geo.items(): - vs = sorted(self._process_values(vs)) - self.geo[k] = GeoValue(k, vs) + for code, values in self.geo.items(): + self.geo[code] = GeoValue(code, values) def _data(self): ret = super(_GeoMixin, self)._data() @@ -281,41 +325,52 @@ class _GeoMixin(_ValuesMixin): class ARecord(_GeoMixin, Record): _type = 'A' + @classmethod + def _validate_value(self, value): + reasons = [] + try: + IPv4Address(unicode(value)) + except Exception: + reasons.append('invalid ip address "{}"'.format(value)) + return reasons + def _process_values(self, values): - for ip in values: - try: - IPv4Address(unicode(ip)) - except Exception: - raise Exception('Invalid record {}, value {} not a valid ip' - .format(self.fqdn, ip)) return values class AaaaRecord(_GeoMixin, Record): _type = 'AAAA' + @classmethod + def _validate_value(self, value): + reasons = [] + try: + IPv6Address(unicode(value)) + except Exception: + reasons.append('invalid ip address "{}"'.format(value)) + return reasons + def _process_values(self, values): - ret = [] - for ip in values: - try: - IPv6Address(unicode(ip)) - ret.append(ip.lower()) - except Exception: - raise Exception('Invalid record {}, value {} not a valid ip' - .format(self.fqdn, ip)) - return ret + return values class _ValueMixin(object): - def __init__(self, zone, name, data, source=None): - super(_ValueMixin, self).__init__(zone, name, data, source=source) + @classmethod + def validate(cls, name, data): + reasons = super(_ValueMixin, cls).validate(name, data) + value = None try: value = data['value'] except KeyError: - raise Exception('Invalid record {}, missing value' - .format(self.fqdn)) - self.value = self._process_value(value) + reasons.append('missing value') + if value: + reasons.extend(cls._validate_value(value)) + return reasons + + def __init__(self, zone, name, data, source=None): + super(_ValueMixin, self).__init__(zone, name, data, source=source) + self.value = self._process_value(data['value']) def changes(self, other, target): if self.value != other.value: @@ -336,62 +391,187 @@ class _ValueMixin(object): class AliasRecord(_ValueMixin, Record): _type = 'ALIAS' - def _process_value(self, value): + @classmethod + def _validate_value(self, value): + reasons = [] if not value.endswith('.'): - raise Exception('Invalid record {}, value ({}) missing trailing .' - .format(self.fqdn, value)) + reasons.append('missing trailing .') + return reasons + + def _process_value(self, value): return value +class CaaValue(object): + # https://tools.ietf.org/html/rfc6844#page-5 + + @classmethod + def _validate_value(cls, value): + reasons = [] + try: + flags = int(value.get('flags', 0)) + if flags < 0 or flags > 255: + reasons.append('invalid flags "{}"'.format(flags)) + except ValueError: + reasons.append('invalid flags "{}"'.format(value['flags'])) + + if 'tag' not in value: + reasons.append('missing tag') + if 'value' not in value: + reasons.append('missing value') + + return reasons + + def __init__(self, value): + self.flags = int(value.get('flags', 0)) + self.tag = value['tag'] + self.value = value['value'] + + @property + def data(self): + return { + 'flags': self.flags, + 'tag': self.tag, + 'value': self.value, + } + + def __cmp__(self, other): + if self.flags == other.flags: + if self.tag == other.tag: + return cmp(self.value, other.value) + return cmp(self.tag, other.tag) + return cmp(self.flags, other.flags) + + def __repr__(self): + return '{} {} "{}"'.format(self.flags, self.tag, self.value) + + +class CaaRecord(_ValuesMixin, Record): + _type = 'CAA' + + @classmethod + def _validate_value(cls, value): + return CaaValue._validate_value(value) + + def _process_values(self, values): + return [CaaValue(v) for v in values] + + class CnameRecord(_ValueMixin, Record): _type = 'CNAME' - def _process_value(self, value): + @classmethod + def validate(cls, name, data): + reasons = [] + if name == '': + reasons.append('root CNAME not allowed') + reasons.extend(super(CnameRecord, cls).validate(name, data)) + return reasons + + @classmethod + def _validate_value(cls, value): + reasons = [] if not value.endswith('.'): - raise Exception('Invalid record {}, value ({}) missing trailing .' - .format(self.fqdn, value)) - return value.lower() + reasons.append('missing trailing .') + return reasons + + def _process_value(self, value): + return value class MxValue(object): + @classmethod + def _validate_value(cls, value): + reasons = [] + try: + int(value.get('preference', None) or value['priority']) + except KeyError: + reasons.append('missing preference') + except ValueError: + reasons.append('invalid preference "{}"' + .format(value['preference'])) + exchange = None + try: + exchange = value.get('exchange', None) or value['value'] + if not exchange.endswith('.'): + reasons.append('missing trailing .') + except KeyError: + reasons.append('missing exchange') + return reasons + def __init__(self, value): - # TODO: rename preference - self.priority = int(value['priority']) - # TODO: rename to exchange? - self.value = value['value'].lower() + # RFC1035 says preference, half the providers use priority + try: + preference = value['preference'] + except KeyError: + preference = value['priority'] + self.preference = int(preference) + # UNTIL 1.0 remove value fallback + try: + exchange = value['exchange'] + except KeyError: + exchange = value['value'] + self.exchange = exchange @property def data(self): return { - 'priority': self.priority, - 'value': self.value, + 'preference': self.preference, + 'exchange': self.exchange, } def __cmp__(self, other): - if self.priority == other.priority: - return cmp(self.value, other.value) - return cmp(self.priority, other.priority) + if self.preference == other.preference: + return cmp(self.exchange, other.exchange) + return cmp(self.preference, other.preference) def __repr__(self): - return "'{} {}'".format(self.priority, self.value) + return "'{} {}'".format(self.preference, self.exchange) class MxRecord(_ValuesMixin, Record): _type = 'MX' + @classmethod + def _validate_value(cls, value): + return MxValue._validate_value(value) + def _process_values(self, values): - ret = [] - for value in values: - try: - ret.append(MxValue(value)) - except KeyError as e: - raise Exception('Invalid value in record {}, missing {}' - .format(self.fqdn, e.args[0])) - return ret + return [MxValue(v) for v in values] class NaptrValue(object): + VALID_FLAGS = ('S', 'A', 'U', 'P') + + @classmethod + def _validate_value(cls, data): + reasons = [] + try: + int(data['order']) + except KeyError: + reasons.append('missing order') + except ValueError: + reasons.append('invalid order "{}"'.format(data['order'])) + try: + int(data['preference']) + except KeyError: + reasons.append('missing preference') + except ValueError: + reasons.append('invalid preference "{}"' + .format(data['preference'])) + try: + flags = data['flags'] + if flags not in cls.VALID_FLAGS: + reasons.append('unrecognized flags "{}"'.format(flags)) + except KeyError: + reasons.append('missing flags') + + # TODO: validate these... they're non-trivial + for k in ('service', 'regexp', 'replacement'): + if k not in data: + reasons.append('missing {}'.format(k)) + return reasons def __init__(self, value): self.order = int(value['order']) @@ -437,41 +617,70 @@ class NaptrValue(object): class NaptrRecord(_ValuesMixin, Record): _type = 'NAPTR' + @classmethod + def _validate_value(cls, value): + return NaptrValue._validate_value(value) + def _process_values(self, values): - ret = [] - for value in values: - try: - ret.append(NaptrValue(value)) - except KeyError as e: - raise Exception('Invalid value in record {}, missing {}' - .format(self.fqdn, e.args[0])) - return ret + return [NaptrValue(v) for v in values] class NsRecord(_ValuesMixin, Record): _type = 'NS' + @classmethod + def _validate_value(cls, value): + reasons = [] + if not value.endswith('.'): + reasons.append('missing trailing .') + return reasons + def _process_values(self, values): - ret = [] - for ns in values: - if not ns.endswith('.'): - raise Exception('Invalid record {}, value {} missing ' - 'trailing .'.format(self.fqdn, ns)) - ret.append(ns.lower()) - return ret + return values class PtrRecord(_ValueMixin, Record): _type = 'PTR' - def _process_value(self, value): + @classmethod + def _validate_value(cls, value): + reasons = [] if not value.endswith('.'): - raise Exception('Invalid record {}, value ({}) missing trailing .' - .format(self.fqdn, value)) - return value.lower() + reasons.append('missing trailing .') + return reasons + + def _process_value(self, value): + return value class SshfpValue(object): + VALID_ALGORITHMS = (1, 2) + VALID_FINGERPRINT_TYPES = (1,) + + @classmethod + def _validate_value(cls, value): + reasons = [] + try: + algorithm = int(value['algorithm']) + if algorithm not in cls.VALID_ALGORITHMS: + reasons.append('unrecognized algorithm "{}"'.format(algorithm)) + except KeyError: + reasons.append('missing algorithm') + except ValueError: + reasons.append('invalid algorithm "{}"'.format(value['algorithm'])) + try: + fingerprint_type = int(value['fingerprint_type']) + if fingerprint_type not in cls.VALID_FINGERPRINT_TYPES: + reasons.append('unrecognized fingerprint_type "{}"' + .format(fingerprint_type)) + except KeyError: + reasons.append('missing fingerprint_type') + except ValueError: + reasons.append('invalid fingerprint_type "{}"' + .format(value['fingerprint_type'])) + if 'fingerprint' not in value: + reasons.append('missing fingerprint') + return reasons def __init__(self, value): self.algorithm = int(value['algorithm']) @@ -501,26 +710,81 @@ class SshfpValue(object): class SshfpRecord(_ValuesMixin, Record): _type = 'SSHFP' + @classmethod + def _validate_value(cls, value): + return SshfpValue._validate_value(value) + def _process_values(self, values): - ret = [] - for value in values: - try: - ret.append(SshfpValue(value)) - except KeyError as e: - raise Exception('Invalid value in record {}, missing {}' - .format(self.fqdn, e.args[0])) - return ret + return [SshfpValue(v) for v in values] -class SpfRecord(_ValuesMixin, Record): - _type = 'SPF' +_unescaped_semicolon_re = re.compile(r'\w;') + + +class _ChunkedValuesMixin(_ValuesMixin): + CHUNK_SIZE = 255 + + @classmethod + def _validate_value(cls, value): + if _unescaped_semicolon_re.search(value): + return ['unescaped ;'] + return [] def _process_values(self, values): + ret = [] + for v in values: + if v and v[0] == '"': + v = v[1:-1] + ret.append(v.replace('" "', '')) + return ret + + @property + def chunked_values(self): + values = [] + for v in self.values: + v = v.replace('"', '\\"') + vs = [v[i:i + self.CHUNK_SIZE] + for i in range(0, len(v), self.CHUNK_SIZE)] + vs = '" "'.join(vs) + values.append('"{}"'.format(vs)) return values +class SpfRecord(_ChunkedValuesMixin, Record): + _type = 'SPF' + + class SrvValue(object): + @classmethod + def _validate_value(self, value): + reasons = [] + # TODO: validate algorithm and fingerprint_type values + try: + int(value['priority']) + except KeyError: + reasons.append('missing priority') + except ValueError: + reasons.append('invalid priority "{}"'.format(value['priority'])) + try: + int(value['weight']) + except KeyError: + reasons.append('missing weight') + except ValueError: + reasons.append('invalid weight "{}"'.format(value['weight'])) + try: + int(value['port']) + except KeyError: + reasons.append('missing port') + except ValueError: + reasons.append('invalid port "{}"'.format(value['port'])) + try: + if not value['target'].endswith('.'): + reasons.append('missing trailing .') + except KeyError: + reasons.append('missing target') + return reasons + def __init__(self, value): self.priority = int(value['priority']) self.weight = int(value['weight']) @@ -554,28 +818,21 @@ class SrvRecord(_ValuesMixin, Record): _type = 'SRV' _name_re = re.compile(r'^_[^\.]+\.[^\.]+') - def __init__(self, zone, name, data, source=None): - if not self._name_re.match(name): - raise Exception('Invalid name {}.{}'.format(name, zone.name)) - super(SrvRecord, self).__init__(zone, name, data, source) + @classmethod + def validate(cls, name, data): + reasons = [] + if not cls._name_re.match(name): + reasons.append('invalid name') + reasons.extend(super(SrvRecord, cls).validate(name, data)) + return reasons + + @classmethod + def _validate_value(cls, value): + return SrvValue._validate_value(value) def _process_values(self, values): - ret = [] - for value in values: - try: - ret.append(SrvValue(value)) - except KeyError as e: - raise Exception('Invalid value in record {}, missing {}' - .format(self.fqdn, e.args[0])) - return ret + return [SrvValue(v) for v in values] -class TxtRecord(_ValuesMixin, Record): +class TxtRecord(_ChunkedValuesMixin, Record): _type = 'TXT' - - def _process_values(self, values): - for value in values: - if _unescaped_semicolon_re.search(value): - raise Exception('Invalid record {}, unescaped ;' - .format(self.fqdn)) - return values diff --git a/octodns/source/base.py b/octodns/source/base.py index 72ebaab..4ace09f 100644 --- a/octodns/source/base.py +++ b/octodns/source/base.py @@ -16,18 +16,27 @@ class BaseSource(object): if not hasattr(self, 'SUPPORTS_GEO'): raise NotImplementedError('Abstract base class, SUPPORTS_GEO ' 'property missing') + if not hasattr(self, 'SUPPORTS'): + raise NotImplementedError('Abstract base class, SUPPORTS ' + 'property missing') - def populate(self, zone, target=False): + def populate(self, zone, target=False, lenient=False): ''' Loads all zones the provider knows about + + When `target` is True the populate call is being made to load the + current state of the provider. + + When `lenient` is True the populate call may skip record validation and + do a "best effort" load of data. That will allow through some common, + but not best practices stuff that we otherwise would reject. E.g. no + trailing . or mising escapes for ;. ''' raise NotImplementedError('Abstract base class, populate method ' 'missing') def supports(self, record): - # Unless overriden and handled appropriaitely we'll assume that all - # record types are supported - return True + return record._type in self.SUPPORTS def __repr__(self): return self.__class__.__name__ diff --git a/octodns/source/tinydns.py b/octodns/source/tinydns.py index 70f0145..63cafa2 100644 --- a/octodns/source/tinydns.py +++ b/octodns/source/tinydns.py @@ -19,6 +19,7 @@ from .base import BaseSource class TinyDnsBaseSource(BaseSource): SUPPORTS_GEO = False + SUPPORTS = set(('A', 'CNAME', 'MX', 'NS')) split_re = re.compile(r':+') @@ -64,8 +65,8 @@ class TinyDnsBaseSource(BaseSource): 'ttl': ttl, 'type': _type, 'values': [{ - 'priority': r[1], - 'value': '{}.'.format(r[0]) + 'preference': r[1], + 'exchange': '{}.'.format(r[0]) } for r in records] } @@ -80,19 +81,21 @@ class TinyDnsBaseSource(BaseSource): 'values': ['{}.'.format(r[0]) for r in records] } - def populate(self, zone, target=False): - self.log.debug('populate: zone=%s', zone.name) + def populate(self, zone, target=False, lenient=False): + self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, + target, lenient) + before = len(zone.records) if zone.name.endswith('in-addr.arpa.'): - self._populate_in_addr_arpa(zone) + self._populate_in_addr_arpa(zone, lenient) else: - self._populate_normal(zone) + self._populate_normal(zone, lenient) self.log.info('populate: found %s records', len(zone.records) - before) - def _populate_normal(self, zone): + def _populate_normal(self, zone, lenient): type_map = { '=': 'A', '^': None, @@ -128,14 +131,15 @@ class TinyDnsBaseSource(BaseSource): data_for = getattr(self, '_data_for_{}'.format(_type)) data = data_for(_type, d) if data: - record = Record.new(zone, name, data, source=self) + record = Record.new(zone, name, data, source=self, + lenient=lenient) try: zone.add_record(record) except SubzoneRecordException: self.log.debug('_populate_normal: skipping subzone ' 'record=%s', record) - def _populate_in_addr_arpa(self, zone): + def _populate_in_addr_arpa(self, zone, lenient): name_re = re.compile('(?P.+)\.{}$'.format(zone.name[:-1])) for line in self._lines(): @@ -169,7 +173,7 @@ class TinyDnsBaseSource(BaseSource): 'ttl': ttl, 'type': 'PTR', 'value': value - }, source=self) + }, source=self, lenient=lenient) try: zone.add_record(record) except DuplicateRecordException: diff --git a/octodns/yaml.py b/octodns/yaml.py index d4ab541..98bafdb 100644 --- a/octodns/yaml.py +++ b/octodns/yaml.py @@ -21,9 +21,13 @@ class SortEnforcingLoader(SafeLoader): self.flatten_mapping(node) ret = self.construct_pairs(node) keys = [d[0] for d in ret] - if keys != sorted(keys, key=_natsort_key): - raise ConstructorError(None, None, "keys out of order: {}" - .format(', '.join(keys)), node.start_mark) + keys_sorted = sorted(keys, key=_natsort_key) + for key in keys: + expected = keys_sorted.pop(0) + if key != expected: + raise ConstructorError(None, None, 'keys out of order: ' + 'expected {} got {} at {}' + .format(expected, key, node.start_mark)) return dict(ret) diff --git a/octodns/zone.py b/octodns/zone.py index 1822fec..74e5d9e 100644 --- a/octodns/zone.py +++ b/octodns/zone.py @@ -5,6 +5,7 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals +from collections import defaultdict from logging import getLogger import re @@ -19,6 +20,10 @@ class DuplicateRecordException(Exception): pass +class InvalidNodeException(Exception): + pass + + def _is_eligible(record): # Should this record be considered when computing changes # We ignore all top-level NS records @@ -35,19 +40,26 @@ class Zone(object): # Force everyting to lowercase just to be safe self.name = str(name).lower() if name else name self.sub_zones = sub_zones - self.records = set() + # We're grouping by node, it allows us to efficently search for + # duplicates and detect when CNAMEs co-exist with other records + self._records = defaultdict(set) # optional leading . to match empty hostname # optional trailing . b/c some sources don't have it on their fqdn self._name_re = re.compile('\.?{}?$'.format(name)) self.log.debug('__init__: zone=%s, sub_zones=%s', self, sub_zones) + @property + def records(self): + return set([r for _, node in self._records.items() for r in node]) + def hostname_from_fqdn(self, fqdn): return self._name_re.sub('', fqdn) - def add_record(self, record): + def add_record(self, record, replace=False): name = record.name last = name.split('.')[-1] + if last in self.sub_zones: if name != last: # it's a record for something under a sub-zone @@ -59,10 +71,30 @@ class Zone(object): raise SubzoneRecordException('Record {} a managed sub-zone ' 'and not of type NS' .format(record.fqdn)) - if record in self.records: + + if replace: + # will remove it if it exists + self._records[name].discard(record) + + node = self._records[name] + if record in node: + # We already have a record at this node of this type raise DuplicateRecordException('Duplicate record {}, type {}' - .format(record.fqdn, record._type)) - self.records.add(record) + .format(record.fqdn, + record._type)) + elif ((record._type == 'CNAME' and len(node) > 0) or + ('CNAME' in map(lambda r: r._type, node))): + # We're adding a CNAME to existing records or adding to an existing + # CNAME + raise InvalidNodeException('Invalid state, CNAME at {} cannot ' + 'coexist with other records' + .format(record.fqdn)) + + node.add(record) + + def _remove_record(self, record): + 'Only for use in tests' + self._records[record.name].discard(record) def changes(self, desired, target): self.log.debug('changes: zone=%s, target=%s', self, target) diff --git a/requirements-dev.txt b/requirements-dev.txt index 265f407..5cdf252 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,3 +4,4 @@ nose pep8 pyflakes requests_mock +setuptools>=36.4.0 diff --git a/requirements.txt b/requirements.txt index b10ca4c..80fbe1e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,18 +1,23 @@ # These are known good versions. You're free to use others and things will # likely work, but no promises are made, especilly if you go older. PyYaml==3.12 -boto3==1.4.4 -botocore==1.5.4 +azure-mgmt-dns==1.0.1 +azure-common==1.1.6 +boto3==1.4.6 +botocore==1.6.8 dnspython==1.15.0 -docutils==0.13.1 -dyn==1.7.10 -futures==3.0.5 +docutils==0.14 +dyn==1.8.0 +futures==3.1.1 +google-cloud==0.27.0 incf.countryutils==1.0 ipaddress==1.0.18 -jmespath==0.9.0 +jmespath==0.9.3 +msrestazure==0.4.10 natsort==5.0.3 -nsone==0.9.10 -python-dateutil==2.6.0 +nsone==0.9.14 +ovh==0.4.7 +python-dateutil==2.6.1 requests==2.13.0 s3transfer==0.1.10 six==1.10.0 diff --git a/script/coverage b/script/coverage index ca5d693..228a772 100755 --- a/script/coverage +++ b/script/coverage @@ -24,6 +24,7 @@ export DNSIMPLE_TOKEN= export DYN_CUSTOMER= export DYN_PASSWORD= export DYN_USERNAME= +export GOOGLE_APPLICATION_CREDENTIALS= coverage run --branch --source=octodns `which nosetests` --with-xunit "$@" coverage html diff --git a/script/release b/script/release new file mode 100755 index 0000000..d8fabf2 --- /dev/null +++ b/script/release @@ -0,0 +1,14 @@ +#!/bin/bash + +set -e + +cd "$(dirname $0)"/.. +ROOT=$(pwd) + +VERSION=$(grep __VERSION__ $ROOT/octodns/__init__.py | sed -e "s/.* = '//" -e "s/'$//") + +git tag -s v$VERSION -m "Release $VERSION" +git push origin v$VERSION +echo "Tagged and pushed v$VERSION" +python setup.py sdist upload +echo "Updloaded $VERSION" diff --git a/script/test b/script/test index 3ee6e9c..41edfd8 100755 --- a/script/test +++ b/script/test @@ -24,5 +24,6 @@ export DNSIMPLE_TOKEN= export DYN_CUSTOMER= export DYN_PASSWORD= export DYN_USERNAME= +export GOOGLE_APPLICATION_CREDENTIALS= nosetests "$@" diff --git a/tests/config/unit.tests.yaml b/tests/config/unit.tests.yaml index d18bf59..5241406 100644 --- a/tests/config/unit.tests.yaml +++ b/tests/config/unit.tests.yaml @@ -31,6 +31,11 @@ values: - 6.2.3.4. - 7.2.3.4. + - type: CAA + values: + - flags: 0 + tag: issue + value: ca.unit.tests _srv._tcp: ttl: 600 type: SRV @@ -60,12 +65,12 @@ mx: ttl: 300 type: MX values: - - priority: 40 - value: smtp-1.unit.tests. - - priority: 20 - value: smtp-2.unit.tests. - - priority: 30 - value: smtp-3.unit.tests. + - exchange: smtp-1.unit.tests. + preference: 40 + - exchange: smtp-2.unit.tests. + preference: 20 + - exchange: smtp-3.unit.tests. + preference: 30 - priority: 10 value: smtp-4.unit.tests. naptr: diff --git a/tests/fixtures/cloudflare-dns_records-page-2.json b/tests/fixtures/cloudflare-dns_records-page-2.json index 24c49d5..195d6de 100644 --- a/tests/fixtures/cloudflare-dns_records-page-2.json +++ b/tests/fixtures/cloudflare-dns_records-page-2.json @@ -118,14 +118,35 @@ "meta": { "auto_added": false } + }, + { + "id": "fc223b34cd5611334422ab3322997667", + "type": "CAA", + "name": "unit.tests", + "data": { + "flags": 0, + "tag": "issue", + "value": "ca.unit.tests" + }, + "proxiable": false, + "proxied": false, + "ttl": 3600, + "locked": false, + "zone_id": "ff12ab34cd5611334422ab3322997650", + "zone_name": "unit.tests", + "modified_on": "2017-03-11T18:01:42.961566Z", + "created_on": "2017-03-11T18:01:42.961566Z", + "meta": { + "auto_added": false + } } ], "result_info": { "page": 2, "per_page": 10, "total_pages": 2, - "count": 7, - "total_count": 17 + "count": 8, + "total_count": 19 }, "success": true, "errors": [], diff --git a/tests/fixtures/dnsimple-page-2.json b/tests/fixtures/dnsimple-page-2.json index f50704b..40aaa48 100644 --- a/tests/fixtures/dnsimple-page-2.json +++ b/tests/fixtures/dnsimple-page-2.json @@ -159,12 +159,28 @@ "system_record": false, "created_at": "2017-03-09T15:55:09Z", "updated_at": "2017-03-09T15:55:09Z" + }, + { + "id": 12188803, + "zone_id": "unit.tests", + "parent_id": null, + "name": "", + "content": "0 issue \"ca.unit.tests\"", + "ttl": 3600, + "priority": null, + "type": "CAA", + "regions": [ + "global" + ], + "system_record": false, + "created_at": "2017-03-09T15:55:09Z", + "updated_at": "2017-03-09T15:55:09Z" } ], "pagination": { "current_page": 2, "per_page": 20, - "total_entries": 29, + "total_entries": 30, "total_pages": 2 } } diff --git a/tests/fixtures/powerdns-full-data.json b/tests/fixtures/powerdns-full-data.json index 72ce016..b8f8bf3 100644 --- a/tests/fixtures/powerdns-full-data.json +++ b/tests/fixtures/powerdns-full-data.json @@ -230,6 +230,18 @@ ], "ttl": 300, "type": "A" + }, + { + "comments": [], + "name": "unit.tests.", + "records": [ + { + "content": "0 issue \"ca.unit.tests\"", + "disabled": false + } + ], + "ttl": 3600, + "type": "CAA" } ], "serial": 2017012803, diff --git a/tests/helpers.py b/tests/helpers.py index df74e84..adac81d 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -17,11 +17,12 @@ class SimpleSource(object): class SimpleProvider(object): SUPPORTS_GEO = False + SUPPORTS = set(('A',)) def __init__(self, id='test'): pass - def populate(self, zone, source=True): + def populate(self, zone, source=False, lenient=False): pass def supports(self, record): @@ -37,7 +38,7 @@ class GeoProvider(object): def __init__(self, id='test'): pass - def populate(self, zone, source=True): + def populate(self, zone, source=False, lenient=False): pass def supports(self, record): diff --git a/tests/test_octodns_manager.py b/tests/test_octodns_manager.py index fa8bdd1..a5f2022 100644 --- a/tests/test_octodns_manager.py +++ b/tests/test_octodns_manager.py @@ -11,6 +11,7 @@ from unittest import TestCase from octodns.record import Record from octodns.manager import _AggregateTarget, MainThreadExecutor, Manager +from octodns.yaml import safe_load from octodns.zone import Zone from helpers import GeoProvider, NoSshFpProvider, SimpleProvider, \ @@ -128,6 +129,12 @@ class TestManager(TestCase): .sync(dry_run=False, force=True) self.assertEquals(19, tc) + # Include meta + tc = Manager(get_config_filename('simple.yaml'), max_workers=1, + include_meta=True) \ + .sync(dry_run=False, force=True) + self.assertEquals(23, tc) + def test_eligible_targets(self): with TemporaryDirectory() as tmpdir: environ['YAML_TMP_DIR'] = tmpdir.dirname @@ -195,15 +202,26 @@ class TestManager(TestCase): manager = Manager(get_config_filename('simple.yaml')) with self.assertRaises(Exception) as ctx: - manager.dump('unit.tests.', tmpdir.dirname, 'nope') + manager.dump('unit.tests.', tmpdir.dirname, False, 'nope') self.assertEquals('Unknown source: nope', ctx.exception.message) - manager.dump('unit.tests.', tmpdir.dirname, 'in') + manager.dump('unit.tests.', tmpdir.dirname, False, 'in') # make sure this fails with an IOError and not a KeyError when # tyring to find sub zones with self.assertRaises(IOError): - manager.dump('unknown.zone.', tmpdir.dirname, 'in') + manager.dump('unknown.zone.', tmpdir.dirname, False, 'in') + + def test_dump_empty(self): + with TemporaryDirectory() as tmpdir: + environ['YAML_TMP_DIR'] = tmpdir.dirname + manager = Manager(get_config_filename('simple.yaml')) + + manager.dump('empty.', tmpdir.dirname, False, 'in') + + with open(join(tmpdir.dirname, 'empty.yaml')) as fh: + data = safe_load(fh, False) + self.assertFalse(data) def test_validate_configs(self): Manager(get_config_filename('simple-validate.yaml')).validate_configs() diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py new file mode 100644 index 0000000..598fe48 --- /dev/null +++ b/tests/test_octodns_provider_azuredns.py @@ -0,0 +1,379 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from octodns.record import Create, Delete, Record +from octodns.provider.azuredns import _AzureRecord, AzureProvider, \ + _check_endswith_dot, _parse_azure_type +from octodns.zone import Zone +from octodns.provider.base import Plan + +from azure.mgmt.dns.models import ARecord, AaaaRecord, CnameRecord, MxRecord, \ + SrvRecord, NsRecord, PtrRecord, TxtRecord, RecordSet, SoaRecord, \ + Zone as AzureZone +from msrestazure.azure_exceptions import CloudError + +from unittest import TestCase +from mock import Mock, patch + + +zone = Zone(name='unit.tests.', sub_zones=[]) +octo_records = [] +octo_records.append(Record.new(zone, '', { + 'ttl': 0, + 'type': 'A', + 'values': ['1.2.3.4', '10.10.10.10']})) +octo_records.append(Record.new(zone, 'a', { + 'ttl': 1, + 'type': 'A', + 'values': ['1.2.3.4', '1.1.1.1']})) +octo_records.append(Record.new(zone, 'aa', { + 'ttl': 9001, + 'type': 'A', + 'values': ['1.2.4.3']})) +octo_records.append(Record.new(zone, 'aaa', { + 'ttl': 2, + 'type': 'A', + 'values': ['1.1.1.3']})) +octo_records.append(Record.new(zone, 'cname', { + 'ttl': 3, + 'type': 'CNAME', + 'value': 'a.unit.tests.'})) +octo_records.append(Record.new(zone, 'mx1', { + 'ttl': 3, + 'type': 'MX', + 'values': [{ + 'priority': 10, + 'value': 'mx1.unit.tests.', + }, { + 'priority': 20, + 'value': 'mx2.unit.tests.', + }]})) +octo_records.append(Record.new(zone, 'mx2', { + 'ttl': 3, + 'type': 'MX', + 'values': [{ + 'priority': 10, + 'value': 'mx1.unit.tests.', + }]})) +octo_records.append(Record.new(zone, '', { + 'ttl': 4, + 'type': 'NS', + 'values': ['ns1.unit.tests.', 'ns2.unit.tests.']})) +octo_records.append(Record.new(zone, 'foo', { + 'ttl': 5, + 'type': 'NS', + 'value': 'ns1.unit.tests.'})) +octo_records.append(Record.new(zone, '_srv._tcp', { + 'ttl': 6, + 'type': 'SRV', + 'values': [{ + 'priority': 10, + 'weight': 20, + 'port': 30, + 'target': 'foo-1.unit.tests.', + }, { + 'priority': 12, + 'weight': 30, + 'port': 30, + 'target': 'foo-2.unit.tests.', + }]})) +octo_records.append(Record.new(zone, '_srv2._tcp', { + 'ttl': 7, + 'type': 'SRV', + 'values': [{ + 'priority': 12, + 'weight': 17, + 'port': 1, + 'target': 'srvfoo.unit.tests.', + }]})) +octo_records.append(Record.new(zone, 'txt1', { + 'ttl': 8, + 'type': 'TXT', + 'value': 'txt singleton test'})) +octo_records.append(Record.new(zone, 'txt2', { + 'ttl': 9, + 'type': 'TXT', + 'values': ['txt multiple test', 'txt multiple test 2']})) + +azure_records = [] +_base0 = _AzureRecord('TestAzure', octo_records[0]) +_base0.zone_name = 'unit.tests' +_base0.relative_record_set_name = '@' +_base0.record_type = 'A' +_base0.params['ttl'] = 0 +_base0.params['arecords'] = [ARecord('1.2.3.4'), ARecord('10.10.10.10')] +azure_records.append(_base0) + +_base1 = _AzureRecord('TestAzure', octo_records[1]) +_base1.zone_name = 'unit.tests' +_base1.relative_record_set_name = 'a' +_base1.record_type = 'A' +_base1.params['ttl'] = 1 +_base1.params['arecords'] = [ARecord('1.2.3.4'), ARecord('1.1.1.1')] +azure_records.append(_base1) + +_base2 = _AzureRecord('TestAzure', octo_records[2]) +_base2.zone_name = 'unit.tests' +_base2.relative_record_set_name = 'aa' +_base2.record_type = 'A' +_base2.params['ttl'] = 9001 +_base2.params['arecords'] = ARecord('1.2.4.3') +azure_records.append(_base2) + +_base3 = _AzureRecord('TestAzure', octo_records[3]) +_base3.zone_name = 'unit.tests' +_base3.relative_record_set_name = 'aaa' +_base3.record_type = 'A' +_base3.params['ttl'] = 2 +_base3.params['arecords'] = ARecord('1.1.1.3') +azure_records.append(_base3) + +_base4 = _AzureRecord('TestAzure', octo_records[4]) +_base4.zone_name = 'unit.tests' +_base4.relative_record_set_name = 'cname' +_base4.record_type = 'CNAME' +_base4.params['ttl'] = 3 +_base4.params['cname_record'] = CnameRecord('a.unit.tests.') +azure_records.append(_base4) + +_base5 = _AzureRecord('TestAzure', octo_records[5]) +_base5.zone_name = 'unit.tests' +_base5.relative_record_set_name = 'mx1' +_base5.record_type = 'MX' +_base5.params['ttl'] = 3 +_base5.params['mx_records'] = [MxRecord(10, 'mx1.unit.tests.'), + MxRecord(20, 'mx2.unit.tests.')] +azure_records.append(_base5) + +_base6 = _AzureRecord('TestAzure', octo_records[6]) +_base6.zone_name = 'unit.tests' +_base6.relative_record_set_name = 'mx2' +_base6.record_type = 'MX' +_base6.params['ttl'] = 3 +_base6.params['mx_records'] = [MxRecord(10, 'mx1.unit.tests.')] +azure_records.append(_base6) + +_base7 = _AzureRecord('TestAzure', octo_records[7]) +_base7.zone_name = 'unit.tests' +_base7.relative_record_set_name = '@' +_base7.record_type = 'NS' +_base7.params['ttl'] = 4 +_base7.params['ns_records'] = [NsRecord('ns1.unit.tests.'), + NsRecord('ns2.unit.tests.')] +azure_records.append(_base7) + +_base8 = _AzureRecord('TestAzure', octo_records[8]) +_base8.zone_name = 'unit.tests' +_base8.relative_record_set_name = 'foo' +_base8.record_type = 'NS' +_base8.params['ttl'] = 5 +_base8.params['ns_records'] = [NsRecord('ns1.unit.tests.')] +azure_records.append(_base8) + +_base9 = _AzureRecord('TestAzure', octo_records[9]) +_base9.zone_name = 'unit.tests' +_base9.relative_record_set_name = '_srv._tcp' +_base9.record_type = 'SRV' +_base9.params['ttl'] = 6 +_base9.params['srv_records'] = [SrvRecord(10, 20, 30, 'foo-1.unit.tests.'), + SrvRecord(12, 30, 30, 'foo-2.unit.tests.')] +azure_records.append(_base9) + +_base10 = _AzureRecord('TestAzure', octo_records[10]) +_base10.zone_name = 'unit.tests' +_base10.relative_record_set_name = '_srv2._tcp' +_base10.record_type = 'SRV' +_base10.params['ttl'] = 7 +_base10.params['srv_records'] = [SrvRecord(12, 17, 1, 'srvfoo.unit.tests.')] +azure_records.append(_base10) + +_base11 = _AzureRecord('TestAzure', octo_records[11]) +_base11.zone_name = 'unit.tests' +_base11.relative_record_set_name = 'txt1' +_base11.record_type = 'TXT' +_base11.params['ttl'] = 8 +_base11.params['txt_records'] = [TxtRecord(['txt singleton test'])] +azure_records.append(_base11) + +_base12 = _AzureRecord('TestAzure', octo_records[12]) +_base12.zone_name = 'unit.tests' +_base12.relative_record_set_name = 'txt2' +_base12.record_type = 'TXT' +_base12.params['ttl'] = 9 +_base12.params['txt_records'] = [TxtRecord(['txt multiple test']), + TxtRecord(['txt multiple test 2'])] +azure_records.append(_base12) + + +class Test_AzureRecord(TestCase): + def test_azure_record(self): + assert(len(azure_records) == len(octo_records)) + for i in range(len(azure_records)): + octo = _AzureRecord('TestAzure', octo_records[i]) + assert(azure_records[i]._equals(octo)) + + +class Test_ParseAzureType(TestCase): + def test_parse_azure_type(self): + for expected, test in [['A', 'Microsoft.Network/dnszones/A'], + ['AAAA', 'Microsoft.Network/dnszones/AAAA'], + ['NS', 'Microsoft.Network/dnszones/NS'], + ['MX', 'Microsoft.Network/dnszones/MX']]: + self.assertEquals(expected, _parse_azure_type(test)) + + +class Test_CheckEndswithDot(TestCase): + def test_check_endswith_dot(self): + for expected, test in [['a.', 'a'], + ['a.', 'a.'], + ['foo.bar.', 'foo.bar.'], + ['foo.bar.', 'foo.bar']]: + self.assertEquals(expected, _check_endswith_dot(test)) + + +class TestAzureDnsProvider(TestCase): + def _provider(self): + return self._get_provider('mock_spc', 'mock_dns_client') + + @patch('octodns.provider.azuredns.DnsManagementClient') + @patch('octodns.provider.azuredns.ServicePrincipalCredentials') + def _get_provider(self, mock_spc, mock_dns_client): + '''Returns a mock AzureProvider object to use in testing. + + :param mock_spc: placeholder + :type mock_spc: str + :param mock_dns_client: placeholder + :type mock_dns_client: str + + :type return: AzureProvider + ''' + return AzureProvider('mock_id', 'mock_client', 'mock_key', + 'mock_directory', 'mock_sub', 'mock_rg') + + def test_populate_records(self): + provider = self._get_provider() + + rs = [] + rs.append(RecordSet(name='a1', ttl=0, type='A', + arecords=[ARecord('1.1.1.1')])) + rs.append(RecordSet(name='a2', ttl=1, type='A', + arecords=[ARecord('1.1.1.1'), + ARecord('2.2.2.2')])) + rs.append(RecordSet(name='aaaa1', ttl=2, type='AAAA', + aaaa_records=[AaaaRecord('1:1ec:1::1')])) + rs.append(RecordSet(name='aaaa2', ttl=3, type='AAAA', + aaaa_records=[AaaaRecord('1:1ec:1::1'), + AaaaRecord('1:1ec:1::2')])) + rs.append(RecordSet(name='cname1', ttl=4, type='CNAME', + cname_record=CnameRecord('cname.unit.test.'))) + rs.append(RecordSet(name='cname2', ttl=5, type='CNAME', + cname_record=None)) + rs.append(RecordSet(name='mx1', ttl=6, type='MX', + mx_records=[MxRecord(10, 'mx1.unit.test.')])) + rs.append(RecordSet(name='mx2', ttl=7, type='MX', + mx_records=[MxRecord(10, 'mx1.unit.test.'), + MxRecord(11, 'mx2.unit.test.')])) + rs.append(RecordSet(name='ns1', ttl=8, type='NS', + ns_records=[NsRecord('ns1.unit.test.')])) + rs.append(RecordSet(name='ns2', ttl=9, type='NS', + ns_records=[NsRecord('ns1.unit.test.'), + NsRecord('ns2.unit.test.')])) + rs.append(RecordSet(name='ptr1', ttl=10, type='PTR', + ptr_records=[PtrRecord('ptr1.unit.test.')])) + rs.append(RecordSet(name='ptr2', ttl=11, type='PTR', + ptr_records=[PtrRecord(None)])) + rs.append(RecordSet(name='_srv1._tcp', ttl=12, type='SRV', + srv_records=[SrvRecord(1, 2, 3, '1unit.tests.')])) + rs.append(RecordSet(name='_srv2._tcp', ttl=13, type='SRV', + srv_records=[SrvRecord(1, 2, 3, '1unit.tests.'), + SrvRecord(4, 5, 6, '2unit.tests.')])) + rs.append(RecordSet(name='txt1', ttl=14, type='TXT', + txt_records=[TxtRecord('sample text1')])) + rs.append(RecordSet(name='txt2', ttl=15, type='TXT', + txt_records=[TxtRecord('sample text1'), + TxtRecord('sample text2')])) + rs.append(RecordSet(name='', ttl=16, type='SOA', + soa_record=[SoaRecord()])) + + record_list = provider._dns_client.record_sets.list_by_dns_zone + record_list.return_value = rs + + provider.populate(zone) + + self.assertEquals(len(zone.records), 16) + + def test_populate_zone(self): + provider = self._get_provider() + + zone_list = provider._dns_client.zones.list_by_resource_group + zone_list.return_value = [AzureZone(location='global'), + AzureZone(location='global')] + + provider._populate_zones() + + self.assertEquals(len(provider._azure_zones), 1) + + def test_bad_zone_response(self): + provider = self._get_provider() + + _get = provider._dns_client.zones.get + _get.side_effect = CloudError(Mock(status=404), 'Azure Error') + trip = False + try: + provider._check_zone('unit.test', create=False) + except CloudError: + trip = True + self.assertEquals(trip, True) + + def test_apply(self): + provider = self._get_provider() + + changes = [] + deletes = [] + for i in octo_records: + changes.append(Create(i)) + deletes.append(Delete(i)) + + self.assertEquals(13, provider.apply(Plan(None, zone, changes))) + self.assertEquals(13, provider.apply(Plan(zone, zone, deletes))) + + def test_create_zone(self): + provider = self._get_provider() + + changes = [] + for i in octo_records: + changes.append(Create(i)) + desired = Zone('unit2.test.', []) + + err_msg = 'The Resource \'Microsoft.Network/dnszones/unit2.test\' ' + err_msg += 'under resource group \'mock_rg\' was not found.' + _get = provider._dns_client.zones.get + _get.side_effect = CloudError(Mock(status=404), err_msg) + + self.assertEquals(13, provider.apply(Plan(None, desired, changes))) + + def test_check_zone_no_create(self): + provider = self._get_provider() + + rs = [] + rs.append(RecordSet(name='a1', ttl=0, type='A', + arecords=[ARecord('1.1.1.1')])) + rs.append(RecordSet(name='a2', ttl=1, type='A', + arecords=[ARecord('1.1.1.1'), + ARecord('2.2.2.2')])) + + record_list = provider._dns_client.record_sets.list_by_dns_zone + record_list.return_value = rs + + err_msg = 'The Resource \'Microsoft.Network/dnszones/unit3.test\' ' + err_msg += 'under resource group \'mock_rg\' was not found.' + _get = provider._dns_client.zones.get + _get.side_effect = CloudError(Mock(status=404), err_msg) + + provider.populate(Zone('unit3.test.', [])) + + self.assertEquals(len(zone.records), 0) diff --git a/tests/test_octodns_provider_base.py b/tests/test_octodns_provider_base.py index d4e153b..510d3d1 100644 --- a/tests/test_octodns_provider_base.py +++ b/tests/test_octodns_provider_base.py @@ -16,13 +16,17 @@ from octodns.zone import Zone class HelperProvider(BaseProvider): log = getLogger('HelperProvider') + SUPPORTS = set(('A',)) + def __init__(self, extra_changes, apply_disabled=False, include_change_callback=None): self.__extra_changes = extra_changes self.apply_disabled = apply_disabled self.include_change_callback = include_change_callback + self.update_pcent_threshold = Plan.MAX_SAFE_UPDATE_PCENT + self.delete_pcent_threshold = Plan.MAX_SAFE_DELETE_PCENT - def populate(self, zone, target=False): + def populate(self, zone, target=False, lenient=False): pass def _include_change(self, change): @@ -58,12 +62,19 @@ class TestBaseProvider(TestCase): zone = Zone('unit.tests.', []) with self.assertRaises(NotImplementedError) as ctx: HasSupportsGeo('hassupportesgeo').populate(zone) + self.assertEquals('Abstract base class, SUPPORTS property missing', + ctx.exception.message) + + class HasSupports(HasSupportsGeo): + SUPPORTS = set(('A',)) + with self.assertRaises(NotImplementedError) as ctx: + HasSupports('hassupportes').populate(zone) self.assertEquals('Abstract base class, populate method missing', ctx.exception.message) - class HasPopulate(HasSupportsGeo): + class HasPopulate(HasSupports): - def populate(self, zone, target=False): + def populate(self, zone, target=False, lenient=False): zone.add_record(Record.new(zone, '', { 'ttl': 60, 'type': 'A', @@ -81,7 +92,7 @@ class TestBaseProvider(TestCase): 'value': '1.2.3.4' })) - self.assertTrue(HasSupportsGeo('hassupportesgeo') + self.assertTrue(HasSupports('hassupportesgeo') .supports(list(zone.records)[0])) plan = HasPopulate('haspopulate').plan(zone) @@ -205,9 +216,11 @@ class TestBaseProvider(TestCase): for i in range(int(Plan.MIN_EXISTING_RECORDS * Plan.MAX_SAFE_UPDATE_PCENT) + 1)] - with self.assertRaises(UnsafePlan): + with self.assertRaises(UnsafePlan) as ctx: Plan(zone, zone, changes).raise_if_unsafe() + self.assertTrue('Too many updates' in ctx.exception.message) + def test_safe_updates_min_existing_pcent(self): # MAX_SAFE_UPDATE_PCENT is safe when more # than MIN_EXISTING_RECORDS exist @@ -251,9 +264,11 @@ class TestBaseProvider(TestCase): for i in range(int(Plan.MIN_EXISTING_RECORDS * Plan.MAX_SAFE_DELETE_PCENT) + 1)] - with self.assertRaises(UnsafePlan): + with self.assertRaises(UnsafePlan) as ctx: Plan(zone, zone, changes).raise_if_unsafe() + self.assertTrue('Too many deletes' in ctx.exception.message) + def test_safe_deletes_min_existing_pcent(self): # MAX_SAFE_DELETE_PCENT is safe when more # than MIN_EXISTING_RECORDS exist @@ -275,3 +290,59 @@ class TestBaseProvider(TestCase): Plan.MAX_SAFE_DELETE_PCENT))] Plan(zone, zone, changes).raise_if_unsafe() + + def test_safe_updates_min_existing_override(self): + safe_pcent = .4 + # 40% + 1 fails when more + # than MIN_EXISTING_RECORDS exist + zone = Zone('unit.tests.', []) + record = Record.new(zone, 'a', { + 'ttl': 30, + 'type': 'A', + 'value': '1.2.3.4', + }) + + for i in range(int(Plan.MIN_EXISTING_RECORDS)): + zone.add_record(Record.new(zone, str(i), { + 'ttl': 60, + 'type': 'A', + 'value': '2.3.4.5' + })) + + changes = [Update(record, record) + for i in range(int(Plan.MIN_EXISTING_RECORDS * + safe_pcent) + 1)] + + with self.assertRaises(UnsafePlan) as ctx: + Plan(zone, zone, changes, + update_pcent_threshold=safe_pcent).raise_if_unsafe() + + self.assertTrue('Too many updates' in ctx.exception.message) + + def test_safe_deletes_min_existing_override(self): + safe_pcent = .4 + # 40% + 1 fails when more + # than MIN_EXISTING_RECORDS exist + zone = Zone('unit.tests.', []) + record = Record.new(zone, 'a', { + 'ttl': 30, + 'type': 'A', + 'value': '1.2.3.4', + }) + + for i in range(int(Plan.MIN_EXISTING_RECORDS)): + zone.add_record(Record.new(zone, str(i), { + 'ttl': 60, + 'type': 'A', + 'value': '2.3.4.5' + })) + + changes = [Delete(record) + for i in range(int(Plan.MIN_EXISTING_RECORDS * + safe_pcent) + 1)] + + with self.assertRaises(UnsafePlan) as ctx: + Plan(zone, zone, changes, + delete_pcent_threshold=safe_pcent).raise_if_unsafe() + + self.assertTrue('Too many deletes' in ctx.exception.message) diff --git a/tests/test_octodns_provider_cloudflare.py b/tests/test_octodns_provider_cloudflare.py index 3f652a1..04a46e0 100644 --- a/tests/test_octodns_provider_cloudflare.py +++ b/tests/test_octodns_provider_cloudflare.py @@ -33,7 +33,7 @@ class TestCloudflareProvider(TestCase): })) for record in list(expected.records): if record.name == 'sub' and record._type == 'NS': - expected.records.remove(record) + expected._remove_record(record) break empty = {'result': [], 'result_info': {'count': 0, 'per_page': 0}} @@ -118,7 +118,7 @@ class TestCloudflareProvider(TestCase): zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals(9, len(zone.records)) + self.assertEquals(10, len(zone.records)) changes = self.expected.changes(zone, provider) self.assertEquals(0, len(changes)) @@ -126,7 +126,7 @@ class TestCloudflareProvider(TestCase): # re-populating the same zone/records comes out of cache, no calls again = Zone('unit.tests.', []) provider.populate(again) - self.assertEquals(9, len(again.records)) + self.assertEquals(10, len(again.records)) def test_apply(self): provider = CloudflareProvider('test', 'email', 'token') @@ -140,12 +140,12 @@ class TestCloudflareProvider(TestCase): 'id': 42, } }, # zone create - ] + [None] * 16 # individual record creates + ] + [None] * 17 # individual record creates # non-existant zone, create everything plan = provider.plan(self.expected) - self.assertEquals(9, len(plan.changes)) - self.assertEquals(9, provider.apply(plan)) + self.assertEquals(10, len(plan.changes)) + self.assertEquals(10, provider.apply(plan)) provider._request.assert_has_calls([ # created the domain @@ -170,7 +170,7 @@ class TestCloudflareProvider(TestCase): }), ], True) # expected number of total calls - self.assertEquals(18, provider._request.call_count) + self.assertEquals(19, provider._request.call_count) provider._request.reset_mock() diff --git a/tests/test_octodns_provider_dnsimple.py b/tests/test_octodns_provider_dnsimple.py index 1f62bfd..950d460 100644 --- a/tests/test_octodns_provider_dnsimple.py +++ b/tests/test_octodns_provider_dnsimple.py @@ -33,7 +33,7 @@ class TestDnsimpleProvider(TestCase): })) for record in list(expected.records): if record.name == 'sub' and record._type == 'NS': - expected.records.remove(record) + expected._remove_record(record) break def test_populate(self): @@ -78,14 +78,14 @@ class TestDnsimpleProvider(TestCase): zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals(14, len(zone.records)) + self.assertEquals(15, len(zone.records)) changes = self.expected.changes(zone, provider) self.assertEquals(0, len(changes)) # 2nd populate makes no network calls/all from cache again = Zone('unit.tests.', []) provider.populate(again) - self.assertEquals(14, len(again.records)) + self.assertEquals(15, len(again.records)) # bust the cache del provider._zone_records[zone.name] @@ -147,7 +147,7 @@ class TestDnsimpleProvider(TestCase): }), ]) # expected number of total calls - self.assertEquals(26, provider._client._request.call_count) + self.assertEquals(27, provider._client._request.call_count) provider._client._request.reset_mock() diff --git a/tests/test_octodns_provider_dyn.py b/tests/test_octodns_provider_dyn.py index 4b9d8b7..50b5289 100644 --- a/tests/test_octodns_provider_dyn.py +++ b/tests/test_octodns_provider_dyn.py @@ -46,11 +46,11 @@ class TestDynProvider(TestCase): 'type': 'MX', 'ttl': 302, 'values': [{ - 'priority': 10, - 'value': 'smtp-1.unit.tests.' + 'preference': 10, + 'exchange': 'smtp-1.unit.tests.' }, { - 'priority': 20, - 'value': 'smtp-2.unit.tests.' + 'preference': 20, + 'exchange': 'smtp-2.unit.tests.' }] }), ('naptr', { @@ -109,6 +109,14 @@ class TestDynProvider(TestCase): 'weight': 22, 'port': 20, 'target': 'foo-2.unit.tests.' + }]}), + ('', { + 'type': 'CAA', + 'ttl': 308, + 'values': [{ + 'flags': 0, + 'tag': 'issue', + 'value': 'ca.unit.tests' }]})): expected.add_record(Record.new(expected, name, data)) @@ -321,6 +329,16 @@ class TestDynProvider(TestCase): 'ttl': 307, 'zone': 'unit.tests', }], + 'caa_records': [{ + 'fqdn': 'unit.tests', + 'rdata': {'flags': 0, + 'tag': 'issue', + 'value': 'ca.unit.tests'}, + 'record_id': 12, + 'record_type': 'cAA', + 'ttl': 308, + 'zone': 'unit.tests', + }], }} ] got = Zone('unit.tests.', []) @@ -414,10 +432,10 @@ class TestDynProvider(TestCase): update_mock.assert_called() add_mock.assert_called() # Once for each dyn record (8 Records, 2 of which have dual values) - self.assertEquals(14, len(add_mock.call_args_list)) + self.assertEquals(15, len(add_mock.call_args_list)) execute_mock.assert_has_calls([call('/Zone/unit.tests/', 'GET', {}), call('/Zone/unit.tests/', 'GET', {})]) - self.assertEquals(9, len(plan.changes)) + self.assertEquals(10, len(plan.changes)) execute_mock.reset_mock() @@ -602,6 +620,7 @@ class TestDynProviderGeo(TestCase): provider = DynProvider('test', 'cust', 'user', 'pass', True) # short-circuit session checking provider._dyn_sess = True + provider.log.warn = MagicMock() # no tds mock.side_effect = [{'data': []}] @@ -650,6 +669,10 @@ class TestDynProviderGeo(TestCase): set(tds.keys())) self.assertEquals(['A'], tds['unit.tests.'].keys()) self.assertEquals(['A'], tds['geo.unit.tests.'].keys()) + provider.log.warn.assert_called_with("Failed to load TraficDirector " + "'%s': %s", 'something else', + 'need more than 1 value to ' + 'unpack') @patch('dyn.core.SessionEngine.execute') def test_traffic_director_monitor(self, mock): diff --git a/tests/test_octodns_provider_googlecloud.py b/tests/test_octodns_provider_googlecloud.py new file mode 100644 index 0000000..adc2112 --- /dev/null +++ b/tests/test_octodns_provider_googlecloud.py @@ -0,0 +1,429 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from octodns.record import Create, Delete, Update, Record +from octodns.provider.googlecloud import GoogleCloudProvider + +from octodns.zone import Zone +from octodns.provider.base import Plan, BaseProvider + +from unittest import TestCase +from mock import Mock, patch, PropertyMock + +zone = Zone(name='unit.tests.', sub_zones=[]) +octo_records = [] +octo_records.append(Record.new(zone, '', { + 'ttl': 0, + 'type': 'A', + 'values': ['1.2.3.4', '10.10.10.10']})) +octo_records.append(Record.new(zone, 'a', { + 'ttl': 1, + 'type': 'A', + 'values': ['1.2.3.4', '1.1.1.1']})) +octo_records.append(Record.new(zone, 'aa', { + 'ttl': 9001, + 'type': 'A', + 'values': ['1.2.4.3']})) +octo_records.append(Record.new(zone, 'aaa', { + 'ttl': 2, + 'type': 'A', + 'values': ['1.1.1.3']})) +octo_records.append(Record.new(zone, 'cname', { + 'ttl': 3, + 'type': 'CNAME', + 'value': 'a.unit.tests.'})) +octo_records.append(Record.new(zone, 'mx1', { + 'ttl': 3, + 'type': 'MX', + 'values': [{ + 'priority': 10, + 'value': 'mx1.unit.tests.', + }, { + 'priority': 20, + 'value': 'mx2.unit.tests.', + }]})) +octo_records.append(Record.new(zone, 'mx2', { + 'ttl': 3, + 'type': 'MX', + 'values': [{ + 'priority': 10, + 'value': 'mx1.unit.tests.', + }]})) +octo_records.append(Record.new(zone, '', { + 'ttl': 4, + 'type': 'NS', + 'values': ['ns1.unit.tests.', 'ns2.unit.tests.']})) +octo_records.append(Record.new(zone, 'foo', { + 'ttl': 5, + 'type': 'NS', + 'value': 'ns1.unit.tests.'})) +octo_records.append(Record.new(zone, '_srv._tcp', { + 'ttl': 6, + 'type': 'SRV', + 'values': [{ + 'priority': 10, + 'weight': 20, + 'port': 30, + 'target': 'foo-1.unit.tests.', + }, { + 'priority': 12, + 'weight': 30, + 'port': 30, + 'target': 'foo-2.unit.tests.', + }]})) +octo_records.append(Record.new(zone, '_srv2._tcp', { + 'ttl': 7, + 'type': 'SRV', + 'values': [{ + 'priority': 12, + 'weight': 17, + 'port': 1, + 'target': 'srvfoo.unit.tests.', + }]})) +octo_records.append(Record.new(zone, 'txt1', { + 'ttl': 8, + 'type': 'TXT', + 'value': 'txt singleton test'})) +octo_records.append(Record.new(zone, 'txt2', { + 'ttl': 9, + 'type': 'TXT', + 'values': ['txt multiple test', 'txt multiple test 2']})) +octo_records.append(Record.new(zone, 'naptr', { + 'ttl': 9, + 'type': 'NAPTR', + 'values': [{ + 'order': 100, + 'preference': 10, + 'flags': 'S', + 'service': 'SIP+D2U', + 'regexp': "!^.*$!sip:customer-service@unit.tests!", + 'replacement': '_sip._udp.unit.tests.' + }]})) +octo_records.append(Record.new(zone, 'caa', { + 'ttl': 9, + 'type': 'CAA', + 'value': { + 'flags': 0, + 'tag': 'issue', + 'value': 'ca.unit.tests', + }})) +for record in octo_records: + zone.add_record(record) + +# This is the format which the google API likes. +resource_record_sets = [ + ('unit.tests.', u'A', 0, [u'1.2.3.4', u'10.10.10.10']), + (u'a.unit.tests.', u'A', 1, [u'1.1.1.1', u'1.2.3.4']), + (u'aa.unit.tests.', u'A', 9001, [u'1.2.4.3']), + (u'aaa.unit.tests.', u'A', 2, [u'1.1.1.3']), + (u'cname.unit.tests.', u'CNAME', 3, [u'a.unit.tests.']), + (u'mx1.unit.tests.', u'MX', 3, + [u'10 mx1.unit.tests.', u'20 mx2.unit.tests.']), + (u'mx2.unit.tests.', u'MX', 3, [u'10 mx1.unit.tests.']), + ('unit.tests.', u'NS', 4, [u'ns1.unit.tests.', u'ns2.unit.tests.']), + (u'foo.unit.tests.', u'NS', 5, [u'ns1.unit.tests.']), + (u'_srv._tcp.unit.tests.', u'SRV', 6, + [u'10 20 30 foo-1.unit.tests.', u'12 30 30 foo-2.unit.tests.']), + (u'_srv2._tcp.unit.tests.', u'SRV', 7, [u'12 17 1 srvfoo.unit.tests.']), + (u'txt1.unit.tests.', u'TXT', 8, [u'txt singleton test']), + (u'txt2.unit.tests.', u'TXT', 9, + [u'txt multiple test', u'txt multiple test 2']), + (u'naptr.unit.tests.', u'NAPTR', 9, [ + u'100 10 "S" "SIP+D2U" "!^.*$!sip:customer-service@unit.tests!"' + u' _sip._udp.unit.tests.']), + (u'caa.unit.tests.', u'CAA', 9, [u'0 issue ca.unit.tests']) +] + + +class DummyResourceRecordSet: + def __init__(self, record_name, record_type, ttl, rrdatas): + self.name = record_name + self.record_type = record_type + self.ttl = ttl + self.rrdatas = rrdatas + + def __eq__(self, other): + try: + return self.name == other.name \ + and self.record_type == other.record_type \ + and self.ttl == other.ttl \ + and sorted(self.rrdatas) == sorted(other.rrdatas) + except: + return False + + def __repr__(self): + return "{} {} {} {!s}"\ + .format(self.name, self.record_type, self.ttl, self.rrdatas) + + def __hash__(self): + return hash(repr(self)) + + +class DummyGoogleCloudZone: + def __init__(self, dns_name, name=""): + self.dns_name = dns_name + self.name = name + + def resource_record_set(self, *args): + return DummyResourceRecordSet(*args) + + def list_resource_record_sets(self, *args): + pass + + def create(self, *args, **kwargs): + pass + + +class DummyIterator: + """Returns a mock DummyIterator object to use in testing. + This is because API calls for google cloud DNS, if paged, contains a + "next_page_token", which can be used to grab a subsequent + iterator with more results. + + :type return: DummyIterator + """ + def __init__(self, list_of_stuff, page_token=None): + self.iterable = iter(list_of_stuff) + self.next_page_token = page_token + + def __iter__(self): + return self + + def next(self): + return self.iterable.next() + + +class TestGoogleCloudProvider(TestCase): + @patch('octodns.provider.googlecloud.dns') + def _get_provider(*args): + '''Returns a mock GoogleCloudProvider object to use in testing. + + :type return: GoogleCloudProvider + ''' + return GoogleCloudProvider(id=1, project="mock") + + @patch('octodns.provider.googlecloud.dns') + def test___init__(self, *_): + self.assertIsInstance(GoogleCloudProvider(id=1, + credentials_file="test", + project="unit test"), + BaseProvider) + + self.assertIsInstance(GoogleCloudProvider(id=1), + BaseProvider) + + @patch('octodns.provider.googlecloud.time.sleep') + @patch('octodns.provider.googlecloud.dns') + def test__apply(self, *_): + class DummyDesired: + def __init__(self, name, changes): + self.name = name + self.changes = changes + + apply_z = Zone("unit.tests.", []) + create_r = Record.new(apply_z, '', { + 'ttl': 0, + 'type': 'A', + 'values': ['1.2.3.4', '10.10.10.10']}) + delete_r = Record.new(apply_z, 'a', { + 'ttl': 1, + 'type': 'A', + 'values': ['1.2.3.4', '1.1.1.1']}) + update_existing_r = Record.new(apply_z, 'aa', { + 'ttl': 9001, + 'type': 'A', + 'values': ['1.2.4.3']}) + update_new_r = Record.new(apply_z, 'aa', { + 'ttl': 666, + 'type': 'A', + 'values': ['1.4.3.2']}) + + gcloud_zone_mock = DummyGoogleCloudZone("unit.tests.", "unit-tests") + status_mock = Mock() + return_values_for_status = iter( + ["pending"] * 11 + ['done', 'done']) + type(status_mock).status = PropertyMock( + side_effect=return_values_for_status.next) + gcloud_zone_mock.changes = Mock(return_value=status_mock) + + provider = self._get_provider() + provider.gcloud_client = Mock() + provider._gcloud_zones = {"unit.tests.": gcloud_zone_mock} + desired = Mock() + desired.name = "unit.tests." + changes = [] + changes.append(Create(create_r)) + changes.append(Delete(delete_r)) + changes.append(Update(existing=update_existing_r, new=update_new_r)) + + provider.apply(Plan( + existing=[update_existing_r, delete_r], + desired=desired, + changes=changes + )) + + calls_mock = gcloud_zone_mock.changes.return_value + mocked_calls = [] + for mock_call in calls_mock.add_record_set.mock_calls: + mocked_calls.append(mock_call[1][0]) + + self.assertEqual(mocked_calls, [ + DummyResourceRecordSet( + 'unit.tests.', 'A', 0, ['1.2.3.4', '10.10.10.10']), + DummyResourceRecordSet( + 'aa.unit.tests.', 'A', 666, ['1.4.3.2']) + ]) + + mocked_calls2 = [] + for mock_call in calls_mock.delete_record_set.mock_calls: + mocked_calls2.append(mock_call[1][0]) + + self.assertEqual(mocked_calls2, [ + DummyResourceRecordSet( + 'a.unit.tests.', 'A', 1, ['1.2.3.4', '1.1.1.1']), + DummyResourceRecordSet( + 'aa.unit.tests.', 'A', 9001, ['1.2.4.3']) + ]) + + type(status_mock).status = "pending" + + with self.assertRaises(RuntimeError): + provider.apply(Plan( + existing=[update_existing_r, delete_r], + desired=desired, + changes=changes + )) + + unsupported_change = Mock() + unsupported_change.__len__ = Mock(return_value=1) + type_mock = Mock() + type_mock._type = "A" + unsupported_change.record = type_mock + + mock_plan = Mock() + type(mock_plan).desired = PropertyMock(return_value=DummyDesired( + "dummy name", [])) + type(mock_plan).changes = [unsupported_change] + + with self.assertRaises(RuntimeError): + provider.apply(mock_plan) + + def test__get_gcloud_client(self): + provider = self._get_provider() + + self.assertIsInstance(provider, GoogleCloudProvider) + + @patch('octodns.provider.googlecloud.dns') + def test_populate(self, _): + def _get_mock_zones(page_token=None): + if not page_token: + return DummyIterator([ + DummyGoogleCloudZone('example.com.'), + ], page_token="MOCK_PAGE_TOKEN") + elif page_token == "MOCK_PAGE_TOKEN": + return DummyIterator([ + DummyGoogleCloudZone('example2.com.'), + ], page_token="MOCK_PAGE_TOKEN2") + + return DummyIterator([ + google_cloud_zone + ]) + + def _get_mock_record_sets(page_token=None): + if not page_token: + return DummyIterator( + [DummyResourceRecordSet(*v) for v in + resource_record_sets[:3]], page_token="MOCK_PAGE_TOKEN") + elif page_token == "MOCK_PAGE_TOKEN": + + return DummyIterator( + [DummyResourceRecordSet(*v) for v in + resource_record_sets[3:5]], page_token="MOCK_PAGE_TOKEN2") + return DummyIterator( + [DummyResourceRecordSet(*v) for v in resource_record_sets[5:]]) + + google_cloud_zone = DummyGoogleCloudZone('unit.tests.') + + provider = self._get_provider() + provider.gcloud_client.list_zones = Mock(side_effect=_get_mock_zones) + google_cloud_zone.list_resource_record_sets = Mock( + side_effect=_get_mock_record_sets) + + self.assertEqual(provider.gcloud_zones.get("unit.tests.").dns_name, + "unit.tests.") + + test_zone = Zone('unit.tests.', []) + provider.populate(test_zone) + + # test_zone gets fed the same records as zone does, except it's in + # the format returned by google API, so after populate they should look + # excactly the same. + self.assertEqual(test_zone.records, zone.records) + + test_zone2 = Zone('nonexistant.zone.', []) + provider.populate(test_zone2, False, False) + + self.assertEqual(len(test_zone2.records), 0, + msg="Zone should not get records from wrong domain") + + provider.SUPPORTS = set() + test_zone3 = Zone('unit.tests.', []) + provider.populate(test_zone3) + self.assertEqual(len(test_zone3.records), 0) + + @patch('octodns.provider.googlecloud.dns') + def test_populate_corner_cases(self, _): + provider = self._get_provider() + test_zone = Zone('unit.tests.', []) + not_same_fqdn = DummyResourceRecordSet( + 'unit.tests.gr', u'A', 0, [u'1.2.3.4']), + + provider._get_gcloud_records = Mock( + side_effect=[not_same_fqdn]) + provider._gcloud_zones = { + "unit.tests.": DummyGoogleCloudZone("unit.tests.", "unit-tests")} + + provider.populate(test_zone) + + self.assertEqual(len(test_zone.records), 1) + + self.assertEqual(test_zone.records.pop().fqdn, + u'unit.tests.gr.unit.tests.') + + def test__get_gcloud_zone(self): + provider = self._get_provider() + + provider.gcloud_client = Mock() + provider.gcloud_client.list_zones = Mock( + return_value=DummyIterator([])) + + self.assertIsNone(provider.gcloud_zones.get("nonexistant.xone"), + msg="Check that nonexistant zones return None when" + "there's no create=True flag") + + def test__get_rrsets(self): + provider = self._get_provider() + dummy_gcloud_zone = DummyGoogleCloudZone("unit.tests") + for octo_record in octo_records: + _rrset_func = getattr( + provider, '_rrset_for_{}'.format(octo_record._type)) + self.assertEqual( + _rrset_func(dummy_gcloud_zone, octo_record).record_type, + octo_record._type + ) + + def test__create_zone(self): + provider = self._get_provider() + + provider.gcloud_client = Mock() + provider.gcloud_client.list_zones = Mock( + return_value=DummyIterator([])) + + mock_zone = provider._create_gcloud_zone("nonexistant.zone.mock") + + mock_zone.create.assert_called() + provider.gcloud_client.zone.assert_called() diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index acb0125..cde23b0 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -6,7 +6,8 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals from mock import Mock, call, patch -from nsone.rest.errors import AuthException, ResourceException +from nsone.rest.errors import AuthException, RateLimitException, \ + ResourceException from unittest import TestCase from octodns.record import Delete, Record, Update @@ -44,11 +45,11 @@ class TestNs1Provider(TestCase): 'ttl': 35, 'type': 'MX', 'values': [{ - 'priority': 10, - 'value': 'mx1.unit.tests.', + 'preference': 10, + 'exchange': 'mx1.unit.tests.', }, { - 'priority': 20, - 'value': 'mx2.unit.tests.', + 'preference': 20, + 'exchange': 'mx2.unit.tests.', }] })) expected.add(Record.new(zone, 'naptr', { @@ -95,6 +96,15 @@ class TestNs1Provider(TestCase): 'type': 'NS', 'values': ['ns3.unit.tests.', 'ns4.unit.tests.'], })) + expected.add(Record.new(zone, '', { + 'ttl': 40, + 'type': 'CAA', + 'value': { + 'flags': 0, + 'tag': 'issue', + 'value': 'ca.unit.tests', + }, + })) nsone_records = [{ 'type': 'A', @@ -140,6 +150,11 @@ class TestNs1Provider(TestCase): 'ttl': 39, 'short_answers': ['ns3.unit.tests.', 'ns4.unit.tests.'], 'domain': 'sub.unit.tests.', + }, { + 'type': 'CAA', + 'ttl': 40, + 'short_answers': ['0 issue ca.unit.tests'], + 'domain': 'unit.tests.', }] @patch('nsone.NSONE.loadZone') @@ -195,7 +210,8 @@ class TestNs1Provider(TestCase): provider = Ns1Provider('test', 'api-key') desired = Zone('unit.tests.', []) - desired.records.update(self.expected) + for r in self.expected: + desired.add_record(r) plan = provider.plan(desired) # everything except the root NS @@ -225,7 +241,15 @@ class TestNs1Provider(TestCase): create_mock.reset_mock() load_mock.side_effect = \ ResourceException('server error: zone not found') - create_mock.side_effect = None + # ugh, need a mock zone with a mock prop since we're using getattr, we + # can actually control side effects on `meth` with that. + mock_zone = Mock() + mock_zone.add_SRV = Mock() + mock_zone.add_SRV.side_effect = [ + RateLimitException('boo', period=0), + None, + ] + create_mock.side_effect = [mock_zone] got_n = provider.apply(plan) self.assertEquals(expected_n, got_n) @@ -245,12 +269,60 @@ class TestNs1Provider(TestCase): self.assertEquals(2, len(plan.changes)) self.assertIsInstance(plan.changes[0], Update) self.assertIsInstance(plan.changes[1], Delete) - + # ugh, we need a mock record that can be returned from loadRecord for + # the update and delete targets, we can add our side effects to that to + # trigger rate limit handling + mock_record = Mock() + mock_record.update.side_effect = [ + RateLimitException('one', period=0), + None, + ] + mock_record.delete.side_effect = [ + RateLimitException('two', period=0), + None, + ] + nsone_zone.loadRecord.side_effect = [mock_record, mock_record] got_n = provider.apply(plan) self.assertEquals(2, got_n) nsone_zone.loadRecord.assert_has_calls([ call('unit.tests', u'A'), - call().update(answers=[u'1.2.3.4'], ttl=32), call('delete-me', u'A'), - call().delete() ]) + mock_record.assert_has_calls([ + call.update(answers=[u'1.2.3.4'], ttl=32), + call.delete() + ]) + + def test_escaping(self): + provider = Ns1Provider('test', 'api-key') + + record = { + 'ttl': 31, + 'short_answers': ['foo; bar baz; blip'] + } + self.assertEquals(['foo\; bar baz\; blip'], + provider._data_for_SPF('SPF', record)['values']) + + record = { + 'ttl': 31, + 'short_answers': ['no', 'foo; bar baz; blip', 'yes'] + } + self.assertEquals(['no', 'foo\; bar baz\; blip', 'yes'], + provider._data_for_TXT('TXT', record)['values']) + + zone = Zone('unit.tests.', []) + record = Record.new(zone, 'spf', { + 'ttl': 34, + 'type': 'SPF', + 'value': 'foo\; bar baz\; blip' + }) + self.assertEquals(['foo; bar baz; blip'], + provider._params_for_SPF(record)['answers']) + + record = Record.new(zone, 'txt', { + 'ttl': 35, + 'type': 'TXT', + 'value': 'foo\; bar baz\; blip' + }) + self.assertEquals(['foo; bar baz; blip'], + provider._params_for_TXT(record)['answers']) diff --git a/tests/test_octodns_provider_ovh.py b/tests/test_octodns_provider_ovh.py new file mode 100644 index 0000000..2816748 --- /dev/null +++ b/tests/test_octodns_provider_ovh.py @@ -0,0 +1,359 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from unittest import TestCase + +from mock import patch, call +from ovh import APIError + +from octodns.provider.ovh import OvhProvider +from octodns.record import Record +from octodns.zone import Zone + + +class TestOvhProvider(TestCase): + api_record = [] + + zone = Zone('unit.tests.', []) + expected = set() + + # A, subdomain='' + api_record.append({ + 'fieldType': 'A', + 'ttl': 100, + 'target': '1.2.3.4', + 'subDomain': '', + 'id': 1 + }) + expected.add(Record.new(zone, '', { + 'ttl': 100, + 'type': 'A', + 'value': '1.2.3.4', + })) + + # A, subdomain='sub + api_record.append({ + 'fieldType': 'A', + 'ttl': 200, + 'target': '1.2.3.4', + 'subDomain': 'sub', + 'id': 2 + }) + expected.add(Record.new(zone, 'sub', { + 'ttl': 200, + 'type': 'A', + 'value': '1.2.3.4', + })) + + # CNAME + api_record.append({ + 'fieldType': 'CNAME', + 'ttl': 300, + 'target': 'unit.tests.', + 'subDomain': 'www2', + 'id': 3 + }) + expected.add(Record.new(zone, 'www2', { + 'ttl': 300, + 'type': 'CNAME', + 'value': 'unit.tests.', + })) + + # MX + api_record.append({ + 'fieldType': 'MX', + 'ttl': 400, + 'target': '10 mx1.unit.tests.', + 'subDomain': '', + 'id': 4 + }) + expected.add(Record.new(zone, '', { + 'ttl': 400, + 'type': 'MX', + 'values': [{ + 'preference': 10, + 'exchange': 'mx1.unit.tests.', + }] + })) + + # NAPTR + api_record.append({ + 'fieldType': 'NAPTR', + 'ttl': 500, + 'target': '10 100 "S" "SIP+D2U" "!^.*$!sip:info@bar.example.com!" .', + 'subDomain': 'naptr', + 'id': 5 + }) + expected.add(Record.new(zone, 'naptr', { + 'ttl': 500, + 'type': 'NAPTR', + 'values': [{ + 'flags': 'S', + 'order': 10, + 'preference': 100, + 'regexp': '!^.*$!sip:info@bar.example.com!', + 'replacement': '.', + 'service': 'SIP+D2U', + }] + })) + + # NS + api_record.append({ + 'fieldType': 'NS', + 'ttl': 600, + 'target': 'ns1.unit.tests.', + 'subDomain': '', + 'id': 6 + }) + api_record.append({ + 'fieldType': 'NS', + 'ttl': 600, + 'target': 'ns2.unit.tests.', + 'subDomain': '', + 'id': 7 + }) + expected.add(Record.new(zone, '', { + 'ttl': 600, + 'type': 'NS', + 'values': ['ns1.unit.tests.', 'ns2.unit.tests.'], + })) + + # NS with sub + api_record.append({ + 'fieldType': 'NS', + 'ttl': 700, + 'target': 'ns3.unit.tests.', + 'subDomain': 'www3', + 'id': 8 + }) + api_record.append({ + 'fieldType': 'NS', + 'ttl': 700, + 'target': 'ns4.unit.tests.', + 'subDomain': 'www3', + 'id': 9 + }) + expected.add(Record.new(zone, 'www3', { + 'ttl': 700, + 'type': 'NS', + 'values': ['ns3.unit.tests.', 'ns4.unit.tests.'], + })) + + api_record.append({ + 'fieldType': 'SRV', + 'ttl': 800, + 'target': '10 20 30 foo-1.unit.tests.', + 'subDomain': '_srv._tcp', + 'id': 10 + }) + api_record.append({ + 'fieldType': 'SRV', + 'ttl': 800, + 'target': '40 50 60 foo-2.unit.tests.', + 'subDomain': '_srv._tcp', + 'id': 11 + }) + expected.add(Record.new(zone, '_srv._tcp', { + 'ttl': 800, + 'type': 'SRV', + 'values': [{ + 'priority': 10, + 'weight': 20, + 'port': 30, + 'target': 'foo-1.unit.tests.', + }, { + 'priority': 40, + 'weight': 50, + 'port': 60, + 'target': 'foo-2.unit.tests.', + }] + })) + + # PTR + api_record.append({ + 'fieldType': 'PTR', + 'ttl': 900, + 'target': 'unit.tests.', + 'subDomain': '4', + 'id': 12 + }) + expected.add(Record.new(zone, '4', { + 'ttl': 900, + 'type': 'PTR', + 'value': 'unit.tests.' + })) + + # SPF + api_record.append({ + 'fieldType': 'SPF', + 'ttl': 1000, + 'target': 'v=spf1 include:unit.texts.rerirect ~all', + 'subDomain': '', + 'id': 13 + }) + expected.add(Record.new(zone, '', { + 'ttl': 1000, + 'type': 'SPF', + 'value': 'v=spf1 include:unit.texts.rerirect ~all' + })) + + # SSHFP + api_record.append({ + 'fieldType': 'SSHFP', + 'ttl': 1100, + 'target': '1 1 bf6b6825d2977c511a475bbefb88aad54a92ac73 ', + 'subDomain': '', + 'id': 14 + }) + expected.add(Record.new(zone, '', { + 'ttl': 1100, + 'type': 'SSHFP', + 'value': { + 'algorithm': 1, + 'fingerprint': 'bf6b6825d2977c511a475bbefb88aad54a92ac73', + 'fingerprint_type': 1 + } + })) + + # AAAA + api_record.append({ + 'fieldType': 'AAAA', + 'ttl': 1200, + 'target': '1:1ec:1::1', + 'subDomain': '', + 'id': 15 + }) + expected.add(Record.new(zone, '', { + 'ttl': 200, + 'type': 'AAAA', + 'value': '1:1ec:1::1', + })) + + @patch('ovh.Client') + def test_populate(self, client_mock): + provider = OvhProvider('test', 'endpoint', 'application_key', + 'application_secret', 'consumer_key') + + with patch.object(provider._client, 'get') as get_mock: + zone = Zone('unit.tests.', []) + get_mock.side_effect = APIError('boom') + with self.assertRaises(APIError) as ctx: + provider.populate(zone) + self.assertEquals(get_mock.side_effect, ctx.exception) + + with patch.object(provider._client, 'get') as get_mock: + zone = Zone('unit.tests.', []) + get_returns = [[record['id'] for record in self.api_record]] + get_returns += self.api_record + get_mock.side_effect = get_returns + provider.populate(zone) + self.assertEquals(self.expected, zone.records) + + @patch('ovh.Client') + def test_apply(self, client_mock): + provider = OvhProvider('test', 'endpoint', 'application_key', + 'application_secret', 'consumer_key') + + desired = Zone('unit.tests.', []) + + for r in self.expected: + desired.add_record(r) + + with patch.object(provider._client, 'post') as get_mock: + plan = provider.plan(desired) + get_mock.side_effect = APIError('boom') + with self.assertRaises(APIError) as ctx: + provider.apply(plan) + self.assertEquals(get_mock.side_effect, ctx.exception) + + with patch.object(provider._client, 'get') as get_mock: + get_returns = [[1, 2], { + 'fieldType': 'A', + 'ttl': 600, + 'target': '5.6.7.8', + 'subDomain': '', + 'id': 100 + }, {'fieldType': 'A', + 'ttl': 600, + 'target': '5.6.7.8', + 'subDomain': 'fake', + 'id': 101 + }] + get_mock.side_effect = get_returns + + plan = provider.plan(desired) + + with patch.object(provider._client, 'post') as post_mock: + with patch.object(provider._client, 'delete') as delete_mock: + with patch.object(provider._client, 'get') as get_mock: + get_mock.side_effect = [[100], [101]] + provider.apply(plan) + wanted_calls = [ + call(u'/domain/zone/unit.tests/record', + fieldType=u'A', + subDomain=u'', target=u'1.2.3.4', ttl=100), + call(u'/domain/zone/unit.tests/record', + fieldType=u'SRV', + subDomain=u'10 20 30 foo-1.unit.tests.', + target='_srv._tcp', ttl=800), + call(u'/domain/zone/unit.tests/record', + fieldType=u'SRV', + subDomain=u'40 50 60 foo-2.unit.tests.', + target='_srv._tcp', ttl=800), + call(u'/domain/zone/unit.tests/record', + fieldType=u'PTR', subDomain='4', + target=u'unit.tests.', ttl=900), + call(u'/domain/zone/unit.tests/record', + fieldType=u'NS', subDomain='www3', + target=u'ns3.unit.tests.', ttl=700), + call(u'/domain/zone/unit.tests/record', + fieldType=u'NS', subDomain='www3', + target=u'ns4.unit.tests.', ttl=700), + call(u'/domain/zone/unit.tests/record', + fieldType=u'SSHFP', + subDomain=u'1 1 bf6b6825d2977c511a475bbefb88a' + u'ad54' + u'a92ac73', + target=u'', ttl=1100), + call(u'/domain/zone/unit.tests/record', + fieldType=u'AAAA', subDomain=u'', + target=u'1:1ec:1::1', ttl=200), + call(u'/domain/zone/unit.tests/record', + fieldType=u'MX', subDomain=u'', + target=u'10 mx1.unit.tests.', ttl=400), + call(u'/domain/zone/unit.tests/record', + fieldType=u'CNAME', subDomain='www2', + target=u'unit.tests.', ttl=300), + call(u'/domain/zone/unit.tests/record', + fieldType=u'SPF', subDomain=u'', + target=u'v=spf1 include:unit.texts.' + u'rerirect ~all', + ttl=1000), + call(u'/domain/zone/unit.tests/record', + fieldType=u'A', + subDomain='sub', target=u'1.2.3.4', ttl=200), + call(u'/domain/zone/unit.tests/record', + fieldType=u'NAPTR', subDomain='naptr', + target=u'10 100 "S" "SIP+D2U" "!^.*$!sip:' + u'info@bar' + u'.example.com!" .', + ttl=500), + call(u'/domain/zone/unit.tests/refresh')] + + post_mock.assert_has_calls(wanted_calls) + + # Get for delete calls + get_mock.assert_has_calls( + [call(u'/domain/zone/unit.tests/record', + fieldType=u'A', subDomain=u''), + call(u'/domain/zone/unit.tests/record', + fieldType=u'A', subDomain='fake')] + ) + # 2 delete calls, one for update + one for delete + delete_mock.assert_has_calls( + [call(u'/domain/zone/unit.tests/record/100'), + call(u'/domain/zone/unit.tests/record/101')]) diff --git a/tests/test_octodns_provider_powerdns.py b/tests/test_octodns_provider_powerdns.py index c7db997..9cf372e 100644 --- a/tests/test_octodns_provider_powerdns.py +++ b/tests/test_octodns_provider_powerdns.py @@ -78,7 +78,7 @@ class TestPowerDnsProvider(TestCase): source = YamlProvider('test', join(dirname(__file__), 'config')) source.populate(expected) expected_n = len(expected.records) - 1 - self.assertEquals(14, expected_n) + self.assertEquals(15, expected_n) # No diffs == no changes with requests_mock() as mock: @@ -86,7 +86,7 @@ class TestPowerDnsProvider(TestCase): zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals(14, len(zone.records)) + self.assertEquals(15, len(zone.records)) changes = expected.changes(zone, provider) self.assertEquals(0, len(changes)) @@ -166,7 +166,7 @@ class TestPowerDnsProvider(TestCase): expected = Zone('unit.tests.', []) source = YamlProvider('test', join(dirname(__file__), 'config')) source.populate(expected) - self.assertEquals(15, len(expected.records)) + self.assertEquals(16, len(expected.records)) # A small change to a single record with requests_mock() as mock: @@ -252,7 +252,7 @@ class TestPowerDnsProvider(TestCase): plan = provider.plan(expected) self.assertFalse(plan) # remove it now that we don't need the unrelated change any longer - expected.records.remove(unrelated_record) + expected._remove_record(unrelated_record) # ttl diff with requests_mock() as mock: diff --git a/tests/test_octodns_provider_route53.py b/tests/test_octodns_provider_route53.py index 4616cfe..10a140e 100644 --- a/tests/test_octodns_provider_route53.py +++ b/tests/test_octodns_provider_route53.py @@ -58,11 +58,11 @@ class TestRoute53Provider(TestCase): 'Goodbye World?']}), ('', {'ttl': 64, 'type': 'MX', 'values': [{ - 'priority': 10, - 'value': 'smtp-1.unit.tests.', + 'preference': 10, + 'exchange': 'smtp-1.unit.tests.', }, { - 'priority': 20, - 'value': 'smtp-2.unit.tests.', + 'preference': 20, + 'exchange': 'smtp-2.unit.tests.', }]}), ('naptr', {'ttl': 65, 'type': 'NAPTR', 'value': { @@ -83,6 +83,12 @@ class TestRoute53Provider(TestCase): {'ttl': 67, 'type': 'NS', 'values': ['8.2.3.4.', '9.2.3.4.']}), ('sub', {'ttl': 68, 'type': 'NS', 'values': ['5.2.3.4.', '6.2.3.4.']}), + ('', + {'ttl': 69, 'type': 'CAA', 'value': { + 'flags': 0, + 'tag': 'issue', + 'value': 'ca.unit.tests' + }}), ): record = Record.new(expected, name, data) expected.add_record(record) @@ -311,6 +317,13 @@ class TestRoute53Provider(TestCase): 'Value': 'ns1.unit.tests.', }], 'TTL': 69, + }, { + 'Name': 'unit.tests.', + 'Type': 'CAA', + 'ResourceRecords': [{ + 'Value': '0 issue "ca.unit.tests"', + }], + 'TTL': 69, }], 'IsTruncated': False, 'MaxItems': '100', @@ -358,7 +371,7 @@ class TestRoute53Provider(TestCase): {'HostedZoneId': 'z42'}) plan = provider.plan(self.expected) - self.assertEquals(8, len(plan.changes)) + self.assertEquals(9, len(plan.changes)) for change in plan.changes: self.assertIsInstance(change, Create) stubber.assert_no_pending_responses() @@ -377,17 +390,17 @@ class TestRoute53Provider(TestCase): 'SubmittedAt': '2017-01-29T01:02:03Z', }}, {'HostedZoneId': 'z42', 'ChangeBatch': ANY}) - self.assertEquals(8, provider.apply(plan)) + self.assertEquals(9, provider.apply(plan)) stubber.assert_no_pending_responses() # Delete by monkey patching in a populate that includes an extra record - def add_extra_populate(existing, target): + def add_extra_populate(existing, target, lenient): for record in self.expected.records: - existing.records.add(record) + existing.add_record(record) record = Record.new(existing, 'extra', {'ttl': 99, 'type': 'A', 'values': ['9.9.9.9']}) - existing.records.add(record) + existing.add_record(record) provider.populate = add_extra_populate change_resource_record_sets_params = { @@ -417,10 +430,10 @@ class TestRoute53Provider(TestCase): # Update by monkey patching in a populate that modifies the A record # with geos - def mod_geo_populate(existing, target): + def mod_geo_populate(existing, target, lenient): for record in self.expected.records: if record._type != 'A' or not record.geo: - existing.records.add(record) + existing.add_record(record) record = Record.new(existing, '', { 'ttl': 61, 'type': 'A', @@ -431,7 +444,7 @@ class TestRoute53Provider(TestCase): 'NA-US-KY': ['7.2.3.4'] } }) - existing.records.add(record) + existing.add_record(record) provider.populate = mod_geo_populate change_resource_record_sets_params = { @@ -513,10 +526,10 @@ class TestRoute53Provider(TestCase): # Update converting to non-geo by monkey patching in a populate that # modifies the A record with geos - def mod_add_geo_populate(existing, target): + def mod_add_geo_populate(existing, target, lenient): for record in self.expected.records: if record._type != 'A' or record.geo: - existing.records.add(record) + existing.add_record(record) record = Record.new(existing, 'simple', { 'ttl': 61, 'type': 'A', @@ -525,7 +538,7 @@ class TestRoute53Provider(TestCase): 'OC': ['3.2.3.4', '4.2.3.4'], } }) - existing.records.add(record) + existing.add_record(record) provider.populate = mod_add_geo_populate change_resource_record_sets_params = { @@ -590,7 +603,7 @@ class TestRoute53Provider(TestCase): {}) plan = provider.plan(self.expected) - self.assertEquals(8, len(plan.changes)) + self.assertEquals(9, len(plan.changes)) for change in plan.changes: self.assertIsInstance(change, Create) stubber.assert_no_pending_responses() @@ -637,7 +650,7 @@ class TestRoute53Provider(TestCase): 'SubmittedAt': '2017-01-29T01:02:03Z', }}, {'HostedZoneId': 'z42', 'ChangeBatch': ANY}) - self.assertEquals(8, provider.apply(plan)) + self.assertEquals(9, provider.apply(plan)) stubber.assert_no_pending_responses() def test_health_checks_pagination(self): @@ -1269,16 +1282,16 @@ class TestRoute53Provider(TestCase): @patch('octodns.provider.route53.Route53Provider._really_apply') def test_apply_1(self, really_apply_mock): - # 17 RRs with max of 18 should only get applied in one call - provider, plan = self._get_test_plan(18) + # 18 RRs with max of 19 should only get applied in one call + provider, plan = self._get_test_plan(19) provider.apply(plan) really_apply_mock.assert_called_once() @patch('octodns.provider.route53.Route53Provider._really_apply') def test_apply_2(self, really_apply_mock): - # 17 RRs with max of 17 should only get applied in two calls - provider, plan = self._get_test_plan(17) + # 18 RRs with max of 17 should only get applied in two calls + provider, plan = self._get_test_plan(18) provider.apply(plan) self.assertEquals(2, really_apply_mock.call_count) @@ -1327,6 +1340,14 @@ class TestRoute53Provider(TestCase): 'Type': 'TXT', })) + def test_client_max_attempts(self): + provider = Route53Provider('test', 'abc', '123', + client_max_attempts=42) + # NOTE: this will break if boto ever changes the impl details... + self.assertEquals(43, provider._conn.meta.events + ._unique_id_handlers['retry-config-route53'] + ['handler']._checker.__dict__['_max_attempts']) + class TestRoute53Records(TestCase): @@ -1355,8 +1376,10 @@ class TestRoute53Records(TestCase): False) self.assertEquals(c, c) d = _Route53Record(None, Record.new(existing, '', - {'ttl': 42, 'type': 'CNAME', - 'value': 'foo.bar.'}), + {'ttl': 42, 'type': 'MX', + 'value': { + 'preference': 10, + 'exchange': 'foo.bar.'}}), False) self.assertEquals(d, d) diff --git a/tests/test_octodns_provider_yaml.py b/tests/test_octodns_provider_yaml.py index 9438f01..36cd8d6 100644 --- a/tests/test_octodns_provider_yaml.py +++ b/tests/test_octodns_provider_yaml.py @@ -30,7 +30,7 @@ class TestYamlProvider(TestCase): # without it we see everything source.populate(zone) - self.assertEquals(15, len(zone.records)) + self.assertEquals(16, len(zone.records)) # Assumption here is that a clean round-trip means that everything # worked as expected, data that went in came back out and could be diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index 9c72413..365ea0f 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -7,9 +7,10 @@ from __future__ import absolute_import, division, print_function, \ from unittest import TestCase -from octodns.record import ARecord, AaaaRecord, AliasRecord, CnameRecord, \ - Create, Delete, GeoValue, MxRecord, NaptrRecord, NaptrValue, NsRecord, \ - PtrRecord, Record, SshfpRecord, SpfRecord, SrvRecord, TxtRecord, Update +from octodns.record import ARecord, AaaaRecord, AliasRecord, CaaRecord, \ + CnameRecord, Create, Delete, GeoValue, MxRecord, NaptrRecord, \ + NaptrValue, NsRecord, Record, SshfpRecord, SpfRecord, SrvRecord, \ + TxtRecord, Update, ValidationError from octodns.zone import Zone from helpers import GeoProvider, SimpleProvider @@ -42,15 +43,6 @@ class TestRecord(TestCase): self.assertEquals([b_value], b.values) self.assertEquals(b_data, b.data) - # missing ttl - with self.assertRaises(Exception) as ctx: - ARecord(self.zone, None, {'value': '1.1.1.1'}) - self.assertTrue('missing ttl' in ctx.exception.message) - # missing values & value - with self.assertRaises(Exception) as ctx: - ARecord(self.zone, None, {'ttl': 42}) - self.assertTrue('missing value(s)' in ctx.exception.message) - # top-level data = {'ttl': 30, 'value': '4.2.3.4'} self.assertEquals(self.zone.name, ARecord(self.zone, '', data).fqdn) @@ -104,20 +96,6 @@ class TestRecord(TestCase): DummyRecord().__repr__() - def test_invalid_a(self): - with self.assertRaises(Exception) as ctx: - ARecord(self.zone, 'a', { - 'ttl': 30, - 'value': 'foo', - }) - self.assertTrue('Invalid record' in ctx.exception.message) - with self.assertRaises(Exception) as ctx: - ARecord(self.zone, 'a', { - 'ttl': 30, - 'values': ['1.2.3.4', 'bar'], - }) - self.assertTrue('Invalid record' in ctx.exception.message) - def test_geo(self): geo_data = {'ttl': 42, 'values': ['5.2.3.4', '6.2.3.4'], 'geo': {'AF': ['1.1.1.1'], @@ -157,19 +135,6 @@ class TestRecord(TestCase): # Geo provider does consider lack of geo diffs to be changes self.assertTrue(geo.changes(other, geo_target)) - # invalid geo code - with self.assertRaises(Exception) as ctx: - ARecord(self.zone, 'geo', {'ttl': 42, - 'values': ['5.2.3.4', '6.2.3.4'], - 'geo': {'abc': ['1.1.1.1']}}) - self.assertEquals('Invalid geo "abc"', ctx.exception.message) - - with self.assertRaises(Exception) as ctx: - ARecord(self.zone, 'geo', {'ttl': 42, - 'values': ['5.2.3.4', '6.2.3.4'], - 'geo': {'NA-US': ['1.1.1']}}) - self.assertTrue('not a valid ip' in ctx.exception.message) - # __repr__ doesn't blow up geo.__repr__() @@ -187,30 +152,12 @@ class TestRecord(TestCase): self.assertEquals([b_value], b.values) self.assertEquals(b_data, b.data) - # missing values & value - with self.assertRaises(Exception) as ctx: - _type(self.zone, None, {'ttl': 42}) - self.assertTrue('missing value(s)' in ctx.exception.message) - def test_aaaa(self): a_values = ['2001:0db8:3c4d:0015:0000:0000:1a2f:1a2b', '2001:0db8:3c4d:0015:0000:0000:1a2f:1a3b'] b_value = '2001:0db8:3c4d:0015:0000:0000:1a2f:1a4b' self.assertMultipleValues(AaaaRecord, a_values, b_value) - with self.assertRaises(Exception) as ctx: - AaaaRecord(self.zone, 'a', { - 'ttl': 30, - 'value': 'foo', - }) - self.assertTrue('Invalid record' in ctx.exception.message) - with self.assertRaises(Exception) as ctx: - AaaaRecord(self.zone, 'a', { - 'ttl': 30, - 'values': [b_value, 'bar'], - }) - self.assertTrue('Invalid record' in ctx.exception.message) - def assertSingleValue(self, _type, a_value, b_value): a_data = {'ttl': 30, 'value': a_value} a = _type(self.zone, 'a', a_data) @@ -225,11 +172,6 @@ class TestRecord(TestCase): self.assertEquals(b_value, b.value) self.assertEquals(b_data, b.data) - # missing value - with self.assertRaises(Exception) as ctx: - _type(self.zone, None, {'ttl': 42}) - self.assertTrue('missing value' in ctx.exception.message) - target = SimpleProvider() # No changes with self self.assertFalse(a.changes(a, target)) @@ -251,15 +193,6 @@ class TestRecord(TestCase): self.assertEquals(a_data['value'], a.value) self.assertEquals(a_data, a.data) - # missing value - with self.assertRaises(Exception) as ctx: - AliasRecord(self.zone, None, {'ttl': 0}) - self.assertTrue('missing value' in ctx.exception.message) - # bad name - with self.assertRaises(Exception) as ctx: - AliasRecord(self.zone, None, {'ttl': 0, 'value': 'www.unit.tests'}) - self.assertTrue('missing trailing .' in ctx.exception.message) - target = SimpleProvider() # No changes with self self.assertFalse(a.changes(a, target)) @@ -273,73 +206,115 @@ class TestRecord(TestCase): # __repr__ doesn't blow up a.__repr__() + def test_caa(self): + a_values = [{ + 'flags': 0, + 'tag': 'issue', + 'value': 'ca.example.net', + }, { + 'flags': 128, + 'tag': 'iodef', + 'value': 'mailto:security@example.com', + }] + a_data = {'ttl': 30, 'values': a_values} + a = CaaRecord(self.zone, 'a', a_data) + self.assertEquals('a', a.name) + self.assertEquals('a.unit.tests.', a.fqdn) + self.assertEquals(30, a.ttl) + self.assertEquals(a_values[0]['flags'], a.values[0].flags) + self.assertEquals(a_values[0]['tag'], a.values[0].tag) + self.assertEquals(a_values[0]['value'], a.values[0].value) + self.assertEquals(a_values[1]['flags'], a.values[1].flags) + self.assertEquals(a_values[1]['tag'], a.values[1].tag) + self.assertEquals(a_values[1]['value'], a.values[1].value) + self.assertEquals(a_data, a.data) + + b_value = { + 'tag': 'iodef', + 'value': 'http://iodef.example.com/', + } + b_data = {'ttl': 30, 'value': b_value} + b = CaaRecord(self.zone, 'b', b_data) + self.assertEquals(0, b.values[0].flags) + self.assertEquals(b_value['tag'], b.values[0].tag) + self.assertEquals(b_value['value'], b.values[0].value) + b_data['value']['flags'] = 0 + self.assertEquals(b_data, b.data) + + target = SimpleProvider() + # No changes with self + self.assertFalse(a.changes(a, target)) + # Diff in flags causes change + other = CaaRecord(self.zone, 'a', {'ttl': 30, 'values': a_values}) + other.values[0].flags = 128 + change = a.changes(other, target) + self.assertEqual(change.existing, a) + self.assertEqual(change.new, other) + # Diff in tag causes change + other.values[0].flags = a.values[0].flags + other.values[0].tag = 'foo' + change = a.changes(other, target) + self.assertEqual(change.existing, a) + self.assertEqual(change.new, other) + # Diff in value causes change + other.values[0].tag = a.values[0].tag + other.values[0].value = 'bar' + change = a.changes(other, target) + self.assertEqual(change.existing, a) + self.assertEqual(change.new, other) + + # __repr__ doesn't blow up + a.__repr__() + def test_cname(self): self.assertSingleValue(CnameRecord, 'target.foo.com.', 'other.foo.com.') - with self.assertRaises(Exception) as ctx: - CnameRecord(self.zone, 'a', { - 'ttl': 30, - 'value': 'foo', - }) - self.assertTrue('Invalid record' in ctx.exception.message) - with self.assertRaises(Exception) as ctx: - CnameRecord(self.zone, 'a', { - 'ttl': 30, - 'values': ['foo.com.', 'bar.com'], - }) - self.assertTrue('Invalid record' in ctx.exception.message) - def test_mx(self): a_values = [{ - 'priority': 10, - 'value': 'smtp1' + 'preference': 10, + 'exchange': 'smtp1.' }, { 'priority': 20, - 'value': 'smtp2' + 'value': 'smtp2.' }] a_data = {'ttl': 30, 'values': a_values} a = MxRecord(self.zone, 'a', a_data) self.assertEquals('a', a.name) self.assertEquals('a.unit.tests.', a.fqdn) self.assertEquals(30, a.ttl) - self.assertEquals(a_values[0]['priority'], a.values[0].priority) - self.assertEquals(a_values[0]['value'], a.values[0].value) - self.assertEquals(a_values[1]['priority'], a.values[1].priority) - self.assertEquals(a_values[1]['value'], a.values[1].value) + self.assertEquals(a_values[0]['preference'], a.values[0].preference) + self.assertEquals(a_values[0]['exchange'], a.values[0].exchange) + self.assertEquals(a_values[1]['priority'], a.values[1].preference) + self.assertEquals(a_values[1]['value'], a.values[1].exchange) + a_data['values'][1] = { + 'preference': 20, + 'exchange': 'smtp2.', + } self.assertEquals(a_data, a.data) b_value = { - 'priority': 12, - 'value': 'smtp3', + 'preference': 12, + 'exchange': 'smtp3.', } b_data = {'ttl': 30, 'value': b_value} b = MxRecord(self.zone, 'b', b_data) - self.assertEquals(b_value['priority'], b.values[0].priority) - self.assertEquals(b_value['value'], b.values[0].value) + self.assertEquals(b_value['preference'], b.values[0].preference) + self.assertEquals(b_value['exchange'], b.values[0].exchange) self.assertEquals(b_data, b.data) - # missing value - with self.assertRaises(Exception) as ctx: - MxRecord(self.zone, None, {'ttl': 42}) - self.assertTrue('missing value(s)' in ctx.exception.message) - # invalid value - with self.assertRaises(Exception) as ctx: - MxRecord(self.zone, None, {'ttl': 42, 'value': {}}) - self.assertTrue('Invalid value' in ctx.exception.message) - target = SimpleProvider() # No changes with self self.assertFalse(a.changes(a, target)) - # Diff in priority causes change + # Diff in preference causes change other = MxRecord(self.zone, 'a', {'ttl': 30, 'values': a_values}) - other.values[0].priority = 22 + other.values[0].preference = 22 change = a.changes(other, target) self.assertEqual(change.existing, a) self.assertEqual(change.new, other) # Diff in value causes change - other.values[0].priority = a.values[0].priority - other.values[0].value = 'smtpX' + other.values[0].preference = a.values[0].preference + other.values[0].exchange = 'smtpX' change = a.changes(other, target) self.assertEqual(change.existing, a) self.assertEqual(change.new, other) @@ -387,15 +362,6 @@ class TestRecord(TestCase): self.assertEquals(b_value[k], getattr(b.values[0], k)) self.assertEquals(b_data, b.data) - # missing value - with self.assertRaises(Exception) as ctx: - NaptrRecord(self.zone, None, {'ttl': 42}) - self.assertTrue('missing value' in ctx.exception.message) - # invalid value - with self.assertRaises(Exception) as ctx: - NaptrRecord(self.zone, None, {'ttl': 42, 'value': {}}) - self.assertTrue('Invalid value' in ctx.exception.message) - target = SimpleProvider() # No changes with self self.assertFalse(a.changes(a, target)) @@ -538,33 +504,6 @@ class TestRecord(TestCase): self.assertEquals([b_value], b.values) self.assertEquals(b_data, b.data) - # missing values & value - with self.assertRaises(Exception) as ctx: - NsRecord(self.zone, None, {'ttl': 42}) - self.assertTrue('missing value(s)' in ctx.exception.message) - - with self.assertRaises(Exception) as ctx: - NsRecord(self.zone, 'a', { - 'ttl': 30, - 'value': 'foo', - }) - self.assertTrue('Invalid record' in ctx.exception.message) - with self.assertRaises(Exception) as ctx: - NsRecord(self.zone, 'a', { - 'ttl': 30, - 'values': ['foo.com.', 'bar.com'], - }) - self.assertTrue('Invalid record' in ctx.exception.message) - - def test_ptr(self): - self.assertSingleValue(PtrRecord, 'foo.bar.com.', 'other.bar.com.') - with self.assertRaises(Exception) as ctx: - PtrRecord(self.zone, 'a', { - 'ttl': 30, - 'value': 'foo', - }) - self.assertTrue('Invalid record' in ctx.exception.message) - def test_sshfp(self): a_values = [{ 'algorithm': 10, @@ -599,15 +538,6 @@ class TestRecord(TestCase): self.assertEquals(b_value['fingerprint'], b.values[0].fingerprint) self.assertEquals(b_data, b.data) - # missing value - with self.assertRaises(Exception) as ctx: - SshfpRecord(self.zone, None, {'ttl': 42}) - self.assertTrue('missing value(s)' in ctx.exception.message) - # invalid value - with self.assertRaises(Exception) as ctx: - SshfpRecord(self.zone, None, {'ttl': 42, 'value': {}}) - self.assertTrue('Invalid value' in ctx.exception.message) - target = SimpleProvider() # No changes with self self.assertFalse(a.changes(a, target)) @@ -677,21 +607,6 @@ class TestRecord(TestCase): self.assertEquals(b_value['target'], b.values[0].target) self.assertEquals(b_data, b.data) - # invalid name - with self.assertRaises(Exception) as ctx: - SrvRecord(self.zone, 'bad', {'ttl': 42}) - self.assertEquals('Invalid name bad.unit.tests.', - ctx.exception.message) - - # missing value - with self.assertRaises(Exception) as ctx: - SrvRecord(self.zone, '_missing._tcp', {'ttl': 42}) - self.assertTrue('missing value(s)' in ctx.exception.message) - # invalid value - with self.assertRaises(Exception) as ctx: - SrvRecord(self.zone, '_missing._udp', {'ttl': 42, 'value': {}}) - self.assertTrue('Invalid value' in ctx.exception.message) - target = SimpleProvider() # No changes with self self.assertFalse(a.changes(a, target)) @@ -729,21 +644,6 @@ class TestRecord(TestCase): b_value = 'b other' self.assertMultipleValues(TxtRecord, a_values, b_value) - Record.new(self.zone, 'txt', { - 'ttl': 44, - 'type': 'TXT', - 'value': 'escaped\; foo', - }) - - with self.assertRaises(Exception) as ctx: - Record.new(self.zone, 'txt', { - 'ttl': 44, - 'type': 'TXT', - 'value': 'un-escaped; foo', - }) - self.assertEquals('Invalid record txt.unit.tests., unescaped ;', - ctx.exception.message) - def test_record_new(self): txt = Record.new(self.zone, 'txt', { 'ttl': 44, @@ -795,6 +695,28 @@ class TestRecord(TestCase): self.assertEquals(values, geo.values) self.assertEquals(['NA-US', 'NA'], list(geo.parents)) + def test_healthcheck(self): + new = Record.new(self.zone, 'a', { + 'ttl': 44, + 'type': 'A', + 'value': '1.2.3.4', + 'octodns': { + 'healthcheck': { + 'path': '/_ready', + 'host': 'bleep.bloop', + } + } + }) + self.assertEquals('/_ready', new.healthcheck_path) + self.assertEquals('bleep.bloop', new.healthcheck_host) + new = Record.new(self.zone, 'a', { + 'ttl': 44, + 'type': 'A', + 'value': '1.2.3.4', + }) + self.assertEquals('/_dns', new.healthcheck_path) + self.assertEquals('a.unit.tests', new.healthcheck_host) + def test_inored(self): new = Record.new(self.zone, 'txt', { 'ttl': 44, @@ -821,24 +743,858 @@ class TestRecord(TestCase): }) self.assertFalse(new.ignored) - def test_healthcheck(self): - new = Record.new(self.zone, 'a', { - 'ttl': 44, + +class TestRecordValidation(TestCase): + zone = Zone('unit.tests.', []) + + def test_base(self): + # no ttl + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'A', + 'value': '1.2.3.4', + }) + self.assertEquals(['missing ttl'], ctx.exception.reasons) + + # invalid ttl + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, 'www', { + 'type': 'A', + 'ttl': -1, + 'value': '1.2.3.4', + }) + self.assertEquals('www.unit.tests.', ctx.exception.fqdn) + self.assertEquals(['invalid ttl'], ctx.exception.reasons) + + # no exception if we're in lenient mode + Record.new(self.zone, 'www', { 'type': 'A', + 'ttl': -1, + 'value': '1.2.3.4', + }, lenient=True) + + # __init__ may still blow up, even if validation is lenient + with self.assertRaises(KeyError) as ctx: + Record.new(self.zone, 'www', { + 'type': 'A', + 'ttl': -1, + }, lenient=True) + self.assertEquals(('value',), ctx.exception.args) + + def test_A_and_values_mixin(self): + # doesn't blow up + Record.new(self.zone, '', { + 'type': 'A', + 'ttl': 600, 'value': '1.2.3.4', - 'octodns': { - 'healthcheck': { - 'path': '/_ready', - 'host': 'bleep.bloop', - } - } }) - self.assertEquals('/_ready', new.healthcheck_path) - self.assertEquals('bleep.bloop', new.healthcheck_host) - new = Record.new(self.zone, 'a', { - 'ttl': 44, + Record.new(self.zone, '', { 'type': 'A', + 'ttl': 600, + 'values': [ + '1.2.3.4', + '1.2.3.5', + ] + }) + + # missing value(s) + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'A', + 'ttl': 600, + }) + self.assertEquals(['missing value(s)'], ctx.exception.reasons) + # missing value(s) & ttl + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'A', + }) + self.assertEquals(['missing ttl', 'missing value(s)'], + ctx.exception.reasons) + + # invalid ip address + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'A', + 'ttl': 600, + 'value': 'hello' + }) + self.assertEquals(['invalid ip address "hello"'], + ctx.exception.reasons) + + # invalid ip addresses + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'A', + 'ttl': 600, + 'values': ['hello', 'goodbye'] + }) + self.assertEquals([ + 'invalid ip address "hello"', + 'invalid ip address "goodbye"' + ], ctx.exception.reasons) + + # invalid & valid ip addresses, no ttl + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'A', + 'values': ['1.2.3.4', 'hello', '5.6.7.8'] + }) + self.assertEquals([ + 'missing ttl', + 'invalid ip address "hello"', + ], ctx.exception.reasons) + + def test_geo(self): + Record.new(self.zone, '', { + 'geo': { + 'NA': ['1.2.3.5'], + 'NA-US': ['1.2.3.5', '1.2.3.6'] + }, + 'type': 'A', + 'ttl': 600, 'value': '1.2.3.4', }) - self.assertEquals('/_dns', new.healthcheck_path) - self.assertEquals('a.unit.tests', new.healthcheck_host) + + # invalid ip address + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'geo': { + 'NA': ['hello'], + 'NA-US': ['1.2.3.5', '1.2.3.6'] + }, + 'type': 'A', + 'ttl': 600, + 'value': '1.2.3.4', + }) + self.assertEquals(['invalid ip address "hello"'], + ctx.exception.reasons) + + # invalid geo code + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'geo': { + 'XYZ': ['1.2.3.4'], + }, + 'type': 'A', + 'ttl': 600, + 'value': '1.2.3.4', + }) + self.assertEquals(['invalid geo "XYZ"'], ctx.exception.reasons) + + # invalid ip address + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'geo': { + 'NA': ['hello'], + 'NA-US': ['1.2.3.5', 'goodbye'] + }, + 'type': 'A', + 'ttl': 600, + 'value': '1.2.3.4', + }) + self.assertEquals([ + 'invalid ip address "hello"', + 'invalid ip address "goodbye"' + ], ctx.exception.reasons) + + def test_AAAA(self): + # doesn't blow up + Record.new(self.zone, '', { + 'type': 'AAAA', + 'ttl': 600, + 'value': '2601:644:500:e210:62f8:1dff:feb8:947a', + }) + Record.new(self.zone, '', { + 'type': 'AAAA', + 'ttl': 600, + 'values': [ + '2601:644:500:e210:62f8:1dff:feb8:947a', + '2601:644:500:e210:62f8:1dff:feb8:947b', + ] + }) + + # invalid ip address + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'AAAA', + 'ttl': 600, + 'value': 'hello' + }) + self.assertEquals(['invalid ip address "hello"'], + ctx.exception.reasons) + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'AAAA', + 'ttl': 600, + 'value': '1.2.3.4' + }) + self.assertEquals(['invalid ip address "1.2.3.4"'], + ctx.exception.reasons) + + # invalid ip addresses + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'AAAA', + 'ttl': 600, + 'values': ['hello', 'goodbye'] + }) + self.assertEquals([ + 'invalid ip address "hello"', + 'invalid ip address "goodbye"' + ], ctx.exception.reasons) + + def test_ALIAS_and_value_mixin(self): + # doesn't blow up + Record.new(self.zone, '', { + 'type': 'ALIAS', + 'ttl': 600, + 'value': 'foo.bar.com.', + }) + + # missing value + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'ALIAS', + 'ttl': 600, + }) + self.assertEquals(['missing value'], ctx.exception.reasons) + + # missing trailing . + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'ALIAS', + 'ttl': 600, + 'value': 'foo.bar.com', + }) + self.assertEquals(['missing trailing .'], ctx.exception.reasons) + + def test_CAA(self): + # doesn't blow up + Record.new(self.zone, '', { + 'type': 'CAA', + 'ttl': 600, + 'value': { + 'flags': 128, + 'tag': 'iodef', + 'value': 'http://foo.bar.com/' + } + }) + + # invalid flags + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'CAA', + 'ttl': 600, + 'value': { + 'flags': -42, + 'tag': 'iodef', + 'value': 'http://foo.bar.com/', + } + }) + self.assertEquals(['invalid flags "-42"'], ctx.exception.reasons) + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'CAA', + 'ttl': 600, + 'value': { + 'flags': 442, + 'tag': 'iodef', + 'value': 'http://foo.bar.com/', + } + }) + self.assertEquals(['invalid flags "442"'], ctx.exception.reasons) + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'CAA', + 'ttl': 600, + 'value': { + 'flags': 'nope', + 'tag': 'iodef', + 'value': 'http://foo.bar.com/', + } + }) + self.assertEquals(['invalid flags "nope"'], ctx.exception.reasons) + + # missing tag + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'CAA', + 'ttl': 600, + 'value': { + 'value': 'http://foo.bar.com/', + } + }) + self.assertEquals(['missing tag'], ctx.exception.reasons) + + # missing value + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'CAA', + 'ttl': 600, + 'value': { + 'tag': 'iodef', + } + }) + self.assertEquals(['missing value'], ctx.exception.reasons) + + def test_CNAME(self): + # doesn't blow up + Record.new(self.zone, 'www', { + 'type': 'CNAME', + 'ttl': 600, + 'value': 'foo.bar.com.', + }) + + # root cname is a no-no + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'CNAME', + 'ttl': 600, + 'value': 'foo.bar.com.', + }) + self.assertEquals(['root CNAME not allowed'], ctx.exception.reasons) + + # missing trailing . + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, 'www', { + 'type': 'CNAME', + 'ttl': 600, + 'value': 'foo.bar.com', + }) + self.assertEquals(['missing trailing .'], ctx.exception.reasons) + + def test_MX(self): + # doesn't blow up + Record.new(self.zone, '', { + 'type': 'MX', + 'ttl': 600, + 'value': { + 'preference': 10, + 'exchange': 'foo.bar.com.' + } + }) + + # missing preference + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'MX', + 'ttl': 600, + 'value': { + 'exchange': 'foo.bar.com.' + } + }) + self.assertEquals(['missing preference'], ctx.exception.reasons) + + # invalid preference + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'MX', + 'ttl': 600, + 'value': { + 'preference': 'nope', + 'exchange': 'foo.bar.com.' + } + }) + self.assertEquals(['invalid preference "nope"'], ctx.exception.reasons) + + # missing exchange + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'MX', + 'ttl': 600, + 'value': { + 'preference': 10, + } + }) + self.assertEquals(['missing exchange'], ctx.exception.reasons) + + # missing trailing . + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'MX', + 'ttl': 600, + 'value': { + 'preference': 10, + 'exchange': 'foo.bar.com' + } + }) + self.assertEquals(['missing trailing .'], ctx.exception.reasons) + + def test_NXPTR(self): + # doesn't blow up + Record.new(self.zone, '', { + 'type': 'NAPTR', + 'ttl': 600, + 'value': { + 'order': 10, + 'preference': 20, + 'flags': 'S', + 'service': 'srv', + 'regexp': '.*', + 'replacement': '.' + } + }) + + # missing X priority + value = { + 'order': 10, + 'preference': 20, + 'flags': 'S', + 'service': 'srv', + 'regexp': '.*', + 'replacement': '.' + } + for k in ('order', 'preference', 'flags', 'service', 'regexp', + 'replacement'): + v = dict(value) + del v[k] + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'NAPTR', + 'ttl': 600, + 'value': v + }) + self.assertEquals(['missing {}'.format(k)], ctx.exception.reasons) + + # non-int order + v = dict(value) + v['order'] = 'boo' + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'NAPTR', + 'ttl': 600, + 'value': v + }) + self.assertEquals(['invalid order "boo"'], ctx.exception.reasons) + + # non-int preference + v = dict(value) + v['preference'] = 'who' + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'NAPTR', + 'ttl': 600, + 'value': v + }) + self.assertEquals(['invalid preference "who"'], ctx.exception.reasons) + + # unrecognized flags + v = dict(value) + v['flags'] = 'X' + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'NAPTR', + 'ttl': 600, + 'value': v + }) + self.assertEquals(['unrecognized flags "X"'], ctx.exception.reasons) + + def test_NS(self): + # doesn't blow up + Record.new(self.zone, '', { + 'type': 'NS', + 'ttl': 600, + 'values': [ + 'foo.bar.com.', + '1.2.3.4.' + ] + }) + + # missing value + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'NS', + 'ttl': 600, + }) + self.assertEquals(['missing value(s)'], ctx.exception.reasons) + + # no trailing . + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'NS', + 'ttl': 600, + 'value': 'foo.bar', + }) + self.assertEquals(['missing trailing .'], ctx.exception.reasons) + + def test_PTR(self): + # doesn't blow up (name & zone here don't make any sense, but not + # important) + Record.new(self.zone, '', { + 'type': 'PTR', + 'ttl': 600, + 'value': 'foo.bar.com.', + }) + + # missing value + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'PTR', + 'ttl': 600, + }) + self.assertEquals(['missing value'], ctx.exception.reasons) + + # no trailing . + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'PTR', + 'ttl': 600, + 'value': 'foo.bar', + }) + self.assertEquals(['missing trailing .'], ctx.exception.reasons) + + def test_SSHFP(self): + # doesn't blow up + Record.new(self.zone, '', { + 'type': 'SSHFP', + 'ttl': 600, + 'value': { + 'algorithm': 1, + 'fingerprint_type': 1, + 'fingerprint': 'bf6b6825d2977c511a475bbefb88aad54a92ac73' + } + }) + + # missing algorithm + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'SSHFP', + 'ttl': 600, + 'value': { + 'fingerprint_type': 1, + 'fingerprint': 'bf6b6825d2977c511a475bbefb88aad54a92ac73' + } + }) + self.assertEquals(['missing algorithm'], ctx.exception.reasons) + + # invalid algorithm + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'SSHFP', + 'ttl': 600, + 'value': { + 'algorithm': 'nope', + 'fingerprint_type': 1, + 'fingerprint': 'bf6b6825d2977c511a475bbefb88aad54a92ac73' + } + }) + self.assertEquals(['invalid algorithm "nope"'], ctx.exception.reasons) + + # unrecognized algorithm + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'SSHFP', + 'ttl': 600, + 'value': { + 'algorithm': 42, + 'fingerprint_type': 1, + 'fingerprint': 'bf6b6825d2977c511a475bbefb88aad54a92ac73' + } + }) + self.assertEquals(['unrecognized algorithm "42"'], + ctx.exception.reasons) + + # missing fingerprint_type + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'SSHFP', + 'ttl': 600, + 'value': { + 'algorithm': 1, + 'fingerprint': 'bf6b6825d2977c511a475bbefb88aad54a92ac73' + } + }) + self.assertEquals(['missing fingerprint_type'], ctx.exception.reasons) + + # invalid fingerprint_type + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'SSHFP', + 'ttl': 600, + 'value': { + 'algorithm': 1, + 'fingerprint_type': 'yeeah', + 'fingerprint': 'bf6b6825d2977c511a475bbefb88aad54a92ac73' + } + }) + self.assertEquals(['invalid fingerprint_type "yeeah"'], + ctx.exception.reasons) + + # unrecognized fingerprint_type + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'SSHFP', + 'ttl': 600, + 'value': { + 'algorithm': 1, + 'fingerprint_type': 42, + 'fingerprint': 'bf6b6825d2977c511a475bbefb88aad54a92ac73' + } + }) + self.assertEquals(['unrecognized fingerprint_type "42"'], + ctx.exception.reasons) + + # missing fingerprint + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'SSHFP', + 'ttl': 600, + 'value': { + 'algorithm': 1, + 'fingerprint_type': 1, + } + }) + self.assertEquals(['missing fingerprint'], ctx.exception.reasons) + + def test_SPF(self): + # doesn't blow up (name & zone here don't make any sense, but not + # important) + Record.new(self.zone, '', { + 'type': 'SPF', + 'ttl': 600, + 'values': [ + 'v=spf1 ip4:192.168.0.1/16-all', + 'v=spf1 ip4:10.1.2.1/24-all', + 'this has some\; semi-colons\; in it', + ] + }) + + # missing value + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'SPF', + 'ttl': 600, + }) + self.assertEquals(['missing value(s)'], ctx.exception.reasons) + + # missing escapes + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'SPF', + 'ttl': 600, + 'value': 'this has some; semi-colons\; in it', + }) + self.assertEquals(['unescaped ;'], ctx.exception.reasons) + + def test_SRV(self): + # doesn't blow up + Record.new(self.zone, '_srv._tcp', { + 'type': 'SRV', + 'ttl': 600, + 'value': { + 'priority': 1, + 'weight': 2, + 'port': 3, + 'target': 'foo.bar.baz.' + } + }) + + # invalid name + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, 'neup', { + 'type': 'SRV', + 'ttl': 600, + 'value': { + 'priority': 1, + 'weight': 2, + 'port': 3, + 'target': 'foo.bar.baz.' + } + }) + self.assertEquals(['invalid name'], ctx.exception.reasons) + + # missing priority + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '_srv._tcp', { + 'type': 'SRV', + 'ttl': 600, + 'value': { + 'weight': 2, + 'port': 3, + 'target': 'foo.bar.baz.' + } + }) + self.assertEquals(['missing priority'], ctx.exception.reasons) + + # invalid priority + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '_srv._tcp', { + 'type': 'SRV', + 'ttl': 600, + 'value': { + 'priority': 'foo', + 'weight': 2, + 'port': 3, + 'target': 'foo.bar.baz.' + } + }) + self.assertEquals(['invalid priority "foo"'], ctx.exception.reasons) + + # missing weight + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '_srv._tcp', { + 'type': 'SRV', + 'ttl': 600, + 'value': { + 'priority': 1, + 'port': 3, + 'target': 'foo.bar.baz.' + } + }) + self.assertEquals(['missing weight'], ctx.exception.reasons) + # invalid weight + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '_srv._tcp', { + 'type': 'SRV', + 'ttl': 600, + 'value': { + 'priority': 1, + 'weight': 'foo', + 'port': 3, + 'target': 'foo.bar.baz.' + } + }) + self.assertEquals(['invalid weight "foo"'], ctx.exception.reasons) + + # missing port + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '_srv._tcp', { + 'type': 'SRV', + 'ttl': 600, + 'value': { + 'priority': 1, + 'weight': 2, + 'target': 'foo.bar.baz.' + } + }) + self.assertEquals(['missing port'], ctx.exception.reasons) + # invalid port + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '_srv._tcp', { + 'type': 'SRV', + 'ttl': 600, + 'value': { + 'priority': 1, + 'weight': 2, + 'port': 'foo', + 'target': 'foo.bar.baz.' + } + }) + self.assertEquals(['invalid port "foo"'], ctx.exception.reasons) + + # missing target + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '_srv._tcp', { + 'type': 'SRV', + 'ttl': 600, + 'value': { + 'priority': 1, + 'weight': 2, + 'port': 3, + } + }) + self.assertEquals(['missing target'], ctx.exception.reasons) + # invalid target + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '_srv._tcp', { + 'type': 'SRV', + 'ttl': 600, + 'value': { + 'priority': 1, + 'weight': 2, + 'port': 3, + 'target': 'foo.bar.baz' + } + }) + self.assertEquals(['missing trailing .'], + ctx.exception.reasons) + + def test_TXT(self): + # doesn't blow up (name & zone here don't make any sense, but not + # important) + Record.new(self.zone, '', { + 'type': 'TXT', + 'ttl': 600, + 'values': [ + 'hello world', + 'this has some\; semi-colons\; in it', + ] + }) + + # missing value + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'TXT', + 'ttl': 600, + }) + self.assertEquals(['missing value(s)'], ctx.exception.reasons) + + # missing escapes + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'TXT', + 'ttl': 600, + 'value': 'this has some; semi-colons\; in it', + }) + self.assertEquals(['unescaped ;'], ctx.exception.reasons) + + def test_TXT_long_value_chunking(self): + expected = '"Lorem ipsum dolor sit amet, consectetur adipiscing ' \ + 'elit, sed do eiusmod tempor incididunt ut labore et dolore ' \ + 'magna aliqua. Ut enim ad minim veniam, quis nostrud ' \ + 'exercitation ullamco laboris nisi ut aliquip ex ea commodo ' \ + 'consequat. Duis aute irure dolor i" "n reprehenderit in ' \ + 'voluptate velit esse cillum dolore eu fugiat nulla pariatur. ' \ + 'Excepteur sint occaecat cupidatat non proident, sunt in culpa ' \ + 'qui officia deserunt mollit anim id est laborum."' + + long_value = 'Lorem ipsum dolor sit amet, consectetur adipiscing ' \ + 'elit, sed do eiusmod tempor incididunt ut labore et dolore ' \ + 'magna aliqua. Ut enim ad minim veniam, quis nostrud ' \ + 'exercitation ullamco laboris nisi ut aliquip ex ea commodo ' \ + 'consequat. Duis aute irure dolor in reprehenderit in ' \ + 'voluptate velit esse cillum dolore eu fugiat nulla ' \ + 'pariatur. Excepteur sint occaecat cupidatat non proident, ' \ + 'sunt in culpa qui officia deserunt mollit anim id est ' \ + 'laborum.' + # Single string + single = Record.new(self.zone, '', { + 'type': 'TXT', + 'ttl': 600, + 'values': [ + 'hello world', + long_value, + 'this has some\; semi-colons\; in it', + ] + }) + self.assertEquals(3, len(single.values)) + self.assertEquals(3, len(single.chunked_values)) + # Note we are checking that this normalizes the chunking, not that we + # get out what we put in. + self.assertEquals(expected, single.chunked_values[0]) + + long_split_value = '"Lorem ipsum dolor sit amet, consectetur ' \ + 'adipiscing elit, sed do eiusmod tempor incididunt ut ' \ + 'labore et dolore magna aliqua. Ut enim ad minim veniam, ' \ + 'quis nostrud exercitation ullamco laboris nisi ut aliquip ' \ + 'ex" " ea commodo consequat. Duis aute irure dolor in ' \ + 'reprehenderit in voluptate velit esse cillum dolore eu ' \ + 'fugiat nulla pariatur. Excepteur sint occaecat cupidatat ' \ + 'non proident, sunt in culpa qui officia deserunt mollit ' \ + 'anim id est laborum."' + # Chunked + chunked = Record.new(self.zone, '', { + 'type': 'TXT', + 'ttl': 600, + 'values': [ + '"hello world"', + long_split_value, + '"this has some\; semi-colons\; in it"', + ] + }) + self.assertEquals(expected, chunked.chunked_values[0]) + # should be single values, no quoting + self.assertEquals(single.values, chunked.values) + # should be chunked values, with quoting + self.assertEquals(single.chunked_values, chunked.chunked_values) diff --git a/tests/test_octodns_source_tinydns.py b/tests/test_octodns_source_tinydns.py index b4cea06..5792b25 100644 --- a/tests/test_octodns_source_tinydns.py +++ b/tests/test_octodns_source_tinydns.py @@ -68,22 +68,22 @@ class TestTinyDnsFileSource(TestCase): 'type': 'MX', 'ttl': 3600, 'values': [{ - 'priority': 10, - 'value': 'smtp-1-host.example.com.', + 'preference': 10, + 'exchange': 'smtp-1-host.example.com.', }, { - 'priority': 20, - 'value': 'smtp-2-host.example.com.', + 'preference': 20, + 'exchange': 'smtp-2-host.example.com.', }] }), ('smtp', { 'type': 'MX', 'ttl': 1800, 'values': [{ - 'priority': 30, - 'value': 'smtp-1-host.example.com.', + 'preference': 30, + 'exchange': 'smtp-1-host.example.com.', }, { - 'priority': 40, - 'value': 'smtp-2-host.example.com.', + 'preference': 40, + 'exchange': 'smtp-2-host.example.com.', }] }), ): diff --git a/tests/test_octodns_yaml.py b/tests/test_octodns_yaml.py index 0f454b3..effe231 100644 --- a/tests/test_octodns_yaml.py +++ b/tests/test_octodns_yaml.py @@ -48,8 +48,8 @@ class TestYaml(TestCase): '*.11.2': 'd' '*.10.1': 'c' ''') - self.assertEquals('keys out of order: *.2.2, *.1.2, *.11.2, *.10.1', - ctx.exception.problem) + self.assertTrue('keys out of order: expected *.1.2 got *.2.2 at' in + ctx.exception.problem) buf = StringIO() safe_dump({ diff --git a/tests/test_octodns_zone.py b/tests/test_octodns_zone.py index 88bbb68..8d75100 100644 --- a/tests/test_octodns_zone.py +++ b/tests/test_octodns_zone.py @@ -8,7 +8,8 @@ from __future__ import absolute_import, division, print_function, \ from unittest import TestCase from octodns.record import ARecord, AaaaRecord, Create, Delete, Record, Update -from octodns.zone import DuplicateRecordException, SubzoneRecordException, Zone +from octodns.zone import DuplicateRecordException, InvalidNodeException, \ + SubzoneRecordException, Zone from helpers import SimpleProvider @@ -38,6 +39,7 @@ class TestZone(TestCase): a = ARecord(zone, 'a', {'ttl': 42, 'value': '1.1.1.1'}) b = ARecord(zone, 'b', {'ttl': 42, 'value': '1.1.1.1'}) + c = ARecord(zone, 'a', {'ttl': 43, 'value': '2.2.2.2'}) zone.add_record(a) self.assertEquals(zone.records, set([a])) @@ -47,6 +49,11 @@ class TestZone(TestCase): self.assertEquals('Duplicate record a.unit.tests., type A', ctx.exception.message) self.assertEquals(zone.records, set([a])) + + # can add duplicate with replace=True + zone.add_record(c, replace=True) + self.assertEquals('2.2.2.2', list(zone.records)[0].values[0]) + # Can add dup name, with different type zone.add_record(b) self.assertEquals(zone.records, set([a, b])) @@ -70,7 +77,7 @@ class TestZone(TestCase): # add a record, delete a record -> [Delete, Create] c = ARecord(before, 'c', {'ttl': 42, 'value': '1.1.1.1'}) after.add_record(c) - after.records.remove(b) + after._remove_record(b) self.assertEquals(after.records, set([a, c])) changes = before.changes(after, target) self.assertEquals(2, len(changes)) @@ -205,3 +212,27 @@ class TestZone(TestCase): self.assertTrue(zone_missing.changes(zone_normal, provider)) self.assertFalse(zone_missing.changes(zone_ignored, provider)) + + def test_cname_coexisting(self): + zone = Zone('unit.tests.', []) + a = Record.new(zone, 'www', { + 'ttl': 60, + 'type': 'A', + 'value': '9.9.9.9', + }) + cname = Record.new(zone, 'www', { + 'ttl': 60, + 'type': 'CNAME', + 'value': 'foo.bar.com.', + }) + + # add cname to a + zone.add_record(a) + with self.assertRaises(InvalidNodeException): + zone.add_record(cname) + + # add a to cname + zone = Zone('unit.tests.', []) + zone.add_record(cname) + with self.assertRaises(InvalidNodeException): + zone.add_record(a)