diff --git a/.dependabot/config.yml b/.dependabot/config.yml new file mode 100644 index 0000000..165af5d --- /dev/null +++ b/.dependabot/config.yml @@ -0,0 +1,6 @@ +version: 1 + +update_configs: + - package_manager: "python" + directory: "/" + update_schedule: "weekly" diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..726e2e8 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,24 @@ +name: OctoDNS +on: [pull_request] + +jobs: + ci: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [2.7, 3.7] + steps: + - uses: actions/checkout@master + - name: Setup python + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + architecture: x64 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install virtualenv + - name: CI Build + run: | + ./script/cibuild diff --git a/.gitignore b/.gitignore index 1efa084..715b687 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,7 @@ +# +# Do not add editor or OS specific ignores here. Have a look at adding +# `excludesfile` to your `~/.gitconfig` to globally ignore such things. +# *.pyc .coverage .env diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index b17ca01..0000000 --- a/.travis.yml +++ /dev/null @@ -1,7 +0,0 @@ -language: python -python: - - 2.7 -script: ./script/cibuild -notifications: - email: - - ross@github.com diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f283ec..491370f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,50 @@ +## v0.9.10 - ????-??-?? - ??? + +* Added support for dynamic records to Ns1Provider, updated client and rate + limiting implementation +* Moved CI to use GitHub Actions +* Set up dependabot to automatically PR requirements updates +* Pass at bumping all of the requirements + +## v0.9.9 - 2019-11-04 - Python 3.7 Support + +* Extensive pass through the whole codebase to support Python 3 + * Tons of updates to replace `def __cmp__` with `__eq__` and friends to + preserve custom equality and ordering behaviors that are essential to + octoDNS's processes. + * Quite a few objects required the addition of `__eq__` and friends so that + they're sortable in Python 3 now that those things are more strict. A few + places this required jumping through hoops of sorts. Thankfully our tests + are pretty thorough and caught a lot of issues and hopefully the whole + plan, review, apply process will backstop that. + * Explicit ordering of changes by (name, type) to address inconsistent + ordering for a number of providers that just convert changes into API + calls as they come. Python 2 sets ordered consistently, Python 3 they do + not. https://github.com/github/octodns/pull/384/commits/7958233fccf9ea22d95e2fd06c48d7d0a4529e26 + * Route53 `_mod_keyer` ordering wasn't 100% complete and thus unreliable and + random in Python 3. This has been addressed and may result in value + reordering on next plan, no actual changes in behavior should occur. + * `incf.countryutils` (in pypi) was last released in 2009 is not python 3 + compatible (it's country data is also pretty stale.) `pycountry_convert` + appears to have the functionality required to replace its usage so it has + been removed as a dependency/requirement. + * Bunch of additional unit tests and supporting config to exercise new code + and verify things that were run into during the Python 3 work + * lots of `six`ing of things +* Validate Record name & fqdn length + +## v0.9.8 - 2019-09-30 - One with no changes b/c PyPi description problems + +* No material changes + +## v0.9.7 - 2019-09-30 - It's about time + +* AkamaiProvider, ConstellixProvider, MythicBeastsProvider, SelectelProvider, + & TransipPovider providers added +* Route53Provider seperator fix +* YamlProvider export error around stringification +* PyPi markdown rendering fix + ## v0.9.6 - 2019-07-16 - The little one that fixes stuff from the big one * Reduced dynamic record value weight range to 0-15 so that Dyn and Route53 diff --git a/README.md b/README.md index 75a006c..6127234 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,9 @@ We start by creating a config file to tell OctoDNS about our providers and the z ```yaml --- +manager: + max_workers: 2 + providers: config: class: octodns.provider.yaml.YamlProvider @@ -80,6 +83,8 @@ zones: Further information can be found in the `docstring` of each source and provider class. +The `max_workers` key in the `manager` section of the config enables threading to parallelize the planning portion of the sync. + Now that we have something to tell OctoDNS about our providers & zones we need to tell it about or records. We'll keep it simple for now and just create a single `A` record at the top-level of the domain. `config/example.com.yaml` @@ -90,8 +95,8 @@ Now that we have something to tell OctoDNS about our providers & zones we need t ttl: 60 type: A values: - - 1.2.3.4 - - 1.2.3.5 + - 1.2.3.4 + - 1.2.3.5 ``` Further information can be found in [Records Documentation](/docs/records.md). @@ -169,9 +174,10 @@ The above command pulled the existing data out of Route53 and placed the results ## Supported providers -| Provider | Requirements | Record Support | Dynamic/Geo Support | Notes | +| Provider | Requirements | Record Support | Dynamic | Notes | |--|--|--|--|--| | [AzureProvider](/octodns/provider/azuredns.py) | azure-mgmt-dns | A, AAAA, CAA, CNAME, MX, NS, PTR, SRV, TXT | No | | +| [Akamai](/octodns/provider/fastdns.py) | edgegrid-python | A, AAAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, SSHFP, TXT | No | | | [CloudflareProvider](/octodns/provider/cloudflare.py) | | A, AAAA, ALIAS, CAA, CNAME, MX, NS, SPF, SRV, TXT | No | CAA tags restricted | | [ConstellixProvider](/octodns/provider/constellix.py) | | A, AAAA, ALIAS (ANAME), CAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | CAA tags restricted | | [DigitalOceanProvider](/octodns/provider/digitalocean.py) | | A, AAAA, CAA, CNAME, MX, NS, TXT, SRV | No | CAA tags restricted | @@ -181,12 +187,13 @@ The above command pulled the existing data out of Route53 and placed the results | [EtcHostsProvider](/octodns/provider/etc_hosts.py) | | A, AAAA, ALIAS, CNAME | No | | | [GoogleCloudProvider](/octodns/provider/googlecloud.py) | google-cloud-dns | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | No | | | [MythicBeastsProvider](/octodns/provider/mythicbeasts.py) | Mythic Beasts | A, AAAA, ALIAS, CNAME, MX, NS, SRV, SSHFP, CAA, TXT | No | | -| [Ns1Provider](/octodns/provider/ns1.py) | nsone | All | Partial Geo | No health checking for GeoDNS | -| [OVH](/octodns/provider/ovh.py) | ovh | A, AAAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, SSHFP, TXT, DKIM | No | | +| [Ns1Provider](/octodns/provider/ns1.py) | ns1-python | All | Yes | No CNAME support, missing `NA` geo target | +| [OVH](/octodns/provider/ovh.py) | ovh | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, SSHFP, TXT, DKIM | No | | | [PowerDnsProvider](/octodns/provider/powerdns.py) | | All | No | | | [Rackspace](/octodns/provider/rackspace.py) | | A, AAAA, ALIAS, CNAME, MX, NS, PTR, SPF, TXT | No | | | [Route53](/octodns/provider/route53.py) | boto3 | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | Both | CNAME health checks don't support a Host header | | [Selectel](/octodns/provider/selectel.py) | | A, AAAA, CNAME, MX, NS, SPF, SRV, TXT | No | | +| [Transip](/octodns/provider/transip.py) | transip | A, AAAA, CNAME, MX, SRV, SPF, TXT, SSHFP, CAA | No | | | [AxfrSource](/octodns/source/axfr.py) | | A, AAAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | read-only | | [ZoneFileSource](/octodns/source/axfr.py) | | A, AAAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | read-only | | [TinyDnsFileSource](/octodns/source/tinydns.py) | | A, CNAME, MX, NS, PTR | No | read-only | diff --git a/octodns/__init__.py b/octodns/__init__.py index 6422577..404d688 100644 --- a/octodns/__init__.py +++ b/octodns/__init__.py @@ -3,4 +3,4 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -__VERSION__ = '0.9.6' +__VERSION__ = '0.9.9' diff --git a/octodns/cmds/report.py b/octodns/cmds/report.py index 2b32e77..3a26052 100755 --- a/octodns/cmds/report.py +++ b/octodns/cmds/report.py @@ -13,6 +13,8 @@ from logging import getLogger from sys import stdout import re +from six import text_type + from octodns.cmds.args import ArgumentParser from octodns.manager import Manager from octodns.zone import Zone @@ -65,7 +67,7 @@ def main(): resolver = AsyncResolver(configure=False, num_workers=int(args.num_workers)) if not ip_addr_re.match(server): - server = unicode(query(server, 'A')[0]) + server = text_type(query(server, 'A')[0]) log.info('server=%s', server) resolver.nameservers = [server] resolver.lifetime = int(args.timeout) @@ -81,12 +83,12 @@ def main(): stdout.write(',') stdout.write(record._type) stdout.write(',') - stdout.write(unicode(record.ttl)) + stdout.write(text_type(record.ttl)) compare = {} for future in futures: stdout.write(',') try: - answers = [unicode(r) for r in future.result()] + answers = [text_type(r) for r in future.result()] except (NoAnswer, NoNameservers): answers = ['*no answer*'] except NXDOMAIN: diff --git a/octodns/equality.py b/octodns/equality.py new file mode 100644 index 0000000..bd22c7d --- /dev/null +++ b/octodns/equality.py @@ -0,0 +1,30 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + + +class EqualityTupleMixin(object): + + def _equality_tuple(self): + raise NotImplementedError('_equality_tuple method not implemented') + + def __eq__(self, other): + return self._equality_tuple() == other._equality_tuple() + + def __ne__(self, other): + return self._equality_tuple() != other._equality_tuple() + + def __lt__(self, other): + return self._equality_tuple() < other._equality_tuple() + + def __le__(self, other): + return self._equality_tuple() <= other._equality_tuple() + + def __gt__(self, other): + return self._equality_tuple() > other._equality_tuple() + + def __ge__(self, other): + return self._equality_tuple() >= other._equality_tuple() diff --git a/octodns/manager.py b/octodns/manager.py index 4952315..f19885f 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -69,6 +69,10 @@ class MainThreadExecutor(object): return MakeThreadFuture(func, args, kwargs) +class ManagerException(Exception): + pass + + class Manager(object): log = logging.getLogger('Manager') @@ -95,7 +99,7 @@ class Manager(object): self.include_meta = include_meta or manager_config.get('include_meta', False) - self.log.info('__init__: max_workers=%s', self.include_meta) + self.log.info('__init__: include_meta=%s', self.include_meta) self.log.debug('__init__: configuring providers') self.providers = {} @@ -105,16 +109,16 @@ class Manager(object): _class = provider_config.pop('class') except KeyError: self.log.exception('Invalid provider class') - raise Exception('Provider {} is missing class' - .format(provider_name)) + raise ManagerException('Provider {} is missing class' + .format(provider_name)) _class = self._get_named_class('provider', _class) kwargs = self._build_kwargs(provider_config) try: self.providers[provider_name] = _class(provider_name, **kwargs) except TypeError: self.log.exception('Invalid provider config') - raise Exception('Incorrect provider config for {}' - .format(provider_name)) + raise ManagerException('Incorrect provider config for {}' + .format(provider_name)) zone_tree = {} # sort by reversed strings so that parent zones always come first @@ -148,8 +152,8 @@ class Manager(object): _class = plan_output_config.pop('class') except KeyError: self.log.exception('Invalid plan_output class') - raise Exception('plan_output {} is missing class' - .format(plan_output_name)) + raise ManagerException('plan_output {} is missing class' + .format(plan_output_name)) _class = self._get_named_class('plan_output', _class) kwargs = self._build_kwargs(plan_output_config) try: @@ -157,8 +161,8 @@ class Manager(object): _class(plan_output_name, **kwargs) except TypeError: self.log.exception('Invalid plan_output config') - raise Exception('Incorrect plan_output config for {}' - .format(plan_output_name)) + raise ManagerException('Incorrect plan_output config for {}' + .format(plan_output_name)) def _get_named_class(self, _type, _class): try: @@ -167,13 +171,15 @@ class Manager(object): except (ImportError, ValueError): self.log.exception('_get_{}_class: Unable to import ' 'module %s', _class) - raise Exception('Unknown {} class: {}'.format(_type, _class)) + raise ManagerException('Unknown {} class: {}' + .format(_type, _class)) try: return getattr(module, class_name) except AttributeError: self.log.exception('_get_{}_class: Unable to get class %s ' 'from module %s', class_name, module) - raise Exception('Unknown {} class: {}'.format(_type, _class)) + raise ManagerException('Unknown {} class: {}' + .format(_type, _class)) def _build_kwargs(self, source): # Build up the arguments we need to pass to the provider @@ -186,9 +192,9 @@ class Manager(object): v = environ[env_var] except KeyError: self.log.exception('Invalid provider config') - raise Exception('Incorrect provider config, ' - 'missing env var {}' - .format(env_var)) + raise ManagerException('Incorrect provider config, ' + 'missing env var {}' + .format(env_var)) except AttributeError: pass kwargs[k] = v @@ -248,7 +254,7 @@ class Manager(object): zones = self.config['zones'].items() if eligible_zones: - zones = filter(lambda d: d[0] in eligible_zones, zones) + zones = [z for z in zones if z[0] in eligible_zones] futures = [] for zone_name, config in zones: @@ -256,14 +262,16 @@ class Manager(object): try: sources = config['sources'] except KeyError: - raise Exception('Zone {} is missing sources'.format(zone_name)) + raise ManagerException('Zone {} is missing sources' + .format(zone_name)) try: targets = config['targets'] except KeyError: - raise Exception('Zone {} is missing targets'.format(zone_name)) + raise ManagerException('Zone {} is missing targets' + .format(zone_name)) if eligible_targets: - targets = filter(lambda d: d in eligible_targets, targets) + targets = [t for t in targets if t in eligible_targets] if not targets: # Don't bother planning (and more importantly populating) zones @@ -275,23 +283,29 @@ class Manager(object): self.log.info('sync: sources=%s -> targets=%s', sources, targets) try: - sources = [self.providers[source] for source in sources] + # rather than using a list comprehension, we break this loop + # out so that the `except` block below can reference the + # `source` + collected = [] + for source in sources: + collected.append(self.providers[source]) + sources = collected except KeyError: - raise Exception('Zone {}, unknown source: {}'.format(zone_name, - source)) + raise ManagerException('Zone {}, unknown source: {}' + .format(zone_name, source)) try: trgs = [] for target in targets: trg = self.providers[target] if not isinstance(trg, BaseProvider): - raise Exception('{} - "{}" does not support targeting' - .format(trg, target)) + raise ManagerException('{} - "{}" does not support ' + 'targeting'.format(trg, target)) trgs.append(trg) targets = trgs except KeyError: - raise Exception('Zone {}, unknown target: {}'.format(zone_name, - target)) + raise ManagerException('Zone {}, unknown target: {}' + .format(zone_name, target)) futures.append(self._executor.submit(self._populate_and_plan, zone_name, sources, targets)) @@ -344,7 +358,7 @@ class Manager(object): a = [self.providers[source] for source in a] b = [self.providers[source] for source in b] except KeyError as e: - raise Exception('Unknown source: {}'.format(e.args[0])) + raise ManagerException('Unknown source: {}'.format(e.args[0])) sub_zones = self.configured_sub_zones(zone) za = Zone(zone, sub_zones) @@ -370,7 +384,7 @@ class Manager(object): try: sources = [self.providers[s] for s in sources] except KeyError as e: - raise Exception('Unknown source: {}'.format(e.args[0])) + raise ManagerException('Unknown source: {}'.format(e.args[0])) clz = YamlProvider if split: @@ -393,13 +407,20 @@ class Manager(object): try: sources = config['sources'] except KeyError: - raise Exception('Zone {} is missing sources'.format(zone_name)) + raise ManagerException('Zone {} is missing sources' + .format(zone_name)) try: - sources = [self.providers[source] for source in sources] + # rather than using a list comprehension, we break this loop + # out so that the `except` block below can reference the + # `source` + collected = [] + for source in sources: + collected.append(self.providers[source]) + sources = collected except KeyError: - raise Exception('Zone {}, unknown source: {}'.format(zone_name, - source)) + raise ManagerException('Zone {}, unknown source: {}' + .format(zone_name, source)) for source in sources: if isinstance(source, YamlProvider): diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 0bca46d..3d8122a 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -175,6 +175,10 @@ class _AzureRecord(object): :type return: bool ''' + + def key_dict(d): + return sum([hash('{}:{}'.format(k, v)) for k, v in d.items()]) + def parse_dict(params): vals = [] for char in params: @@ -185,7 +189,7 @@ class _AzureRecord(object): vals.append(record.__dict__) except: vals.append(list_records.__dict__) - vals.sort() + vals.sort(key=key_dict) return vals return (self.resource_group == b.resource_group) & \ @@ -373,13 +377,13 @@ class AzureProvider(BaseProvider): self._populate_zones() self._check_zone(zone_name) - _records = set() + _records = [] records = self._dns_client.record_sets.list_by_dns_zone if self._check_zone(zone_name): exists = True for azrecord in records(self._resource_group, zone_name): if _parse_azure_type(azrecord.type) in self.SUPPORTS: - _records.add(azrecord) + _records.append(azrecord) for azrecord in _records: record_name = azrecord.name if azrecord.name != '@' else '' typ = _parse_azure_type(azrecord.type) diff --git a/octodns/provider/base.py b/octodns/provider/base.py index 2c93e49..ae87844 100644 --- a/octodns/provider/base.py +++ b/octodns/provider/base.py @@ -5,6 +5,8 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals +from six import text_type + from ..source.base import BaseSource from ..zone import Zone from .plan import Plan @@ -58,7 +60,7 @@ class BaseProvider(BaseSource): # allow the provider to filter out false positives before = len(changes) - changes = filter(self._include_change, changes) + changes = [c for c in changes if self._include_change(c)] after = len(changes) if before != after: self.log.info('plan: filtered out %s changes', before - after) @@ -68,7 +70,7 @@ class BaseProvider(BaseSource): changes=changes) if extra: self.log.info('plan: extra changes\n %s', '\n ' - .join([unicode(c) for c in extra])) + .join([text_type(c) for c in extra])) changes += extra if changes: diff --git a/octodns/provider/cloudflare.py b/octodns/provider/cloudflare.py index 491f84d..1f0d1ea 100644 --- a/octodns/provider/cloudflare.py +++ b/octodns/provider/cloudflare.py @@ -449,7 +449,7 @@ class CloudflareProvider(BaseProvider): # Round trip the single value through a record to contents flow # to get a consistent _gen_data result that matches what # went in to new_contents - data = self._gen_data(r).next() + data = next(self._gen_data(r)) # Record the record_id and data for this existing record key = self._gen_key(data) diff --git a/octodns/provider/constellix.py b/octodns/provider/constellix.py index 939284d..5ca89e1 100644 --- a/octodns/provider/constellix.py +++ b/octodns/provider/constellix.py @@ -9,6 +9,7 @@ from collections import defaultdict from requests import Session from base64 import b64encode from ipaddress import ip_address +from six import string_types import hashlib import hmac import logging @@ -87,7 +88,7 @@ class ConstellixClient(object): if self._domains is None: zones = [] - resp = self._request('GET', '/').json() + resp = self._request('GET', '').json() zones += resp self._domains = {'{}.'.format(z['name']): z['id'] for z in zones} @@ -95,11 +96,16 @@ class ConstellixClient(object): return self._domains def domain(self, name): - path = '/{}'.format(self.domains.get(name)) + zone_id = self.domains.get(name, False) + if not zone_id: + raise ConstellixClientNotFound() + path = '/{}'.format(zone_id) return self._request('GET', path).json() def domain_create(self, name): - self._request('POST', '/', data={'names': [name]}) + resp = self._request('POST', '/', data={'names': [name]}) + # Add newly created zone to domain cache + self._domains['{}.'.format(name)] = resp.json()[0]['id'] def _absolutize_value(self, value, zone_name): if value == '': @@ -111,6 +117,8 @@ class ConstellixClient(object): def records(self, zone_name): zone_id = self.domains.get(zone_name, False) + if not zone_id: + raise ConstellixClientNotFound() path = '/{}/records'.format(zone_id) resp = self._request('GET', path).json() @@ -122,7 +130,7 @@ class ConstellixClient(object): # change relative values to absolute value = record['value'] if record['type'] in ['ALIAS', 'CNAME', 'MX', 'NS', 'SRV']: - if isinstance(value, unicode): + if isinstance(value, string_types): record['value'] = self._absolutize_value(value, zone_name) if isinstance(value, list): @@ -148,6 +156,10 @@ class ConstellixClient(object): self._request('POST', path, data=params) def record_delete(self, zone_name, record_type, record_id): + # change ALIAS records to ANAME + if record_type == 'ALIAS': + record_type = 'ANAME' + zone_id = self.domains.get(zone_name, False) path = '/{}/records/{}/{}'.format(zone_id, record_type, record_id) self._request('DELETE', path) @@ -424,8 +436,8 @@ class ConstellixProvider(BaseProvider): for record in self.zone_records(zone): if existing.name == record['name'] and \ existing._type == record['type']: - self._client.record_delete(zone.name, record['type'], - record['id']) + self._client.record_delete(zone.name, record['type'], + record['id']) def _apply(self, plan): desired = plan.desired diff --git a/octodns/provider/dnsmadeeasy.py b/octodns/provider/dnsmadeeasy.py index cc10c9a..0bf05a0 100644 --- a/octodns/provider/dnsmadeeasy.py +++ b/octodns/provider/dnsmadeeasy.py @@ -374,7 +374,7 @@ class DnsMadeEasyProvider(BaseProvider): for record in self.zone_records(zone): if existing.name == record['name'] and \ existing._type == record['type']: - self._client.record_delete(zone.name, record['id']) + self._client.record_delete(zone.name, record['id']) def _apply(self, plan): desired = plan.desired diff --git a/octodns/provider/dyn.py b/octodns/provider/dyn.py index c306238..f4f8e53 100644 --- a/octodns/provider/dyn.py +++ b/octodns/provider/dyn.py @@ -903,10 +903,10 @@ class DynProvider(BaseProvider): # Sort the values for consistent ordering so that we can compare values = sorted(values, key=_dynamic_value_sort_key) # Ensure that weight is included and if not use the default - values = map(lambda v: { + values = [{ 'value': v['value'], 'weight': v.get('weight', 1), - }, values) + } for v in values] # Walk through our existing pools looking for a match we can use for pool in pools: diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index f851303..8f651f0 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -7,7 +7,7 @@ from __future__ import absolute_import, division, print_function, \ from requests import Session from akamai.edgegrid import EdgeGridAuth -from urlparse import urljoin +from six.moves.urllib.parse import urljoin from collections import defaultdict from logging import getLogger diff --git a/octodns/provider/mythicbeasts.py b/octodns/provider/mythicbeasts.py index 17029db..b255a74 100644 --- a/octodns/provider/mythicbeasts.py +++ b/octodns/provider/mythicbeasts.py @@ -328,7 +328,7 @@ class MythicBeastsProvider(BaseProvider): exists = True for line in resp.content.splitlines(): - match = MythicBeastsProvider.RE_POPLINE.match(line) + match = MythicBeastsProvider.RE_POPLINE.match(line.decode("utf-8")) if match is None: self.log.debug('failed to match line: %s', line) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 5fdf5b0..a25911d 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -8,37 +8,251 @@ from __future__ import absolute_import, division, print_function, \ from logging import getLogger from itertools import chain from collections import OrderedDict, defaultdict -from nsone import NSONE -from nsone.rest.errors import RateLimitException, ResourceException -from incf.countryutils import transformations +from ns1 import NS1 +from ns1.rest.errors import RateLimitException, ResourceException +from pycountry_convert import country_alpha2_to_continent_code from time import sleep +from uuid import uuid4 -from ..record import Record +from six import text_type + +from ..record import Record, Update from .base import BaseProvider +class Ns1Exception(Exception): + pass + + +class Ns1Client(object): + log = getLogger('NS1Client') + + def __init__(self, api_key, retry_count=4): + self.log.debug('__init__: retry_count=%d', retry_count) + self.retry_count = retry_count + + client = NS1(apiKey=api_key) + self._records = client.records() + self._zones = client.zones() + self._monitors = client.monitors() + self._notifylists = client.notifylists() + self._datasource = client.datasource() + self._datafeed = client.datafeed() + + self._datasource_id = None + self._feeds_for_monitors = None + self._monitors_cache = None + + @property + def datasource_id(self): + if self._datasource_id is None: + name = 'octoDNS NS1 Data Source' + source = None + for candidate in self.datasource_list(): + if candidate['name'] == name: + # Found it + source = candidate + break + + if source is None: + self.log.info('datasource_id: creating datasource %s', name) + # We need to create it + source = self.datasource_create(name=name, + sourcetype='nsone_monitoring') + self.log.info('datasource_id: id=%s', source['id']) + + self._datasource_id = source['id'] + + return self._datasource_id + + @property + def feeds_for_monitors(self): + if self._feeds_for_monitors is None: + self.log.debug('feeds_for_monitors: fetching & building') + self._feeds_for_monitors = { + f['config']['jobid']: f['id'] + for f in self.datafeed_list(self.datasource_id) + } + + return self._feeds_for_monitors + + @property + def monitors(self): + if self._monitors_cache is None: + self.log.debug('monitors: fetching & building') + self._monitors_cache = \ + {m['id']: m for m in self.monitors_list()} + return self._monitors_cache + + def datafeed_create(self, sourceid, name, config): + ret = self._try(self._datafeed.create, sourceid, name, config) + self.feeds_for_monitors[config['jobid']] = ret['id'] + return ret + + def datafeed_delete(self, sourceid, feedid): + ret = self._try(self._datafeed.delete, sourceid, feedid) + self._feeds_for_monitors = { + k: v for k, v in self._feeds_for_monitors.items() if v != feedid + } + return ret + + def datafeed_list(self, sourceid): + return self._try(self._datafeed.list, sourceid) + + def datasource_create(self, **body): + return self._try(self._datasource.create, **body) + + def datasource_list(self): + return self._try(self._datasource.list) + + def monitors_create(self, **params): + body = {} + ret = self._try(self._monitors.create, body, **params) + self.monitors[ret['id']] = ret + return ret + + def monitors_delete(self, jobid): + ret = self._try(self._monitors.delete, jobid) + self.monitors.pop(jobid) + return ret + + def monitors_list(self): + return self._try(self._monitors.list) + + def monitors_update(self, job_id, **params): + body = {} + ret = self._try(self._monitors.update, job_id, body, **params) + self.monitors[ret['id']] = ret + return ret + + def notifylists_delete(self, nlid): + return self._try(self._notifylists.delete, nlid) + + def notifylists_create(self, **body): + return self._try(self._notifylists.create, body) + + def notifylists_list(self): + return self._try(self._notifylists.list) + + def records_create(self, zone, domain, _type, **params): + return self._try(self._records.create, zone, domain, _type, **params) + + def records_delete(self, zone, domain, _type): + return self._try(self._records.delete, zone, domain, _type) + + def records_retrieve(self, zone, domain, _type): + return self._try(self._records.retrieve, zone, domain, _type) + + def records_update(self, zone, domain, _type, **params): + return self._try(self._records.update, zone, domain, _type, **params) + + def zones_create(self, name): + return self._try(self._zones.create, name) + + def zones_retrieve(self, name): + return self._try(self._zones.retrieve, name) + + def _try(self, method, *args, **kwargs): + tries = self.retry_count + while True: # We'll raise to break after our tries expire + try: + return method(*args, **kwargs) + except RateLimitException as e: + if tries <= 1: + raise + period = float(e.period) + self.log.warn('rate limit encountered, pausing ' + 'for %ds and trying again, %d remaining', + period, tries) + sleep(period) + tries -= 1 + + class Ns1Provider(BaseProvider): ''' Ns1 provider - nsone: + ns1: class: octodns.provider.ns1.Ns1Provider api_key: env/NS1_API_KEY + # Only required if using dynamic records + monitor_regions: + - lga ''' SUPPORTS_GEO = True - SUPPORTS_DYNAMIC = False + SUPPORTS_DYNAMIC = True SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CAA', 'CNAME', 'MX', 'NAPTR', 'NS', 'PTR', 'SPF', 'SRV', 'TXT')) ZONE_NOT_FOUND_MESSAGE = 'server error: zone not found' - def __init__(self, id, api_key, *args, **kwargs): + _DYNAMIC_FILTERS = [{ + 'config': {}, + 'filter': 'up' + }, { + 'config': {}, + 'filter': u'geofence_regional' + }, { + 'config': {}, + 'filter': u'select_first_region' + }, { + 'config': { + 'eliminate': u'1' + }, + 'filter': 'priority' + }, { + 'config': {}, + 'filter': u'weighted_shuffle' + }, { + 'config': { + 'N': u'1' + }, + 'filter': u'select_first_n' + }] + _REGION_TO_CONTINENT = { + 'AFRICA': 'AF', + 'ASIAPAC': 'AS', + 'EUROPE': 'EU', + 'SOUTH-AMERICA': 'SA', + 'US-CENTRAL': 'NA', + 'US-EAST': 'NA', + 'US-WEST': 'NA', + } + _CONTINENT_TO_REGIONS = { + 'AF': ('AFRICA',), + 'AS': ('ASIAPAC',), + 'EU': ('EUROPE',), + 'SA': ('SOUTH-AMERICA',), + # TODO: what about CA, MX, and all the other NA countries? + 'NA': ('US-CENTRAL', 'US-EAST', 'US-WEST'), + } + + def __init__(self, id, api_key, retry_count=4, monitor_regions=None, *args, + **kwargs): self.log = getLogger('Ns1Provider[{}]'.format(id)) - self.log.debug('__init__: id=%s, api_key=***', id) + self.log.debug('__init__: id=%s, api_key=***, retry_count=%d, ' + 'monitor_regions=%s', id, retry_count, monitor_regions) super(Ns1Provider, self).__init__(id, *args, **kwargs) - self._client = NSONE(apiKey=api_key) + self.monitor_regions = monitor_regions + + self._client = Ns1Client(api_key, retry_count) + + def _encode_notes(self, data): + return ' '.join(['{}:{}'.format(k, v) + for k, v in sorted(data.items())]) + + def _parse_notes(self, note): + data = {} + if note: + for piece in note.split(' '): + try: + k, v = piece.split(':', 1) + data[k] = v + except ValueError: + pass + return data - def _data_for_A(self, _type, record): + def _data_for_geo_A(self, _type, record): # record meta (which would include geo information is only # returned when getting a record's detail, not from zone detail geo = defaultdict(list) @@ -47,8 +261,6 @@ class Ns1Provider(BaseProvider): 'type': _type, } values, codes = [], [] - if 'answers' not in record: - values = record['short_answers'] for answer in record.get('answers', []): meta = answer.get('meta', {}) if meta: @@ -60,8 +272,7 @@ class Ns1Provider(BaseProvider): us_state = meta.get('us_state', []) ca_province = meta.get('ca_province', []) for cntry in country: - cn = transformations.cc_to_cn(cntry) - con = transformations.cn_to_ctca2(cn) + con = country_alpha2_to_continent_code(cntry) key = '{}-{}'.format(con, cntry) geo[key].extend(answer['answer']) for state in us_state: @@ -76,14 +287,124 @@ class Ns1Provider(BaseProvider): else: values.extend(answer['answer']) codes.append([]) - values = [unicode(x) for x in values] + values = [text_type(x) for x in values] geo = OrderedDict( - {unicode(k): [unicode(x) for x in v] for k, v in geo.items()} + {text_type(k): [text_type(x) for x in v] for k, v in geo.items()} ) data['values'] = values data['geo'] = geo return data + def _data_for_dynamic_A(self, _type, record): + # First make sure we have the expected filters config + if self._DYNAMIC_FILTERS != record['filters']: + self.log.error('_data_for_dynamic_A: %s %s has unsupported ' + 'filters', record['domain'], _type) + raise Ns1Exception('Unrecognized advanced record') + + # All regions (pools) will include the list of default values + # (eventually) at higher priorities, we'll just add them to this set to + # we'll have the complete collection. + default = set() + # Fill out the pools by walking the answers and looking at their + # region. + pools = defaultdict(lambda: {'fallback': None, 'values': []}) + for answer in record['answers']: + # region (group name in the UI) is the pool name + pool_name = answer['region'] + pool = pools[answer['region']] + + meta = answer['meta'] + value = text_type(answer['answer'][0]) + if meta['priority'] == 1: + # priority 1 means this answer is part of the pools own values + pool['values'].append({ + 'value': value, + 'weight': int(meta.get('weight', 1)), + }) + else: + # It's a fallback, we only care about it if it's a + # final/default + notes = self._parse_notes(meta.get('note', '')) + if notes.get('from', False) == '--default--': + default.add(value) + + # The regions objects map to rules, but it's a bit fuzzy since they're + # tied to pools on the NS1 side, e.g. we can only have 1 rule per pool, + # that may eventually run into problems, but I don't have any use-cases + # examples currently where it would + rules = [] + for pool_name, region in sorted(record['regions'].items()): + meta = region['meta'] + notes = self._parse_notes(meta.get('note', '')) + + # The group notes field in the UI is a `note` on the region here, + # that's where we can find our pool's fallback. + if 'fallback' in notes: + # set the fallback pool name + pools[pool_name]['fallback'] = notes['fallback'] + + geos = set() + + # continents are mapped (imperfectly) to regions, but what about + # Canada/North America + for georegion in meta.get('georegion', []): + geos.add(self._REGION_TO_CONTINENT[georegion]) + + # Countries are easy enough to map, we just have ot find their + # continent + for country in meta.get('country', []): + con = country_alpha2_to_continent_code(country) + geos.add('{}-{}'.format(con, country)) + + # States are easy too, just assume NA-US (CA providences aren't + # supported by octoDNS currently) + for state in meta.get('us_state', []): + geos.add('NA-US-{}'.format(state)) + + rule = { + 'pool': pool_name, + '_order': notes['rule-order'], + } + if geos: + rule['geos'] = sorted(geos) + rules.append(rule) + + # Order and convert to a list + default = sorted(default) + # Order + rules.sort(key=lambda r: (r['_order'], r['pool'])) + + return { + 'dynamic': { + 'pools': pools, + 'rules': rules, + }, + 'ttl': record['ttl'], + 'type': _type, + 'values': sorted(default), + } + + def _data_for_A(self, _type, record): + if record.get('tier', 1) > 1: + # Advanced record, see if it's first answer has a note + try: + first_answer_note = record['answers'][0]['meta']['note'] + except (IndexError, KeyError): + first_answer_note = '' + # If that note includes a `from` (pool name) it's a dynamic record + if 'from:' in first_answer_note: + return self._data_for_dynamic_A(_type, record) + # If not it's an old geo record + return self._data_for_geo_A(_type, record) + + # This is a basic record, just convert it + return { + 'ttl': record['ttl'], + 'type': _type, + 'values': [text_type(x) for x in record['short_answers']] + } + _data_for_AAAA = _data_for_A def _data_for_SPF(self, _type, record): @@ -188,18 +509,29 @@ class Ns1Provider(BaseProvider): target, lenient) try: - nsone_zone = self._client.loadZone(zone.name[:-1]) - records = nsone_zone.data['records'] + ns1_zone_name = zone.name[:-1] + ns1_zone = self._client.zones_retrieve(ns1_zone_name) + + records = [] + geo_records = [] # change answers for certain types to always be absolute - for record in records: + for record in ns1_zone['records']: if record['type'] in ['ALIAS', 'CNAME', 'MX', 'NS', 'PTR', 'SRV']: for i, a in enumerate(record['short_answers']): if not a.endswith('.'): record['short_answers'][i] = '{}.'.format(a) - geo_records = nsone_zone.search(has_geo=True) + if record.get('tier', 1) > 1: + # Need to get the full record data for geo records + record = self._client.records_retrieve(ns1_zone_name, + record['domain'], + record['type']) + geo_records.append(record) + else: + records.append(record) + exists = True except ResourceException as e: if e.message != self.ZONE_NOT_FOUND_MESSAGE: @@ -218,49 +550,343 @@ class Ns1Provider(BaseProvider): continue data_for = getattr(self, '_data_for_{}'.format(_type)) name = zone.hostname_from_fqdn(record['domain']) - record = Record.new(zone, name, data_for(_type, record), - source=self, lenient=lenient) + data = data_for(_type, record) + record = Record.new(zone, name, data, source=self, lenient=lenient) zone_hash[(_type, name)] = record [zone.add_record(r, lenient=lenient) for r in zone_hash.values()] self.log.info('populate: found %s records, exists=%s', len(zone.records) - before, exists) return exists - def _params_for_A(self, record): - params = {'answers': record.values, 'ttl': record.ttl} - if hasattr(record, 'geo'): - # purposefully set non-geo answers to have an empty meta, - # so that we know we did this on purpose if/when troubleshooting - params['answers'] = [{"answer": [x], "meta": {}} - for x in record.values] - has_country = False - for iso_region, target in record.geo.items(): - key = 'iso_region_code' - value = iso_region - if not has_country and \ - len(value.split('-')) > 1: # pragma: nocover - has_country = True - for answer in target.values: - params['answers'].append( - { - 'answer': [answer], - 'meta': {key: [value]}, - }, - ) - params['filters'] = [] - if has_country: - params['filters'].append( - {"filter": "shuffle", "config": {}} - ) - params['filters'].append( - {"filter": "geotarget_country", "config": {}} - ) - params['filters'].append( - {"filter": "select_first_n", - "config": {"N": 1}} + def _params_for_geo_A(self, record): + # purposefully set non-geo answers to have an empty meta, + # so that we know we did this on purpose if/when troubleshooting + params = { + 'answers': [{"answer": [x], "meta": {}} for x in record.values], + 'ttl': record.ttl, + } + + has_country = False + for iso_region, target in record.geo.items(): + key = 'iso_region_code' + value = iso_region + if not has_country and \ + len(value.split('-')) > 1: # pragma: nocover + has_country = True + for answer in target.values: + params['answers'].append( + { + 'answer': [answer], + 'meta': {key: [value]}, + }, ) - self.log.debug("params for A: %s", params) - return params + + params['filters'] = [] + if has_country: + params['filters'].append( + {"filter": "shuffle", "config": {}} + ) + params['filters'].append( + {"filter": "geotarget_country", "config": {}} + ) + params['filters'].append( + {"filter": "select_first_n", + "config": {"N": 1}} + ) + + return params, None + + def _monitors_for(self, record): + monitors = {} + + if getattr(record, 'dynamic', False): + expected_host = record.fqdn[:-1] + expected_type = record._type + + for monitor in self._client.monitors.values(): + data = self._parse_notes(monitor['notes']) + if expected_host == data['host'] and \ + expected_type == data['type']: + # This monitor does not belong to this record + config = monitor['config'] + value = config['host'] + monitors[value] = monitor + + return monitors + + def _uuid(self): + return uuid4().hex + + def _feed_create(self, monitor): + monitor_id = monitor['id'] + self.log.debug('_feed_create: monitor=%s', monitor_id) + # TODO: looks like length limit is 64 char + name = '{} - {}'.format(monitor['name'], self._uuid()[:6]) + + # Create the data feed + config = { + 'jobid': monitor_id, + } + feed = self._client.datafeed_create(self._client.datasource_id, name, + config) + feed_id = feed['id'] + self.log.debug('_feed_create: feed=%s', feed_id) + + return feed_id + + def _monitor_create(self, monitor): + self.log.debug('_monitor_create: monitor="%s"', monitor['name']) + # Create the notify list + notify_list = [{ + 'config': { + 'sourceid': self._client.datasource_id, + }, + 'type': 'datafeed', + }] + nl = self._client.notifylists_create(name=monitor['name'], + notify_list=notify_list) + nl_id = nl['id'] + self.log.debug('_monitor_create: notify_list=%s', nl_id) + + # Create the monitor + monitor['notify_list'] = nl_id + monitor = self._client.monitors_create(**monitor) + monitor_id = monitor['id'] + self.log.debug('_monitor_create: monitor=%s', monitor_id) + + return monitor_id, self._feed_create(monitor) + + def _monitor_gen(self, record, value): + host = record.fqdn[:-1] + _type = record._type + + request = r'GET {path} HTTP/1.0\r\nHost: {host}\r\n' \ + r'User-agent: NS1\r\n\r\n'.format(path=record.healthcheck_path, + host=record.healthcheck_host) + + return { + 'active': True, + 'config': { + 'connect_timeout': 2000, + 'host': value, + 'port': record.healthcheck_port, + 'response_timeout': 10000, + 'send': request, + 'ssl': record.healthcheck_protocol == 'HTTPS', + }, + 'frequency': 60, + 'job_type': 'tcp', + 'name': '{} - {} - {}'.format(host, _type, value), + 'notes': self._encode_notes({ + 'host': host, + 'type': _type, + }), + 'policy': 'quorum', + 'rapid_recheck': False, + 'region_scope': 'fixed', + 'regions': self.monitor_regions, + 'rules': [{ + 'comparison': 'contains', + 'key': 'output', + 'value': '200 OK', + }], + } + + def _monitor_is_match(self, expected, have): + # Make sure what we have matches what's in expected exactly. Anything + # else in have will be ignored. + for k, v in expected.items(): + if have.get(k, '--missing--') != v: + return False + + return True + + def _monitor_sync(self, record, value, existing): + self.log.debug('_monitor_sync: record=%s, value=%s', record.fqdn, + value) + expected = self._monitor_gen(record, value) + + if existing: + self.log.debug('_monitor_sync: existing=%s', existing['id']) + monitor_id = existing['id'] + + if not self._monitor_is_match(expected, existing): + self.log.debug('_monitor_sync: existing needs update') + # Update the monitor to match expected, everything else will be + # left alone and assumed correct + self._client.monitors_update(monitor_id, **expected) + + feed_id = self._client.feeds_for_monitors.get(monitor_id) + if feed_id is None: + self.log.warn('_monitor_sync: %s (%s) missing feed, creating', + existing['name'], monitor_id) + feed_id = self._feed_create(existing) + else: + self.log.debug('_monitor_sync: needs create') + # We don't have an existing monitor create it (and related bits) + monitor_id, feed_id = self._monitor_create(expected) + + return monitor_id, feed_id + + def _monitors_gc(self, record, active_monitor_ids=None): + self.log.debug('_monitors_gc: record=%s, active_monitor_ids=%s', + record.fqdn, active_monitor_ids) + + if active_monitor_ids is None: + active_monitor_ids = set() + + for monitor in self._monitors_for(record).values(): + monitor_id = monitor['id'] + if monitor_id in active_monitor_ids: + continue + + self.log.debug('_monitors_gc: deleting %s', monitor_id) + + feed_id = self._client.feeds_for_monitors.get(monitor_id) + if feed_id: + self._client.datafeed_delete(self._client.datasource_id, + feed_id) + + self._client.monitors_delete(monitor_id) + + notify_list_id = monitor['notify_list'] + self._client.notifylists_delete(notify_list_id) + + def _params_for_dynamic_A(self, record): + pools = record.dynamic.pools + + # Convert rules to regions + regions = {} + for i, rule in enumerate(record.dynamic.rules): + pool_name = rule.data['pool'] + + notes = { + 'rule-order': i, + } + + fallback = pools[pool_name].data.get('fallback', None) + if fallback: + notes['fallback'] = fallback + + country = set() + georegion = set() + us_state = set() + + for geo in rule.data.get('geos', []): + n = len(geo) + if n == 8: + # US state, e.g. NA-US-KY + us_state.add(geo[-2:]) + elif n == 5: + # Country, e.g. EU-FR + country.add(geo[-2:]) + else: + # Continent, e.g. AS + georegion.update(self._CONTINENT_TO_REGIONS[geo]) + + meta = { + 'note': self._encode_notes(notes), + } + if georegion: + meta['georegion'] = sorted(georegion) + if country: + meta['country'] = sorted(country) + if us_state: + meta['us_state'] = sorted(us_state) + + regions[pool_name] = { + 'meta': meta, + } + + existing_monitors = self._monitors_for(record) + active_monitors = set() + + # Build a list of primary values for each pool, including their + # feed_id (monitor) + pool_answers = defaultdict(list) + for pool_name, pool in sorted(pools.items()): + for value in pool.data['values']: + weight = value['weight'] + value = value['value'] + existing = existing_monitors.get(value) + monitor_id, feed_id = self._monitor_sync(record, value, + existing) + active_monitors.add(monitor_id) + pool_answers[pool_name].append({ + 'answer': [value], + 'weight': weight, + 'feed_id': feed_id, + }) + + default_answers = [{ + 'answer': [v], + 'weight': 1, + } for v in record.values] + + # Build our list of answers + answers = [] + for pool_name in sorted(pools.keys()): + priority = 1 + + # Dynamic/health checked + current_pool_name = pool_name + seen = set() + while current_pool_name and current_pool_name not in seen: + seen.add(current_pool_name) + pool = pools[current_pool_name] + for answer in pool_answers[current_pool_name]: + answer = { + 'answer': answer['answer'], + 'meta': { + 'priority': priority, + 'note': self._encode_notes({ + 'from': current_pool_name, + }), + 'up': { + 'feed': answer['feed_id'], + }, + 'weight': answer['weight'], + }, + 'region': pool_name, # the one we're answering + } + answers.append(answer) + + current_pool_name = pool.data.get('fallback', None) + priority += 1 + + # Static/default + for answer in default_answers: + answer = { + 'answer': answer['answer'], + 'meta': { + 'priority': priority, + 'note': self._encode_notes({ + 'from': '--default--', + }), + 'up': True, + 'weight': 1, + }, + 'region': pool_name, # the one we're answering + } + answers.append(answer) + + return { + 'answers': answers, + 'filters': self._DYNAMIC_FILTERS, + 'regions': regions, + 'ttl': record.ttl, + }, active_monitors + + def _params_for_A(self, record): + if getattr(record, 'dynamic', False): + return self._params_for_dynamic_A(record) + elif hasattr(record, 'geo'): + return self._params_for_geo_A(record) + + return { + 'answers': record.values, + 'ttl': record.ttl, + }, None _params_for_AAAA = _params_for_A _params_for_NS = _params_for_A @@ -270,81 +896,97 @@ class Ns1Provider(BaseProvider): # escaped in values so we have to strip them here and add # them when going the other way values = [v.replace('\\;', ';') for v in record.values] - return {'answers': values, 'ttl': record.ttl} + return {'answers': values, 'ttl': record.ttl}, None _params_for_TXT = _params_for_SPF def _params_for_CAA(self, record): values = [(v.flags, v.tag, v.value) for v in record.values] - return {'answers': values, 'ttl': record.ttl} + return {'answers': values, 'ttl': record.ttl}, None + # TODO: dynamic CNAME support def _params_for_CNAME(self, record): - return {'answers': [record.value], 'ttl': record.ttl} + return {'answers': [record.value], 'ttl': record.ttl}, None _params_for_ALIAS = _params_for_CNAME _params_for_PTR = _params_for_CNAME def _params_for_MX(self, record): values = [(v.preference, v.exchange) for v in record.values] - return {'answers': values, 'ttl': record.ttl} + return {'answers': values, 'ttl': record.ttl}, None def _params_for_NAPTR(self, record): values = [(v.order, v.preference, v.flags, v.service, v.regexp, v.replacement) for v in record.values] - return {'answers': values, 'ttl': record.ttl} + return {'answers': values, 'ttl': record.ttl}, None def _params_for_SRV(self, record): values = [(v.priority, v.weight, v.port, v.target) for v in record.values] - return {'answers': values, 'ttl': record.ttl} + return {'answers': values, 'ttl': record.ttl}, None + + def _extra_changes(self, desired, changes, **kwargs): + self.log.debug('_extra_changes: desired=%s', desired.name) - def _get_name(self, record): - return record.fqdn[:-1] if record.name == '' else record.name + changed = set([c.record for c in changes]) - def _apply_Create(self, nsone_zone, change): + extra = [] + for record in desired.records: + if record in changed or not getattr(record, 'dynamic', False): + # Already changed, or no dynamic , no need to check it + continue + + for have in self._monitors_for(record).values(): + value = have['config']['host'] + expected = self._monitor_gen(record, value) + # TODO: find values which have missing monitors + if not self._monitor_is_match(expected, have): + self.log.info('_extra_changes: monitor mis-match for %s', + expected['name']) + extra.append(Update(record, record)) + break + if not have.get('notify_list'): + self.log.info('_extra_changes: broken monitor no notify ' + 'list %s (%s)', have['name'], have['id']) + extra.append(Update(record, record)) + break + + return extra + + def _apply_Create(self, ns1_zone, change): new = change.new - name = self._get_name(new) + zone = new.zone.name[:-1] + domain = new.fqdn[:-1] _type = new._type - params = getattr(self, '_params_for_{}'.format(_type))(new) - meth = getattr(nsone_zone, 'add_{}'.format(_type)) - try: - meth(name, **params) - except RateLimitException as e: - period = float(e.period) - self.log.warn('_apply_Create: rate limit encountered, pausing ' - 'for %ds and trying again', period) - sleep(period) - meth(name, **params) - - def _apply_Update(self, nsone_zone, change): - existing = change.existing - name = self._get_name(existing) - _type = existing._type - record = nsone_zone.loadRecord(name, _type) + params, active_monitor_ids = \ + getattr(self, '_params_for_{}'.format(_type))(new) + self._client.records_create(zone, domain, _type, **params) + self._monitors_gc(new, active_monitor_ids) + + def _apply_Update(self, ns1_zone, change): new = change.new - params = getattr(self, '_params_for_{}'.format(_type))(new) - try: - record.update(**params) - except RateLimitException as e: - period = float(e.period) - self.log.warn('_apply_Update: rate limit encountered, pausing ' - 'for %ds and trying again', period) - sleep(period) - record.update(**params) - - def _apply_Delete(self, nsone_zone, change): + zone = new.zone.name[:-1] + domain = new.fqdn[:-1] + _type = new._type + params, active_monitor_ids = \ + getattr(self, '_params_for_{}'.format(_type))(new) + self._client.records_update(zone, domain, _type, **params) + self._monitors_gc(new, active_monitor_ids) + + def _apply_Delete(self, ns1_zone, change): existing = change.existing - name = self._get_name(existing) + zone = existing.zone.name[:-1] + domain = existing.fqdn[:-1] _type = existing._type - record = nsone_zone.loadRecord(name, _type) - try: - record.delete() - except RateLimitException as e: - period = float(e.period) - self.log.warn('_apply_Delete: rate limit encountered, pausing ' - 'for %ds and trying again', period) - sleep(period) - record.delete() + self._client.records_delete(zone, domain, _type) + self._monitors_gc(existing) + + def _has_dynamic(self, changes): + for change in changes: + if getattr(change.record, 'dynamic', False): + return True + + return False def _apply(self, plan): desired = plan.desired @@ -352,16 +994,22 @@ class Ns1Provider(BaseProvider): self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name, len(changes)) + # Make sure that if we're going to make any dynamic changes that we + # have monitor_regions configured before touching anything so we can + # abort early and not half-apply + if self._has_dynamic(changes) and self.monitor_regions is None: + raise Ns1Exception('Monitored record, but monitor_regions not set') + domain_name = desired.name[:-1] try: - nsone_zone = self._client.loadZone(domain_name) + ns1_zone = self._client.zones_retrieve(domain_name) except ResourceException as e: if e.message != self.ZONE_NOT_FOUND_MESSAGE: raise self.log.debug('_apply: no matching zone, creating') - nsone_zone = self._client.createZone(domain_name) + ns1_zone = self._client.zones_create(domain_name) for change in changes: class_name = change.__class__.__name__ - getattr(self, '_apply_{}'.format(class_name))(nsone_zone, + getattr(self, '_apply_{}'.format(class_name))(ns1_zone, change) diff --git a/octodns/provider/ovh.py b/octodns/provider/ovh.py index d968da4..54f62ac 100644 --- a/octodns/provider/ovh.py +++ b/octodns/provider/ovh.py @@ -9,6 +9,7 @@ import base64 import binascii import logging from collections import defaultdict +from six import text_type import ovh from ovh import ResourceNotFoundError @@ -39,8 +40,8 @@ class OvhProvider(BaseProvider): # This variable is also used in populate method to filter which OVH record # types are supported by octodns - SUPPORTS = set(('A', 'AAAA', 'CNAME', 'DKIM', 'MX', 'NAPTR', 'NS', 'PTR', - 'SPF', 'SRV', 'SSHFP', 'TXT')) + SUPPORTS = set(('A', 'AAAA', 'CAA', 'CNAME', 'DKIM', 'MX', 'NAPTR', 'NS', + 'PTR', 'SPF', 'SRV', 'SSHFP', 'TXT')) def __init__(self, id, endpoint, application_key, application_secret, consumer_key, *args, **kwargs): @@ -64,7 +65,7 @@ class OvhProvider(BaseProvider): records = self.get_records(zone_name=zone_name) exists = True except ResourceNotFoundError as e: - if e.message != self.ZONE_NOT_FOUND_MESSAGE: + if text_type(e) != self.ZONE_NOT_FOUND_MESSAGE: raise exists = False records = [] @@ -138,6 +139,22 @@ class OvhProvider(BaseProvider): 'value': record['target'] } + @staticmethod + def _data_for_CAA(_type, records): + values = [] + for record in records: + flags, tag, value = record['target'].split(' ', 2) + values.append({ + 'flags': flags, + 'tag': tag, + 'value': value[1:-1] + }) + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': values + } + @staticmethod def _data_for_MX(_type, records): values = [] @@ -243,6 +260,17 @@ class OvhProvider(BaseProvider): 'fieldType': record._type } + @staticmethod + def _params_for_CAA(record): + for value in record.values: + yield { + 'target': '{} {} "{}"'.format(value.flags, value.tag, + value.value), + 'subDomain': record.name, + 'ttl': record.ttl, + 'fieldType': record._type + } + @staticmethod def _params_for_MX(record): for value in record.values: @@ -322,10 +350,10 @@ class OvhProvider(BaseProvider): 'n': lambda _: True, 'g': lambda _: True} - splitted = value.split('\\;') + splitted = [v for v in value.split('\\;') if v] found_key = False for splitted_value in splitted: - sub_split = map(lambda x: x.strip(), splitted_value.split("=", 1)) + sub_split = [x.strip() for x in splitted_value.split("=", 1)] if len(sub_split) < 2: return False key, value = sub_split[0], sub_split[1] @@ -343,7 +371,7 @@ class OvhProvider(BaseProvider): @staticmethod def _is_valid_dkim_key(key): try: - base64.decodestring(key) + base64.decodestring(bytearray(key, 'utf-8')) except binascii.Error: return False return True diff --git a/octodns/provider/plan.py b/octodns/provider/plan.py index bae244f..af6863a 100644 --- a/octodns/provider/plan.py +++ b/octodns/provider/plan.py @@ -5,10 +5,11 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -from StringIO import StringIO from logging import DEBUG, ERROR, INFO, WARN, getLogger from sys import stdout +from six import StringIO, text_type + class UnsafePlan(Exception): pass @@ -26,7 +27,11 @@ class Plan(object): delete_pcent_threshold=MAX_SAFE_DELETE_PCENT): self.existing = existing self.desired = desired - self.changes = changes + # Sort changes to ensure we always have a consistent ordering for + # things that make assumptions about that. Many providers will do their + # own ordering to ensure things happen in a way that makes sense to + # them and/or is as safe as possible. + self.changes = sorted(changes) self.exists = exists self.update_pcent_threshold = update_pcent_threshold self.delete_pcent_threshold = delete_pcent_threshold @@ -122,7 +127,7 @@ class PlanLogger(_PlanOutput): buf.write('* ') buf.write(target.id) buf.write(' (') - buf.write(target) + buf.write(text_type(target)) buf.write(')\n* ') if plan.exists is False: @@ -135,7 +140,7 @@ class PlanLogger(_PlanOutput): buf.write('\n* ') buf.write('Summary: ') - buf.write(plan) + buf.write(text_type(plan)) buf.write('\n') else: buf.write(hr) @@ -147,11 +152,11 @@ class PlanLogger(_PlanOutput): def _value_stringifier(record, sep): try: - values = [unicode(v) for v in record.values] + values = [text_type(v) for v in record.values] except AttributeError: values = [record.value] for code, gv in sorted(getattr(record, 'geo', {}).items()): - vs = ', '.join([unicode(v) for v in gv.values]) + vs = ', '.join([text_type(v) for v in gv.values]) values.append('{}: {}'.format(code, vs)) return sep.join(values) @@ -193,7 +198,7 @@ class PlanMarkdown(_PlanOutput): fh.write(' | ') # TTL if existing: - fh.write(unicode(existing.ttl)) + fh.write(text_type(existing.ttl)) fh.write(' | ') fh.write(_value_stringifier(existing, '; ')) fh.write(' | |\n') @@ -201,7 +206,7 @@ class PlanMarkdown(_PlanOutput): fh.write('| | | | ') if new: - fh.write(unicode(new.ttl)) + fh.write(text_type(new.ttl)) fh.write(' | ') fh.write(_value_stringifier(new, '; ')) fh.write(' | ') @@ -210,7 +215,7 @@ class PlanMarkdown(_PlanOutput): fh.write(' |\n') fh.write('\nSummary: ') - fh.write(unicode(plan)) + fh.write(text_type(plan)) fh.write('\n\n') else: fh.write('## No changes were planned\n') @@ -261,7 +266,7 @@ class PlanHtml(_PlanOutput): # TTL if existing: fh.write(' ') - fh.write(unicode(existing.ttl)) + fh.write(text_type(existing.ttl)) fh.write('\n ') fh.write(_value_stringifier(existing, '
')) fh.write('\n \n \n') @@ -270,7 +275,7 @@ class PlanHtml(_PlanOutput): if new: fh.write(' ') - fh.write(unicode(new.ttl)) + fh.write(text_type(new.ttl)) fh.write('\n ') fh.write(_value_stringifier(new, '
')) fh.write('\n ') @@ -279,7 +284,7 @@ class PlanHtml(_PlanOutput): fh.write('\n \n') fh.write(' \n Summary: ') - fh.write(unicode(plan)) + fh.write(text_type(plan)) fh.write('\n \n\n') else: fh.write('No changes were planned') diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py index 5038929..7fed05b 100644 --- a/octodns/provider/rackspace.py +++ b/octodns/provider/rackspace.py @@ -7,13 +7,16 @@ from __future__ import absolute_import, division, print_function, \ from requests import HTTPError, Session, post from collections import defaultdict import logging -import string import time from ..record import Record from .base import BaseProvider +def _value_keyer(v): + return (v.get('type', ''), v['name'], v.get('data', '')) + + def add_trailing_dot(s): assert s assert s[-1] != '.' @@ -28,12 +31,12 @@ def remove_trailing_dot(s): def escape_semicolon(s): assert s - return string.replace(s, ';', '\\;') + return s.replace(';', '\\;') def unescape_semicolon(s): assert s - return string.replace(s, '\\;', ';') + return s.replace('\\;', ';') class RackspaceProvider(BaseProvider): @@ -367,11 +370,9 @@ class RackspaceProvider(BaseProvider): self._delete('domains/{}/records?{}'.format(domain_id, params)) if updates: - data = {"records": sorted(updates, key=lambda v: v['name'])} + data = {"records": sorted(updates, key=_value_keyer)} self._put('domains/{}/records'.format(domain_id), data=data) if creates: - data = {"records": sorted(creates, key=lambda v: v['type'] + - v['name'] + - v.get('data', ''))} + data = {"records": sorted(creates, key=_value_keyer)} self._post('domains/{}/records'.format(domain_id), data=data) diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index 81e366a..89fb7a8 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -8,17 +8,19 @@ from __future__ import absolute_import, division, print_function, \ from boto3 import client from botocore.config import Config from collections import defaultdict -from incf.countryutils.transformations import cca_to_ctca2 from ipaddress import AddressValueError, ip_address +from pycountry_convert import country_alpha2_to_continent_code from uuid import uuid4 import logging import re +from six import text_type + +from ..equality import EqualityTupleMixin from ..record import Record, Update from ..record.geo import GeoCodes from .base import BaseProvider - octal_re = re.compile(r'\\(\d\d\d)') @@ -28,7 +30,7 @@ def _octal_replace(s): return octal_re.sub(lambda m: chr(int(m.group(1), 8)), s) -class _Route53Record(object): +class _Route53Record(EqualityTupleMixin): @classmethod def _new_dynamic(cls, provider, record, hosted_zone_id, creating): @@ -147,7 +149,7 @@ class _Route53Record(object): } } - # NOTE: we're using __hash__ and __cmp__ methods that consider + # NOTE: we're using __hash__ and ordering methods that consider # _Route53Records equivalent if they have the same class, fqdn, and _type. # Values are ignored. This is useful when computing diffs/changes. @@ -155,17 +157,10 @@ class _Route53Record(object): 'sub-classes should never use this method' return '{}:{}'.format(self.fqdn, self._type).__hash__() - def __cmp__(self, other): - '''sub-classes should call up to this and return its value if non-zero. - When it's zero they should compute their own __cmp__''' - if self.__class__ != other.__class__: - return cmp(self.__class__, other.__class__) - elif self.fqdn != other.fqdn: - return cmp(self.fqdn, other.fqdn) - elif self._type != other._type: - return cmp(self._type, other._type) - # We're ignoring ttl, it's not an actual differentiator - return 0 + def _equality_tuple(self): + '''Sub-classes should call up to this and return its value and add + any additional fields they need to hav considered.''' + return (self.__class__.__name__, self.fqdn, self._type) def __repr__(self): return '_Route53Record<{} {} {} {}>'.format(self.fqdn, self._type, @@ -506,11 +501,9 @@ class _Route53GeoRecord(_Route53Record): return '{}:{}:{}'.format(self.fqdn, self._type, self.geo.code).__hash__() - def __cmp__(self, other): - ret = super(_Route53GeoRecord, self).__cmp__(other) - if ret != 0: - return ret - return cmp(self.geo.code, other.geo.code) + def _equality_tuple(self): + return super(_Route53GeoRecord, self)._equality_tuple() + \ + (self.geo.code,) def __repr__(self): return '_Route53GeoRecord<{} {} {} {} {}>'.format(self.fqdn, @@ -553,7 +546,10 @@ def _mod_keyer(mod): if rrset.get('GeoLocation', False): unique_id = rrset['SetIdentifier'] else: - unique_id = rrset['Name'] + if 'SetIdentifier' in rrset: + unique_id = '{}-{}'.format(rrset['Name'], rrset['SetIdentifier']) + else: + unique_id = rrset['Name'] # Prioritise within the action_priority, ensuring targets come first. if rrset.get('GeoLocation', False): @@ -622,8 +618,9 @@ class Route53Provider(BaseProvider): def __init__(self, id, access_key_id=None, secret_access_key=None, max_changes=1000, client_max_attempts=None, - session_token=None, *args, **kwargs): + session_token=None, delegation_set_id=None, *args, **kwargs): self.max_changes = max_changes + self.delegation_set_id = delegation_set_id _msg = 'access_key_id={}, secret_access_key=***, ' \ 'session_token=***'.format(access_key_id) use_fallback_auth = access_key_id is None and \ @@ -678,10 +675,16 @@ class Route53Provider(BaseProvider): return id if create: ref = uuid4().hex + del_set = self.delegation_set_id self.log.debug('_get_zone_id: no matching zone, creating, ' 'ref=%s', ref) - resp = self._conn.create_hosted_zone(Name=name, - CallerReference=ref) + if del_set: + resp = self._conn.create_hosted_zone(Name=name, + CallerReference=ref, + DelegationSetId=del_set) + else: + resp = self._conn.create_hosted_zone(Name=name, + CallerReference=ref) self.r53_zones[name] = id = resp['HostedZone']['Id'] return id return None @@ -700,7 +703,7 @@ class Route53Provider(BaseProvider): if cc == '*': # This is the default return - cn = cca_to_ctca2(cc) + cn = country_alpha2_to_continent_code(cc) try: return '{}-{}-{}'.format(cn, cc, loc['SubdivisionCode']) except KeyError: @@ -1037,8 +1040,8 @@ class Route53Provider(BaseProvider): # ip_address's returned object for equivalence # E.g 2001:4860:4860::8842 -> 2001:4860:4860:0:0:0:0:8842 if value: - value = ip_address(unicode(value)) - config_ip_address = ip_address(unicode(config['IPAddress'])) + value = ip_address(text_type(value)) + config_ip_address = ip_address(text_type(config['IPAddress'])) else: # No value so give this a None to match value's config_ip_address = None @@ -1059,7 +1062,7 @@ class Route53Provider(BaseProvider): fqdn, record._type, value) try: - ip_address(unicode(value)) + ip_address(text_type(value)) # We're working with an IP, host is the Host header healthcheck_host = record.healthcheck_host except (AddressValueError, ValueError): diff --git a/octodns/provider/transip.py b/octodns/provider/transip.py new file mode 100644 index 0000000..7458e36 --- /dev/null +++ b/octodns/provider/transip.py @@ -0,0 +1,353 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from suds import WebFault + +from collections import defaultdict +from .base import BaseProvider +from logging import getLogger +from ..record import Record +from transip.service.domain import DomainService +from transip.service.objects import DnsEntry + + +class TransipException(Exception): + pass + + +class TransipConfigException(TransipException): + pass + + +class TransipNewZoneException(TransipException): + pass + + +class TransipProvider(BaseProvider): + ''' + Transip DNS provider + + transip: + class: octodns.provider.transip.TransipProvider + # Your Transip account name (required) + account: yourname + # Path to a private key file (required if key is not used) + key_file: /path/to/file + # The api key as string (required if key_file is not used) + key: | + \''' + -----BEGIN PRIVATE KEY----- + ... + -----END PRIVATE KEY----- + \''' + # if both `key_file` and `key` are presented `key_file` is used + + ''' + SUPPORTS_GEO = False + SUPPORTS_DYNAMIC = False + SUPPORTS = set( + ('A', 'AAAA', 'CNAME', 'MX', 'SRV', 'SPF', 'TXT', 'SSHFP', 'CAA')) + # unsupported by OctoDNS: 'TLSA' + MIN_TTL = 120 + TIMEOUT = 15 + ROOT_RECORD = '@' + + def __init__(self, id, account, key=None, key_file=None, *args, **kwargs): + self.log = getLogger('TransipProvider[{}]'.format(id)) + self.log.debug('__init__: id=%s, account=%s, token=***', id, + account) + super(TransipProvider, self).__init__(id, *args, **kwargs) + + if key_file is not None: + self._client = DomainService(account, private_key_file=key_file) + elif key is not None: + self._client = DomainService(account, private_key=key) + else: + raise TransipConfigException( + 'Missing `key` of `key_file` parameter in config' + ) + + self.account = account + self.key = key + + self._currentZone = {} + + def populate(self, zone, target=False, lenient=False): + + exists = False + self._currentZone = zone + self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, + target, lenient) + + before = len(zone.records) + try: + zoneInfo = self._client.get_info(zone.name[:-1]) + except WebFault as e: + if e.fault.faultcode == '102' and target is False: + # Zone not found in account, and not a target so just + # leave an empty zone. + return exists + elif e.fault.faultcode == '102' and target is True: + self.log.warning('populate: Transip can\'t create new zones') + raise TransipNewZoneException( + ('populate: ({}) Transip used ' + + 'as target for non-existing zone: {}').format( + e.fault.faultcode, zone.name)) + else: + self.log.error('populate: (%s) %s ', e.fault.faultcode, + e.fault.faultstring) + raise e + + self.log.debug('populate: found %s records for zone %s', + len(zoneInfo.dnsEntries), zone.name) + exists = True + if zoneInfo.dnsEntries: + values = defaultdict(lambda: defaultdict(list)) + for record in zoneInfo.dnsEntries: + name = zone.hostname_from_fqdn(record['name']) + if name == self.ROOT_RECORD: + name = '' + + if record['type'] in self.SUPPORTS: + values[name][record['type']].append(record) + + for name, types in values.items(): + for _type, records in types.items(): + data_for = getattr(self, '_data_for_{}'.format(_type)) + record = Record.new(zone, name, data_for(_type, records), + source=self, lenient=lenient) + zone.add_record(record, lenient=lenient) + self.log.info('populate: found %s records, exists = %s', + len(zone.records) - before, exists) + + self._currentZone = {} + return exists + + def _apply(self, plan): + desired = plan.desired + changes = plan.changes + self.log.debug('apply: zone=%s, changes=%d', desired.name, + len(changes)) + + self._currentZone = plan.desired + try: + self._client.get_info(plan.desired.name[:-1]) + except WebFault as e: + self.log.exception('_apply: get_info failed') + raise e + + _dns_entries = [] + for record in plan.desired.records: + if record._type in self.SUPPORTS: + entries_for = getattr(self, + '_entries_for_{}'.format(record._type)) + + # Root records have '@' as name + name = record.name + if name == '': + name = self.ROOT_RECORD + + _dns_entries.extend(entries_for(name, record)) + + try: + self._client.set_dns_entries(plan.desired.name[:-1], _dns_entries) + except WebFault as e: + self.log.warning(('_apply: Set DNS returned ' + + 'one or more errors: {}').format( + e.fault.faultstring)) + raise TransipException(200, e.fault.faultstring) + + self._currentZone = {} + + def _entries_for_multiple(self, name, record): + _entries = [] + + for value in record.values: + _entries.append(DnsEntry(name, record.ttl, record._type, value)) + + return _entries + + def _entries_for_single(self, name, record): + + return [DnsEntry(name, record.ttl, record._type, record.value)] + + _entries_for_A = _entries_for_multiple + _entries_for_AAAA = _entries_for_multiple + _entries_for_NS = _entries_for_multiple + _entries_for_SPF = _entries_for_multiple + _entries_for_CNAME = _entries_for_single + + def _entries_for_MX(self, name, record): + _entries = [] + + for value in record.values: + content = "{} {}".format(value.preference, value.exchange) + _entries.append(DnsEntry(name, record.ttl, record._type, content)) + + return _entries + + def _entries_for_SRV(self, name, record): + _entries = [] + + for value in record.values: + content = "{} {} {} {}".format(value.priority, value.weight, + value.port, value.target) + _entries.append(DnsEntry(name, record.ttl, record._type, content)) + + return _entries + + def _entries_for_SSHFP(self, name, record): + _entries = [] + + for value in record.values: + content = "{} {} {}".format(value.algorithm, + value.fingerprint_type, + value.fingerprint) + _entries.append(DnsEntry(name, record.ttl, record._type, content)) + + return _entries + + def _entries_for_CAA(self, name, record): + _entries = [] + + for value in record.values: + content = "{} {} {}".format(value.flags, value.tag, + value.value) + _entries.append(DnsEntry(name, record.ttl, record._type, content)) + + return _entries + + def _entries_for_TXT(self, name, record): + _entries = [] + + for value in record.values: + value = value.replace('\\;', ';') + _entries.append(DnsEntry(name, record.ttl, record._type, value)) + + return _entries + + def _parse_to_fqdn(self, value): + + # Enforce switch from suds.sax.text.Text to string + value = str(value) + + # TransIP allows '@' as value to alias the root record. + # this provider won't set an '@' value, but can be an existing record + if value == self.ROOT_RECORD: + value = self._currentZone.name + + if value[-1] != '.': + self.log.debug('parseToFQDN: changed %s to %s', value, + '{}.{}'.format(value, self._currentZone.name)) + value = '{}.{}'.format(value, self._currentZone.name) + + return value + + def _get_lowest_ttl(self, records): + _ttl = 100000 + for record in records: + _ttl = min(_ttl, record['expire']) + return _ttl + + def _data_for_multiple(self, _type, records): + + _values = [] + for record in records: + # Enforce switch from suds.sax.text.Text to string + _values.append(str(record['content'])) + + return { + 'ttl': self._get_lowest_ttl(records), + 'type': _type, + 'values': _values + } + + _data_for_A = _data_for_multiple + _data_for_AAAA = _data_for_multiple + _data_for_NS = _data_for_multiple + _data_for_SPF = _data_for_multiple + + def _data_for_CNAME(self, _type, records): + return { + 'ttl': records[0]['expire'], + 'type': _type, + 'value': self._parse_to_fqdn(records[0]['content']) + } + + def _data_for_MX(self, _type, records): + _values = [] + for record in records: + preference, exchange = record['content'].split(" ", 1) + _values.append({ + 'preference': preference, + 'exchange': self._parse_to_fqdn(exchange) + }) + return { + 'ttl': self._get_lowest_ttl(records), + 'type': _type, + 'values': _values + } + + def _data_for_SRV(self, _type, records): + _values = [] + for record in records: + priority, weight, port, target = record['content'].split(' ', 3) + _values.append({ + 'port': port, + 'priority': priority, + 'target': self._parse_to_fqdn(target), + 'weight': weight + }) + + return { + 'type': _type, + 'ttl': self._get_lowest_ttl(records), + 'values': _values + } + + def _data_for_SSHFP(self, _type, records): + _values = [] + for record in records: + algorithm, fp_type, fingerprint = record['content'].split(' ', 2) + _values.append({ + 'algorithm': algorithm, + 'fingerprint': fingerprint.lower(), + 'fingerprint_type': fp_type + }) + + return { + 'type': _type, + 'ttl': self._get_lowest_ttl(records), + 'values': _values + } + + def _data_for_CAA(self, _type, records): + _values = [] + for record in records: + flags, tag, value = record['content'].split(' ', 2) + _values.append({ + 'flags': flags, + 'tag': tag, + 'value': value + }) + + return { + 'type': _type, + 'ttl': self._get_lowest_ttl(records), + 'values': _values + } + + def _data_for_TXT(self, _type, records): + _values = [] + for record in records: + _values.append(record['content'].replace(';', '\\;')) + + return { + 'type': _type, + 'ttl': self._get_lowest_ttl(records), + 'values': _values + } diff --git a/octodns/provider/yaml.py b/octodns/provider/yaml.py index 966e96e..10add5a 100644 --- a/octodns/provider/yaml.py +++ b/octodns/provider/yaml.py @@ -28,7 +28,79 @@ class YamlProvider(BaseProvider): default_ttl: 3600 # Whether or not to enforce sorting order on the yaml config # (optional, default True) - enforce_order: True + enforce_order: true + # Whether duplicate records should replace rather than error + # (optiona, default False) + populate_should_replace: false + + Overriding values can be accomplished using multiple yaml providers in the + `sources` list where subsequent providers have `populate_should_replace` + set to `true`. An example use of this would be a zone that you want to push + to external DNS providers and internally, but you want to modify some of + the records in the internal version. + + config/octodns.com.yaml + --- + other: + type: A + values: + - 192.30.252.115 + - 192.30.252.116 + www: + type: A + values: + - 192.30.252.113 + - 192.30.252.114 + + + internal/octodns.com.yaml + --- + 'www': + type: A + values: + - 10.0.0.12 + - 10.0.0.13 + + external.yaml + --- + providers: + config: + class: octodns.provider.yaml.YamlProvider + directory: ./config + + zones: + + octodns.com.: + sources: + - config + targets: + - route53 + + internal.yaml + --- + providers: + config: + class: octodns.provider.yaml.YamlProvider + directory: ./config + + internal: + class: octodns.provider.yaml.YamlProvider + directory: ./internal + populate_should_replace: true + + zones: + + octodns.com.: + sources: + - config + - internal + targets: + - pdns + + You can then sync our records eternally with `--config-file=external.yaml` + and internally (with the custom overrides) with + `--config-file=internal.yaml` + ''' SUPPORTS_GEO = True SUPPORTS_DYNAMIC = True @@ -36,16 +108,18 @@ class YamlProvider(BaseProvider): 'PTR', 'SSHFP', 'SPF', 'SRV', 'TXT')) def __init__(self, id, directory, default_ttl=3600, enforce_order=True, - *args, **kwargs): + populate_should_replace=False, *args, **kwargs): self.log = logging.getLogger('{}[{}]'.format( self.__class__.__name__, id)) self.log.debug('__init__: id=%s, directory=%s, default_ttl=%d, ' - 'enforce_order=%d', id, directory, default_ttl, - enforce_order) + 'enforce_order=%d, populate_should_replace=%d', + id, directory, default_ttl, enforce_order, + populate_should_replace) super(YamlProvider, self).__init__(id, *args, **kwargs) self.directory = directory self.default_ttl = default_ttl self.enforce_order = enforce_order + self.populate_should_replace = populate_should_replace def _populate_from_file(self, filename, zone, lenient): with open(filename, 'r') as fh: @@ -59,9 +133,10 @@ class YamlProvider(BaseProvider): d['ttl'] = self.default_ttl record = Record.new(zone, name, d, source=self, lenient=lenient) - zone.add_record(record, lenient=lenient) - self.log.debug( - '_populate_from_file: successfully loaded "%s"', filename) + zone.add_record(record, lenient=lenient, + replace=self.populate_should_replace) + self.log.debug('_populate_from_file: successfully loaded "%s"', + filename) def populate(self, zone, target=False, lenient=False): self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, diff --git a/octodns/record/__init__.py b/octodns/record/__init__.py index 83632bc..98c1836 100644 --- a/octodns/record/__init__.py +++ b/octodns/record/__init__.py @@ -9,6 +9,9 @@ from ipaddress import IPv4Address, IPv6Address from logging import getLogger import re +from six import string_types, text_type + +from ..equality import EqualityTupleMixin from .geo import GeoCodes @@ -23,6 +26,12 @@ class Change(object): 'Returns new if we have one, existing otherwise' return self.new or self.existing + def __lt__(self, other): + self_record = self.record + other_record = other.record + return ((self_record.name, self_record._type) < + (other_record.name, other_record._type)) + class Create(Change): @@ -68,11 +77,12 @@ class ValidationError(Exception): self.reasons = reasons -class Record(object): +class Record(EqualityTupleMixin): log = getLogger('Record') @classmethod def new(cls, zone, name, data, source=None, lenient=False): + name = text_type(name) fqdn = '{}.{}'.format(name, zone.name) if name else zone.name try: _type = data['type'] @@ -96,7 +106,7 @@ class Record(object): }[_type] except KeyError: raise Exception('Unknown record type: "{}"'.format(_type)) - reasons = _class.validate(name, data) + reasons = _class.validate(name, fqdn, data) try: lenient |= data['octodns']['lenient'] except KeyError: @@ -109,8 +119,16 @@ class Record(object): return _class(zone, name, data, source=source) @classmethod - def validate(cls, name, data): + def validate(cls, name, fqdn, data): reasons = [] + n = len(fqdn) + if n > 253: + reasons.append('invalid fqdn, "{}" is too long at {} chars, max ' + 'is 253'.format(fqdn, n)) + n = len(name) + if n > 63: + reasons.append('invalid name, "{}" is too long at {} chars, max ' + 'is 63'.format(name, n)) try: ttl = int(data['ttl']) if ttl < 0: @@ -130,7 +148,7 @@ class Record(object): self.__class__.__name__, name) self.zone = zone # force everything lower-case just to be safe - self.name = unicode(name).lower() if name else name + self.name = text_type(name).lower() if name else name self.source = source self.ttl = int(data['ttl']) @@ -194,24 +212,22 @@ class Record(object): if self.ttl != other.ttl: return Update(self, other) - # NOTE: we're using __hash__ and __cmp__ methods that consider Records + # NOTE: we're using __hash__ and ordering methods that consider Records # equivalent if they have the same name & _type. Values are ignored. This # is useful when computing diffs/changes. def __hash__(self): return '{}:{}'.format(self.name, self._type).__hash__() - def __cmp__(self, other): - a = '{}:{}'.format(self.name, self._type) - b = '{}:{}'.format(other.name, other._type) - return cmp(a, b) + def _equality_tuple(self): + return (self.name, self._type) def __repr__(self): # Make sure this is always overridden raise NotImplementedError('Abstract base class, __repr__ required') -class GeoValue(object): +class GeoValue(EqualityTupleMixin): geo_re = re.compile(r'^(?P\w\w)(-(?P\w\w)' r'(-(?P\w\w))?)?$') @@ -238,11 +254,9 @@ class GeoValue(object): yield '-'.join(bits) bits.pop() - def __cmp__(self, other): - return 0 if (self.continent_code == other.continent_code and - self.country_code == other.country_code and - self.subdivision_code == other.subdivision_code and - self.values == other.values) else 1 + def _equality_tuple(self): + return (self.continent_code, self.country_code, self.subdivision_code, + self.values) def __repr__(self): return "'Geo {} {} {} {}'".format(self.continent_code, @@ -253,8 +267,8 @@ class GeoValue(object): class _ValuesMixin(object): @classmethod - def validate(cls, name, data): - reasons = super(_ValuesMixin, cls).validate(name, data) + def validate(cls, name, fqdn, data): + reasons = super(_ValuesMixin, cls).validate(name, fqdn, data) values = data.get('values', data.get('value', [])) @@ -268,7 +282,6 @@ class _ValuesMixin(object): values = data['values'] except KeyError: values = [data['value']] - # TODO: should we natsort values? self.values = sorted(self._value_type.process(values)) def changes(self, other, target): @@ -292,7 +305,7 @@ class _ValuesMixin(object): return ret def __repr__(self): - values = "['{}']".format("', '".join([unicode(v) + values = "['{}']".format("', '".join([text_type(v) for v in self.values])) return '<{} {} {}, {}, {}>'.format(self.__class__.__name__, self._type, self.ttl, @@ -307,8 +320,8 @@ class _GeoMixin(_ValuesMixin): ''' @classmethod - def validate(cls, name, data): - reasons = super(_GeoMixin, cls).validate(name, data) + def validate(cls, name, fqdn, data): + reasons = super(_GeoMixin, cls).validate(name, fqdn, data) try: geo = dict(data['geo']) for code, values in geo.items(): @@ -354,8 +367,8 @@ class _GeoMixin(_ValuesMixin): class _ValueMixin(object): @classmethod - def validate(cls, name, data): - reasons = super(_ValueMixin, cls).validate(name, data) + def validate(cls, name, fqdn, data): + reasons = super(_ValueMixin, cls).validate(name, fqdn, data) reasons.extend(cls._value_type.validate(data.get('value', None), cls._type)) return reasons @@ -481,8 +494,8 @@ class _DynamicMixin(object): r'(-(?P\w\w))?)?$') @classmethod - def validate(cls, name, data): - reasons = super(_DynamicMixin, cls).validate(name, data) + def validate(cls, name, fqdn, data): + reasons = super(_DynamicMixin, cls).validate(name, fqdn, data) if 'dynamic' not in data: return reasons @@ -574,7 +587,7 @@ class _DynamicMixin(object): reasons.append('rule {} missing pool'.format(rule_num)) continue - if not isinstance(pool, basestring): + if not isinstance(pool, string_types): reasons.append('rule {} invalid pool "{}"' .format(rule_num, pool)) elif pool not in pools: @@ -671,13 +684,13 @@ class _IpList(object): return ['missing value(s)'] reasons = [] for value in data: - if value is '': + if value == '': reasons.append('empty value') elif value is None: reasons.append('missing value(s)') else: try: - cls._address_type(unicode(value)) + cls._address_type(text_type(value)) except Exception: reasons.append('invalid {} address "{}"' .format(cls._address_name, value)) @@ -685,7 +698,8 @@ class _IpList(object): @classmethod def process(cls, values): - return values + # Translating None into '' so that the list will be sortable in python3 + return [v if v is not None else '' for v in values] class Ipv4List(_IpList): @@ -742,7 +756,7 @@ class AliasRecord(_ValueMixin, Record): _value_type = AliasValue -class CaaValue(object): +class CaaValue(EqualityTupleMixin): # https://tools.ietf.org/html/rfc6844#page-5 @classmethod @@ -781,12 +795,8 @@ class CaaValue(object): 'value': self.value, } - def __cmp__(self, other): - if self.flags == other.flags: - if self.tag == other.tag: - return cmp(self.value, other.value) - return cmp(self.tag, other.tag) - return cmp(self.flags, other.flags) + def _equality_tuple(self): + return (self.flags, self.tag, self.value) def __repr__(self): return '{} {} "{}"'.format(self.flags, self.tag, self.value) @@ -802,15 +812,15 @@ class CnameRecord(_DynamicMixin, _ValueMixin, Record): _value_type = CnameValue @classmethod - def validate(cls, name, data): + def validate(cls, name, fqdn, data): reasons = [] if name == '': reasons.append('root CNAME not allowed') - reasons.extend(super(CnameRecord, cls).validate(name, data)) + reasons.extend(super(CnameRecord, cls).validate(name, fqdn, data)) return reasons -class MxValue(object): +class MxValue(EqualityTupleMixin): @classmethod def validate(cls, data, _type): @@ -863,10 +873,11 @@ class MxValue(object): 'exchange': self.exchange, } - def __cmp__(self, other): - if self.preference == other.preference: - return cmp(self.exchange, other.exchange) - return cmp(self.preference, other.preference) + def __hash__(self): + return hash((self.preference, self.exchange)) + + def _equality_tuple(self): + return (self.preference, self.exchange) def __repr__(self): return "'{} {}'".format(self.preference, self.exchange) @@ -877,7 +888,7 @@ class MxRecord(_ValuesMixin, Record): _value_type = MxValue -class NaptrValue(object): +class NaptrValue(EqualityTupleMixin): VALID_FLAGS = ('S', 'A', 'U', 'P') @classmethod @@ -936,18 +947,12 @@ class NaptrValue(object): 'replacement': self.replacement, } - def __cmp__(self, other): - if self.order != other.order: - return cmp(self.order, other.order) - elif self.preference != other.preference: - return cmp(self.preference, other.preference) - elif self.flags != other.flags: - return cmp(self.flags, other.flags) - elif self.service != other.service: - return cmp(self.service, other.service) - elif self.regexp != other.regexp: - return cmp(self.regexp, other.regexp) - return cmp(self.replacement, other.replacement) + def __hash__(self): + return hash(self.__repr__()) + + def _equality_tuple(self): + return (self.order, self.preference, self.flags, self.service, + self.regexp, self.replacement) def __repr__(self): flags = self.flags if self.flags is not None else '' @@ -997,7 +1002,7 @@ class PtrRecord(_ValueMixin, Record): _value_type = PtrValue -class SshfpValue(object): +class SshfpValue(EqualityTupleMixin): VALID_ALGORITHMS = (1, 2, 3, 4) VALID_FINGERPRINT_TYPES = (1, 2) @@ -1048,12 +1053,11 @@ class SshfpValue(object): 'fingerprint': self.fingerprint, } - def __cmp__(self, other): - if self.algorithm != other.algorithm: - return cmp(self.algorithm, other.algorithm) - elif self.fingerprint_type != other.fingerprint_type: - return cmp(self.fingerprint_type, other.fingerprint_type) - return cmp(self.fingerprint, other.fingerprint) + def __hash__(self): + return hash(self.__repr__()) + + def _equality_tuple(self): + return (self.algorithm, self.fingerprint_type, self.fingerprint) def __repr__(self): return "'{} {} {}'".format(self.algorithm, self.fingerprint_type, @@ -1114,7 +1118,7 @@ class SpfRecord(_ChunkedValuesMixin, Record): _value_type = _ChunkedValue -class SrvValue(object): +class SrvValue(EqualityTupleMixin): @classmethod def validate(cls, data, _type): @@ -1169,14 +1173,11 @@ class SrvValue(object): 'target': self.target, } - def __cmp__(self, other): - if self.priority != other.priority: - return cmp(self.priority, other.priority) - elif self.weight != other.weight: - return cmp(self.weight, other.weight) - elif self.port != other.port: - return cmp(self.port, other.port) - return cmp(self.target, other.target) + def __hash__(self): + return hash(self.__repr__()) + + def _equality_tuple(self): + return (self.priority, self.weight, self.port, self.target) def __repr__(self): return "'{} {} {} {}'".format(self.priority, self.weight, self.port, @@ -1189,11 +1190,11 @@ class SrvRecord(_ValuesMixin, Record): _name_re = re.compile(r'^_[^\.]+\.[^\.]+') @classmethod - def validate(cls, name, data): + def validate(cls, name, fqdn, data): reasons = [] if not cls._name_re.match(name): reasons.append('invalid name') - reasons.extend(super(SrvRecord, cls).validate(name, data)) + reasons.extend(super(SrvRecord, cls).validate(name, fqdn, data)) return reasons diff --git a/octodns/source/axfr.py b/octodns/source/axfr.py index f35c4b3..be2acf5 100644 --- a/octodns/source/axfr.py +++ b/octodns/source/axfr.py @@ -15,6 +15,7 @@ from dns.exception import DNSException from collections import defaultdict from os import listdir from os.path import join +from six import text_type import logging from ..record import Record @@ -179,8 +180,7 @@ class ZoneFileSourceNotFound(ZoneFileSourceException): class ZoneFileSourceLoadFailure(ZoneFileSourceException): def __init__(self, error): - super(ZoneFileSourceLoadFailure, self).__init__( - error.message) + super(ZoneFileSourceLoadFailure, self).__init__(text_type(error)) class ZoneFileSource(AxfrBaseSource): diff --git a/octodns/source/tinydns.py b/octodns/source/tinydns.py index dc2bc1b..9c44ed8 100755 --- a/octodns/source/tinydns.py +++ b/octodns/source/tinydns.py @@ -67,7 +67,8 @@ class TinyDnsBaseSource(BaseSource): values = [] for record in records: - new_value = record[0].decode('unicode-escape').replace(";", "\\;") + new_value = record[0].encode('latin1').decode('unicode-escape') \ + .replace(";", "\\;") values.append(new_value) try: @@ -252,7 +253,7 @@ class TinyDnsFileSource(TinyDnsBaseSource): # Ignore hidden files continue with open(join(self.directory, filename), 'r') as fh: - lines += filter(lambda l: l, fh.read().split('\n')) + lines += [l for l in fh.read().split('\n') if l] self._cache = lines diff --git a/octodns/yaml.py b/octodns/yaml.py index 98bafdb..4187199 100644 --- a/octodns/yaml.py +++ b/octodns/yaml.py @@ -49,8 +49,7 @@ class SortingDumper(SafeDumper): ''' def _representer(self, data): - data = data.items() - data.sort(key=lambda d: _natsort_key(d[0])) + data = sorted(data.items(), key=lambda d: _natsort_key(d[0])) return self.represent_mapping(self.DEFAULT_MAPPING_TAG, data) diff --git a/octodns/zone.py b/octodns/zone.py index 916f81b..5f099ac 100644 --- a/octodns/zone.py +++ b/octodns/zone.py @@ -9,6 +9,8 @@ from collections import defaultdict from logging import getLogger import re +from six import text_type + from .record import Create, Delete @@ -38,7 +40,7 @@ class Zone(object): raise Exception('Invalid zone name {}, missing ending dot' .format(name)) # Force everything to lowercase just to be safe - self.name = unicode(name).lower() if name else name + self.name = text_type(name).lower() if name else name self.sub_zones = sub_zones # We're grouping by node, it allows us to efficiently search for # duplicates and detect when CNAMEs co-exist with other records @@ -82,8 +84,8 @@ class Zone(object): raise DuplicateRecordException('Duplicate record {}, type {}' .format(record.fqdn, record._type)) - elif not lenient and (((record._type == 'CNAME' and len(node) > 0) or - ('CNAME' in map(lambda r: r._type, node)))): + elif not lenient and ((record._type == 'CNAME' and len(node) > 0) or + ('CNAME' in [r._type for r in node])): # We're adding a CNAME to existing records or adding to an existing # CNAME raise InvalidNodeException('Invalid state, CNAME at {} cannot ' diff --git a/requirements-dev.txt b/requirements-dev.txt index 77dd50c..3ad1b04 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,9 +1,8 @@ coverage mock nose -pycodestyle==2.4.0 -pycountry>=18.12.8 -pycountry_convert>=0.7.2 -pyflakes==1.6.0 +pycodestyle==2.5.0 +pyflakes==2.1.1 +readme_renderer[md]==24.0 requests_mock -twine==1.13.0 +twine==1.15.0 diff --git a/requirements.txt b/requirements.txt index 75dc1df..eadad34 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,24 +1,26 @@ -PyYaml==4.2b1 -azure-common==1.1.18 -azure-mgmt-dns==2.1.0 -boto3==1.7.5 -botocore==1.10.5 -dnspython==1.15.0 -docutils==0.14 +PyYaml==5.3 +azure-common==1.1.24 +azure-mgmt-dns==3.0.0 +boto3==1.11.9 +botocore==1.14.9 +dnspython==1.16.0 +docutils==0.16 dyn==1.8.1 edgegrid-python==1.1.1 -futures==3.2.0 -google-cloud-core==0.28.1 -google-cloud-dns==0.29.0 -incf.countryutils==1.0 -ipaddress==1.0.22 -jmespath==0.9.3 -msrestazure==0.6.0 -natsort==5.5.0 -nsone==0.9.100 -ovh==0.4.8 -python-dateutil==2.6.1 -requests==2.20.0 -s3transfer==0.1.13 -six==1.11.0 -setuptools==38.5.2 +futures==3.2.0; python_version < '3.0' +google-cloud-core==1.3.0 +google-cloud-dns==0.31.0 +ipaddress==1.0.23 +jmespath==0.9.4 +msrestazure==0.6.2 +natsort==6.2.1 +ns1-python==0.13.0 +ovh==0.5.0 +pycountry-convert==0.7.2 +pycountry==19.8.18 +python-dateutil==2.8.1 +requests==2.22.0 +s3transfer==0.3.2 +setuptools==44.0.0 +six==1.14.0 +transip==2.0.0 diff --git a/script/cibuild b/script/cibuild index d048e8e..a2dc527 100755 --- a/script/cibuild +++ b/script/cibuild @@ -27,4 +27,6 @@ echo "## lint ################################################################## script/lint echo "## tests/coverage ##############################################################" script/coverage +echo "## validate setup.py build #####################################################" +python setup.py build echo "## complete ####################################################################" diff --git a/script/coverage b/script/coverage index 8552eba..ad8189e 100755 --- a/script/coverage +++ b/script/coverage @@ -29,7 +29,7 @@ export GOOGLE_APPLICATION_CREDENTIALS= coverage run --branch --source=octodns --omit=octodns/cmds/* "$(command -v nosetests)" --with-xunit "$@" coverage html coverage xml -coverage report +coverage report --show-missing coverage report | grep ^TOTAL | grep -qv 100% && { echo "Incomplete code coverage" >&2 exit 1 diff --git a/script/release b/script/release index dd3e1b1..f2c90bf 100755 --- a/script/release +++ b/script/release @@ -22,5 +22,6 @@ git tag -s "v$VERSION" -m "Release $VERSION" git push origin "v$VERSION" echo "Tagged and pushed v$VERSION" python setup.py sdist +twine check dist/*$VERSION.tar.gz twine upload dist/*$VERSION.tar.gz echo "Uploaded $VERSION" diff --git a/setup.py b/setup.py index 75a39d7..c56aa82 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,9 @@ #!/usr/bin/env python +try: + from StringIO import StringIO +except ImportError: + from io import StringIO from os.path import dirname, join import octodns @@ -21,6 +25,39 @@ console_scripts = { for name in cmds } + +def long_description(): + buf = StringIO() + yaml_block = False + supported_providers = False + with open('README.md') as fh: + for line in fh: + if line == '```yaml\n': + yaml_block = True + continue + elif yaml_block and line == '---\n': + # skip the line + continue + elif yaml_block and line == '```\n': + yaml_block = False + continue + elif supported_providers: + if line.startswith('## '): + supported_providers = False + # write this line out, no continue + else: + # We're ignoring this one + continue + elif line == '## Supported providers\n': + supported_providers = True + continue + buf.write(line) + buf = buf.getvalue() + with open('/tmp/mod', 'w') as fh: + fh.write(buf) + return buf + + setup( author='Ross McFarland', author_email='rwmcfa1@gmail.com', @@ -31,16 +68,16 @@ setup( install_requires=[ 'PyYaml>=4.2b1', 'dnspython>=1.15.0', - 'futures>=3.2.0', - 'incf.countryutils>=1.0', + 'futures>=3.2.0; python_version<"3.2"', 'ipaddress>=1.0.22', 'natsort>=5.5.0', - # botocore doesn't like >=2.7.0 for some reason - 'python-dateutil>=2.6.0,<2.7.0', + 'pycountry>=19.8.18', + 'pycountry-convert>=0.7.2', + 'python-dateutil>=2.8.1', 'requests>=2.20.0' ], license='MIT', - long_description=open('README.md').read(), + long_description=long_description(), long_description_content_type='text/markdown', name='octodns', packages=find_packages(), diff --git a/tests/config/override/dynamic.tests.yaml b/tests/config/override/dynamic.tests.yaml new file mode 100644 index 0000000..d79e092 --- /dev/null +++ b/tests/config/override/dynamic.tests.yaml @@ -0,0 +1,13 @@ +--- +# Replace 'a' with a generic record +a: + type: A + values: + - 4.4.4.4 + - 5.5.5.5 +# Add another record +added: + type: A + values: + - 6.6.6.6 + - 7.7.7.7 diff --git a/tests/config/provider-problems.yaml b/tests/config/provider-problems.yaml new file mode 100644 index 0000000..9071046 --- /dev/null +++ b/tests/config/provider-problems.yaml @@ -0,0 +1,28 @@ +providers: + yaml: + class: octodns.provider.yaml.YamlProvider + directory: ./config + simple_source: + class: helpers.SimpleSource +zones: + missing.sources.: + targets: + - yaml + missing.targets.: + sources: + - yaml + unknown.source.: + sources: + - not-there + targets: + - yaml + unknown.target.: + sources: + - yaml + targets: + - not-there-either + not.targetable.: + sources: + - yaml + targets: + - simple_source diff --git a/tests/config/unknown-provider.yaml b/tests/config/unknown-provider.yaml index 9071046..a0e9f55 100644 --- a/tests/config/unknown-provider.yaml +++ b/tests/config/unknown-provider.yaml @@ -5,24 +5,8 @@ providers: simple_source: class: helpers.SimpleSource zones: - missing.sources.: - targets: - - yaml - missing.targets.: - sources: - - yaml unknown.source.: sources: - not-there targets: - yaml - unknown.target.: - sources: - - yaml - targets: - - not-there-either - not.targetable.: - sources: - - yaml - targets: - - simple_source diff --git a/tests/test_octodns_equality.py b/tests/test_octodns_equality.py new file mode 100644 index 0000000..dcdc460 --- /dev/null +++ b/tests/test_octodns_equality.py @@ -0,0 +1,68 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from unittest import TestCase + +from octodns.equality import EqualityTupleMixin + + +class TestEqualityTupleMixin(TestCase): + + def test_basics(self): + + class Simple(EqualityTupleMixin): + + def __init__(self, a, b, c): + self.a = a + self.b = b + self.c = c + + def _equality_tuple(self): + return (self.a, self.b) + + one = Simple(1, 2, 3) + same = Simple(1, 2, 3) + matches = Simple(1, 2, 'ignored') + doesnt = Simple(2, 3, 4) + + # equality + self.assertEquals(one, one) + self.assertEquals(one, same) + self.assertEquals(same, one) + # only a & c are considered + self.assertEquals(one, matches) + self.assertEquals(matches, one) + self.assertNotEquals(one, doesnt) + self.assertNotEquals(doesnt, one) + + # lt + self.assertTrue(one < doesnt) + self.assertFalse(doesnt < one) + self.assertFalse(one < same) + + # le + self.assertTrue(one <= doesnt) + self.assertFalse(doesnt <= one) + self.assertTrue(one <= same) + + # gt + self.assertFalse(one > doesnt) + self.assertTrue(doesnt > one) + self.assertFalse(one > same) + + # ge + self.assertFalse(one >= doesnt) + self.assertTrue(doesnt >= one) + self.assertTrue(one >= same) + + def test_not_implemented(self): + + class MissingMethod(EqualityTupleMixin): + pass + + with self.assertRaises(NotImplementedError): + MissingMethod() == MissingMethod() diff --git a/tests/test_octodns_manager.py b/tests/test_octodns_manager.py index 0dd3514..13eea95 100644 --- a/tests/test_octodns_manager.py +++ b/tests/test_octodns_manager.py @@ -7,10 +7,12 @@ from __future__ import absolute_import, division, print_function, \ from os import environ from os.path import dirname, join +from six import text_type from unittest import TestCase from octodns.record import Record -from octodns.manager import _AggregateTarget, MainThreadExecutor, Manager +from octodns.manager import _AggregateTarget, MainThreadExecutor, Manager, \ + ManagerException from octodns.yaml import safe_load from octodns.zone import Zone @@ -27,80 +29,81 @@ def get_config_filename(which): class TestManager(TestCase): def test_missing_provider_class(self): - with self.assertRaises(Exception) as ctx: + with self.assertRaises(ManagerException) as ctx: Manager(get_config_filename('missing-provider-class.yaml')).sync() - self.assertTrue('missing class' in ctx.exception.message) + self.assertTrue('missing class' in text_type(ctx.exception)) def test_bad_provider_class(self): - with self.assertRaises(Exception) as ctx: + with self.assertRaises(ManagerException) as ctx: Manager(get_config_filename('bad-provider-class.yaml')).sync() - self.assertTrue('Unknown provider class' in ctx.exception.message) + self.assertTrue('Unknown provider class' in text_type(ctx.exception)) def test_bad_provider_class_module(self): - with self.assertRaises(Exception) as ctx: + with self.assertRaises(ManagerException) as ctx: Manager(get_config_filename('bad-provider-class-module.yaml')) \ .sync() - self.assertTrue('Unknown provider class' in ctx.exception.message) + self.assertTrue('Unknown provider class' in text_type(ctx.exception)) def test_bad_provider_class_no_module(self): - with self.assertRaises(Exception) as ctx: + with self.assertRaises(ManagerException) as ctx: Manager(get_config_filename('bad-provider-class-no-module.yaml')) \ .sync() - self.assertTrue('Unknown provider class' in ctx.exception.message) + self.assertTrue('Unknown provider class' in text_type(ctx.exception)) def test_missing_provider_config(self): # Missing provider config - with self.assertRaises(Exception) as ctx: + with self.assertRaises(ManagerException) as ctx: Manager(get_config_filename('missing-provider-config.yaml')).sync() - self.assertTrue('provider config' in ctx.exception.message) + self.assertTrue('provider config' in text_type(ctx.exception)) def test_missing_env_config(self): - with self.assertRaises(Exception) as ctx: + with self.assertRaises(ManagerException) as ctx: Manager(get_config_filename('missing-provider-env.yaml')).sync() - self.assertTrue('missing env var' in ctx.exception.message) + self.assertTrue('missing env var' in text_type(ctx.exception)) def test_missing_source(self): - with self.assertRaises(Exception) as ctx: - Manager(get_config_filename('unknown-provider.yaml')) \ + with self.assertRaises(ManagerException) as ctx: + Manager(get_config_filename('provider-problems.yaml')) \ .sync(['missing.sources.']) - self.assertTrue('missing sources' in ctx.exception.message) + self.assertTrue('missing sources' in text_type(ctx.exception)) def test_missing_targets(self): - with self.assertRaises(Exception) as ctx: - Manager(get_config_filename('unknown-provider.yaml')) \ + with self.assertRaises(ManagerException) as ctx: + Manager(get_config_filename('provider-problems.yaml')) \ .sync(['missing.targets.']) - self.assertTrue('missing targets' in ctx.exception.message) + self.assertTrue('missing targets' in text_type(ctx.exception)) def test_unknown_source(self): - with self.assertRaises(Exception) as ctx: - Manager(get_config_filename('unknown-provider.yaml')) \ + with self.assertRaises(ManagerException) as ctx: + Manager(get_config_filename('provider-problems.yaml')) \ .sync(['unknown.source.']) - self.assertTrue('unknown source' in ctx.exception.message) + self.assertTrue('unknown source' in text_type(ctx.exception)) def test_unknown_target(self): - with self.assertRaises(Exception) as ctx: - Manager(get_config_filename('unknown-provider.yaml')) \ + with self.assertRaises(ManagerException) as ctx: + Manager(get_config_filename('provider-problems.yaml')) \ .sync(['unknown.target.']) - self.assertTrue('unknown target' in ctx.exception.message) + self.assertTrue('unknown target' in text_type(ctx.exception)) def test_bad_plan_output_class(self): - with self.assertRaises(Exception) as ctx: + with self.assertRaises(ManagerException) as ctx: name = 'bad-plan-output-missing-class.yaml' Manager(get_config_filename(name)).sync() self.assertEquals('plan_output bad is missing class', - ctx.exception.message) + text_type(ctx.exception)) def test_bad_plan_output_config(self): - with self.assertRaises(Exception) as ctx: + with self.assertRaises(ManagerException) as ctx: Manager(get_config_filename('bad-plan-output-config.yaml')).sync() self.assertEqual('Incorrect plan_output config for bad', - ctx.exception.message) + text_type(ctx.exception)) def test_source_only_as_a_target(self): - with self.assertRaises(Exception) as ctx: - Manager(get_config_filename('unknown-provider.yaml')) \ + with self.assertRaises(ManagerException) as ctx: + Manager(get_config_filename('provider-problems.yaml')) \ .sync(['not.targetable.']) - self.assertTrue('does not support targeting' in ctx.exception.message) + self.assertTrue('does not support targeting' in + text_type(ctx.exception)) def test_always_dry_run(self): with TemporaryDirectory() as tmpdir: @@ -180,9 +183,9 @@ class TestManager(TestCase): 'unit.tests.') self.assertEquals(14, len(changes)) - with self.assertRaises(Exception) as ctx: + with self.assertRaises(ManagerException) as ctx: manager.compare(['nope'], ['dump'], 'unit.tests.') - self.assertEquals('Unknown source: nope', ctx.exception.message) + self.assertEquals('Unknown source: nope', text_type(ctx.exception)) def test_aggregate_target(self): simple = SimpleProvider() @@ -220,10 +223,10 @@ class TestManager(TestCase): environ['YAML_TMP_DIR'] = tmpdir.dirname manager = Manager(get_config_filename('simple.yaml')) - with self.assertRaises(Exception) as ctx: + with self.assertRaises(ManagerException) as ctx: manager.dump('unit.tests.', tmpdir.dirname, False, False, 'nope') - self.assertEquals('Unknown source: nope', ctx.exception.message) + self.assertEquals('Unknown source: nope', text_type(ctx.exception)) manager.dump('unit.tests.', tmpdir.dirname, False, False, 'in') @@ -249,10 +252,10 @@ class TestManager(TestCase): environ['YAML_TMP_DIR'] = tmpdir.dirname manager = Manager(get_config_filename('simple-split.yaml')) - with self.assertRaises(Exception) as ctx: + with self.assertRaises(ManagerException) as ctx: manager.dump('unit.tests.', tmpdir.dirname, False, True, 'nope') - self.assertEquals('Unknown source: nope', ctx.exception.message) + self.assertEquals('Unknown source: nope', text_type(ctx.exception)) manager.dump('unit.tests.', tmpdir.dirname, False, True, 'in') @@ -265,15 +268,15 @@ class TestManager(TestCase): def test_validate_configs(self): Manager(get_config_filename('simple-validate.yaml')).validate_configs() - with self.assertRaises(Exception) as ctx: + with self.assertRaises(ManagerException) as ctx: Manager(get_config_filename('missing-sources.yaml')) \ .validate_configs() - self.assertTrue('missing sources' in ctx.exception.message) + self.assertTrue('missing sources' in text_type(ctx.exception)) - with self.assertRaises(Exception) as ctx: + with self.assertRaises(ManagerException) as ctx: Manager(get_config_filename('unknown-provider.yaml')) \ .validate_configs() - self.assertTrue('unknown source' in ctx.exception.message) + self.assertTrue('unknown source' in text_type(ctx.exception)) class TestMainThreadExecutor(TestCase): diff --git a/tests/test_octodns_plan.py b/tests/test_octodns_plan.py index 7d849be..9cf812d 100644 --- a/tests/test_octodns_plan.py +++ b/tests/test_octodns_plan.py @@ -5,8 +5,8 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -from StringIO import StringIO from logging import getLogger +from six import StringIO, text_type from unittest import TestCase from octodns.provider.plan import Plan, PlanHtml, PlanLogger, PlanMarkdown @@ -59,7 +59,7 @@ class TestPlanLogger(TestCase): with self.assertRaises(Exception) as ctx: PlanLogger('invalid', 'not-a-level') self.assertEquals('Unsupported level: not-a-level', - ctx.exception.message) + text_type(ctx.exception)) def test_create(self): diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index 920c502..1769cef 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -321,7 +321,7 @@ class Test_ParseAzureType(TestCase): ['AAAA', 'Microsoft.Network/dnszones/AAAA'], ['NS', 'Microsoft.Network/dnszones/NS'], ['MX', 'Microsoft.Network/dnszones/MX']]: - self.assertEquals(expected, _parse_azure_type(test)) + self.assertEquals(expected, _parse_azure_type(test)) class Test_CheckEndswithDot(TestCase): diff --git a/tests/test_octodns_provider_base.py b/tests/test_octodns_provider_base.py index e28850a..f33db0f 100644 --- a/tests/test_octodns_provider_base.py +++ b/tests/test_octodns_provider_base.py @@ -6,6 +6,7 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals from logging import getLogger +from six import text_type from unittest import TestCase from octodns.record import Create, Delete, Record, Update @@ -48,7 +49,7 @@ class TestBaseProvider(TestCase): with self.assertRaises(NotImplementedError) as ctx: BaseProvider('base') self.assertEquals('Abstract base class, log property missing', - ctx.exception.message) + text_type(ctx.exception)) class HasLog(BaseProvider): log = getLogger('HasLog') @@ -56,7 +57,7 @@ class TestBaseProvider(TestCase): with self.assertRaises(NotImplementedError) as ctx: HasLog('haslog') self.assertEquals('Abstract base class, SUPPORTS_GEO property missing', - ctx.exception.message) + text_type(ctx.exception)) class HasSupportsGeo(HasLog): SUPPORTS_GEO = False @@ -65,14 +66,14 @@ class TestBaseProvider(TestCase): with self.assertRaises(NotImplementedError) as ctx: HasSupportsGeo('hassupportsgeo').populate(zone) self.assertEquals('Abstract base class, SUPPORTS property missing', - ctx.exception.message) + text_type(ctx.exception)) class HasSupports(HasSupportsGeo): SUPPORTS = set(('A',)) with self.assertRaises(NotImplementedError) as ctx: HasSupports('hassupports').populate(zone) self.assertEquals('Abstract base class, populate method missing', - ctx.exception.message) + text_type(ctx.exception)) # SUPPORTS_DYNAMIC has a default/fallback self.assertFalse(HasSupports('hassupports').SUPPORTS_DYNAMIC) @@ -118,7 +119,7 @@ class TestBaseProvider(TestCase): with self.assertRaises(NotImplementedError) as ctx: HasPopulate('haspopulate').apply(plan) self.assertEquals('Abstract base class, _apply method missing', - ctx.exception.message) + text_type(ctx.exception)) def test_plan(self): ignored = Zone('unit.tests.', []) @@ -193,7 +194,7 @@ class TestBaseProvider(TestCase): }) for i in range(int(Plan.MIN_EXISTING_RECORDS)): - zone.add_record(Record.new(zone, unicode(i), { + zone.add_record(Record.new(zone, text_type(i), { 'ttl': 60, 'type': 'A', 'value': '2.3.4.5' @@ -225,7 +226,7 @@ class TestBaseProvider(TestCase): }) for i in range(int(Plan.MIN_EXISTING_RECORDS)): - zone.add_record(Record.new(zone, unicode(i), { + zone.add_record(Record.new(zone, text_type(i), { 'ttl': 60, 'type': 'A', 'value': '2.3.4.5' @@ -238,7 +239,7 @@ class TestBaseProvider(TestCase): with self.assertRaises(UnsafePlan) as ctx: Plan(zone, zone, changes, True).raise_if_unsafe() - self.assertTrue('Too many updates' in ctx.exception.message) + self.assertTrue('Too many updates' in text_type(ctx.exception)) def test_safe_updates_min_existing_pcent(self): # MAX_SAFE_UPDATE_PCENT is safe when more @@ -251,7 +252,7 @@ class TestBaseProvider(TestCase): }) for i in range(int(Plan.MIN_EXISTING_RECORDS)): - zone.add_record(Record.new(zone, unicode(i), { + zone.add_record(Record.new(zone, text_type(i), { 'ttl': 60, 'type': 'A', 'value': '2.3.4.5' @@ -273,7 +274,7 @@ class TestBaseProvider(TestCase): }) for i in range(int(Plan.MIN_EXISTING_RECORDS)): - zone.add_record(Record.new(zone, unicode(i), { + zone.add_record(Record.new(zone, text_type(i), { 'ttl': 60, 'type': 'A', 'value': '2.3.4.5' @@ -286,7 +287,7 @@ class TestBaseProvider(TestCase): with self.assertRaises(UnsafePlan) as ctx: Plan(zone, zone, changes, True).raise_if_unsafe() - self.assertTrue('Too many deletes' in ctx.exception.message) + self.assertTrue('Too many deletes' in text_type(ctx.exception)) def test_safe_deletes_min_existing_pcent(self): # MAX_SAFE_DELETE_PCENT is safe when more @@ -299,7 +300,7 @@ class TestBaseProvider(TestCase): }) for i in range(int(Plan.MIN_EXISTING_RECORDS)): - zone.add_record(Record.new(zone, unicode(i), { + zone.add_record(Record.new(zone, text_type(i), { 'ttl': 60, 'type': 'A', 'value': '2.3.4.5' @@ -322,7 +323,7 @@ class TestBaseProvider(TestCase): }) for i in range(int(Plan.MIN_EXISTING_RECORDS)): - zone.add_record(Record.new(zone, unicode(i), { + zone.add_record(Record.new(zone, text_type(i), { 'ttl': 60, 'type': 'A', 'value': '2.3.4.5' @@ -336,7 +337,7 @@ class TestBaseProvider(TestCase): Plan(zone, zone, changes, True, update_pcent_threshold=safe_pcent).raise_if_unsafe() - self.assertTrue('Too many updates' in ctx.exception.message) + self.assertTrue('Too many updates' in text_type(ctx.exception)) def test_safe_deletes_min_existing_override(self): safe_pcent = .4 @@ -350,7 +351,7 @@ class TestBaseProvider(TestCase): }) for i in range(int(Plan.MIN_EXISTING_RECORDS)): - zone.add_record(Record.new(zone, unicode(i), { + zone.add_record(Record.new(zone, text_type(i), { 'ttl': 60, 'type': 'A', 'value': '2.3.4.5' @@ -364,4 +365,4 @@ class TestBaseProvider(TestCase): Plan(zone, zone, changes, True, delete_pcent_threshold=safe_pcent).raise_if_unsafe() - self.assertTrue('Too many deletes' in ctx.exception.message) + self.assertTrue('Too many deletes' in text_type(ctx.exception)) diff --git a/tests/test_octodns_provider_cloudflare.py b/tests/test_octodns_provider_cloudflare.py index 5c6d503..3581033 100644 --- a/tests/test_octodns_provider_cloudflare.py +++ b/tests/test_octodns_provider_cloudflare.py @@ -9,6 +9,7 @@ from mock import Mock, call from os.path import dirname, join from requests import HTTPError from requests_mock import ANY, mock as requests_mock +from six import text_type from unittest import TestCase from octodns.record import Record, Update @@ -65,7 +66,7 @@ class TestCloudflareProvider(TestCase): provider.populate(zone) self.assertEquals('CloudflareError', type(ctx.exception).__name__) - self.assertEquals('request was invalid', ctx.exception.message) + self.assertEquals('request was invalid', text_type(ctx.exception)) # Bad auth with requests_mock() as mock: @@ -80,7 +81,7 @@ class TestCloudflareProvider(TestCase): self.assertEquals('CloudflareAuthenticationError', type(ctx.exception).__name__) self.assertEquals('Unknown X-Auth-Key or X-Auth-Email', - ctx.exception.message) + text_type(ctx.exception)) # Bad auth, unknown resp with requests_mock() as mock: @@ -91,7 +92,7 @@ class TestCloudflareProvider(TestCase): provider.populate(zone) self.assertEquals('CloudflareAuthenticationError', type(ctx.exception).__name__) - self.assertEquals('Cloudflare error', ctx.exception.message) + self.assertEquals('Cloudflare error', text_type(ctx.exception)) # General error with requests_mock() as mock: @@ -742,23 +743,25 @@ class TestCloudflareProvider(TestCase): # the CDN. self.assertEquals(3, len(zone.records)) - record = list(zone.records)[0] - self.assertEquals('multi', record.name) - self.assertEquals('multi.unit.tests.', record.fqdn) + ordered = sorted(zone.records, key=lambda r: r.name) + + record = ordered[0] + self.assertEquals('a', record.name) + self.assertEquals('a.unit.tests.', record.fqdn) self.assertEquals('CNAME', record._type) - self.assertEquals('multi.unit.tests.cdn.cloudflare.net.', record.value) + self.assertEquals('a.unit.tests.cdn.cloudflare.net.', record.value) - record = list(zone.records)[1] + record = ordered[1] self.assertEquals('cname', record.name) self.assertEquals('cname.unit.tests.', record.fqdn) self.assertEquals('CNAME', record._type) self.assertEquals('cname.unit.tests.cdn.cloudflare.net.', record.value) - record = list(zone.records)[2] - self.assertEquals('a', record.name) - self.assertEquals('a.unit.tests.', record.fqdn) + record = ordered[2] + self.assertEquals('multi', record.name) + self.assertEquals('multi.unit.tests.', record.fqdn) self.assertEquals('CNAME', record._type) - self.assertEquals('a.unit.tests.cdn.cloudflare.net.', record.value) + self.assertEquals('multi.unit.tests.cdn.cloudflare.net.', record.value) # CDN enabled records can't be updated, we don't know the real values # never point a Cloudflare record to itself. @@ -950,7 +953,7 @@ class TestCloudflareProvider(TestCase): 'value': 'ns1.unit.tests.' }) - data = provider._gen_data(record).next() + data = next(provider._gen_data(record)) self.assertFalse('proxied' in data) @@ -965,7 +968,7 @@ class TestCloudflareProvider(TestCase): }), False ) - data = provider._gen_data(record).next() + data = next(provider._gen_data(record)) self.assertFalse(data['proxied']) @@ -980,7 +983,7 @@ class TestCloudflareProvider(TestCase): }), True ) - data = provider._gen_data(record).next() + data = next(provider._gen_data(record)) self.assertTrue(data['proxied']) diff --git a/tests/test_octodns_provider_constellix.py b/tests/test_octodns_provider_constellix.py index 346bb17..151d0d4 100644 --- a/tests/test_octodns_provider_constellix.py +++ b/tests/test_octodns_provider_constellix.py @@ -10,16 +10,15 @@ from mock import Mock, call from os.path import dirname, join from requests import HTTPError from requests_mock import ANY, mock as requests_mock +from six import text_type from unittest import TestCase from octodns.record import Record -from octodns.provider.constellix import ConstellixClientNotFound, \ +from octodns.provider.constellix import \ ConstellixProvider from octodns.provider.yaml import YamlProvider from octodns.zone import Zone -import json - class TestConstellixProvider(TestCase): expected = Zone('unit.tests.', []) @@ -65,7 +64,7 @@ class TestConstellixProvider(TestCase): with self.assertRaises(Exception) as ctx: zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals('Unauthorized', ctx.exception.message) + self.assertEquals('Unauthorized', text_type(ctx.exception)) # Bad request with requests_mock() as mock: @@ -77,7 +76,7 @@ class TestConstellixProvider(TestCase): zone = Zone('unit.tests.', []) provider.populate(zone) self.assertEquals('\n - "unittests" is not a valid domain name', - ctx.exception.message) + text_type(ctx.exception)) # General error with requests_mock() as mock: @@ -101,7 +100,7 @@ class TestConstellixProvider(TestCase): with requests_mock() as mock: base = 'https://api.dns.constellix.com/v1/domains' with open('tests/fixtures/constellix-domains.json') as fh: - mock.get('{}{}'.format(base, '/'), text=fh.read()) + mock.get('{}{}'.format(base, ''), text=fh.read()) with open('tests/fixtures/constellix-records.json') as fh: mock.get('{}{}'.format(base, '/123123/records'), text=fh.read()) @@ -127,15 +126,15 @@ class TestConstellixProvider(TestCase): resp.json = Mock() provider._client._request = Mock(return_value=resp) - with open('tests/fixtures/constellix-domains.json') as fh: - domains = json.load(fh) - # non-existent domain, create everything resp.json.side_effect = [ - ConstellixClientNotFound, # no zone in populate - ConstellixClientNotFound, # no domain during apply - domains + [], # no domains returned during populate + [{ + 'id': 123123, + 'name': 'unit.tests' + }], # domain created in apply ] + plan = provider.plan(self.expected) # No root NS, no ignored, no excluded, no unsupported @@ -144,10 +143,15 @@ class TestConstellixProvider(TestCase): self.assertEquals(n, provider.apply(plan)) provider._client._request.assert_has_calls([ - # created the domain - call('POST', '/', data={'names': ['unit.tests']}), # get all domains to build the cache - call('GET', '/'), + call('GET', ''), + # created the domain + call('POST', '/', data={'names': ['unit.tests']}) + ]) + # These two checks are broken up so that ordering doesn't break things. + # Python3 doesn't make the calls in a consistent order so different + # things follow the GET / on different runs + provider._client._request.assert_has_calls([ call('POST', '/123123/records/SRV', data={ 'roundRobin': [{ 'priority': 10, @@ -165,7 +169,7 @@ class TestConstellixProvider(TestCase): }), ]) - self.assertEquals(20, provider._client._request.call_count) + self.assertEquals(18, provider._client._request.call_count) provider._client._request.reset_mock() @@ -187,6 +191,14 @@ class TestConstellixProvider(TestCase): 'value': [ '3.2.3.4' ] + }, { + 'id': 11189899, + 'type': 'ALIAS', + 'name': 'alias', + 'ttl': 600, + 'value': [{ + 'value': 'aname.unit.tests.' + }] } ]) @@ -201,8 +213,8 @@ class TestConstellixProvider(TestCase): })) plan = provider.plan(wanted) - self.assertEquals(2, len(plan.changes)) - self.assertEquals(2, provider.apply(plan)) + self.assertEquals(3, len(plan.changes)) + self.assertEquals(3, provider.apply(plan)) # recreate for update, and deletes for the 2 parts of the other provider._client._request.assert_has_calls([ @@ -214,5 +226,6 @@ class TestConstellixProvider(TestCase): 'ttl': 300 }), call('DELETE', '/123123/records/A/11189897'), - call('DELETE', '/123123/records/A/11189898') + call('DELETE', '/123123/records/A/11189898'), + call('DELETE', '/123123/records/ANAME/11189899') ], any_order=True) diff --git a/tests/test_octodns_provider_digitalocean.py b/tests/test_octodns_provider_digitalocean.py index fce62b1..ebb5319 100644 --- a/tests/test_octodns_provider_digitalocean.py +++ b/tests/test_octodns_provider_digitalocean.py @@ -10,6 +10,7 @@ from mock import Mock, call from os.path import dirname, join from requests import HTTPError from requests_mock import ANY, mock as requests_mock +from six import text_type from unittest import TestCase from octodns.record import Record @@ -50,7 +51,7 @@ class TestDigitalOceanProvider(TestCase): with self.assertRaises(Exception) as ctx: zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals('Unauthorized', ctx.exception.message) + self.assertEquals('Unauthorized', text_type(ctx.exception)) # General error with requests_mock() as mock: @@ -175,7 +176,20 @@ class TestDigitalOceanProvider(TestCase): call('GET', '/domains/unit.tests/records', {'page': 1}), # delete the initial A record call('DELETE', '/domains/unit.tests/records/11189877'), - # created at least one of the record with expected data + # created at least some of the record with expected data + call('POST', '/domains/unit.tests/records', data={ + 'data': '1.2.3.4', + 'name': '@', + 'ttl': 300, 'type': 'A'}), + call('POST', '/domains/unit.tests/records', data={ + 'data': '1.2.3.5', + 'name': '@', + 'ttl': 300, 'type': 'A'}), + call('POST', '/domains/unit.tests/records', data={ + 'data': 'ca.unit.tests.', + 'flags': 0, 'name': '@', + 'tag': 'issue', + 'ttl': 3600, 'type': 'CAA'}), call('POST', '/domains/unit.tests/records', data={ 'name': '_srv._tcp', 'weight': 20, diff --git a/tests/test_octodns_provider_dnsimple.py b/tests/test_octodns_provider_dnsimple.py index b458ad7..e3a9b8d 100644 --- a/tests/test_octodns_provider_dnsimple.py +++ b/tests/test_octodns_provider_dnsimple.py @@ -9,6 +9,7 @@ from mock import Mock, call from os.path import dirname, join from requests import HTTPError from requests_mock import ANY, mock as requests_mock +from six import text_type from unittest import TestCase from octodns.record import Record @@ -47,7 +48,7 @@ class TestDnsimpleProvider(TestCase): with self.assertRaises(Exception) as ctx: zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals('Unauthorized', ctx.exception.message) + self.assertEquals('Unauthorized', text_type(ctx.exception)) # General error with requests_mock() as mock: @@ -138,7 +139,32 @@ class TestDnsimpleProvider(TestCase): provider._client._request.assert_has_calls([ # created the domain call('POST', '/domains', data={'name': 'unit.tests'}), - # created at least one of the record with expected data + # created at least some of the record with expected data + call('POST', '/zones/unit.tests/records', data={ + 'content': '1.2.3.4', + 'type': 'A', + 'name': '', + 'ttl': 300}), + call('POST', '/zones/unit.tests/records', data={ + 'content': '1.2.3.5', + 'type': 'A', + 'name': '', + 'ttl': 300}), + call('POST', '/zones/unit.tests/records', data={ + 'content': '0 issue "ca.unit.tests"', + 'type': 'CAA', + 'name': '', + 'ttl': 3600}), + call('POST', '/zones/unit.tests/records', data={ + 'content': '1 1 7491973e5f8b39d5327cd4e08bc81b05f7710b49', + 'type': 'SSHFP', + 'name': '', + 'ttl': 3600}), + call('POST', '/zones/unit.tests/records', data={ + 'content': '1 1 bf6b6825d2977c511a475bbefb88aad54a92ac73', + 'type': 'SSHFP', + 'name': '', + 'ttl': 3600}), call('POST', '/zones/unit.tests/records', data={ 'content': '20 30 foo-1.unit.tests.', 'priority': 10, diff --git a/tests/test_octodns_provider_dnsmadeeasy.py b/tests/test_octodns_provider_dnsmadeeasy.py index bb663fb..ba61b94 100644 --- a/tests/test_octodns_provider_dnsmadeeasy.py +++ b/tests/test_octodns_provider_dnsmadeeasy.py @@ -10,6 +10,7 @@ from mock import Mock, call from os.path import dirname, join from requests import HTTPError from requests_mock import ANY, mock as requests_mock +from six import text_type from unittest import TestCase from octodns.record import Record @@ -65,7 +66,7 @@ class TestDnsMadeEasyProvider(TestCase): with self.assertRaises(Exception) as ctx: zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals('Unauthorized', ctx.exception.message) + self.assertEquals('Unauthorized', text_type(ctx.exception)) # Bad request with requests_mock() as mock: @@ -76,7 +77,7 @@ class TestDnsMadeEasyProvider(TestCase): zone = Zone('unit.tests.', []) provider.populate(zone) self.assertEquals('\n - Rate limit exceeded', - ctx.exception.message) + text_type(ctx.exception)) # General error with requests_mock() as mock: @@ -148,7 +149,27 @@ class TestDnsMadeEasyProvider(TestCase): call('POST', '/', data={'name': 'unit.tests'}), # get all domains to build the cache call('GET', '/'), - # created at least one of the record with expected data + # created at least some of the record with expected data + call('POST', '/123123/records', data={ + 'type': 'A', + 'name': '', + 'value': '1.2.3.4', + 'ttl': 300}), + call('POST', '/123123/records', data={ + 'type': 'A', + 'name': '', + 'value': '1.2.3.5', + 'ttl': 300}), + call('POST', '/123123/records', data={ + 'type': 'ANAME', + 'name': '', + 'value': 'aname.unit.tests.', + 'ttl': 1800}), + call('POST', '/123123/records', data={ + 'name': '', + 'value': 'ca.unit.tests', + 'issuerCritical': 0, 'caaType': 'issue', + 'ttl': 3600, 'type': 'CAA'}), call('POST', '/123123/records', data={ 'name': '_srv._tcp', 'weight': 20, diff --git a/tests/test_octodns_provider_dyn.py b/tests/test_octodns_provider_dyn.py index 4f224fc..7c023fd 100644 --- a/tests/test_octodns_provider_dyn.py +++ b/tests/test_octodns_provider_dyn.py @@ -670,8 +670,8 @@ class TestDynProviderGeo(TestCase): tds = provider.traffic_directors self.assertEquals(set(['unit.tests.', 'geo.unit.tests.']), set(tds.keys())) - self.assertEquals(['A'], tds['unit.tests.'].keys()) - self.assertEquals(['A'], tds['geo.unit.tests.'].keys()) + self.assertEquals(['A'], list(tds['unit.tests.'].keys())) + self.assertEquals(['A'], list(tds['geo.unit.tests.'].keys())) provider.log.warn.assert_called_with("Unsupported TrafficDirector " "'%s'", 'something else') diff --git a/tests/test_octodns_provider_fastdns.py b/tests/test_octodns_provider_fastdns.py index 5f503c7..a8bed74 100644 --- a/tests/test_octodns_provider_fastdns.py +++ b/tests/test_octodns_provider_fastdns.py @@ -9,6 +9,7 @@ from __future__ import absolute_import, division, print_function, \ from os.path import dirname, join from requests import HTTPError from requests_mock import ANY, mock as requests_mock +from six import text_type from unittest import TestCase from octodns.record import Record @@ -147,4 +148,4 @@ class TestFastdnsProvider(TestCase): changes = provider.apply(plan) except NameError as e: expected = "contractId not specified to create zone" - self.assertEquals(e.message, expected) + self.assertEquals(text_type(e), expected) diff --git a/tests/test_octodns_provider_googlecloud.py b/tests/test_octodns_provider_googlecloud.py index 3a3e600..e642668 100644 --- a/tests/test_octodns_provider_googlecloud.py +++ b/tests/test_octodns_provider_googlecloud.py @@ -193,8 +193,13 @@ class DummyIterator: def __iter__(self): return self + # python2 def next(self): - return self.iterable.next() + return next(self.iterable) + + # python3 + def __next__(self): + return next(self.iterable) class TestGoogleCloudProvider(TestCase): @@ -247,7 +252,7 @@ class TestGoogleCloudProvider(TestCase): return_values_for_status = iter( ["pending"] * 11 + ['done', 'done']) type(status_mock).status = PropertyMock( - side_effect=return_values_for_status.next) + side_effect=lambda: next(return_values_for_status)) gcloud_zone_mock.changes = Mock(return_value=status_mock) provider = self._get_provider() diff --git a/tests/test_octodns_provider_mythicbeasts.py b/tests/test_octodns_provider_mythicbeasts.py index 5acbc55..960bd65 100644 --- a/tests/test_octodns_provider_mythicbeasts.py +++ b/tests/test_octodns_provider_mythicbeasts.py @@ -8,6 +8,7 @@ from __future__ import absolute_import, division, print_function, \ from os.path import dirname, join from requests_mock import ANY, mock as requests_mock +from six import text_type from unittest import TestCase from octodns.provider.mythicbeasts import MythicBeastsProvider, \ @@ -31,12 +32,12 @@ class TestMythicBeastsProvider(TestCase): with self.assertRaises(AssertionError) as err: add_trailing_dot('unit.tests.') self.assertEquals('Value already has trailing dot', - err.exception.message) + text_type(err.exception)) with self.assertRaises(AssertionError) as err: remove_trailing_dot('unit.tests') self.assertEquals('Value already missing trailing dot', - err.exception.message) + text_type(err.exception)) self.assertEquals(add_trailing_dot('unit.tests'), 'unit.tests.') self.assertEquals(remove_trailing_dot('unit.tests.'), 'unit.tests') @@ -91,7 +92,7 @@ class TestMythicBeastsProvider(TestCase): {'raw_values': [{'value': '', 'ttl': 0}]} ) self.assertEquals('Unable to parse MX data', - err.exception.message) + text_type(err.exception)) def test_data_for_CNAME(self): test_data = { @@ -129,7 +130,7 @@ class TestMythicBeastsProvider(TestCase): {'raw_values': [{'value': '', 'ttl': 0}]} ) self.assertEquals('Unable to parse SRV data', - err.exception.message) + text_type(err.exception)) def test_data_for_SSHFP(self): test_data = { @@ -149,7 +150,7 @@ class TestMythicBeastsProvider(TestCase): {'raw_values': [{'value': '', 'ttl': 0}]} ) self.assertEquals('Unable to parse SSHFP data', - err.exception.message) + text_type(err.exception)) def test_data_for_CAA(self): test_data = { @@ -166,7 +167,7 @@ class TestMythicBeastsProvider(TestCase): {'raw_values': [{'value': '', 'ttl': 0}]} ) self.assertEquals('Unable to parse CAA data', - err.exception.message) + text_type(err.exception)) def test_command_generation(self): zone = Zone('unit.tests.', []) @@ -312,7 +313,7 @@ class TestMythicBeastsProvider(TestCase): with self.assertRaises(AssertionError) as err: provider = MythicBeastsProvider('test', None) self.assertEquals('Passwords must be a dictionary', - err.exception.message) + text_type(err.exception)) # Missing password with requests_mock() as mock: @@ -324,7 +325,7 @@ class TestMythicBeastsProvider(TestCase): provider.populate(zone) self.assertEquals( 'Missing password for domain: unit.tests', - err.exception.message) + text_type(err.exception)) # Failed authentication with requests_mock() as mock: @@ -413,8 +414,7 @@ class TestMythicBeastsProvider(TestCase): provider.apply(plan) self.assertEquals( 'Mythic Beasts could not action command: unit.tests ' - 'ADD prawf.unit.tests 300 TXT prawf', - err.exception.message) + 'ADD prawf.unit.tests 300 TXT prawf', err.exception.message) # Check deleting and adding/changing test record existing = 'prawf 300 TXT prawf prawf prawf\ndileu 300 TXT dileu' @@ -441,11 +441,11 @@ class TestMythicBeastsProvider(TestCase): plan = provider.plan(wanted) # Octo ignores NS records (15-1) - self.assertEquals(1, len(filter(lambda u: isinstance(u, Update), - plan.changes))) - self.assertEquals(1, len(filter(lambda d: isinstance(d, Delete), - plan.changes))) - self.assertEquals(14, len(filter(lambda c: isinstance(c, Create), - plan.changes))) + self.assertEquals(1, len([c for c in plan.changes + if isinstance(c, Update)])) + self.assertEquals(1, len([c for c in plan.changes + if isinstance(c, Delete)])) + self.assertEquals(14, len([c for c in plan.changes + if isinstance(c, Create)])) self.assertEquals(16, provider.apply(plan)) self.assertTrue(plan.exists) diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index 178ce53..1d4e8f9 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -5,24 +5,19 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -from mock import Mock, call, patch -from nsone.rest.errors import AuthException, RateLimitException, \ +from collections import defaultdict +from mock import call, patch +from ns1.rest.errors import AuthException, RateLimitException, \ ResourceException +from six import text_type from unittest import TestCase from octodns.record import Delete, Record, Update -from octodns.provider.ns1 import Ns1Provider +from octodns.provider.ns1 import Ns1Client, Ns1Exception, Ns1Provider +from octodns.provider.plan import Plan from octodns.zone import Zone -class DummyZone(object): - - def __init__(self, records): - self.data = { - 'records': records - } - - class TestNs1Provider(TestCase): zone = Zone('unit.tests.', []) expected = set() @@ -115,7 +110,7 @@ class TestNs1Provider(TestCase): }, })) - nsone_records = [{ + ns1_records = [{ 'type': 'A', 'ttl': 32, 'short_answers': ['1.2.3.4'], @@ -171,43 +166,42 @@ class TestNs1Provider(TestCase): 'domain': 'unit.tests.', }] - @patch('nsone.NSONE.loadZone') - def test_populate(self, load_mock): + @patch('ns1.rest.records.Records.retrieve') + @patch('ns1.rest.zones.Zones.retrieve') + def test_populate(self, zone_retrieve_mock, record_retrieve_mock): provider = Ns1Provider('test', 'api-key') # Bad auth - load_mock.side_effect = AuthException('unauthorized') + zone_retrieve_mock.side_effect = AuthException('unauthorized') zone = Zone('unit.tests.', []) with self.assertRaises(AuthException) as ctx: provider.populate(zone) - self.assertEquals(load_mock.side_effect, ctx.exception) + self.assertEquals(zone_retrieve_mock.side_effect, ctx.exception) # General error - load_mock.reset_mock() - load_mock.side_effect = ResourceException('boom') + zone_retrieve_mock.reset_mock() + zone_retrieve_mock.side_effect = ResourceException('boom') zone = Zone('unit.tests.', []) with self.assertRaises(ResourceException) as ctx: provider.populate(zone) - self.assertEquals(load_mock.side_effect, ctx.exception) - self.assertEquals(('unit.tests',), load_mock.call_args[0]) + self.assertEquals(zone_retrieve_mock.side_effect, ctx.exception) + self.assertEquals(('unit.tests',), zone_retrieve_mock.call_args[0]) # Non-existent zone doesn't populate anything - load_mock.reset_mock() - load_mock.side_effect = \ + zone_retrieve_mock.reset_mock() + zone_retrieve_mock.side_effect = \ ResourceException('server error: zone not found') zone = Zone('unit.tests.', []) exists = provider.populate(zone) self.assertEquals(set(), zone.records) - self.assertEquals(('unit.tests',), load_mock.call_args[0]) + self.assertEquals(('unit.tests',), zone_retrieve_mock.call_args[0]) self.assertFalse(exists) # Existing zone w/o records - load_mock.reset_mock() - nsone_zone = DummyZone([]) - load_mock.side_effect = [nsone_zone] - zone_search = Mock() - zone_search.return_value = [ - { + zone_retrieve_mock.reset_mock() + record_retrieve_mock.reset_mock() + ns1_zone = { + 'records': [{ "domain": "geo.unit.tests", "zone": "unit.tests", "type": "A", @@ -220,22 +214,25 @@ class TestNs1Provider(TestCase): {'answer': ['4.5.6.7'], 'meta': {'iso_region_code': ['NA-US-WA']}}, ], + 'tier': 3, 'ttl': 34, - }, - ] - nsone_zone.search = zone_search + }], + } + zone_retrieve_mock.side_effect = [ns1_zone] + # Its tier 3 so we'll do a full lookup + record_retrieve_mock.side_effect = ns1_zone['records'] zone = Zone('unit.tests.', []) provider.populate(zone) self.assertEquals(1, len(zone.records)) - self.assertEquals(('unit.tests',), load_mock.call_args[0]) + self.assertEquals(('unit.tests',), zone_retrieve_mock.call_args[0]) + record_retrieve_mock.assert_has_calls([call('unit.tests', + 'geo.unit.tests', 'A')]) # Existing zone w/records - load_mock.reset_mock() - nsone_zone = DummyZone(self.nsone_records) - load_mock.side_effect = [nsone_zone] - zone_search = Mock() - zone_search.return_value = [ - { + zone_retrieve_mock.reset_mock() + record_retrieve_mock.reset_mock() + ns1_zone = { + 'records': self.ns1_records + [{ "domain": "geo.unit.tests", "zone": "unit.tests", "type": "A", @@ -248,27 +245,30 @@ class TestNs1Provider(TestCase): {'answer': ['4.5.6.7'], 'meta': {'iso_region_code': ['NA-US-WA']}}, ], + 'tier': 3, 'ttl': 34, - }, - ] - nsone_zone.search = zone_search + }], + } + zone_retrieve_mock.side_effect = [ns1_zone] + # Its tier 3 so we'll do a full lookup + record_retrieve_mock.side_effect = ns1_zone['records'] zone = Zone('unit.tests.', []) provider.populate(zone) self.assertEquals(self.expected, zone.records) - self.assertEquals(('unit.tests',), load_mock.call_args[0]) + self.assertEquals(('unit.tests',), zone_retrieve_mock.call_args[0]) + record_retrieve_mock.assert_has_calls([call('unit.tests', + 'geo.unit.tests', 'A')]) # Test skipping unsupported record type - load_mock.reset_mock() - nsone_zone = DummyZone(self.nsone_records + [{ - 'type': 'UNSUPPORTED', - 'ttl': 42, - 'short_answers': ['unsupported'], - 'domain': 'unsupported.unit.tests.', - }]) - load_mock.side_effect = [nsone_zone] - zone_search = Mock() - zone_search.return_value = [ - { + zone_retrieve_mock.reset_mock() + record_retrieve_mock.reset_mock() + ns1_zone = { + 'records': self.ns1_records + [{ + 'type': 'UNSUPPORTED', + 'ttl': 42, + 'short_answers': ['unsupported'], + 'domain': 'unsupported.unit.tests.', + }, { "domain": "geo.unit.tests", "zone": "unit.tests", "type": "A", @@ -281,18 +281,27 @@ class TestNs1Provider(TestCase): {'answer': ['4.5.6.7'], 'meta': {'iso_region_code': ['NA-US-WA']}}, ], + 'tier': 3, 'ttl': 34, - }, - ] - nsone_zone.search = zone_search + }], + } + zone_retrieve_mock.side_effect = [ns1_zone] zone = Zone('unit.tests.', []) provider.populate(zone) self.assertEquals(self.expected, zone.records) - self.assertEquals(('unit.tests',), load_mock.call_args[0]) + self.assertEquals(('unit.tests',), zone_retrieve_mock.call_args[0]) + record_retrieve_mock.assert_has_calls([call('unit.tests', + 'geo.unit.tests', 'A')]) - @patch('nsone.NSONE.createZone') - @patch('nsone.NSONE.loadZone') - def test_sync(self, load_mock, create_mock): + @patch('ns1.rest.records.Records.delete') + @patch('ns1.rest.records.Records.update') + @patch('ns1.rest.records.Records.create') + @patch('ns1.rest.records.Records.retrieve') + @patch('ns1.rest.zones.Zones.create') + @patch('ns1.rest.zones.Zones.retrieve') + def test_sync(self, zone_retrieve_mock, zone_create_mock, + record_retrieve_mock, record_create_mock, + record_update_mock, record_delete_mock): provider = Ns1Provider('test', 'api-key') desired = Zone('unit.tests.', []) @@ -306,124 +315,142 @@ class TestNs1Provider(TestCase): self.assertTrue(plan.exists) # Fails, general error - load_mock.reset_mock() - create_mock.reset_mock() - load_mock.side_effect = ResourceException('boom') + zone_retrieve_mock.reset_mock() + zone_create_mock.reset_mock() + zone_retrieve_mock.side_effect = ResourceException('boom') with self.assertRaises(ResourceException) as ctx: provider.apply(plan) - self.assertEquals(load_mock.side_effect, ctx.exception) + self.assertEquals(zone_retrieve_mock.side_effect, ctx.exception) # Fails, bad auth - load_mock.reset_mock() - create_mock.reset_mock() - load_mock.side_effect = \ + zone_retrieve_mock.reset_mock() + zone_create_mock.reset_mock() + zone_retrieve_mock.side_effect = \ ResourceException('server error: zone not found') - create_mock.side_effect = AuthException('unauthorized') + zone_create_mock.side_effect = AuthException('unauthorized') with self.assertRaises(AuthException) as ctx: provider.apply(plan) - self.assertEquals(create_mock.side_effect, ctx.exception) + self.assertEquals(zone_create_mock.side_effect, ctx.exception) # non-existent zone, create - load_mock.reset_mock() - create_mock.reset_mock() - load_mock.side_effect = \ + zone_retrieve_mock.reset_mock() + zone_create_mock.reset_mock() + zone_retrieve_mock.side_effect = \ ResourceException('server error: zone not found') - # ugh, need a mock zone with a mock prop since we're using getattr, we - # can actually control side effects on `meth` with that. - mock_zone = Mock() - mock_zone.add_SRV = Mock() - mock_zone.add_SRV.side_effect = [ + + zone_create_mock.side_effect = ['foo'] + # Test out the create rate-limit handling, then 9 successes + record_create_mock.side_effect = [ RateLimitException('boo', period=0), - None, - ] - create_mock.side_effect = [mock_zone] + ] + ([None] * 9) + got_n = provider.apply(plan) self.assertEquals(expected_n, got_n) + # Zone was created + zone_create_mock.assert_has_calls([call('unit.tests')]) + # Checking that we got some of the expected records too + record_create_mock.assert_has_calls([ + call('unit.tests', 'unit.tests', 'A', answers=[ + {'answer': ['1.2.3.4'], 'meta': {}} + ], filters=[], ttl=32), + call('unit.tests', 'unit.tests', 'CAA', answers=[ + (0, 'issue', 'ca.unit.tests') + ], ttl=40), + call('unit.tests', 'unit.tests', 'MX', answers=[ + (10, 'mx1.unit.tests.'), (20, 'mx2.unit.tests.') + ], ttl=35), + ]) + # Update & delete - load_mock.reset_mock() - create_mock.reset_mock() - nsone_zone = DummyZone(self.nsone_records + [{ - 'type': 'A', - 'ttl': 42, - 'short_answers': ['9.9.9.9'], - 'domain': 'delete-me.unit.tests.', - }]) - nsone_zone.data['records'][0]['short_answers'][0] = '2.2.2.2' - nsone_zone.loadRecord = Mock() - zone_search = Mock() - zone_search.return_value = [ - { + zone_retrieve_mock.reset_mock() + zone_create_mock.reset_mock() + + ns1_zone = { + 'records': self.ns1_records + [{ + 'type': 'A', + 'ttl': 42, + 'short_answers': ['9.9.9.9'], + 'domain': 'delete-me.unit.tests.', + }, { "domain": "geo.unit.tests", "zone": "unit.tests", "type": "A", - "answers": [ - {'answer': ['1.1.1.1'], 'meta': {}}, - {'answer': ['1.2.3.4'], - 'meta': {'ca_province': ['ON']}}, - {'answer': ['2.3.4.5'], 'meta': {'us_state': ['NY']}}, - {'answer': ['3.4.5.6'], 'meta': {'country': ['US']}}, - {'answer': ['4.5.6.7'], - 'meta': {'iso_region_code': ['NA-US-WA']}}, + "short_answers": [ + '1.1.1.1', + '1.2.3.4', + '2.3.4.5', + '3.4.5.6', + '4.5.6.7', ], + 'tier': 3, # This flags it as advacned, full load required 'ttl': 34, - }, - ] - nsone_zone.search = zone_search - load_mock.side_effect = [nsone_zone, nsone_zone] + }], + } + ns1_zone['records'][0]['short_answers'][0] = '2.2.2.2' + + record_retrieve_mock.side_effect = [{ + "domain": "geo.unit.tests", + "zone": "unit.tests", + "type": "A", + "answers": [ + {'answer': ['1.1.1.1'], 'meta': {}}, + {'answer': ['1.2.3.4'], + 'meta': {'ca_province': ['ON']}}, + {'answer': ['2.3.4.5'], 'meta': {'us_state': ['NY']}}, + {'answer': ['3.4.5.6'], 'meta': {'country': ['US']}}, + {'answer': ['4.5.6.7'], + 'meta': {'iso_region_code': ['NA-US-WA']}}, + ], + 'tier': 3, + 'ttl': 34, + }] + + zone_retrieve_mock.side_effect = [ns1_zone, ns1_zone] plan = provider.plan(desired) self.assertEquals(3, len(plan.changes)) - self.assertIsInstance(plan.changes[0], Update) - self.assertIsInstance(plan.changes[2], Delete) - # ugh, we need a mock record that can be returned from loadRecord for - # the update and delete targets, we can add our side effects to that to - # trigger rate limit handling - mock_record = Mock() - mock_record.update.side_effect = [ + # Shouldn't rely on order so just count classes + classes = defaultdict(lambda: 0) + for change in plan.changes: + classes[change.__class__] += 1 + self.assertEquals(1, classes[Delete]) + self.assertEquals(2, classes[Update]) + + record_update_mock.side_effect = [ RateLimitException('one', period=0), None, None, ] - mock_record.delete.side_effect = [ + record_delete_mock.side_effect = [ RateLimitException('two', period=0), None, None, ] - nsone_zone.loadRecord.side_effect = [mock_record, mock_record, - mock_record] + got_n = provider.apply(plan) self.assertEquals(3, got_n) - nsone_zone.loadRecord.assert_has_calls([ - call('unit.tests', u'A'), - call('geo', u'A'), - call('delete-me', u'A'), - ]) - mock_record.assert_has_calls([ - call.update(answers=[{'answer': [u'1.2.3.4'], 'meta': {}}], - filters=[], - ttl=32), - call.update(answers=[{u'answer': [u'1.2.3.4'], u'meta': {}}], - filters=[], - ttl=32), - call.update( - answers=[ - {u'answer': [u'101.102.103.104'], u'meta': {}}, - {u'answer': [u'101.102.103.105'], u'meta': {}}, - { - u'answer': [u'201.202.203.204'], - u'meta': { - u'iso_region_code': [u'NA-US-NY'] - }, - }, - ], + + record_update_mock.assert_has_calls([ + call('unit.tests', 'unit.tests', 'A', answers=[ + {'answer': ['1.2.3.4'], 'meta': {}}], + filters=[], + ttl=32), + call('unit.tests', 'unit.tests', 'A', answers=[ + {'answer': ['1.2.3.4'], 'meta': {}}], + filters=[], + ttl=32), + call('unit.tests', 'geo.unit.tests', 'A', answers=[ + {'answer': ['101.102.103.104'], 'meta': {}}, + {'answer': ['101.102.103.105'], 'meta': {}}, + { + 'answer': ['201.202.203.204'], + 'meta': {'iso_region_code': ['NA-US-NY']} + }], filters=[ - {u'filter': u'shuffle', u'config': {}}, - {u'filter': u'geotarget_country', u'config': {}}, - {u'filter': u'select_first_n', u'config': {u'N': 1}}, - ], - ttl=34), - call.delete(), - call.delete() + {'filter': 'shuffle', 'config': {}}, + {'filter': 'geotarget_country', 'config': {}}, + {'filter': 'select_first_n', 'config': {'N': 1}}], + ttl=34) ]) def test_escaping(self): @@ -448,21 +475,21 @@ class TestNs1Provider(TestCase): 'type': 'SPF', 'value': 'foo\\; bar baz\\; blip' }) - self.assertEquals(['foo; bar baz; blip'], - provider._params_for_SPF(record)['answers']) + params, _ = provider._params_for_SPF(record) + self.assertEquals(['foo; bar baz; blip'], params['answers']) record = Record.new(zone, 'txt', { 'ttl': 35, 'type': 'TXT', 'value': 'foo\\; bar baz\\; blip' }) - self.assertEquals(['foo; bar baz; blip'], - provider._params_for_TXT(record)['answers']) + params, _ = provider._params_for_SPF(record) + self.assertEquals(['foo; bar baz; blip'], params['answers']) def test_data_for_CNAME(self): provider = Ns1Provider('test', 'api-key') - # answers from nsone + # answers from ns1 a_record = { 'ttl': 31, 'type': 'CNAME', @@ -476,7 +503,7 @@ class TestNs1Provider(TestCase): self.assertEqual(a_expected, provider._data_for_CNAME(a_record['type'], a_record)) - # no answers from nsone + # no answers from ns1 b_record = { 'ttl': 32, 'type': 'CNAME', @@ -489,3 +516,1039 @@ class TestNs1Provider(TestCase): } self.assertEqual(b_expected, provider._data_for_CNAME(b_record['type'], b_record)) + + +class TestNs1ProviderDynamic(TestCase): + zone = Zone('unit.tests.', []) + + record = Record.new(zone, '', { + 'dynamic': { + 'pools': { + 'lhr': { + 'fallback': 'iad', + 'values': [{ + 'value': '3.4.5.6', + }], + }, + 'iad': { + 'values': [{ + 'value': '1.2.3.4', + }, { + 'value': '2.3.4.5', + }], + }, + }, + 'rules': [{ + 'geos': [ + 'AF', + 'EU-GB', + 'NA-US-FL' + ], + 'pool': 'lhr', + }, { + 'pool': 'iad', + }], + }, + 'octodns': { + 'healthcheck': { + 'host': 'send.me', + 'path': '/_ping', + 'port': 80, + 'protocol': 'HTTP', + } + }, + 'ttl': 32, + 'type': 'A', + 'value': '1.2.3.4', + 'meta': {}, + }) + + def test_notes(self): + provider = Ns1Provider('test', 'api-key') + + self.assertEquals({}, provider._parse_notes(None)) + self.assertEquals({}, provider._parse_notes('')) + self.assertEquals({}, provider._parse_notes('blah-blah-blah')) + + # Round tripping + data = { + 'key': 'value', + 'priority': '1', + } + notes = provider._encode_notes(data) + self.assertEquals(data, provider._parse_notes(notes)) + + def test_monitors_for(self): + provider = Ns1Provider('test', 'api-key') + + # pre-populate the client's monitors cache + monitor_one = { + 'config': { + 'host': '1.2.3.4', + }, + 'notes': 'host:unit.tests type:A', + } + monitor_four = { + 'config': { + 'host': '2.3.4.5', + }, + 'notes': 'host:unit.tests type:A', + } + provider._client._monitors_cache = { + 'one': monitor_one, + 'two': { + 'config': { + 'host': '8.8.8.8', + }, + 'notes': 'host:unit.tests type:AAAA', + }, + 'three': { + 'config': { + 'host': '9.9.9.9', + }, + 'notes': 'host:other.unit.tests type:A', + }, + 'four': monitor_four, + } + + # Would match, but won't get there b/c it's not dynamic + record = Record.new(self.zone, '', { + 'ttl': 32, + 'type': 'A', + 'value': '1.2.3.4', + 'meta': {}, + }) + self.assertEquals({}, provider._monitors_for(record)) + + # Will match some records + self.assertEquals({ + '1.2.3.4': monitor_one, + '2.3.4.5': monitor_four, + }, provider._monitors_for(self.record)) + + def test_uuid(self): + # Just a smoke test/for coverage + provider = Ns1Provider('test', 'api-key') + self.assertTrue(provider._uuid()) + + @patch('octodns.provider.ns1.Ns1Provider._uuid') + @patch('ns1.rest.data.Feed.create') + def test_feed_create(self, datafeed_create_mock, uuid_mock): + provider = Ns1Provider('test', 'api-key') + + # pre-fill caches to avoid extranious calls (things we're testing + # elsewhere) + provider._client._datasource_id = 'foo' + provider._client._feeds_for_monitors = {} + + uuid_mock.reset_mock() + datafeed_create_mock.reset_mock() + uuid_mock.side_effect = ['xxxxxxxxxxxxxx'] + feed = { + 'id': 'feed', + } + datafeed_create_mock.side_effect = [feed] + monitor = { + 'id': 'one', + 'name': 'one name', + 'config': { + 'host': '1.2.3.4', + }, + 'notes': 'host:unit.tests type:A', + } + self.assertEquals('feed', provider._feed_create(monitor)) + datafeed_create_mock.assert_has_calls([call('foo', 'one name - xxxxxx', + {'jobid': 'one'})]) + + @patch('octodns.provider.ns1.Ns1Provider._feed_create') + @patch('octodns.provider.ns1.Ns1Client.monitors_create') + @patch('octodns.provider.ns1.Ns1Client.notifylists_create') + def test_monitor_create(self, notifylists_create_mock, + monitors_create_mock, feed_create_mock): + provider = Ns1Provider('test', 'api-key') + + # pre-fill caches to avoid extranious calls (things we're testing + # elsewhere) + provider._client._datasource_id = 'foo' + provider._client._feeds_for_monitors = {} + + notifylists_create_mock.reset_mock() + monitors_create_mock.reset_mock() + feed_create_mock.reset_mock() + notifylists_create_mock.side_effect = [{ + 'id': 'nl-id', + }] + monitors_create_mock.side_effect = [{ + 'id': 'mon-id', + }] + feed_create_mock.side_effect = ['feed-id'] + monitor = { + 'name': 'test monitor', + } + monitor_id, feed_id = provider._monitor_create(monitor) + self.assertEquals('mon-id', monitor_id) + self.assertEquals('feed-id', feed_id) + monitors_create_mock.assert_has_calls([call(name='test monitor', + notify_list='nl-id')]) + + def test_monitor_gen(self): + provider = Ns1Provider('test', 'api-key') + + value = '3.4.5.6' + monitor = provider._monitor_gen(self.record, value) + self.assertEquals(value, monitor['config']['host']) + self.assertTrue('\\nHost: send.me\\r' in monitor['config']['send']) + self.assertFalse(monitor['config']['ssl']) + self.assertEquals('host:unit.tests type:A', monitor['notes']) + + self.record._octodns['healthcheck']['protocol'] = 'HTTPS' + monitor = provider._monitor_gen(self.record, value) + self.assertTrue(monitor['config']['ssl']) + + def test_monitor_is_match(self): + provider = Ns1Provider('test', 'api-key') + + # Empty matches empty + self.assertTrue(provider._monitor_is_match({}, {})) + + # Anything matches empty + self.assertTrue(provider._monitor_is_match({}, { + 'anything': 'goes' + })) + + # Missing doesn't match + self.assertFalse(provider._monitor_is_match({ + 'exepct': 'this', + }, { + 'anything': 'goes' + })) + + # Identical matches + self.assertTrue(provider._monitor_is_match({ + 'exepct': 'this', + }, { + 'exepct': 'this', + })) + + # Different values don't match + self.assertFalse(provider._monitor_is_match({ + 'exepct': 'this', + }, { + 'exepct': 'that', + })) + + # Different sub-values don't match + self.assertFalse(provider._monitor_is_match({ + 'exepct': { + 'this': 'to-be', + }, + }, { + 'exepct': { + 'this': 'something-else', + }, + })) + + @patch('octodns.provider.ns1.Ns1Provider._feed_create') + @patch('octodns.provider.ns1.Ns1Client.monitors_update') + @patch('octodns.provider.ns1.Ns1Provider._monitor_create') + @patch('octodns.provider.ns1.Ns1Provider._monitor_gen') + def test_monitor_sync(self, monitor_gen_mock, monitor_create_mock, + monitors_update_mock, feed_create_mock): + provider = Ns1Provider('test', 'api-key') + + # pre-fill caches to avoid extranious calls (things we're testing + # elsewhere) + provider._client._datasource_id = 'foo' + provider._client._feeds_for_monitors = { + 'mon-id': 'feed-id', + } + + # No existing monitor + monitor_gen_mock.reset_mock() + monitor_create_mock.reset_mock() + monitors_update_mock.reset_mock() + feed_create_mock.reset_mock() + monitor_gen_mock.side_effect = [{'key': 'value'}] + monitor_create_mock.side_effect = [('mon-id', 'feed-id')] + value = '1.2.3.4' + monitor_id, feed_id = provider._monitor_sync(self.record, value, None) + self.assertEquals('mon-id', monitor_id) + self.assertEquals('feed-id', feed_id) + monitor_gen_mock.assert_has_calls([call(self.record, value)]) + monitor_create_mock.assert_has_calls([call({'key': 'value'})]) + monitors_update_mock.assert_not_called() + feed_create_mock.assert_not_called() + + # Existing monitor that doesn't need updates + monitor_gen_mock.reset_mock() + monitor_create_mock.reset_mock() + monitors_update_mock.reset_mock() + feed_create_mock.reset_mock() + monitor = { + 'id': 'mon-id', + 'key': 'value', + 'name': 'monitor name', + } + monitor_gen_mock.side_effect = [monitor] + monitor_id, feed_id = provider._monitor_sync(self.record, value, + monitor) + self.assertEquals('mon-id', monitor_id) + self.assertEquals('feed-id', feed_id) + monitor_gen_mock.assert_called_once() + monitor_create_mock.assert_not_called() + monitors_update_mock.assert_not_called() + feed_create_mock.assert_not_called() + + # Existing monitor that doesn't need updates, but is missing its feed + monitor_gen_mock.reset_mock() + monitor_create_mock.reset_mock() + monitors_update_mock.reset_mock() + feed_create_mock.reset_mock() + monitor = { + 'id': 'mon-id2', + 'key': 'value', + 'name': 'monitor name', + } + monitor_gen_mock.side_effect = [monitor] + feed_create_mock.side_effect = ['feed-id2'] + monitor_id, feed_id = provider._monitor_sync(self.record, value, + monitor) + self.assertEquals('mon-id2', monitor_id) + self.assertEquals('feed-id2', feed_id) + monitor_gen_mock.assert_called_once() + monitor_create_mock.assert_not_called() + monitors_update_mock.assert_not_called() + feed_create_mock.assert_has_calls([call(monitor)]) + + # Existing monitor that needs updates + monitor_gen_mock.reset_mock() + monitor_create_mock.reset_mock() + monitors_update_mock.reset_mock() + feed_create_mock.reset_mock() + monitor = { + 'id': 'mon-id', + 'key': 'value', + 'name': 'monitor name', + } + gened = { + 'other': 'thing', + } + monitor_gen_mock.side_effect = [gened] + monitor_id, feed_id = provider._monitor_sync(self.record, value, + monitor) + self.assertEquals('mon-id', monitor_id) + self.assertEquals('feed-id', feed_id) + monitor_gen_mock.assert_called_once() + monitor_create_mock.assert_not_called() + monitors_update_mock.assert_has_calls([call('mon-id', other='thing')]) + feed_create_mock.assert_not_called() + + @patch('octodns.provider.ns1.Ns1Client.notifylists_delete') + @patch('octodns.provider.ns1.Ns1Client.monitors_delete') + @patch('octodns.provider.ns1.Ns1Client.datafeed_delete') + @patch('octodns.provider.ns1.Ns1Provider._monitors_for') + def test_monitors_gc(self, monitors_for_mock, datafeed_delete_mock, + monitors_delete_mock, notifylists_delete_mock): + provider = Ns1Provider('test', 'api-key') + + # pre-fill caches to avoid extranious calls (things we're testing + # elsewhere) + provider._client._datasource_id = 'foo' + provider._client._feeds_for_monitors = { + 'mon-id': 'feed-id', + } + + # No active monitors and no existing, nothing will happen + monitors_for_mock.reset_mock() + datafeed_delete_mock.reset_mock() + monitors_delete_mock.reset_mock() + notifylists_delete_mock.reset_mock() + monitors_for_mock.side_effect = [{}] + provider._monitors_gc(self.record) + monitors_for_mock.assert_has_calls([call(self.record)]) + datafeed_delete_mock.assert_not_called() + monitors_delete_mock.assert_not_called() + notifylists_delete_mock.assert_not_called() + + # No active monitors and one existing, delete all the things + monitors_for_mock.reset_mock() + datafeed_delete_mock.reset_mock() + monitors_delete_mock.reset_mock() + notifylists_delete_mock.reset_mock() + monitors_for_mock.side_effect = [{ + 'x': { + 'id': 'mon-id', + 'notify_list': 'nl-id', + } + }] + provider._monitors_gc(self.record) + monitors_for_mock.assert_has_calls([call(self.record)]) + datafeed_delete_mock.assert_has_calls([call('foo', 'feed-id')]) + monitors_delete_mock.assert_has_calls([call('mon-id')]) + notifylists_delete_mock.assert_has_calls([call('nl-id')]) + + # Same existing, this time in active list, should be noop + monitors_for_mock.reset_mock() + datafeed_delete_mock.reset_mock() + monitors_delete_mock.reset_mock() + notifylists_delete_mock.reset_mock() + monitors_for_mock.side_effect = [{ + 'x': { + 'id': 'mon-id', + 'notify_list': 'nl-id', + } + }] + provider._monitors_gc(self.record, {'mon-id'}) + monitors_for_mock.assert_has_calls([call(self.record)]) + datafeed_delete_mock.assert_not_called() + monitors_delete_mock.assert_not_called() + notifylists_delete_mock.assert_not_called() + + # Non-active monitor w/o a feed, and another monitor that's left alone + # b/c it's active + monitors_for_mock.reset_mock() + datafeed_delete_mock.reset_mock() + monitors_delete_mock.reset_mock() + notifylists_delete_mock.reset_mock() + monitors_for_mock.side_effect = [{ + 'x': { + 'id': 'mon-id', + 'notify_list': 'nl-id', + }, + 'y': { + 'id': 'mon-id2', + 'notify_list': 'nl-id2', + }, + }] + provider._monitors_gc(self.record, {'mon-id'}) + monitors_for_mock.assert_has_calls([call(self.record)]) + datafeed_delete_mock.assert_not_called() + monitors_delete_mock.assert_has_calls([call('mon-id2')]) + notifylists_delete_mock.assert_has_calls([call('nl-id2')]) + + @patch('octodns.provider.ns1.Ns1Provider._monitor_sync') + @patch('octodns.provider.ns1.Ns1Provider._monitors_for') + def test_params_for_dynamic(self, monitors_for_mock, monitors_sync_mock): + provider = Ns1Provider('test', 'api-key') + + # pre-fill caches to avoid extranious calls (things we're testing + # elsewhere) + provider._client._datasource_id = 'foo' + provider._client._feeds_for_monitors = { + 'mon-id': 'feed-id', + } + + monitors_for_mock.reset_mock() + monitors_sync_mock.reset_mock() + monitors_for_mock.side_effect = [{ + '3.4.5.6': 'mid-3', + }] + monitors_sync_mock.side_effect = [ + ('mid-1', 'fid-1'), + ('mid-2', 'fid-2'), + ('mid-3', 'fid-3'), + ] + # This indirectly calls into _params_for_dynamic_A and tests the + # handling to get there + provider._params_for_A(self.record) + monitors_for_mock.assert_has_calls([call(self.record)]) + monitors_sync_mock.assert_has_calls([ + call(self.record, '1.2.3.4', None), + call(self.record, '2.3.4.5', None), + call(self.record, '3.4.5.6', 'mid-3'), + ]) + + def test_data_for_dynamic_A(self): + provider = Ns1Provider('test', 'api-key') + + # Unexpected filters throws an error + ns1_record = { + 'domain': 'unit.tests', + 'filters': [], + } + with self.assertRaises(Ns1Exception) as ctx: + provider._data_for_dynamic_A('A', ns1_record) + self.assertEquals('Unrecognized advanced record', + text_type(ctx.exception)) + + # empty record turns into empty data + ns1_record = { + 'answers': [], + 'domain': 'unit.tests', + 'filters': Ns1Provider._DYNAMIC_FILTERS, + 'regions': {}, + 'ttl': 42, + } + data = provider._data_for_dynamic_A('A', ns1_record) + self.assertEquals({ + 'dynamic': { + 'pools': {}, + 'rules': [], + }, + 'ttl': 42, + 'type': 'A', + 'values': [], + }, data) + + # Test out a small, but realistic setup that covers all the options + ns1_record = { + 'answers': [{ + 'answer': ['3.4.5.6'], + 'meta': { + 'priority': 1, + 'note': 'from:lhr', + }, + 'region': 'lhr', + }, { + 'answer': ['2.3.4.5'], + 'meta': { + 'priority': 2, + 'weight': 12, + 'note': 'from:iad', + }, + 'region': 'lhr', + }, { + 'answer': ['1.2.3.4'], + 'meta': { + 'priority': 3, + 'note': 'from:--default--', + }, + 'region': 'lhr', + }, { + 'answer': ['2.3.4.5'], + 'meta': { + 'priority': 1, + 'weight': 12, + 'note': 'from:iad', + }, + 'region': 'iad', + }, { + 'answer': ['1.2.3.4'], + 'meta': { + 'priority': 2, + 'note': 'from:--default--', + }, + 'region': 'iad', + }], + 'domain': 'unit.tests', + 'filters': Ns1Provider._DYNAMIC_FILTERS, + 'regions': { + 'lhr': { + 'meta': { + 'note': 'rule-order:1 fallback:iad', + 'country': ['CA'], + 'georegion': ['AFRICA'], + 'us_state': ['OR'], + }, + }, + 'iad': { + 'meta': { + 'note': 'rule-order:2', + }, + } + }, + 'tier': 3, + 'ttl': 42, + } + data = provider._data_for_dynamic_A('A', ns1_record) + self.assertEquals({ + 'dynamic': { + 'pools': { + 'iad': { + 'fallback': None, + 'values': [{ + 'value': '2.3.4.5', + 'weight': 12, + }], + }, + 'lhr': { + 'fallback': 'iad', + 'values': [{ + 'weight': 1, + 'value': '3.4.5.6', + }], + }, + }, + 'rules': [{ + '_order': '1', + 'geos': [ + 'AF', + 'NA-CA', + 'NA-US-OR', + ], + 'pool': 'lhr', + }, { + '_order': '2', + 'pool': 'iad', + }], + }, + 'ttl': 42, + 'type': 'A', + 'values': ['1.2.3.4'], + }, data) + + # Same answer if we go through _data_for_A which out sources the job to + # _data_for_dynamic_A + data2 = provider._data_for_A('A', ns1_record) + self.assertEquals(data, data2) + + @patch('octodns.provider.ns1.Ns1Provider._monitors_for') + def test_extra_changes(self, monitors_for_mock): + provider = Ns1Provider('test', 'api-key') + + desired = Zone('unit.tests.', []) + + # Empty zone and no changes + monitors_for_mock.reset_mock() + extra = provider._extra_changes(desired, []) + self.assertFalse(extra) + monitors_for_mock.assert_not_called() + + # Simple record, ignored + monitors_for_mock.reset_mock() + simple = Record.new(desired, '', { + 'ttl': 32, + 'type': 'A', + 'value': '1.2.3.4', + 'meta': {}, + }) + desired.add_record(simple) + extra = provider._extra_changes(desired, []) + self.assertFalse(extra) + monitors_for_mock.assert_not_called() + + # Dynamic record, inspectable + dynamic = Record.new(desired, 'dyn', { + 'dynamic': { + 'pools': { + 'iad': { + 'values': [{ + 'value': '1.2.3.4', + }], + }, + }, + 'rules': [{ + 'pool': 'iad', + }], + }, + 'octodns': { + 'healthcheck': { + 'host': 'send.me', + 'path': '/_ping', + 'port': 80, + 'protocol': 'HTTP', + } + }, + 'ttl': 32, + 'type': 'A', + 'value': '1.2.3.4', + 'meta': {}, + }) + desired.add_record(dynamic) + + # untouched, but everything in sync so no change needed + monitors_for_mock.reset_mock() + # Generate what we expect to have + gend = provider._monitor_gen(dynamic, '1.2.3.4') + gend.update({ + 'id': 'mid', # need to add an id + 'notify_list': 'xyz', # need to add a notify list (for now) + }) + monitors_for_mock.side_effect = [{ + '1.2.3.4': gend, + }] + extra = provider._extra_changes(desired, []) + self.assertFalse(extra) + monitors_for_mock.assert_has_calls([call(dynamic)]) + + update = Update(dynamic, dynamic) + + # If we don't have a notify list we're broken and we'll expect to see + # an Update + monitors_for_mock.reset_mock() + del gend['notify_list'] + monitors_for_mock.side_effect = [{ + '1.2.3.4': gend, + }] + extra = provider._extra_changes(desired, []) + self.assertEquals(1, len(extra)) + extra = list(extra)[0] + self.assertIsInstance(extra, Update) + self.assertEquals(dynamic, extra.new) + monitors_for_mock.assert_has_calls([call(dynamic)]) + + # Add notify_list back and change the healthcheck protocol, we'll still + # expect to see an update + monitors_for_mock.reset_mock() + gend['notify_list'] = 'xyz' + dynamic._octodns['healthcheck']['protocol'] = 'HTTPS' + del gend['notify_list'] + monitors_for_mock.side_effect = [{ + '1.2.3.4': gend, + }] + extra = provider._extra_changes(desired, []) + self.assertEquals(1, len(extra)) + extra = list(extra)[0] + self.assertIsInstance(extra, Update) + self.assertEquals(dynamic, extra.new) + monitors_for_mock.assert_has_calls([call(dynamic)]) + + # If it's in the changed list, it'll be ignored + monitors_for_mock.reset_mock() + extra = provider._extra_changes(desired, [update]) + self.assertFalse(extra) + monitors_for_mock.assert_not_called() + + DESIRED = Zone('unit.tests.', []) + + SIMPLE = Record.new(DESIRED, 'sim', { + 'ttl': 33, + 'type': 'A', + 'value': '1.2.3.4', + }) + + # Dynamic record, inspectable + DYNAMIC = Record.new(DESIRED, 'dyn', { + 'dynamic': { + 'pools': { + 'iad': { + 'values': [{ + 'value': '1.2.3.4', + }], + }, + }, + 'rules': [{ + 'pool': 'iad', + }], + }, + 'octodns': { + 'healthcheck': { + 'host': 'send.me', + 'path': '/_ping', + 'port': 80, + 'protocol': 'HTTP', + } + }, + 'ttl': 32, + 'type': 'A', + 'value': '1.2.3.4', + 'meta': {}, + }) + + def test_has_dynamic(self): + provider = Ns1Provider('test', 'api-key') + + simple_update = Update(self.SIMPLE, self.SIMPLE) + dynamic_update = Update(self.DYNAMIC, self.DYNAMIC) + + self.assertFalse(provider._has_dynamic([simple_update])) + self.assertTrue(provider._has_dynamic([dynamic_update])) + self.assertTrue(provider._has_dynamic([simple_update, dynamic_update])) + + @patch('octodns.provider.ns1.Ns1Client.zones_retrieve') + @patch('octodns.provider.ns1.Ns1Provider._apply_Update') + def test_apply_monitor_regions(self, apply_update_mock, + zones_retrieve_mock): + provider = Ns1Provider('test', 'api-key') + + simple_update = Update(self.SIMPLE, self.SIMPLE) + simple_plan = Plan(self.DESIRED, self.DESIRED, [simple_update], True) + dynamic_update = Update(self.DYNAMIC, self.DYNAMIC) + dynamic_update = Update(self.DYNAMIC, self.DYNAMIC) + dynamic_plan = Plan(self.DESIRED, self.DESIRED, [dynamic_update], + True) + both_plan = Plan(self.DESIRED, self.DESIRED, [simple_update, + dynamic_update], True) + + # always return foo, we aren't testing this part here + zones_retrieve_mock.side_effect = [ + 'foo', + 'foo', + 'foo', + 'foo', + ] + + # Doesn't blow up, and calls apply once + apply_update_mock.reset_mock() + provider._apply(simple_plan) + apply_update_mock.assert_has_calls([call('foo', simple_update)]) + + # Blows up and apply not called + apply_update_mock.reset_mock() + with self.assertRaises(Ns1Exception) as ctx: + provider._apply(dynamic_plan) + self.assertTrue('monitor_regions not set' in text_type(ctx.exception)) + apply_update_mock.assert_not_called() + + # Blows up and apply not called even though there's a simple + apply_update_mock.reset_mock() + with self.assertRaises(Ns1Exception) as ctx: + provider._apply(both_plan) + self.assertTrue('monitor_regions not set' in text_type(ctx.exception)) + apply_update_mock.assert_not_called() + + # with monitor_regions set + provider.monitor_regions = ['lga'] + + apply_update_mock.reset_mock() + provider._apply(both_plan) + apply_update_mock.assert_has_calls([ + call('foo', dynamic_update), + call('foo', simple_update), + ]) + + +class TestNs1Client(TestCase): + + @patch('ns1.rest.zones.Zones.retrieve') + def test_retry_behavior(self, zone_retrieve_mock): + client = Ns1Client('dummy-key') + + # No retry required, just calls and is returned + zone_retrieve_mock.reset_mock() + zone_retrieve_mock.side_effect = ['foo'] + self.assertEquals('foo', client.zones_retrieve('unit.tests')) + zone_retrieve_mock.assert_has_calls([call('unit.tests')]) + + # One retry required + zone_retrieve_mock.reset_mock() + zone_retrieve_mock.side_effect = [ + RateLimitException('boo', period=0), + 'foo' + ] + self.assertEquals('foo', client.zones_retrieve('unit.tests')) + zone_retrieve_mock.assert_has_calls([call('unit.tests')]) + + # Two retries required + zone_retrieve_mock.reset_mock() + zone_retrieve_mock.side_effect = [ + RateLimitException('boo', period=0), + 'foo' + ] + self.assertEquals('foo', client.zones_retrieve('unit.tests')) + zone_retrieve_mock.assert_has_calls([call('unit.tests')]) + + # Exhaust our retries + zone_retrieve_mock.reset_mock() + zone_retrieve_mock.side_effect = [ + RateLimitException('first', period=0), + RateLimitException('boo', period=0), + RateLimitException('boo', period=0), + RateLimitException('last', period=0), + ] + with self.assertRaises(RateLimitException) as ctx: + client.zones_retrieve('unit.tests') + self.assertEquals('last', text_type(ctx.exception)) + + @patch('ns1.rest.data.Source.list') + @patch('ns1.rest.data.Source.create') + def test_datasource_id(self, datasource_create_mock, datasource_list_mock): + client = Ns1Client('dummy-key') + + # First invocation with an empty list create + datasource_list_mock.reset_mock() + datasource_create_mock.reset_mock() + datasource_list_mock.side_effect = [[]] + datasource_create_mock.side_effect = [{ + 'id': 'foo', + }] + self.assertEquals('foo', client.datasource_id) + name = 'octoDNS NS1 Data Source' + source_type = 'nsone_monitoring' + datasource_create_mock.assert_has_calls([call(name=name, + sourcetype=source_type)]) + datasource_list_mock.assert_called_once() + + # 2nd invocation is cached + datasource_list_mock.reset_mock() + datasource_create_mock.reset_mock() + self.assertEquals('foo', client.datasource_id) + datasource_create_mock.assert_not_called() + datasource_list_mock.assert_not_called() + + # Reset the client's cache + client._datasource_id = None + + # First invocation with a match in the list finds it and doesn't call + # create + datasource_list_mock.reset_mock() + datasource_create_mock.reset_mock() + datasource_list_mock.side_effect = [[{ + 'id': 'other', + 'name': 'not a match', + }, { + 'id': 'bar', + 'name': name, + }]] + self.assertEquals('bar', client.datasource_id) + datasource_create_mock.assert_not_called() + datasource_list_mock.assert_called_once() + + @patch('ns1.rest.data.Feed.delete') + @patch('ns1.rest.data.Feed.create') + @patch('ns1.rest.data.Feed.list') + def test_feeds_for_monitors(self, datafeed_list_mock, + datafeed_create_mock, + datafeed_delete_mock): + client = Ns1Client('dummy-key') + + # pre-cache datasource_id + client._datasource_id = 'foo' + + # Populate the cache and check the results + datafeed_list_mock.reset_mock() + datafeed_list_mock.side_effect = [[{ + 'config': { + 'jobid': 'the-job', + }, + 'id': 'the-feed', + }, { + 'config': { + 'jobid': 'the-other-job', + }, + 'id': 'the-other-feed', + }]] + expected = { + 'the-job': 'the-feed', + 'the-other-job': 'the-other-feed', + } + self.assertEquals(expected, client.feeds_for_monitors) + datafeed_list_mock.assert_called_once() + + # 2nd call uses cache + datafeed_list_mock.reset_mock() + self.assertEquals(expected, client.feeds_for_monitors) + datafeed_list_mock.assert_not_called() + + # create a feed and make sure it's in the cache/map + datafeed_create_mock.reset_mock() + datafeed_create_mock.side_effect = [{ + 'id': 'new-feed', + }] + client.datafeed_create(client.datasource_id, 'new-name', { + 'jobid': 'new-job', + }) + datafeed_create_mock.assert_has_calls([call('foo', 'new-name', { + 'jobid': 'new-job', + })]) + new_expected = expected.copy() + new_expected['new-job'] = 'new-feed' + self.assertEquals(new_expected, client.feeds_for_monitors) + datafeed_create_mock.assert_called_once() + + # Delete a feed and make sure it's out of the cache/map + datafeed_delete_mock.reset_mock() + client.datafeed_delete(client.datasource_id, 'new-feed') + self.assertEquals(expected, client.feeds_for_monitors) + datafeed_delete_mock.assert_called_once() + + @patch('ns1.rest.monitoring.Monitors.delete') + @patch('ns1.rest.monitoring.Monitors.update') + @patch('ns1.rest.monitoring.Monitors.create') + @patch('ns1.rest.monitoring.Monitors.list') + def test_monitors(self, monitors_list_mock, monitors_create_mock, + monitors_update_mock, monitors_delete_mock): + client = Ns1Client('dummy-key') + + one = { + 'id': 'one', + 'key': 'value', + } + two = { + 'id': 'two', + 'key': 'other-value', + } + + # Populate the cache and check the results + monitors_list_mock.reset_mock() + monitors_list_mock.side_effect = [[one, two]] + expected = { + 'one': one, + 'two': two, + } + self.assertEquals(expected, client.monitors) + monitors_list_mock.assert_called_once() + + # 2nd round pulls it from cache + monitors_list_mock.reset_mock() + self.assertEquals(expected, client.monitors) + monitors_list_mock.assert_not_called() + + # Create a monitor, make sure it's in the list + monitors_create_mock.reset_mock() + monitor = { + 'id': 'new-id', + 'key': 'new-value', + } + monitors_create_mock.side_effect = [monitor] + self.assertEquals(monitor, client.monitors_create(param='eter')) + monitors_create_mock.assert_has_calls([call({}, param='eter')]) + new_expected = expected.copy() + new_expected['new-id'] = monitor + self.assertEquals(new_expected, client.monitors) + + # Update a monitor, make sure it's updated in the cache + monitors_update_mock.reset_mock() + monitor = { + 'id': 'new-id', + 'key': 'changed-value', + } + monitors_update_mock.side_effect = [monitor] + self.assertEquals(monitor, client.monitors_update('new-id', + key='changed-value')) + monitors_update_mock \ + .assert_has_calls([call('new-id', {}, key='changed-value')]) + new_expected['new-id'] = monitor + self.assertEquals(new_expected, client.monitors) + + # Delete a monitor, make sure it's out of the list + monitors_delete_mock.reset_mock() + monitors_delete_mock.side_effect = ['deleted'] + self.assertEquals('deleted', client.monitors_delete('new-id')) + monitors_delete_mock.assert_has_calls([call('new-id')]) + self.assertEquals(expected, client.monitors) + + @patch('ns1.rest.monitoring.NotifyLists.delete') + @patch('ns1.rest.monitoring.NotifyLists.create') + @patch('ns1.rest.monitoring.NotifyLists.list') + def test_notifylists(self, notifylists_list_mock, notifylists_create_mock, + notifylists_delete_mock): + client = Ns1Client('dummy-key') + + notifylists_list_mock.reset_mock() + notifylists_create_mock.reset_mock() + notifylists_delete_mock.reset_mock() + notifylists_create_mock.side_effect = ['bar'] + notify_list = [{ + 'config': { + 'sourceid': 'foo', + }, + 'type': 'datafeed', + }] + nl = client.notifylists_create(name='some name', + notify_list=notify_list) + self.assertEquals('bar', nl) + notifylists_list_mock.assert_not_called() + notifylists_create_mock.assert_has_calls([ + call({'name': 'some name', 'notify_list': notify_list}) + ]) + notifylists_delete_mock.assert_not_called() + + notifylists_list_mock.reset_mock() + notifylists_create_mock.reset_mock() + notifylists_delete_mock.reset_mock() + client.notifylists_delete('nlid') + notifylists_list_mock.assert_not_called() + notifylists_create_mock.assert_not_called() + notifylists_delete_mock.assert_has_calls([call('nlid')]) + + notifylists_list_mock.reset_mock() + notifylists_create_mock.reset_mock() + notifylists_delete_mock.reset_mock() + expected = ['one', 'two', 'three'] + notifylists_list_mock.side_effect = [expected] + nls = client.notifylists_list() + self.assertEquals(expected, nls) + notifylists_list_mock.assert_has_calls([call()]) + notifylists_create_mock.assert_not_called() + notifylists_delete_mock.assert_not_called() diff --git a/tests/test_octodns_provider_ovh.py b/tests/test_octodns_provider_ovh.py index d3f468d..3da4276 100644 --- a/tests/test_octodns_provider_ovh.py +++ b/tests/test_octodns_provider_ovh.py @@ -279,6 +279,24 @@ class TestOvhProvider(TestCase): 'id': 18 }) + # CAA + api_record.append({ + 'fieldType': 'CAA', + 'ttl': 1600, + 'target': '0 issue "ca.unit.tests"', + 'subDomain': 'caa', + 'id': 19 + }) + expected.add(Record.new(zone, 'caa', { + 'ttl': 1600, + 'type': 'CAA', + 'values': [{ + 'flags': 0, + 'tag': 'issue', + 'value': 'ca.unit.tests' + }] + })) + valid_dkim = [valid_dkim_key, 'v=DKIM1 \\; %s' % valid_dkim_key, 'h=sha256 \\; %s' % valid_dkim_key, @@ -382,64 +400,66 @@ class TestOvhProvider(TestCase): get_mock.side_effect = [[100], [101], [102], [103]] provider.apply(plan) wanted_calls = [ - call(u'/domain/zone/unit.tests/record', fieldType=u'TXT', - subDomain='txt', target=u'TXT text', ttl=1400), - call(u'/domain/zone/unit.tests/record', fieldType=u'DKIM', - subDomain='dkim', target=self.valid_dkim_key, - ttl=1300), - call(u'/domain/zone/unit.tests/record', fieldType=u'A', - subDomain=u'', target=u'1.2.3.4', ttl=100), - call(u'/domain/zone/unit.tests/record', fieldType=u'SRV', + call('/domain/zone/unit.tests/record', fieldType='A', + subDomain='', target='1.2.3.4', ttl=100), + call('/domain/zone/unit.tests/record', fieldType='AAAA', + subDomain='', target='1:1ec:1::1', ttl=200), + call('/domain/zone/unit.tests/record', fieldType='MX', + subDomain='', target='10 mx1.unit.tests.', ttl=400), + call('/domain/zone/unit.tests/record', fieldType='SPF', + subDomain='', + target='v=spf1 include:unit.texts.redirect ~all', + ttl=1000), + call('/domain/zone/unit.tests/record', fieldType='SSHFP', + subDomain='', + target='1 1 bf6b6825d2977c511a475bbefb88aad54a92ac73', + ttl=1100), + call('/domain/zone/unit.tests/record', fieldType='PTR', + subDomain='4', target='unit.tests.', ttl=900), + call('/domain/zone/unit.tests/record', fieldType='SRV', subDomain='_srv._tcp', - target=u'10 20 30 foo-1.unit.tests.', ttl=800), - call(u'/domain/zone/unit.tests/record', fieldType=u'SRV', + target='10 20 30 foo-1.unit.tests.', ttl=800), + call('/domain/zone/unit.tests/record', fieldType='SRV', subDomain='_srv._tcp', - target=u'40 50 60 foo-2.unit.tests.', ttl=800), - call(u'/domain/zone/unit.tests/record', fieldType=u'PTR', - subDomain='4', target=u'unit.tests.', ttl=900), - call(u'/domain/zone/unit.tests/record', fieldType=u'NS', - subDomain='www3', target=u'ns3.unit.tests.', ttl=700), - call(u'/domain/zone/unit.tests/record', fieldType=u'NS', - subDomain='www3', target=u'ns4.unit.tests.', ttl=700), - call(u'/domain/zone/unit.tests/record', - fieldType=u'SSHFP', subDomain=u'', ttl=1100, - target=u'1 1 bf6b6825d2977c511a475bbefb88a' - u'ad54' - u'a92ac73', - ), - call(u'/domain/zone/unit.tests/record', fieldType=u'AAAA', - subDomain=u'', target=u'1:1ec:1::1', ttl=200), - call(u'/domain/zone/unit.tests/record', fieldType=u'MX', - subDomain=u'', target=u'10 mx1.unit.tests.', ttl=400), - call(u'/domain/zone/unit.tests/record', fieldType=u'CNAME', - subDomain='www2', target=u'unit.tests.', ttl=300), - call(u'/domain/zone/unit.tests/record', fieldType=u'SPF', - subDomain=u'', ttl=1000, - target=u'v=spf1 include:unit.texts.' - u'redirect ~all', - ), - call(u'/domain/zone/unit.tests/record', fieldType=u'A', - subDomain='sub', target=u'1.2.3.4', ttl=200), - call(u'/domain/zone/unit.tests/record', fieldType=u'NAPTR', - subDomain='naptr', ttl=500, - target=u'10 100 "S" "SIP+D2U" "!^.*$!sip:' - u'info@bar' - u'.example.com!" .' - ), - call(u'/domain/zone/unit.tests/refresh')] + target='40 50 60 foo-2.unit.tests.', ttl=800), + call('/domain/zone/unit.tests/record', fieldType='CAA', + subDomain='caa', target='0 issue "ca.unit.tests"', + ttl=1600), + call('/domain/zone/unit.tests/record', fieldType='DKIM', + subDomain='dkim', + target='p=MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCxLaG' + '16G4SaEcXVdiIxTg7gKSGbHKQLm30CHib1h9FzS9nkcyvQSyQj1r' + 'MFyqC//tft3ohx3nvJl+bGCWxdtLYDSmir9PW54e5CTdxEh8MWRk' + 'BO3StF6QG/tAh3aTGDmkqhIJGLb87iHvpmVKqURmEUzJPv5KPJfW' + 'LofADI+q9lQIDAQAB', ttl=1300), + call('/domain/zone/unit.tests/record', fieldType='NAPTR', + subDomain='naptr', + target='10 100 "S" "SIP+D2U" "!^.*$!sip:info@bar.exam' + 'ple.com!" .', ttl=500), + call('/domain/zone/unit.tests/record', fieldType='A', + subDomain='sub', target='1.2.3.4', ttl=200), + call('/domain/zone/unit.tests/record', fieldType='TXT', + subDomain='txt', target='TXT text', ttl=1400), + call('/domain/zone/unit.tests/record', fieldType='CNAME', + subDomain='www2', target='unit.tests.', ttl=300), + call('/domain/zone/unit.tests/record', fieldType='NS', + subDomain='www3', target='ns3.unit.tests.', ttl=700), + call('/domain/zone/unit.tests/record', fieldType='NS', + subDomain='www3', target='ns4.unit.tests.', ttl=700), + call('/domain/zone/unit.tests/refresh')] post_mock.assert_has_calls(wanted_calls) # Get for delete calls wanted_get_calls = [ - call(u'/domain/zone/unit.tests/record', fieldType=u'TXT', - subDomain='txt'), - call(u'/domain/zone/unit.tests/record', fieldType=u'DKIM', - subDomain='dkim'), call(u'/domain/zone/unit.tests/record', fieldType=u'A', subDomain=u''), + call(u'/domain/zone/unit.tests/record', fieldType=u'DKIM', + subDomain='dkim'), call(u'/domain/zone/unit.tests/record', fieldType=u'A', - subDomain='fake')] + subDomain='fake'), + call(u'/domain/zone/unit.tests/record', fieldType=u'TXT', + subDomain='txt')] get_mock.assert_has_calls(wanted_get_calls) # 4 delete calls for update and delete delete_mock.assert_has_calls( diff --git a/tests/test_octodns_provider_powerdns.py b/tests/test_octodns_provider_powerdns.py index 2ce8519..6baee6c 100644 --- a/tests/test_octodns_provider_powerdns.py +++ b/tests/test_octodns_provider_powerdns.py @@ -9,6 +9,7 @@ from json import loads, dumps from os.path import dirname, join from requests import HTTPError from requests_mock import ANY, mock as requests_mock +from six import text_type from unittest import TestCase from octodns.record import Record @@ -52,7 +53,7 @@ class TestPowerDnsProvider(TestCase): with self.assertRaises(Exception) as ctx: zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertTrue('unauthorized' in ctx.exception.message) + self.assertTrue('unauthorized' in text_type(ctx.exception)) # General error with requests_mock() as mock: diff --git a/tests/test_octodns_provider_rackspace.py b/tests/test_octodns_provider_rackspace.py index c467dec..0a6564d 100644 --- a/tests/test_octodns_provider_rackspace.py +++ b/tests/test_octodns_provider_rackspace.py @@ -7,8 +7,9 @@ from __future__ import absolute_import, division, print_function, \ import json import re +from six import text_type +from six.moves.urllib.parse import urlparse from unittest import TestCase -from urlparse import urlparse from requests import HTTPError from requests_mock import ANY, mock as requests_mock @@ -39,7 +40,6 @@ with open('./tests/fixtures/rackspace-sample-recordset-page2.json') as fh: class TestRackspaceProvider(TestCase): def setUp(self): - self.maxDiff = 1000 with requests_mock() as mock: mock.post(ANY, status_code=200, text=AUTH_RESPONSE) self.provider = RackspaceProvider('identity', 'test', 'api-key', @@ -53,7 +53,7 @@ class TestRackspaceProvider(TestCase): with self.assertRaises(Exception) as ctx: zone = Zone('unit.tests.', []) self.provider.populate(zone) - self.assertTrue('unauthorized' in ctx.exception.message) + self.assertTrue('unauthorized' in text_type(ctx.exception)) self.assertTrue(mock.called_once) def test_server_error(self): @@ -792,13 +792,13 @@ class TestRackspaceProvider(TestCase): ExpectedUpdates = { "records": [{ "name": "unit.tests", - "id": "A-222222", - "data": "1.2.3.5", + "id": "A-111111", + "data": "1.2.3.4", "ttl": 3600 }, { "name": "unit.tests", - "id": "A-111111", - "data": "1.2.3.4", + "id": "A-222222", + "data": "1.2.3.5", "ttl": 3600 }, { "name": "unit.tests", diff --git a/tests/test_octodns_provider_route53.py b/tests/test_octodns_provider_route53.py index ceafd13..6a079dd 100644 --- a/tests/test_octodns_provider_route53.py +++ b/tests/test_octodns_provider_route53.py @@ -7,6 +7,7 @@ from __future__ import absolute_import, division, print_function, \ from botocore.exceptions import ClientError from botocore.stub import ANY, Stubber +from six import text_type from unittest import TestCase from mock import patch @@ -369,6 +370,16 @@ class TestRoute53Provider(TestCase): return (provider, stubber) + def _get_stubbed_delegation_set_provider(self): + provider = Route53Provider('test', 'abc', '123', + delegation_set_id="ABCDEFG123456") + + # Use the stubber + stubber = Stubber(provider._conn) + stubber.activate() + + return (provider, stubber) + def _get_stubbed_fallback_auth_provider(self): provider = Route53Provider('test') @@ -912,6 +923,92 @@ class TestRoute53Provider(TestCase): self.assertEquals(9, provider.apply(plan)) stubber.assert_no_pending_responses() + def test_sync_create_with_delegation_set(self): + provider, stubber = self._get_stubbed_delegation_set_provider() + + got = Zone('unit.tests.', []) + + list_hosted_zones_resp = { + 'HostedZones': [], + 'Marker': 'm', + 'IsTruncated': False, + 'MaxItems': '100', + } + stubber.add_response('list_hosted_zones', list_hosted_zones_resp, + {}) + + plan = provider.plan(self.expected) + self.assertEquals(9, len(plan.changes)) + self.assertFalse(plan.exists) + for change in plan.changes: + self.assertIsInstance(change, Create) + stubber.assert_no_pending_responses() + + create_hosted_zone_resp = { + 'HostedZone': { + 'Name': 'unit.tests.', + 'Id': 'z42', + 'CallerReference': 'abc', + }, + 'ChangeInfo': { + 'Id': 'a12', + 'Status': 'PENDING', + 'SubmittedAt': '2017-01-29T01:02:03Z', + 'Comment': 'hrm', + }, + 'DelegationSet': { + 'Id': 'b23', + 'CallerReference': 'blip', + 'NameServers': [ + 'n12.unit.tests.', + ], + }, + 'Location': 'us-east-1', + } + stubber.add_response('create_hosted_zone', + create_hosted_zone_resp, { + 'Name': got.name, + 'CallerReference': ANY, + 'DelegationSetId': 'ABCDEFG123456' + }) + + list_resource_record_sets_resp = { + 'ResourceRecordSets': [{ + 'Name': 'a.unit.tests.', + 'Type': 'A', + 'GeoLocation': { + 'ContinentCode': 'NA', + }, + 'ResourceRecords': [{ + 'Value': '2.2.3.4', + }], + 'TTL': 61, + }], + 'IsTruncated': False, + 'MaxItems': '100', + } + stubber.add_response('list_resource_record_sets', + list_resource_record_sets_resp, + {'HostedZoneId': 'z42'}) + + stubber.add_response('list_health_checks', + { + 'HealthChecks': self.health_checks, + 'IsTruncated': False, + 'MaxItems': '100', + 'Marker': '', + }) + + stubber.add_response('change_resource_record_sets', + {'ChangeInfo': { + 'Id': 'id', + 'Status': 'PENDING', + 'SubmittedAt': '2017-01-29T01:02:03Z', + }}, {'HostedZoneId': 'z42', 'ChangeBatch': ANY}) + + self.assertEquals(9, provider.apply(plan)) + stubber.assert_no_pending_responses() + def test_health_checks_pagination(self): provider, stubber = self._get_stubbed_provider() @@ -1881,10 +1978,10 @@ class TestRoute53Provider(TestCase): @patch('octodns.provider.route53.Route53Provider._really_apply') def test_apply_3(self, really_apply_mock, _): - # with a max of seven modifications, four calls + # with a max of seven modifications, three calls provider, plan = self._get_test_plan(7) provider.apply(plan) - self.assertEquals(4, really_apply_mock.call_count) + self.assertEquals(3, really_apply_mock.call_count) @patch('octodns.provider.route53.Route53Provider._load_records') @patch('octodns.provider.route53.Route53Provider._really_apply') @@ -1903,7 +2000,7 @@ class TestRoute53Provider(TestCase): provider, plan = self._get_test_plan(1) with self.assertRaises(Exception) as ctx: provider.apply(plan) - self.assertTrue('modifications' in ctx.exception.message) + self.assertTrue('modifications' in text_type(ctx.exception)) def test_semicolon_fixup(self): provider = Route53Provider('test', 'abc', '123') @@ -1929,9 +2026,8 @@ class TestRoute53Provider(TestCase): provider = Route53Provider('test', 'abc', '123', client_max_attempts=42) # NOTE: this will break if boto ever changes the impl details... - self.assertEquals(43, provider._conn.meta.events - ._unique_id_handlers['retry-config-route53'] - ['handler']._checker.__dict__['_max_attempts']) + self.assertEquals(42, provider._conn._client_config + .retries['max_attempts']) def test_data_for_dynamic(self): provider = Route53Provider('test', 'abc', '123') @@ -2090,6 +2186,58 @@ class TestRoute53Records(TestCase): e.__repr__() f.__repr__() + def test_route53_record_ordering(self): + # Matches + a = _Route53Record(None, self.record_a, False) + b = _Route53Record(None, self.record_a, False) + self.assertTrue(a == b) + self.assertFalse(a != b) + self.assertFalse(a < b) + self.assertTrue(a <= b) + self.assertFalse(a > b) + self.assertTrue(a >= b) + + # Change the fqdn is greater + fqdn = _Route53Record(None, self.record_a, False, + fqdn_override='other') + self.assertFalse(a == fqdn) + self.assertTrue(a != fqdn) + self.assertFalse(a < fqdn) + self.assertFalse(a <= fqdn) + self.assertTrue(a > fqdn) + self.assertTrue(a >= fqdn) + + provider = DummyProvider() + geo_a = _Route53GeoRecord(provider, self.record_a, 'NA-US', + self.record_a.geo['NA-US'], False) + geo_b = _Route53GeoRecord(provider, self.record_a, 'NA-US', + self.record_a.geo['NA-US'], False) + self.assertTrue(geo_a == geo_b) + self.assertFalse(geo_a != geo_b) + self.assertFalse(geo_a < geo_b) + self.assertTrue(geo_a <= geo_b) + self.assertFalse(geo_a > geo_b) + self.assertTrue(geo_a >= geo_b) + + # Other base + geo_fqdn = _Route53GeoRecord(provider, self.record_a, 'NA-US', + self.record_a.geo['NA-US'], False) + geo_fqdn.fqdn = 'other' + self.assertFalse(geo_a == geo_fqdn) + self.assertTrue(geo_a != geo_fqdn) + self.assertFalse(geo_a < geo_fqdn) + self.assertFalse(geo_a <= geo_fqdn) + self.assertTrue(geo_a > geo_fqdn) + self.assertTrue(geo_a >= geo_fqdn) + + # Other class + self.assertFalse(a == geo_a) + self.assertTrue(a != geo_a) + self.assertFalse(a < geo_a) + self.assertFalse(a <= geo_a) + self.assertTrue(a > geo_a) + self.assertTrue(a >= geo_a) + def test_dynamic_value_delete(self): provider = DummyProvider() geo = _Route53DynamicValue(provider, self.record_a, 'iad', '2.2.2.2', @@ -2206,70 +2354,112 @@ class TestRoute53Records(TestCase): creating=True) self.assertEquals(18, len(route53_records)) + expected_mods = [r.mod('CREATE', []) for r in route53_records] + # Sort so that we get a consistent order and don't rely on set ordering + expected_mods.sort(key=_mod_keyer) + # Convert the route53_records into mods self.assertEquals([{ 'Action': 'CREATE', 'ResourceRecordSet': { 'HealthCheckId': 'hc42', 'Name': '_octodns-ap-southeast-1-value.unit.tests.', - 'ResourceRecords': [{ - 'Value': '1.4.1.2'}], - 'SetIdentifier': 'ap-southeast-1-001', + 'ResourceRecords': [{'Value': '1.4.1.1'}], + 'SetIdentifier': 'ap-southeast-1-000', 'TTL': 60, 'Type': 'A', - 'Weight': 2 - } + 'Weight': 2} }, { 'Action': 'CREATE', 'ResourceRecordSet': { 'HealthCheckId': 'hc42', 'Name': '_octodns-ap-southeast-1-value.unit.tests.', - 'ResourceRecords': [{ - 'Value': '1.4.1.1'}], - 'SetIdentifier': 'ap-southeast-1-000', + 'ResourceRecords': [{'Value': '1.4.1.2'}], + 'SetIdentifier': 'ap-southeast-1-001', 'TTL': 60, 'Type': 'A', - 'Weight': 2 - } + 'Weight': 2} + }, { + 'Action': 'CREATE', + 'ResourceRecordSet': { + 'Name': '_octodns-default-pool.unit.tests.', + 'ResourceRecords': [ + {'Value': '1.1.2.1'}, + {'Value': '1.1.2.2'}], + 'TTL': 60, + 'Type': 'A'} + }, { + 'Action': 'CREATE', + 'ResourceRecordSet': { + 'HealthCheckId': 'hc42', + 'Name': '_octodns-eu-central-1-value.unit.tests.', + 'ResourceRecords': [{'Value': '1.3.1.1'}], + 'SetIdentifier': 'eu-central-1-000', + 'TTL': 60, + 'Type': 'A', + 'Weight': 1} + }, { + 'Action': 'CREATE', + 'ResourceRecordSet': { + 'HealthCheckId': 'hc42', + 'Name': '_octodns-eu-central-1-value.unit.tests.', + 'ResourceRecords': [{'Value': '1.3.1.2'}], + 'SetIdentifier': 'eu-central-1-001', + 'TTL': 60, + 'Type': 'A', + 'Weight': 1} + }, { + 'Action': 'CREATE', + 'ResourceRecordSet': { + 'HealthCheckId': 'hc42', + 'Name': '_octodns-us-east-1-value.unit.tests.', + 'ResourceRecords': [{'Value': '1.5.1.1'}], + 'SetIdentifier': 'us-east-1-000', + 'TTL': 60, + 'Type': 'A', + 'Weight': 1} + }, { + 'Action': 'CREATE', + 'ResourceRecordSet': { + 'HealthCheckId': 'hc42', + 'Name': '_octodns-us-east-1-value.unit.tests.', + 'ResourceRecords': [{'Value': '1.5.1.2'}], + 'SetIdentifier': 'us-east-1-001', + 'TTL': 60, + 'Type': 'A', + 'Weight': 1} }, { 'Action': 'CREATE', 'ResourceRecordSet': { 'AliasTarget': { - 'DNSName': '_octodns-ap-southeast-1-pool.unit.tests.', + 'DNSName': '_octodns-ap-southeast-1-value.unit.tests.', 'EvaluateTargetHealth': True, - 'HostedZoneId': 'z45' - }, - 'GeoLocation': { - 'CountryCode': 'JP'}, - 'Name': 'unit.tests.', - 'SetIdentifier': '0-ap-southeast-1-AS-JP', - 'Type': 'A' - } + 'HostedZoneId': 'z45'}, + 'Failover': 'PRIMARY', + 'Name': '_octodns-ap-southeast-1-pool.unit.tests.', + 'SetIdentifier': 'ap-southeast-1-Primary', + 'Type': 'A'} }, { 'Action': 'CREATE', 'ResourceRecordSet': { 'AliasTarget': { - 'DNSName': '_octodns-eu-central-1-pool.unit.tests.', + 'DNSName': '_octodns-eu-central-1-value.unit.tests.', 'EvaluateTargetHealth': True, 'HostedZoneId': 'z45'}, - 'GeoLocation': { - 'CountryCode': 'US', - 'SubdivisionCode': 'FL', - }, - 'Name': 'unit.tests.', - 'SetIdentifier': '1-eu-central-1-NA-US-FL', + 'Failover': 'PRIMARY', + 'Name': '_octodns-eu-central-1-pool.unit.tests.', + 'SetIdentifier': 'eu-central-1-Primary', 'Type': 'A'} }, { 'Action': 'CREATE', 'ResourceRecordSet': { 'AliasTarget': { - 'DNSName': '_octodns-us-east-1-pool.unit.tests.', + 'DNSName': '_octodns-us-east-1-value.unit.tests.', 'EvaluateTargetHealth': True, 'HostedZoneId': 'z45'}, - 'GeoLocation': { - 'CountryCode': '*'}, - 'Name': 'unit.tests.', - 'SetIdentifier': '2-us-east-1-None', + 'Failover': 'PRIMARY', + 'Name': '_octodns-us-east-1-pool.unit.tests.', + 'SetIdentifier': 'us-east-1-Primary', 'Type': 'A'} }, { 'Action': 'CREATE', @@ -2286,123 +2476,72 @@ class TestRoute53Records(TestCase): 'Action': 'CREATE', 'ResourceRecordSet': { 'AliasTarget': { - 'DNSName': '_octodns-ap-southeast-1-pool.unit.tests.', + 'DNSName': '_octodns-us-east-1-pool.unit.tests.', 'EvaluateTargetHealth': True, 'HostedZoneId': 'z45'}, - 'GeoLocation': { - 'CountryCode': 'CN'}, - 'Name': 'unit.tests.', - 'SetIdentifier': '0-ap-southeast-1-AS-CN', + 'Failover': 'SECONDARY', + 'Name': '_octodns-eu-central-1-pool.unit.tests.', + 'SetIdentifier': 'eu-central-1-Secondary-us-east-1', 'Type': 'A'} }, { 'Action': 'CREATE', 'ResourceRecordSet': { 'AliasTarget': { - 'DNSName': '_octodns-us-east-1-value.unit.tests.', + 'DNSName': '_octodns-default-pool.unit.tests.', 'EvaluateTargetHealth': True, 'HostedZoneId': 'z45'}, - 'Failover': 'PRIMARY', + 'Failover': 'SECONDARY', 'Name': '_octodns-us-east-1-pool.unit.tests.', - 'SetIdentifier': 'us-east-1-Primary', + 'SetIdentifier': 'us-east-1-Secondary-default', 'Type': 'A'} }, { 'Action': 'CREATE', 'ResourceRecordSet': { 'AliasTarget': { - 'DNSName': '_octodns-eu-central-1-pool.unit.tests.', + 'DNSName': '_octodns-ap-southeast-1-pool.unit.tests.', 'EvaluateTargetHealth': True, 'HostedZoneId': 'z45'}, 'GeoLocation': { - 'ContinentCode': 'EU'}, + 'CountryCode': 'CN'}, 'Name': 'unit.tests.', - 'SetIdentifier': '1-eu-central-1-EU', + 'SetIdentifier': '0-ap-southeast-1-AS-CN', 'Type': 'A'} }, { 'Action': 'CREATE', 'ResourceRecordSet': { 'AliasTarget': { - 'DNSName': '_octodns-eu-central-1-value.unit.tests.', + 'DNSName': '_octodns-ap-southeast-1-pool.unit.tests.', 'EvaluateTargetHealth': True, 'HostedZoneId': 'z45'}, - 'Failover': 'PRIMARY', - 'Name': '_octodns-eu-central-1-pool.unit.tests.', - 'SetIdentifier': 'eu-central-1-Primary', - 'Type': 'A'} - }, { - 'Action': 'CREATE', - 'ResourceRecordSet': { - 'Name': '_octodns-default-pool.unit.tests.', - 'ResourceRecords': [{ - 'Value': '1.1.2.1'}, - { - 'Value': '1.1.2.2'}], - 'TTL': 60, + 'GeoLocation': { + 'CountryCode': 'JP'}, + 'Name': 'unit.tests.', + 'SetIdentifier': '0-ap-southeast-1-AS-JP', 'Type': 'A'} - }, { - 'Action': 'CREATE', - 'ResourceRecordSet': { - 'HealthCheckId': 'hc42', - 'Name': '_octodns-eu-central-1-value.unit.tests.', - 'ResourceRecords': [{ - 'Value': '1.3.1.2'}], - 'SetIdentifier': 'eu-central-1-001', - 'TTL': 60, - 'Type': 'A', - 'Weight': 1} - }, { - 'Action': 'CREATE', - 'ResourceRecordSet': { - 'HealthCheckId': 'hc42', - 'Name': '_octodns-eu-central-1-value.unit.tests.', - 'ResourceRecords': [{ - 'Value': '1.3.1.1'}], - 'SetIdentifier': 'eu-central-1-000', - 'TTL': 60, - 'Type': 'A', - 'Weight': 1} }, { 'Action': 'CREATE', 'ResourceRecordSet': { 'AliasTarget': { - 'DNSName': '_octodns-default-pool.unit.tests.', + 'DNSName': '_octodns-eu-central-1-pool.unit.tests.', 'EvaluateTargetHealth': True, 'HostedZoneId': 'z45'}, - 'Failover': 'SECONDARY', - 'Name': '_octodns-us-east-1-pool.unit.tests.', - 'SetIdentifier': 'us-east-1-Secondary-default', + 'GeoLocation': { + 'ContinentCode': 'EU'}, + 'Name': 'unit.tests.', + 'SetIdentifier': '1-eu-central-1-EU', 'Type': 'A'} - }, { - 'Action': 'CREATE', - 'ResourceRecordSet': { - 'HealthCheckId': 'hc42', - 'Name': '_octodns-us-east-1-value.unit.tests.', - 'ResourceRecords': [{ - 'Value': '1.5.1.2'}], - 'SetIdentifier': 'us-east-1-001', - 'TTL': 60, - 'Type': 'A', - 'Weight': 1} - }, { - 'Action': 'CREATE', - 'ResourceRecordSet': { - 'HealthCheckId': 'hc42', - 'Name': '_octodns-us-east-1-value.unit.tests.', - 'ResourceRecords': [{ - 'Value': '1.5.1.1'}], - 'SetIdentifier': 'us-east-1-000', - 'TTL': 60, - 'Type': 'A', - 'Weight': 1} }, { 'Action': 'CREATE', 'ResourceRecordSet': { 'AliasTarget': { - 'DNSName': '_octodns-ap-southeast-1-value.unit.tests.', + 'DNSName': '_octodns-eu-central-1-pool.unit.tests.', 'EvaluateTargetHealth': True, 'HostedZoneId': 'z45'}, - 'Failover': 'PRIMARY', - 'Name': '_octodns-ap-southeast-1-pool.unit.tests.', - 'SetIdentifier': 'ap-southeast-1-Primary', + 'GeoLocation': { + 'CountryCode': 'US', + 'SubdivisionCode': 'FL'}, + 'Name': 'unit.tests.', + 'SetIdentifier': '1-eu-central-1-NA-US-FL', 'Type': 'A'} }, { 'Action': 'CREATE', @@ -2411,11 +2550,12 @@ class TestRoute53Records(TestCase): 'DNSName': '_octodns-us-east-1-pool.unit.tests.', 'EvaluateTargetHealth': True, 'HostedZoneId': 'z45'}, - 'Failover': 'SECONDARY', - 'Name': '_octodns-eu-central-1-pool.unit.tests.', - 'SetIdentifier': 'eu-central-1-Secondary-us-east-1', + 'GeoLocation': { + 'CountryCode': '*'}, + 'Name': 'unit.tests.', + 'SetIdentifier': '2-us-east-1-None', 'Type': 'A'} - }], [r.mod('CREATE', []) for r in route53_records]) + }], expected_mods) for route53_record in route53_records: # Smoke test stringification diff --git a/tests/test_octodns_provider_selectel.py b/tests/test_octodns_provider_selectel.py index a2ba39e..7ad1e6b 100644 --- a/tests/test_octodns_provider_selectel.py +++ b/tests/test_octodns_provider_selectel.py @@ -6,6 +6,7 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals from unittest import TestCase +from six import text_type import requests_mock @@ -288,7 +289,7 @@ class TestSelectelProvider(TestCase): with self.assertRaises(Exception) as ctx: SelectelProvider(123, 'fail_token') - self.assertEquals(ctx.exception.message, + self.assertEquals(text_type(ctx.exception), 'Authorization failed. Invalid or empty token.') @requests_mock.Mocker() diff --git a/tests/test_octodns_provider_transip.py b/tests/test_octodns_provider_transip.py new file mode 100644 index 0000000..3fbfc44 --- /dev/null +++ b/tests/test_octodns_provider_transip.py @@ -0,0 +1,275 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from os.path import dirname, join +from six import text_type + +from suds import WebFault + +from unittest import TestCase + +from octodns.provider.transip import TransipProvider +from octodns.provider.yaml import YamlProvider +from octodns.zone import Zone +from transip.service.domain import DomainService +from transip.service.objects import DnsEntry + + +class MockFault(object): + faultstring = "" + faultcode = "" + + def __init__(self, code, string, *args, **kwargs): + self.faultstring = string + self.faultcode = code + + +class MockResponse(object): + dnsEntries = [] + + +class MockDomainService(DomainService): + + def __init__(self, *args, **kwargs): + super(MockDomainService, self).__init__('MockDomainService', *args, + **kwargs) + self.mockupEntries = [] + + def mockup(self, records): + + provider = TransipProvider('', '', '') + + _dns_entries = [] + for record in records: + if record._type in provider.SUPPORTS: + entries_for = getattr(provider, + '_entries_for_{}'.format(record._type)) + + # Root records have '@' as name + name = record.name + if name == '': + name = provider.ROOT_RECORD + + _dns_entries.extend(entries_for(name, record)) + + # NS is not supported as a DNS Entry, + # so it should cover the if statement + _dns_entries.append( + DnsEntry('@', '3600', 'NS', 'ns01.transip.nl.')) + + self.mockupEntries = _dns_entries + + # Skips authentication layer and returns the entries loaded by "Mockup" + def get_info(self, domain_name): + + # Special 'domain' to trigger error + if str(domain_name) == str('notfound.unit.tests'): + self.raiseZoneNotFound() + + result = MockResponse() + result.dnsEntries = self.mockupEntries + return result + + def set_dns_entries(self, domain_name, dns_entries): + + # Special 'domain' to trigger error + if str(domain_name) == str('failsetdns.unit.tests'): + self.raiseSaveError() + + return True + + def raiseZoneNotFound(self): + fault = MockFault(str('102'), '102 is zone not found') + document = {} + raise WebFault(fault, document) + + def raiseInvalidAuth(self): + fault = MockFault(str('200'), '200 is invalid auth') + document = {} + raise WebFault(fault, document) + + def raiseSaveError(self): + fault = MockFault(str('200'), '202 random error') + document = {} + raise WebFault(fault, document) + + +class TestTransipProvider(TestCase): + bogus_key = str("""-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEA0U5HGCkLrz423IyUf3u4cKN2WrNz1x5KNr6PvH2M/zxas+zB +elbxkdT3AQ+wmfcIvOuTmFRTHv35q2um1aBrPxVw+2s+lWo28VwIRttwIB1vIeWu +lSBnkEZQRLyPI2tH0i5QoMX4CVPf9rvij3Uslimi84jdzDfPFIh6jZ6C8nLipOTG +0IMhge1ofVfB0oSy5H+7PYS2858QLAf5ruYbzbAxZRivS402wGmQ0d0Lc1KxraAj +kiMM5yj/CkH/Vm2w9I6+tLFeASE4ub5HCP5G/ig4dbYtqZMQMpqyAbGxd5SOVtyn +UHagAJUxf8DT3I8PyjEHjxdOPUsxNyRtepO/7QIDAQABAoIBAQC7fiZ7gxE/ezjD +2n6PsHFpHVTBLS2gzzZl0dCKZeFvJk6ODJDImaeuHhrh7X8ifMNsEI9XjnojMhl8 +MGPzy88mZHugDNK0H8B19x5G8v1/Fz7dG5WHas660/HFkS+b59cfdXOugYiOOn9O +08HBBpLZNRUOmVUuQfQTjapSwGLG8PocgpyRD4zx0LnldnJcqYCxwCdev+AAsPnq +ibNtOd/MYD37w9MEGcaxLE8wGgkv8yd97aTjkgE+tp4zsM4QE4Rag133tsLLNznT +4Qr/of15M3NW/DXq/fgctyRcJjZpU66eCXLCz2iRTnLyyxxDC2nwlxKbubV+lcS0 +S4hbfd/BAoGBAO8jXxEaiybR0aIhhSR5esEc3ymo8R8vBN3ZMJ+vr5jEPXr/ZuFj +/R4cZ2XV3VoQJG0pvIOYVPZ5DpJM7W+zSXtJ/7bLXy4Bnmh/rc+YYgC+AXQoLSil +iD2OuB2xAzRAK71DVSO0kv8gEEXCersPT2i6+vC2GIlJvLcYbOdRKWGxAoGBAOAQ +aJbRLtKujH+kMdoMI7tRlL8XwI+SZf0FcieEu//nFyerTePUhVgEtcE+7eQ7hyhG +fIXUFx/wALySoqFzdJDLc8U8pTLhbUaoLOTjkwnCTKQVprhnISqQqqh/0U5u47IE +RWzWKN6OHb0CezNTq80Dr6HoxmPCnJHBHn5LinT9AoGAQSpvZpbIIqz8pmTiBl2A +QQ2gFpcuFeRXPClKYcmbXVLkuhbNL1BzEniFCLAt4LQTaRf9ghLJ3FyCxwVlkpHV +zV4N6/8hkcTpKOraL38D/dXJSaEFJVVuee/hZl3tVJjEEpA9rDwx7ooLRSdJEJ6M +ciq55UyKBSdt4KssSiDI2RECgYBL3mJ7xuLy5bWfNsrGiVvD/rC+L928/5ZXIXPw +26oI0Yfun7ulDH4GOroMcDF/GYT/Zzac3h7iapLlR0WYI47xxGI0A//wBZLJ3QIu +krxkDo2C9e3Y/NqnHgsbOQR3aWbiDT4wxydZjIeXS3LKA2fl6Hyc90PN3cTEOb8I +hq2gRQKBgEt0SxhhtyB93SjgTzmUZZ7PiEf0YJatfM6cevmjWHexrZH+x31PB72s +fH2BQyTKKzoCLB1k/6HRaMnZdrWyWSZ7JKz3AHJ8+58d0Hr8LTrzDM1L6BbjeDct +N4OiVz1I3rbZGYa396lpxO6ku8yCglisL1yrSP6DdEUp66ntpKVd +-----END RSA PRIVATE KEY-----""") + + def make_expected(self): + expected = Zone('unit.tests.', []) + source = YamlProvider('test', join(dirname(__file__), 'config')) + source.populate(expected) + return expected + + def test_init(self): + with self.assertRaises(Exception) as ctx: + TransipProvider('test', 'unittest') + + self.assertEquals( + str('Missing `key` of `key_file` parameter in config'), + str(ctx.exception)) + + TransipProvider('test', 'unittest', key=self.bogus_key) + + # Existence and content of the key is tested in the SDK on client call + TransipProvider('test', 'unittest', key_file='/fake/path') + + def test_populate(self): + _expected = self.make_expected() + + # Unhappy Plan - Not authenticated + # Live test against API, will fail in an unauthorized error + with self.assertRaises(WebFault) as ctx: + provider = TransipProvider('test', 'unittest', self.bogus_key) + zone = Zone('unit.tests.', []) + provider.populate(zone, True) + + self.assertEquals(str('WebFault'), + str(ctx.exception.__class__.__name__)) + + self.assertEquals(str('200'), ctx.exception.fault.faultcode) + + # Unhappy Plan - Zone does not exists + # Will trigger an exception if provider is used as a target for a + # non-existing zone + with self.assertRaises(Exception) as ctx: + provider = TransipProvider('test', 'unittest', self.bogus_key) + provider._client = MockDomainService('unittest', self.bogus_key) + zone = Zone('notfound.unit.tests.', []) + provider.populate(zone, True) + + self.assertEquals(str('TransipNewZoneException'), + str(ctx.exception.__class__.__name__)) + + self.assertEquals( + 'populate: (102) Transip used as target' + + ' for non-existing zone: notfound.unit.tests.', + text_type(ctx.exception)) + + # Happy Plan - Zone does not exists + # Won't trigger an exception if provider is NOT used as a target for a + # non-existing zone. + provider = TransipProvider('test', 'unittest', self.bogus_key) + provider._client = MockDomainService('unittest', self.bogus_key) + zone = Zone('notfound.unit.tests.', []) + provider.populate(zone, False) + + # Happy Plan - Populate with mockup records + provider = TransipProvider('test', 'unittest', self.bogus_key) + provider._client = MockDomainService('unittest', self.bogus_key) + provider._client.mockup(_expected.records) + zone = Zone('unit.tests.', []) + provider.populate(zone, False) + + # Transip allows relative values for types like cname, mx. + # Test is these are correctly appended with the domain + provider._currentZone = zone + self.assertEquals("www.unit.tests.", provider._parse_to_fqdn("www")) + self.assertEquals("www.unit.tests.", + provider._parse_to_fqdn("www.unit.tests.")) + self.assertEquals("www.sub.sub.sub.unit.tests.", + provider._parse_to_fqdn("www.sub.sub.sub")) + self.assertEquals("unit.tests.", + provider._parse_to_fqdn("@")) + + # Happy Plan - Even if the zone has no records the zone should exist + provider = TransipProvider('test', 'unittest', self.bogus_key) + provider._client = MockDomainService('unittest', self.bogus_key) + zone = Zone('unit.tests.', []) + exists = provider.populate(zone, True) + self.assertTrue(exists, 'populate should return true') + + return + + def test_plan(self): + _expected = self.make_expected() + + # Test Happy plan, only create + provider = TransipProvider('test', 'unittest', self.bogus_key) + provider._client = MockDomainService('unittest', self.bogus_key) + plan = provider.plan(_expected) + + self.assertEqual(12, plan.change_counts['Create']) + self.assertEqual(0, plan.change_counts['Update']) + self.assertEqual(0, plan.change_counts['Delete']) + + return + + def test_apply(self): + _expected = self.make_expected() + + # Test happy flow. Create all supoorted records + provider = TransipProvider('test', 'unittest', self.bogus_key) + provider._client = MockDomainService('unittest', self.bogus_key) + plan = provider.plan(_expected) + self.assertEqual(12, len(plan.changes)) + changes = provider.apply(plan) + self.assertEqual(changes, len(plan.changes)) + + # Test unhappy flow. Trigger 'not found error' in apply stage + # This should normally not happen as populate will capture it first + # but just in case. + changes = [] # reset changes + with self.assertRaises(Exception) as ctx: + provider = TransipProvider('test', 'unittest', self.bogus_key) + provider._client = MockDomainService('unittest', self.bogus_key) + plan = provider.plan(_expected) + plan.desired.name = 'notfound.unit.tests.' + changes = provider.apply(plan) + + # Changes should not be set due to an Exception + self.assertEqual([], changes) + + self.assertEquals(str('WebFault'), + str(ctx.exception.__class__.__name__)) + + self.assertEquals(str('102'), ctx.exception.fault.faultcode) + + # Test unhappy flow. Trigger a unrecoverable error while saving + _expected = self.make_expected() # reset expected + changes = [] # reset changes + + with self.assertRaises(Exception) as ctx: + provider = TransipProvider('test', 'unittest', self.bogus_key) + provider._client = MockDomainService('unittest', self.bogus_key) + plan = provider.plan(_expected) + plan.desired.name = 'failsetdns.unit.tests.' + changes = provider.apply(plan) + + # Changes should not be set due to an Exception + self.assertEqual([], changes) + + self.assertEquals(str('TransipException'), + str(ctx.exception.__class__.__name__)) diff --git a/tests/test_octodns_provider_yaml.py b/tests/test_octodns_provider_yaml.py index d5d5e37..f858c05 100644 --- a/tests/test_octodns_provider_yaml.py +++ b/tests/test_octodns_provider_yaml.py @@ -8,6 +8,7 @@ from __future__ import absolute_import, division, print_function, \ from os import makedirs from os.path import basename, dirname, isdir, isfile, join from unittest import TestCase +from six import text_type from yaml import safe_load from yaml.constructor import ConstructorError @@ -57,8 +58,8 @@ class TestYamlProvider(TestCase): # We add everything plan = target.plan(zone) - self.assertEquals(15, len(filter(lambda c: isinstance(c, Create), - plan.changes))) + self.assertEquals(15, len([c for c in plan.changes + if isinstance(c, Create)])) self.assertFalse(isfile(yaml_file)) # Now actually do it @@ -67,8 +68,8 @@ class TestYamlProvider(TestCase): # Dynamic plan plan = target.plan(dynamic_zone) - self.assertEquals(5, len(filter(lambda c: isinstance(c, Create), - plan.changes))) + self.assertEquals(5, len([c for c in plan.changes + if isinstance(c, Create)])) self.assertFalse(isfile(dynamic_yaml_file)) # Apply it self.assertEquals(5, target.apply(plan)) @@ -79,16 +80,15 @@ class TestYamlProvider(TestCase): target.populate(reloaded) self.assertDictEqual( {'included': ['test']}, - filter( - lambda x: x.name == 'included', reloaded.records - )[0]._octodns) + [x for x in reloaded.records + if x.name == 'included'][0]._octodns) self.assertFalse(zone.changes(reloaded, target=source)) # A 2nd sync should still create everything plan = target.plan(zone) - self.assertEquals(15, len(filter(lambda c: isinstance(c, Create), - plan.changes))) + self.assertEquals(15, len([c for c in plan.changes + if isinstance(c, Create)])) with open(yaml_file) as fh: data = safe_load(fh.read()) @@ -116,7 +116,7 @@ class TestYamlProvider(TestCase): self.assertTrue('value' in data.pop('www.sub')) # make sure nothing is left - self.assertEquals([], data.keys()) + self.assertEquals([], list(data.keys())) with open(dynamic_yaml_file) as fh: data = safe_load(fh.read()) @@ -145,7 +145,7 @@ class TestYamlProvider(TestCase): # self.assertTrue('dynamic' in dyna) # make sure nothing is left - self.assertEquals([], data.keys()) + self.assertEquals([], list(data.keys())) def test_empty(self): source = YamlProvider('test', join(dirname(__file__), 'config')) @@ -178,7 +178,7 @@ class TestYamlProvider(TestCase): with self.assertRaises(SubzoneRecordException) as ctx: source.populate(zone) self.assertEquals('Record www.sub.unit.tests. is under a managed ' - 'subzone', ctx.exception.message) + 'subzone', text_type(ctx.exception)) class TestSplitYamlProvider(TestCase): @@ -201,9 +201,8 @@ class TestSplitYamlProvider(TestCase): # This isn't great, but given the variable nature of the temp dir # names, it's necessary. - self.assertItemsEqual( - yaml_files, - (basename(f) for f in _list_all_yaml_files(directory))) + d = list(basename(f) for f in _list_all_yaml_files(directory)) + self.assertEqual(len(yaml_files), len(d)) def test_zone_directory(self): source = SplitYamlProvider( @@ -252,8 +251,8 @@ class TestSplitYamlProvider(TestCase): # We add everything plan = target.plan(zone) - self.assertEquals(15, len(filter(lambda c: isinstance(c, Create), - plan.changes))) + self.assertEquals(15, len([c for c in plan.changes + if isinstance(c, Create)])) self.assertFalse(isdir(zone_dir)) # Now actually do it @@ -261,8 +260,8 @@ class TestSplitYamlProvider(TestCase): # Dynamic plan plan = target.plan(dynamic_zone) - self.assertEquals(5, len(filter(lambda c: isinstance(c, Create), - plan.changes))) + self.assertEquals(5, len([c for c in plan.changes + if isinstance(c, Create)])) self.assertFalse(isdir(dynamic_zone_dir)) # Apply it self.assertEquals(5, target.apply(plan)) @@ -273,16 +272,15 @@ class TestSplitYamlProvider(TestCase): target.populate(reloaded) self.assertDictEqual( {'included': ['test']}, - filter( - lambda x: x.name == 'included', reloaded.records - )[0]._octodns) + [x for x in reloaded.records + if x.name == 'included'][0]._octodns) self.assertFalse(zone.changes(reloaded, target=source)) # A 2nd sync should still create everything plan = target.plan(zone) - self.assertEquals(15, len(filter(lambda c: isinstance(c, Create), - plan.changes))) + self.assertEquals(15, len([c for c in plan.changes + if isinstance(c, Create)])) yaml_file = join(zone_dir, '$unit.tests.yaml') self.assertTrue(isfile(yaml_file)) @@ -371,4 +369,37 @@ class TestSplitYamlProvider(TestCase): with self.assertRaises(SubzoneRecordException) as ctx: source.populate(zone) self.assertEquals('Record www.sub.unit.tests. is under a managed ' - 'subzone', ctx.exception.message) + 'subzone', text_type(ctx.exception)) + + +class TestOverridingYamlProvider(TestCase): + + def test_provider(self): + config = join(dirname(__file__), 'config') + override_config = join(dirname(__file__), 'config', 'override') + base = YamlProvider('base', config, populate_should_replace=False) + override = YamlProvider('test', override_config, + populate_should_replace=True) + + zone = Zone('dynamic.tests.', []) + + # Load the base, should see the 5 records + base.populate(zone) + got = {r.name: r for r in zone.records} + self.assertEquals(5, len(got)) + # We get the "dynamic" A from the bae config + self.assertTrue('dynamic' in got['a'].data) + # No added + self.assertFalse('added' in got) + + # Load the overrides, should replace one and add 1 + override.populate(zone) + got = {r.name: r for r in zone.records} + self.assertEquals(6, len(got)) + # 'a' was replaced with a generic record + self.assertEquals({ + 'ttl': 3600, + 'values': ['4.4.4.4', '5.5.5.5'] + }, got['a'].data) + # And we have the new one + self.assertTrue('added' in got) diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index d6ed2d9..f313342 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -5,12 +5,14 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals +from six import text_type from unittest import TestCase from octodns.record import ARecord, AaaaRecord, AliasRecord, CaaRecord, \ - CnameRecord, Create, Delete, GeoValue, MxRecord, NaptrRecord, NaptrValue, \ - NsRecord, PtrRecord, Record, SshfpRecord, SpfRecord, SrvRecord, \ - TxtRecord, Update, ValidationError, _Dynamic, _DynamicPool, _DynamicRule + CaaValue, CnameRecord, Create, Delete, GeoValue, MxRecord, MxValue, \ + NaptrRecord, NaptrValue, NsRecord, PtrRecord, Record, SshfpRecord, \ + SshfpValue, SpfRecord, SrvRecord, SrvValue, TxtRecord, Update, \ + ValidationError, _Dynamic, _DynamicPool, _DynamicRule from octodns.zone import Zone from helpers import DynamicProvider, GeoProvider, SimpleProvider @@ -482,113 +484,140 @@ class TestRecord(TestCase): # full sorting # equivalent b_naptr_value = b.values[0] - self.assertEquals(0, b_naptr_value.__cmp__(b_naptr_value)) + self.assertTrue(b_naptr_value == b_naptr_value) + self.assertFalse(b_naptr_value != b_naptr_value) + self.assertTrue(b_naptr_value <= b_naptr_value) + self.assertTrue(b_naptr_value >= b_naptr_value) # by order - self.assertEquals(1, b_naptr_value.__cmp__(NaptrValue({ + self.assertTrue(b_naptr_value > NaptrValue({ 'order': 10, 'preference': 31, 'flags': 'M', 'service': 'N', 'regexp': 'O', 'replacement': 'x', - }))) - self.assertEquals(-1, b_naptr_value.__cmp__(NaptrValue({ + })) + self.assertTrue(b_naptr_value < NaptrValue({ 'order': 40, 'preference': 31, 'flags': 'M', 'service': 'N', 'regexp': 'O', 'replacement': 'x', - }))) + })) # by preference - self.assertEquals(1, b_naptr_value.__cmp__(NaptrValue({ + self.assertTrue(b_naptr_value > NaptrValue({ 'order': 30, 'preference': 10, 'flags': 'M', 'service': 'N', 'regexp': 'O', 'replacement': 'x', - }))) - self.assertEquals(-1, b_naptr_value.__cmp__(NaptrValue({ + })) + self.assertTrue(b_naptr_value < NaptrValue({ 'order': 30, 'preference': 40, 'flags': 'M', 'service': 'N', 'regexp': 'O', 'replacement': 'x', - }))) + })) # by flags - self.assertEquals(1, b_naptr_value.__cmp__(NaptrValue({ + self.assertTrue(b_naptr_value > NaptrValue({ 'order': 30, 'preference': 31, 'flags': 'A', 'service': 'N', 'regexp': 'O', 'replacement': 'x', - }))) - self.assertEquals(-1, b_naptr_value.__cmp__(NaptrValue({ + })) + self.assertTrue(b_naptr_value < NaptrValue({ 'order': 30, 'preference': 31, 'flags': 'Z', 'service': 'N', 'regexp': 'O', 'replacement': 'x', - }))) + })) # by service - self.assertEquals(1, b_naptr_value.__cmp__(NaptrValue({ + self.assertTrue(b_naptr_value > NaptrValue({ 'order': 30, 'preference': 31, 'flags': 'M', 'service': 'A', 'regexp': 'O', 'replacement': 'x', - }))) - self.assertEquals(-1, b_naptr_value.__cmp__(NaptrValue({ + })) + self.assertTrue(b_naptr_value < NaptrValue({ 'order': 30, 'preference': 31, 'flags': 'M', 'service': 'Z', 'regexp': 'O', 'replacement': 'x', - }))) + })) # by regexp - self.assertEquals(1, b_naptr_value.__cmp__(NaptrValue({ + self.assertTrue(b_naptr_value > NaptrValue({ 'order': 30, 'preference': 31, 'flags': 'M', 'service': 'N', 'regexp': 'A', 'replacement': 'x', - }))) - self.assertEquals(-1, b_naptr_value.__cmp__(NaptrValue({ + })) + self.assertTrue(b_naptr_value < NaptrValue({ 'order': 30, 'preference': 31, 'flags': 'M', 'service': 'N', 'regexp': 'Z', 'replacement': 'x', - }))) + })) # by replacement - self.assertEquals(1, b_naptr_value.__cmp__(NaptrValue({ + self.assertTrue(b_naptr_value > NaptrValue({ 'order': 30, 'preference': 31, 'flags': 'M', 'service': 'N', 'regexp': 'O', 'replacement': 'a', - }))) - self.assertEquals(-1, b_naptr_value.__cmp__(NaptrValue({ + })) + self.assertTrue(b_naptr_value < NaptrValue({ 'order': 30, 'preference': 31, 'flags': 'M', 'service': 'N', 'regexp': 'O', 'replacement': 'z', - }))) + })) # __repr__ doesn't blow up a.__repr__() + # Hash + v = NaptrValue({ + 'order': 30, + 'preference': 31, + 'flags': 'M', + 'service': 'N', + 'regexp': 'O', + 'replacement': 'z', + }) + o = NaptrValue({ + 'order': 30, + 'preference': 32, + 'flags': 'M', + 'service': 'N', + 'regexp': 'O', + 'replacement': 'z', + }) + values = set() + values.add(v) + self.assertTrue(v in values) + self.assertFalse(o in values) + values.add(o) + self.assertTrue(o in values) + def test_ns(self): a_values = ['5.6.7.8.', '6.7.8.9.', '7.8.9.0.'] a_data = {'ttl': 30, 'values': a_values} @@ -758,14 +787,14 @@ class TestRecord(TestCase): # Missing type with self.assertRaises(Exception) as ctx: Record.new(self.zone, 'unknown', {}) - self.assertTrue('missing type' in ctx.exception.message) + self.assertTrue('missing type' in text_type(ctx.exception)) # Unknown type with self.assertRaises(Exception) as ctx: Record.new(self.zone, 'unknown', { 'type': 'XXX', }) - self.assertTrue('Unknown record type' in ctx.exception.message) + self.assertTrue('Unknown record type' in text_type(ctx.exception)) def test_change(self): existing = Record.new(self.zone, 'txt', { @@ -796,6 +825,38 @@ class TestRecord(TestCase): self.assertEquals(values, geo.values) self.assertEquals(['NA-US', 'NA'], list(geo.parents)) + a = GeoValue('NA-US-CA', values) + b = GeoValue('AP-JP', values) + c = GeoValue('NA-US-CA', ['2.3.4.5']) + + self.assertEqual(a, a) + self.assertEqual(b, b) + self.assertEqual(c, c) + + self.assertNotEqual(a, b) + self.assertNotEqual(a, c) + self.assertNotEqual(b, a) + self.assertNotEqual(b, c) + self.assertNotEqual(c, a) + self.assertNotEqual(c, b) + + self.assertTrue(a > b) + self.assertTrue(a < c) + self.assertTrue(b < a) + self.assertTrue(b < c) + self.assertTrue(c > a) + self.assertTrue(c > b) + + self.assertTrue(a >= a) + self.assertTrue(a >= b) + self.assertTrue(a <= c) + self.assertTrue(b <= a) + self.assertTrue(b <= b) + self.assertTrue(b <= c) + self.assertTrue(c > a) + self.assertTrue(c > b) + self.assertTrue(c >= b) + def test_healthcheck(self): new = Record.new(self.zone, 'a', { 'ttl': 44, @@ -851,11 +912,339 @@ class TestRecord(TestCase): }) self.assertFalse(new.ignored) + def test_ordering_functions(self): + a = Record.new(self.zone, 'a', { + 'ttl': 44, + 'type': 'A', + 'value': '1.2.3.4', + }) + b = Record.new(self.zone, 'b', { + 'ttl': 44, + 'type': 'A', + 'value': '1.2.3.4', + }) + c = Record.new(self.zone, 'c', { + 'ttl': 44, + 'type': 'A', + 'value': '1.2.3.4', + }) + aaaa = Record.new(self.zone, 'a', { + 'ttl': 44, + 'type': 'AAAA', + 'value': '2601:644:500:e210:62f8:1dff:feb8:947a', + }) + + self.assertEquals(a, a) + self.assertEquals(b, b) + self.assertEquals(c, c) + self.assertEquals(aaaa, aaaa) + + self.assertNotEqual(a, b) + self.assertNotEqual(a, c) + self.assertNotEqual(a, aaaa) + self.assertNotEqual(b, a) + self.assertNotEqual(b, c) + self.assertNotEqual(b, aaaa) + self.assertNotEqual(c, a) + self.assertNotEqual(c, b) + self.assertNotEqual(c, aaaa) + self.assertNotEqual(aaaa, a) + self.assertNotEqual(aaaa, b) + self.assertNotEqual(aaaa, c) + + self.assertTrue(a < b) + self.assertTrue(a < c) + self.assertTrue(a < aaaa) + self.assertTrue(b > a) + self.assertTrue(b < c) + self.assertTrue(b > aaaa) + self.assertTrue(c > a) + self.assertTrue(c > b) + self.assertTrue(c > aaaa) + self.assertTrue(aaaa > a) + self.assertTrue(aaaa < b) + self.assertTrue(aaaa < c) + + self.assertTrue(a <= a) + self.assertTrue(a <= b) + self.assertTrue(a <= c) + self.assertTrue(a <= aaaa) + self.assertTrue(b >= a) + self.assertTrue(b >= b) + self.assertTrue(b <= c) + self.assertTrue(b >= aaaa) + self.assertTrue(c >= a) + self.assertTrue(c >= b) + self.assertTrue(c >= c) + self.assertTrue(c >= aaaa) + self.assertTrue(aaaa >= a) + self.assertTrue(aaaa <= b) + self.assertTrue(aaaa <= c) + self.assertTrue(aaaa <= aaaa) + + def test_caa_value(self): + a = CaaValue({'flags': 0, 'tag': 'a', 'value': 'v'}) + b = CaaValue({'flags': 1, 'tag': 'a', 'value': 'v'}) + c = CaaValue({'flags': 0, 'tag': 'c', 'value': 'v'}) + d = CaaValue({'flags': 0, 'tag': 'a', 'value': 'z'}) + + self.assertEqual(a, a) + self.assertEqual(b, b) + self.assertEqual(c, c) + self.assertEqual(d, d) + + self.assertNotEqual(a, b) + self.assertNotEqual(a, c) + self.assertNotEqual(a, d) + self.assertNotEqual(b, a) + self.assertNotEqual(b, c) + self.assertNotEqual(b, d) + self.assertNotEqual(c, a) + self.assertNotEqual(c, b) + self.assertNotEqual(c, d) + + self.assertTrue(a < b) + self.assertTrue(a < c) + self.assertTrue(a < d) + + self.assertTrue(b > a) + self.assertTrue(b > c) + self.assertTrue(b > d) + + self.assertTrue(c > a) + self.assertTrue(c < b) + self.assertTrue(c > d) + + self.assertTrue(d > a) + self.assertTrue(d < b) + self.assertTrue(d < c) + + self.assertTrue(a <= b) + self.assertTrue(a <= c) + self.assertTrue(a <= d) + self.assertTrue(a <= a) + self.assertTrue(a >= a) + + self.assertTrue(b >= a) + self.assertTrue(b >= c) + self.assertTrue(b >= d) + self.assertTrue(b >= b) + self.assertTrue(b <= b) + + self.assertTrue(c >= a) + self.assertTrue(c <= b) + self.assertTrue(c >= d) + self.assertTrue(c >= c) + self.assertTrue(c <= c) + + self.assertTrue(d >= a) + self.assertTrue(d <= b) + self.assertTrue(d <= c) + self.assertTrue(d >= d) + self.assertTrue(d <= d) + + def test_mx_value(self): + a = MxValue({'preference': 0, 'priority': 'a', 'exchange': 'v', + 'value': '1'}) + b = MxValue({'preference': 10, 'priority': 'a', 'exchange': 'v', + 'value': '2'}) + c = MxValue({'preference': 0, 'priority': 'b', 'exchange': 'z', + 'value': '3'}) + + self.assertEqual(a, a) + self.assertEqual(b, b) + self.assertEqual(c, c) + + self.assertNotEqual(a, b) + self.assertNotEqual(a, c) + self.assertNotEqual(b, a) + self.assertNotEqual(b, c) + self.assertNotEqual(c, a) + self.assertNotEqual(c, b) + + self.assertTrue(a < b) + self.assertTrue(a < c) + + self.assertTrue(b > a) + self.assertTrue(b > c) + + self.assertTrue(c > a) + self.assertTrue(c < b) + + self.assertTrue(a <= b) + self.assertTrue(a <= c) + self.assertTrue(a <= a) + self.assertTrue(a >= a) + + self.assertTrue(b >= a) + self.assertTrue(b >= c) + self.assertTrue(b >= b) + self.assertTrue(b <= b) + + self.assertTrue(c >= a) + self.assertTrue(c <= b) + self.assertTrue(c >= c) + self.assertTrue(c <= c) + + def test_sshfp_value(self): + a = SshfpValue({'algorithm': 0, 'fingerprint_type': 0, + 'fingerprint': 'abcd'}) + b = SshfpValue({'algorithm': 1, 'fingerprint_type': 0, + 'fingerprint': 'abcd'}) + c = SshfpValue({'algorithm': 0, 'fingerprint_type': 1, + 'fingerprint': 'abcd'}) + d = SshfpValue({'algorithm': 0, 'fingerprint_type': 0, + 'fingerprint': 'bcde'}) + + self.assertEqual(a, a) + self.assertEqual(b, b) + self.assertEqual(c, c) + self.assertEqual(d, d) + + self.assertNotEqual(a, b) + self.assertNotEqual(a, c) + self.assertNotEqual(a, d) + self.assertNotEqual(b, a) + self.assertNotEqual(b, c) + self.assertNotEqual(b, d) + self.assertNotEqual(c, a) + self.assertNotEqual(c, b) + self.assertNotEqual(c, d) + self.assertNotEqual(d, a) + self.assertNotEqual(d, b) + self.assertNotEqual(d, c) + + self.assertTrue(a < b) + self.assertTrue(a < c) + + self.assertTrue(b > a) + self.assertTrue(b > c) + + self.assertTrue(c > a) + self.assertTrue(c < b) + + self.assertTrue(a <= b) + self.assertTrue(a <= c) + self.assertTrue(a <= a) + self.assertTrue(a >= a) + + self.assertTrue(b >= a) + self.assertTrue(b >= c) + self.assertTrue(b >= b) + self.assertTrue(b <= b) + + self.assertTrue(c >= a) + self.assertTrue(c <= b) + self.assertTrue(c >= c) + self.assertTrue(c <= c) + + # Hash + values = set() + values.add(a) + self.assertTrue(a in values) + self.assertFalse(b in values) + values.add(b) + self.assertTrue(b in values) + + def test_srv_value(self): + a = SrvValue({'priority': 0, 'weight': 0, 'port': 0, 'target': 'foo.'}) + b = SrvValue({'priority': 1, 'weight': 0, 'port': 0, 'target': 'foo.'}) + c = SrvValue({'priority': 0, 'weight': 2, 'port': 0, 'target': 'foo.'}) + d = SrvValue({'priority': 0, 'weight': 0, 'port': 3, 'target': 'foo.'}) + e = SrvValue({'priority': 0, 'weight': 0, 'port': 0, 'target': 'mmm.'}) + + self.assertEqual(a, a) + self.assertEqual(b, b) + self.assertEqual(c, c) + self.assertEqual(d, d) + self.assertEqual(e, e) + + self.assertNotEqual(a, b) + self.assertNotEqual(a, c) + self.assertNotEqual(a, d) + self.assertNotEqual(a, e) + self.assertNotEqual(b, a) + self.assertNotEqual(b, c) + self.assertNotEqual(b, d) + self.assertNotEqual(b, e) + self.assertNotEqual(c, a) + self.assertNotEqual(c, b) + self.assertNotEqual(c, d) + self.assertNotEqual(c, e) + self.assertNotEqual(d, a) + self.assertNotEqual(d, b) + self.assertNotEqual(d, c) + self.assertNotEqual(d, e) + self.assertNotEqual(e, a) + self.assertNotEqual(e, b) + self.assertNotEqual(e, c) + self.assertNotEqual(e, d) + + self.assertTrue(a < b) + self.assertTrue(a < c) + + self.assertTrue(b > a) + self.assertTrue(b > c) + + self.assertTrue(c > a) + self.assertTrue(c < b) + + self.assertTrue(a <= b) + self.assertTrue(a <= c) + self.assertTrue(a <= a) + self.assertTrue(a >= a) + + self.assertTrue(b >= a) + self.assertTrue(b >= c) + self.assertTrue(b >= b) + self.assertTrue(b <= b) + + self.assertTrue(c >= a) + self.assertTrue(c <= b) + self.assertTrue(c >= c) + self.assertTrue(c <= c) + + # Hash + values = set() + values.add(a) + self.assertTrue(a in values) + self.assertFalse(b in values) + values.add(b) + self.assertTrue(b in values) + class TestRecordValidation(TestCase): zone = Zone('unit.tests.', []) def test_base(self): + # fqdn length, DNS defins max as 253 + with self.assertRaises(ValidationError) as ctx: + # The . will put this over the edge + name = 'x' * (253 - len(self.zone.name)) + Record.new(self.zone, name, { + 'ttl': 300, + 'type': 'A', + 'value': '1.2.3.4', + }) + reason = ctx.exception.reasons[0] + self.assertTrue(reason.startswith('invalid fqdn, "xxxx')) + self.assertTrue(reason.endswith('.unit.tests." is too long at 254' + ' chars, max is 253')) + + # label length, DNS defins max as 63 + with self.assertRaises(ValidationError) as ctx: + # The . will put this over the edge + name = 'x' * 64 + Record.new(self.zone, name, { + 'ttl': 300, + 'type': 'A', + 'value': '1.2.3.4', + }) + reason = ctx.exception.reasons[0] + self.assertTrue(reason.startswith('invalid name, "xxxx')) + self.assertTrue(reason.endswith('xxx" is too long at 64' + ' chars, max is 63')) + # no ttl with self.assertRaises(ValidationError) as ctx: Record.new(self.zone, '', { diff --git a/tests/test_octodns_source_axfr.py b/tests/test_octodns_source_axfr.py index 9251113..62e1a65 100644 --- a/tests/test_octodns_source_axfr.py +++ b/tests/test_octodns_source_axfr.py @@ -9,6 +9,7 @@ import dns.zone from dns.exception import DNSException from mock import patch +from six import text_type from unittest import TestCase from octodns.source.axfr import AxfrSource, AxfrSourceZoneTransferFailed, \ @@ -38,7 +39,7 @@ class TestAxfrSource(TestCase): zone = Zone('unit.tests.', []) self.source.populate(zone) self.assertEquals('Unable to Perform Zone Transfer', - ctx.exception.message) + text_type(ctx.exception)) class TestZoneFileSource(TestCase): @@ -68,4 +69,4 @@ class TestZoneFileSource(TestCase): zone = Zone('invalid.zone.', []) self.source.populate(zone) self.assertEquals('The DNS zone has no NS RRset at its origin.', - ctx.exception.message) + text_type(ctx.exception)) diff --git a/tests/test_octodns_yaml.py b/tests/test_octodns_yaml.py index effe231..f211854 100644 --- a/tests/test_octodns_yaml.py +++ b/tests/test_octodns_yaml.py @@ -5,7 +5,7 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -from StringIO import StringIO +from six import StringIO from unittest import TestCase from yaml.constructor import ConstructorError diff --git a/tests/test_octodns_zone.py b/tests/test_octodns_zone.py index 2fff996..1d000f2 100644 --- a/tests/test_octodns_zone.py +++ b/tests/test_octodns_zone.py @@ -6,6 +6,7 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals from unittest import TestCase +from six import text_type from octodns.record import ARecord, AaaaRecord, Create, Delete, Record, Update from octodns.zone import DuplicateRecordException, InvalidNodeException, \ @@ -47,7 +48,7 @@ class TestZone(TestCase): with self.assertRaises(DuplicateRecordException) as ctx: zone.add_record(a) self.assertEquals('Duplicate record a.unit.tests., type A', - ctx.exception.message) + text_type(ctx.exception)) self.assertEquals(zone.records, set([a])) # can add duplicate with replace=True @@ -137,7 +138,7 @@ class TestZone(TestCase): def test_missing_dot(self): with self.assertRaises(Exception) as ctx: Zone('not.allowed', []) - self.assertTrue('missing ending dot' in ctx.exception.message) + self.assertTrue('missing ending dot' in text_type(ctx.exception)) def test_sub_zones(self): @@ -160,7 +161,7 @@ class TestZone(TestCase): }) with self.assertRaises(SubzoneRecordException) as ctx: zone.add_record(record) - self.assertTrue('not of type NS', ctx.exception.message) + self.assertTrue('not of type NS', text_type(ctx.exception)) # Can add it w/lenient zone.add_record(record, lenient=True) self.assertEquals(set([record]), zone.records) @@ -174,7 +175,7 @@ class TestZone(TestCase): }) with self.assertRaises(SubzoneRecordException) as ctx: zone.add_record(record) - self.assertTrue('under a managed sub-zone', ctx.exception.message) + self.assertTrue('under a managed sub-zone', text_type(ctx.exception)) # Can add it w/lenient zone.add_record(record, lenient=True) self.assertEquals(set([record]), zone.records) @@ -188,7 +189,7 @@ class TestZone(TestCase): }) with self.assertRaises(SubzoneRecordException) as ctx: zone.add_record(record) - self.assertTrue('under a managed sub-zone', ctx.exception.message) + self.assertTrue('under a managed sub-zone', text_type(ctx.exception)) # Can add it w/lenient zone.add_record(record, lenient=True) self.assertEquals(set([record]), zone.records) diff --git a/tests/zones/invalid.zone. b/tests/zones/invalid.zone. index c814af6..04748a1 100644 --- a/tests/zones/invalid.zone. +++ b/tests/zones/invalid.zone. @@ -1,5 +1,5 @@ $ORIGIN invalid.zone. -@ IN SOA ns1.invalid.zone. root.invalid.zone. ( +@ 3600 IN SOA ns1.invalid.zone. root.invalid.zone. ( 2018071501 ; Serial 3600 ; Refresh (1 hour) 600 ; Retry (10 minutes) diff --git a/tests/zones/unit.tests. b/tests/zones/unit.tests. index 95828ad..0305e05 100644 --- a/tests/zones/unit.tests. +++ b/tests/zones/unit.tests. @@ -1,5 +1,5 @@ $ORIGIN unit.tests. -@ IN SOA ns1.unit.tests. root.unit.tests. ( +@ 3600 IN SOA ns1.unit.tests. root.unit.tests. ( 2018071501 ; Serial 3600 ; Refresh (1 hour) 600 ; Retry (10 minutes)