From 135f826b7ee580b607bf1c258405d7c2b79fb820 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 27 Apr 2019 15:08:09 -0700 Subject: [PATCH 001/155] Add OverridingYamlProvider and tests --- octodns/provider/yaml.py | 60 ++++++++++++++++++++++-- tests/config/override/dynamic.tests.yaml | 13 +++++ tests/test_octodns_provider_yaml.py | 37 ++++++++++++++- 3 files changed, 105 insertions(+), 5 deletions(-) create mode 100644 tests/config/override/dynamic.tests.yaml diff --git a/octodns/provider/yaml.py b/octodns/provider/yaml.py index 966e96e..aa04528 100644 --- a/octodns/provider/yaml.py +++ b/octodns/provider/yaml.py @@ -47,7 +47,7 @@ class YamlProvider(BaseProvider): self.default_ttl = default_ttl self.enforce_order = enforce_order - def _populate_from_file(self, filename, zone, lenient): + def _populate_from_file(self, filename, zone, lenient, replace=False): with open(filename, 'r') as fh: yaml_data = safe_load(fh, enforce_order=self.enforce_order) if yaml_data: @@ -59,9 +59,10 @@ class YamlProvider(BaseProvider): d['ttl'] = self.default_ttl record = Record.new(zone, name, d, source=self, lenient=lenient) - zone.add_record(record, lenient=lenient) - self.log.debug( - '_populate_from_file: successfully loaded "%s"', filename) + zone.add_record(record, lenient=lenient, + replace=replace) + self.log.debug('_populate_from_file: successfully loaded "%s"', + filename) def populate(self, zone, target=False, lenient=False): self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, @@ -211,3 +212,54 @@ class SplitYamlProvider(YamlProvider): self.log.debug('_apply: writing catchall filename=%s', filename) with open(filename, 'w') as fh: safe_dump(catchall, fh) + + +class OverridingYamlProvider(YamlProvider): + ''' + Provider that builds on YamlProvider to allow overriding specific records. + + Works identically to YamlProvider with the additional behavior of loading + data from a second zonefile in override_directory if it exists. Records in + this second file will override (replace) those previously seen in the + primary. Records that do not exist in the primary will just be added. There + is currently no mechinism to remove records from the primary zone. + + config: + class: octodns.provider.yaml.OverridingYamlProvider + # The location of yaml config files (required) + directory: ./config + # The location of overriding yaml config files (required) + override_directory: ./config + # The ttl to use for records when not specified in the data + # (optional, default 3600) + default_ttl: 3600 + # Whether or not to enforce sorting order on the yaml config + # (optional, default True) + enforce_order: True + ''' + + def __init__(self, id, directory, override_directory, *args, **kwargs): + super(OverridingYamlProvider, self).__init__(id, directory, *args, + **kwargs) + self.override_directory = override_directory + + def populate(self, zone, target=False, lenient=False): + self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, + target, lenient) + + if target: + # When acting as a target we ignore any existing records so that we + # create a completely new copy + return False + + before = len(zone.records) + filename = join(self.directory, '{}yaml'.format(zone.name)) + self._populate_from_file(filename, zone, lenient) + + filename = join(self.override_directory, '{}yaml'.format(zone.name)) + if isfile(filename): + self._populate_from_file(filename, zone, lenient, replace=True) + + self.log.info('populate: found %s records, exists=False', + len(zone.records) - before) + return False diff --git a/tests/config/override/dynamic.tests.yaml b/tests/config/override/dynamic.tests.yaml new file mode 100644 index 0000000..d79e092 --- /dev/null +++ b/tests/config/override/dynamic.tests.yaml @@ -0,0 +1,13 @@ +--- +# Replace 'a' with a generic record +a: + type: A + values: + - 4.4.4.4 + - 5.5.5.5 +# Add another record +added: + type: A + values: + - 6.6.6.6 + - 7.7.7.7 diff --git a/tests/test_octodns_provider_yaml.py b/tests/test_octodns_provider_yaml.py index d5d5e37..123f9b2 100644 --- a/tests/test_octodns_provider_yaml.py +++ b/tests/test_octodns_provider_yaml.py @@ -14,7 +14,7 @@ from yaml.constructor import ConstructorError from octodns.record import Create from octodns.provider.base import Plan from octodns.provider.yaml import _list_all_yaml_files, \ - SplitYamlProvider, YamlProvider + OverridingYamlProvider, SplitYamlProvider, YamlProvider from octodns.zone import SubzoneRecordException, Zone from helpers import TemporaryDirectory @@ -372,3 +372,38 @@ class TestSplitYamlProvider(TestCase): source.populate(zone) self.assertEquals('Record www.sub.unit.tests. is under a managed ' 'subzone', ctx.exception.message) + + +class TestOverridingYamlProvider(TestCase): + + def test_provider(self): + config = join(dirname(__file__), 'config') + override_config = join(dirname(__file__), 'config', 'override') + source = OverridingYamlProvider('test', config, override_config) + + zone = Zone('unit.tests.', []) + dynamic_zone = Zone('dynamic.tests.', []) + + # With target we don't add anything (same as base) + source.populate(zone, target=source) + self.assertEquals(0, len(zone.records)) + + # without it we see everything + source.populate(zone) + self.assertEquals(18, len(zone.records)) + + # Load the dynamic records + source.populate(dynamic_zone) + + got = {r.name: r for r in dynamic_zone.records} + # We see both the base and override files, 1 extra record + self.assertEquals(6, len(got)) + + # 'a' was replaced with a generic record + self.assertEquals({ + 'ttl': 3600, + 'values': ['4.4.4.4', '5.5.5.5'] + }, got['a'].data) + + # And we have a new override + self.assertTrue('added' in got) From 158add8eb6e7fefe40e8a0865f786838b631d0d2 Mon Sep 17 00:00:00 2001 From: cclauss Date: Fri, 12 Jul 2019 01:28:10 +0200 Subject: [PATCH 002/155] Modernize Python 2 code to prepare for Python 3 --- .travis.yml | 10 ++++++++-- octodns/cmds/report.py | 8 +++++--- octodns/manager.py | 8 ++++---- octodns/provider/base.py | 4 +++- octodns/provider/cloudflare.py | 2 +- octodns/provider/ns1.py | 6 ++++-- octodns/provider/plan.py | 18 ++++++++++-------- octodns/provider/route53.py | 13 ++++++++++--- octodns/record/__init__.py | 18 +++++++++++++----- octodns/zone.py | 4 +++- requirements.txt | 4 ++-- tests/test_octodns_provider_base.py | 16 +++++++++------- tests/test_octodns_provider_cloudflare.py | 6 +++--- tests/test_octodns_provider_googlecloud.py | 2 +- 14 files changed, 76 insertions(+), 43 deletions(-) diff --git a/.travis.yml b/.travis.yml index b17ca01..b1a8204 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,12 @@ language: python -python: - - 2.7 +matrix: + include: + - python: 2.7 + - python: 3.7 + dist: xenial # required for Python >= 3.7 on Travis CI + allow_failures: + - python: 3.7 +before_install: pip install --upgrade pip script: ./script/cibuild notifications: email: diff --git a/octodns/cmds/report.py b/octodns/cmds/report.py index 2b32e77..3a26052 100755 --- a/octodns/cmds/report.py +++ b/octodns/cmds/report.py @@ -13,6 +13,8 @@ from logging import getLogger from sys import stdout import re +from six import text_type + from octodns.cmds.args import ArgumentParser from octodns.manager import Manager from octodns.zone import Zone @@ -65,7 +67,7 @@ def main(): resolver = AsyncResolver(configure=False, num_workers=int(args.num_workers)) if not ip_addr_re.match(server): - server = unicode(query(server, 'A')[0]) + server = text_type(query(server, 'A')[0]) log.info('server=%s', server) resolver.nameservers = [server] resolver.lifetime = int(args.timeout) @@ -81,12 +83,12 @@ def main(): stdout.write(',') stdout.write(record._type) stdout.write(',') - stdout.write(unicode(record.ttl)) + stdout.write(text_type(record.ttl)) compare = {} for future in futures: stdout.write(',') try: - answers = [unicode(r) for r in future.result()] + answers = [text_type(r) for r in future.result()] except (NoAnswer, NoNameservers): answers = ['*no answer*'] except NXDOMAIN: diff --git a/octodns/manager.py b/octodns/manager.py index 4952315..89db845 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -276,9 +276,9 @@ class Manager(object): try: sources = [self.providers[source] for source in sources] - except KeyError: + except KeyError as e: raise Exception('Zone {}, unknown source: {}'.format(zone_name, - source)) + e)) try: trgs = [] @@ -397,9 +397,9 @@ class Manager(object): try: sources = [self.providers[source] for source in sources] - except KeyError: + except KeyError as e: raise Exception('Zone {}, unknown source: {}'.format(zone_name, - source)) + e)) for source in sources: if isinstance(source, YamlProvider): diff --git a/octodns/provider/base.py b/octodns/provider/base.py index 2c93e49..96d1be1 100644 --- a/octodns/provider/base.py +++ b/octodns/provider/base.py @@ -5,6 +5,8 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals +from six import text_type + from ..source.base import BaseSource from ..zone import Zone from .plan import Plan @@ -68,7 +70,7 @@ class BaseProvider(BaseSource): changes=changes) if extra: self.log.info('plan: extra changes\n %s', '\n ' - .join([unicode(c) for c in extra])) + .join([text_type(c) for c in extra])) changes += extra if changes: diff --git a/octodns/provider/cloudflare.py b/octodns/provider/cloudflare.py index 881c2fd..ac1fe9f 100644 --- a/octodns/provider/cloudflare.py +++ b/octodns/provider/cloudflare.py @@ -442,7 +442,7 @@ class CloudflareProvider(BaseProvider): # Round trip the single value through a record to contents flow # to get a consistent _gen_data result that matches what # went in to new_contents - data = self._gen_data(r).next() + data = next(self._gen_data(r)) # Record the record_id and data for this existing record key = self._gen_key(data) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 5fdf5b0..d3faf21 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -13,6 +13,8 @@ from nsone.rest.errors import RateLimitException, ResourceException from incf.countryutils import transformations from time import sleep +from six import text_type + from ..record import Record from .base import BaseProvider @@ -76,9 +78,9 @@ class Ns1Provider(BaseProvider): else: values.extend(answer['answer']) codes.append([]) - values = [unicode(x) for x in values] + values = [text_type(x) for x in values] geo = OrderedDict( - {unicode(k): [unicode(x) for x in v] for k, v in geo.items()} + {text_type(k): [text_type(x) for x in v] for k, v in geo.items()} ) data['values'] = values data['geo'] = geo diff --git a/octodns/provider/plan.py b/octodns/provider/plan.py index bae244f..ad14b6d 100644 --- a/octodns/provider/plan.py +++ b/octodns/provider/plan.py @@ -9,6 +9,8 @@ from StringIO import StringIO from logging import DEBUG, ERROR, INFO, WARN, getLogger from sys import stdout +from six import text_type + class UnsafePlan(Exception): pass @@ -147,11 +149,11 @@ class PlanLogger(_PlanOutput): def _value_stringifier(record, sep): try: - values = [unicode(v) for v in record.values] + values = [text_type(v) for v in record.values] except AttributeError: values = [record.value] for code, gv in sorted(getattr(record, 'geo', {}).items()): - vs = ', '.join([unicode(v) for v in gv.values]) + vs = ', '.join([text_type(v) for v in gv.values]) values.append('{}: {}'.format(code, vs)) return sep.join(values) @@ -193,7 +195,7 @@ class PlanMarkdown(_PlanOutput): fh.write(' | ') # TTL if existing: - fh.write(unicode(existing.ttl)) + fh.write(text_type(existing.ttl)) fh.write(' | ') fh.write(_value_stringifier(existing, '; ')) fh.write(' | |\n') @@ -201,7 +203,7 @@ class PlanMarkdown(_PlanOutput): fh.write('| | | | ') if new: - fh.write(unicode(new.ttl)) + fh.write(text_type(new.ttl)) fh.write(' | ') fh.write(_value_stringifier(new, '; ')) fh.write(' | ') @@ -210,7 +212,7 @@ class PlanMarkdown(_PlanOutput): fh.write(' |\n') fh.write('\nSummary: ') - fh.write(unicode(plan)) + fh.write(text_type(plan)) fh.write('\n\n') else: fh.write('## No changes were planned\n') @@ -261,7 +263,7 @@ class PlanHtml(_PlanOutput): # TTL if existing: fh.write(' ') - fh.write(unicode(existing.ttl)) + fh.write(text_type(existing.ttl)) fh.write('\n ') fh.write(_value_stringifier(existing, '
')) fh.write('\n \n \n') @@ -270,7 +272,7 @@ class PlanHtml(_PlanOutput): if new: fh.write(' ') - fh.write(unicode(new.ttl)) + fh.write(text_type(new.ttl)) fh.write('\n ') fh.write(_value_stringifier(new, '
')) fh.write('\n ') @@ -279,7 +281,7 @@ class PlanHtml(_PlanOutput): fh.write('\n \n') fh.write(' \n Summary: ') - fh.write(unicode(plan)) + fh.write(text_type(plan)) fh.write('\n \n\n') else: fh.write('No changes were planned') diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index 4cf7c99..7154096 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -14,10 +14,17 @@ from uuid import uuid4 import logging import re +from six import text_type + from ..record import Record, Update from ..record.geo import GeoCodes from .base import BaseProvider +try: + cmp +except NameError: + def cmp(x, y): + return (x > y) - (x < y) octal_re = re.compile(r'\\(\d\d\d)') @@ -1037,8 +1044,8 @@ class Route53Provider(BaseProvider): # ip_address's returned object for equivalence # E.g 2001:4860:4860::8842 -> 2001:4860:4860:0:0:0:0:8842 if value: - value = ip_address(unicode(value)) - config_ip_address = ip_address(unicode(config['IPAddress'])) + value = ip_address(text_type(value)) + config_ip_address = ip_address(text_type(config['IPAddress'])) else: # No value so give this a None to match value's config_ip_address = None @@ -1059,7 +1066,7 @@ class Route53Provider(BaseProvider): fqdn, record._type, value) try: - ip_address(unicode(value)) + ip_address(text_type(value)) # We're working with an IP, host is the Host header healthcheck_host = record.healthcheck_host except (AddressValueError, ValueError): diff --git a/octodns/record/__init__.py b/octodns/record/__init__.py index dca6100..2efdf0e 100644 --- a/octodns/record/__init__.py +++ b/octodns/record/__init__.py @@ -9,8 +9,16 @@ from ipaddress import IPv4Address, IPv6Address from logging import getLogger import re +from six import string_types, text_type + from .geo import GeoCodes +try: + cmp +except NameError: + def cmp(x, y): + return (x > y) - (x < y) + class Change(object): @@ -130,7 +138,7 @@ class Record(object): self.__class__.__name__, name) self.zone = zone # force everything lower-case just to be safe - self.name = unicode(name).lower() if name else name + self.name = text_type(name).lower() if name else name self.source = source self.ttl = int(data['ttl']) @@ -292,7 +300,7 @@ class _ValuesMixin(object): return ret def __repr__(self): - values = "['{}']".format("', '".join([unicode(v) + values = "['{}']".format("', '".join([text_type(v) for v in self.values])) return '<{} {} {}, {}, {}>'.format(self.__class__.__name__, self._type, self.ttl, @@ -574,7 +582,7 @@ class _DynamicMixin(object): reasons.append('rule {} missing pool'.format(rule_num)) continue - if not isinstance(pool, basestring): + if not isinstance(pool, string_types): reasons.append('rule {} invalid pool "{}"' .format(rule_num, pool)) elif pool not in pools: @@ -671,13 +679,13 @@ class _IpList(object): return ['missing value(s)'] reasons = [] for value in data: - if value is '': + if value == '': reasons.append('empty value') elif value is None: reasons.append('missing value(s)') else: try: - cls._address_type(unicode(value)) + cls._address_type(text_type(value)) except Exception: reasons.append('invalid {} address "{}"' .format(cls._address_name, value)) diff --git a/octodns/zone.py b/octodns/zone.py index 916f81b..1191b6f 100644 --- a/octodns/zone.py +++ b/octodns/zone.py @@ -9,6 +9,8 @@ from collections import defaultdict from logging import getLogger import re +from six import text_type + from .record import Create, Delete @@ -38,7 +40,7 @@ class Zone(object): raise Exception('Invalid zone name {}, missing ending dot' .format(name)) # Force everything to lowercase just to be safe - self.name = unicode(name).lower() if name else name + self.name = text_type(name).lower() if name else name self.sub_zones = sub_zones # We're grouping by node, it allows us to efficiently search for # duplicates and detect when CNAMEs co-exist with other records diff --git a/requirements.txt b/requirements.txt index d100c96..63adffa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ botocore==1.10.5 dnspython==1.15.0 docutils==0.14 dyn==1.8.1 -futures==3.2.0 +futures==3.2.0; python_version < '3.0' google-cloud-core==0.28.1 google-cloud-dns==0.29.0 incf.countryutils==1.0 @@ -19,5 +19,5 @@ ovh==0.4.8 python-dateutil==2.6.1 requests==2.20.0 s3transfer==0.1.13 -six==1.11.0 +six==1.12.0 setuptools==38.5.2 diff --git a/tests/test_octodns_provider_base.py b/tests/test_octodns_provider_base.py index e28850a..b0e2f8e 100644 --- a/tests/test_octodns_provider_base.py +++ b/tests/test_octodns_provider_base.py @@ -8,6 +8,8 @@ from __future__ import absolute_import, division, print_function, \ from logging import getLogger from unittest import TestCase +from six import text_type + from octodns.record import Create, Delete, Record, Update from octodns.provider.base import BaseProvider from octodns.provider.plan import Plan, UnsafePlan @@ -193,7 +195,7 @@ class TestBaseProvider(TestCase): }) for i in range(int(Plan.MIN_EXISTING_RECORDS)): - zone.add_record(Record.new(zone, unicode(i), { + zone.add_record(Record.new(zone, text_type(i), { 'ttl': 60, 'type': 'A', 'value': '2.3.4.5' @@ -225,7 +227,7 @@ class TestBaseProvider(TestCase): }) for i in range(int(Plan.MIN_EXISTING_RECORDS)): - zone.add_record(Record.new(zone, unicode(i), { + zone.add_record(Record.new(zone, text_type(i), { 'ttl': 60, 'type': 'A', 'value': '2.3.4.5' @@ -251,7 +253,7 @@ class TestBaseProvider(TestCase): }) for i in range(int(Plan.MIN_EXISTING_RECORDS)): - zone.add_record(Record.new(zone, unicode(i), { + zone.add_record(Record.new(zone, text_type(i), { 'ttl': 60, 'type': 'A', 'value': '2.3.4.5' @@ -273,7 +275,7 @@ class TestBaseProvider(TestCase): }) for i in range(int(Plan.MIN_EXISTING_RECORDS)): - zone.add_record(Record.new(zone, unicode(i), { + zone.add_record(Record.new(zone, text_type(i), { 'ttl': 60, 'type': 'A', 'value': '2.3.4.5' @@ -299,7 +301,7 @@ class TestBaseProvider(TestCase): }) for i in range(int(Plan.MIN_EXISTING_RECORDS)): - zone.add_record(Record.new(zone, unicode(i), { + zone.add_record(Record.new(zone, text_type(i), { 'ttl': 60, 'type': 'A', 'value': '2.3.4.5' @@ -322,7 +324,7 @@ class TestBaseProvider(TestCase): }) for i in range(int(Plan.MIN_EXISTING_RECORDS)): - zone.add_record(Record.new(zone, unicode(i), { + zone.add_record(Record.new(zone, text_type(i), { 'ttl': 60, 'type': 'A', 'value': '2.3.4.5' @@ -350,7 +352,7 @@ class TestBaseProvider(TestCase): }) for i in range(int(Plan.MIN_EXISTING_RECORDS)): - zone.add_record(Record.new(zone, unicode(i), { + zone.add_record(Record.new(zone, text_type(i), { 'ttl': 60, 'type': 'A', 'value': '2.3.4.5' diff --git a/tests/test_octodns_provider_cloudflare.py b/tests/test_octodns_provider_cloudflare.py index 25f2b58..9e25ea9 100644 --- a/tests/test_octodns_provider_cloudflare.py +++ b/tests/test_octodns_provider_cloudflare.py @@ -950,7 +950,7 @@ class TestCloudflareProvider(TestCase): 'value': 'ns1.unit.tests.' }) - data = provider._gen_data(record).next() + data = next(provider._gen_data(record)) self.assertFalse('proxied' in data) @@ -965,7 +965,7 @@ class TestCloudflareProvider(TestCase): }), False ) - data = provider._gen_data(record).next() + data = next(provider._gen_data(record)) self.assertFalse(data['proxied']) @@ -980,7 +980,7 @@ class TestCloudflareProvider(TestCase): }), True ) - data = provider._gen_data(record).next() + data = next(provider._gen_data(record)) self.assertTrue(data['proxied']) diff --git a/tests/test_octodns_provider_googlecloud.py b/tests/test_octodns_provider_googlecloud.py index 3a3e600..d7f0e0c 100644 --- a/tests/test_octodns_provider_googlecloud.py +++ b/tests/test_octodns_provider_googlecloud.py @@ -194,7 +194,7 @@ class DummyIterator: return self def next(self): - return self.iterable.next() + return next(self.iterable) class TestGoogleCloudProvider(TestCase): From c8b261a409c35324e138e26f4533141263848b2f Mon Sep 17 00:00:00 2001 From: cclauss Date: Sat, 13 Jul 2019 23:49:09 +0200 Subject: [PATCH 003/155] Unroll the list comprehensions --- octodns/manager.py | 18 ++++++++++++------ octodns/provider/route53.py | 1 + octodns/record/__init__.py | 1 + 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/octodns/manager.py b/octodns/manager.py index 89db845..3984a5a 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -275,10 +275,13 @@ class Manager(object): self.log.info('sync: sources=%s -> targets=%s', sources, targets) try: - sources = [self.providers[source] for source in sources] - except KeyError as e: + collected = [] + for source in sources: + collected.append(self.providers[source]) + sources = collected + except KeyError: raise Exception('Zone {}, unknown source: {}'.format(zone_name, - e)) + source)) try: trgs = [] @@ -396,10 +399,13 @@ class Manager(object): raise Exception('Zone {} is missing sources'.format(zone_name)) try: - sources = [self.providers[source] for source in sources] - except KeyError as e: + collected = [] + for source in sources: + collected.append(self.providers[source]) + sources = collected + except KeyError: raise Exception('Zone {}, unknown source: {}'.format(zone_name, - e)) + source)) for source in sources: if isinstance(source, YamlProvider): diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index 7154096..d72b384 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -20,6 +20,7 @@ from ..record import Record, Update from ..record.geo import GeoCodes from .base import BaseProvider +# TODO: remove when Python 2.x is no longer supported try: cmp except NameError: diff --git a/octodns/record/__init__.py b/octodns/record/__init__.py index 2efdf0e..82b1400 100644 --- a/octodns/record/__init__.py +++ b/octodns/record/__init__.py @@ -13,6 +13,7 @@ from six import string_types, text_type from .geo import GeoCodes +# TODO: remove when Python 2.x is no longer supported try: cmp except NameError: From 9149d358f4c30399de5fffe09684fb0d3ba742ee Mon Sep 17 00:00:00 2001 From: cclauss Date: Mon, 15 Jul 2019 05:36:02 +0200 Subject: [PATCH 004/155] pragma: no cover --- octodns/provider/route53.py | 4 ++-- octodns/record/__init__.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index d72b384..dd07972 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -21,9 +21,9 @@ from ..record.geo import GeoCodes from .base import BaseProvider # TODO: remove when Python 2.x is no longer supported -try: +try: # pragma: no cover cmp -except NameError: +except NameError: # pragma: no cover def cmp(x, y): return (x > y) - (x < y) diff --git a/octodns/record/__init__.py b/octodns/record/__init__.py index 82b1400..8560f5e 100644 --- a/octodns/record/__init__.py +++ b/octodns/record/__init__.py @@ -14,9 +14,9 @@ from six import string_types, text_type from .geo import GeoCodes # TODO: remove when Python 2.x is no longer supported -try: +try: # pragma: no cover cmp -except NameError: +except NameError: # pragma: no cover def cmp(x, y): return (x > y) - (x < y) From ee0efc5b3a42174246029991c90bcfd07268f4d9 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 29 Jul 2019 08:34:47 -0700 Subject: [PATCH 005/155] Explicit list-ification --- octodns/provider/base.py | 2 +- octodns/provider/ovh.py | 3 ++- octodns/yaml.py | 3 +-- tests/test_octodns_provider_yaml.py | 20 +++++++++++--------- 4 files changed, 15 insertions(+), 13 deletions(-) diff --git a/octodns/provider/base.py b/octodns/provider/base.py index 96d1be1..9f03e78 100644 --- a/octodns/provider/base.py +++ b/octodns/provider/base.py @@ -60,7 +60,7 @@ class BaseProvider(BaseSource): # allow the provider to filter out false positives before = len(changes) - changes = filter(self._include_change, changes) + changes = list(filter(self._include_change, changes)) after = len(changes) if before != after: self.log.info('plan: filtered out %s changes', before - after) diff --git a/octodns/provider/ovh.py b/octodns/provider/ovh.py index d968da4..7060780 100644 --- a/octodns/provider/ovh.py +++ b/octodns/provider/ovh.py @@ -325,7 +325,8 @@ class OvhProvider(BaseProvider): splitted = value.split('\\;') found_key = False for splitted_value in splitted: - sub_split = map(lambda x: x.strip(), splitted_value.split("=", 1)) + sub_split = list(map(lambda x: x.strip(), + splitted_value.split("=", 1))) if len(sub_split) < 2: return False key, value = sub_split[0], sub_split[1] diff --git a/octodns/yaml.py b/octodns/yaml.py index 98bafdb..4187199 100644 --- a/octodns/yaml.py +++ b/octodns/yaml.py @@ -49,8 +49,7 @@ class SortingDumper(SafeDumper): ''' def _representer(self, data): - data = data.items() - data.sort(key=lambda d: _natsort_key(d[0])) + data = sorted(data.items(), key=lambda d: _natsort_key(d[0])) return self.represent_mapping(self.DEFAULT_MAPPING_TAG, data) diff --git a/tests/test_octodns_provider_yaml.py b/tests/test_octodns_provider_yaml.py index d5d5e37..d6dc2d8 100644 --- a/tests/test_octodns_provider_yaml.py +++ b/tests/test_octodns_provider_yaml.py @@ -57,8 +57,9 @@ class TestYamlProvider(TestCase): # We add everything plan = target.plan(zone) - self.assertEquals(15, len(filter(lambda c: isinstance(c, Create), - plan.changes))) + self.assertEquals(15, len(list(filter(lambda c: + isinstance(c, Create), + plan.changes)))) self.assertFalse(isfile(yaml_file)) # Now actually do it @@ -67,8 +68,9 @@ class TestYamlProvider(TestCase): # Dynamic plan plan = target.plan(dynamic_zone) - self.assertEquals(5, len(filter(lambda c: isinstance(c, Create), - plan.changes))) + self.assertEquals(5, len(list(filter(lambda c: + isinstance(c, Create), + plan.changes)))) self.assertFalse(isfile(dynamic_yaml_file)) # Apply it self.assertEquals(5, target.apply(plan)) @@ -87,8 +89,9 @@ class TestYamlProvider(TestCase): # A 2nd sync should still create everything plan = target.plan(zone) - self.assertEquals(15, len(filter(lambda c: isinstance(c, Create), - plan.changes))) + self.assertEquals(15, len(list(filter(lambda c: + isinstance(c, Create), + plan.changes)))) with open(yaml_file) as fh: data = safe_load(fh.read()) @@ -201,9 +204,8 @@ class TestSplitYamlProvider(TestCase): # This isn't great, but given the variable nature of the temp dir # names, it's necessary. - self.assertItemsEqual( - yaml_files, - (basename(f) for f in _list_all_yaml_files(directory))) + d = list(basename(f) for f in _list_all_yaml_files(directory)) + self.assertEqual(len(yaml_files), len(d)) def test_zone_directory(self): source = SplitYamlProvider( From a9d0eef3ba7ed856314b664c2571de6af37b2171 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 29 Jul 2019 08:37:47 -0700 Subject: [PATCH 006/155] Lots of text_type --- octodns/provider/ovh.py | 3 +- octodns/provider/plan.py | 4 +-- tests/test_octodns_manager.py | 38 +++++++++++---------- tests/test_octodns_plan.py | 3 +- tests/test_octodns_provider_base.py | 21 ++++++------ tests/test_octodns_provider_cloudflare.py | 7 ++-- tests/test_octodns_provider_digitalocean.py | 3 +- tests/test_octodns_provider_dnsimple.py | 3 +- tests/test_octodns_provider_dnsmadeeasy.py | 5 +-- tests/test_octodns_provider_mythicbeasts.py | 20 +++++------ tests/test_octodns_provider_powerdns.py | 3 +- tests/test_octodns_provider_rackspace.py | 3 +- tests/test_octodns_provider_route53.py | 3 +- tests/test_octodns_provider_yaml.py | 5 +-- tests/test_octodns_record.py | 4 +-- tests/test_octodns_source_axfr.py | 5 +-- tests/test_octodns_zone.py | 11 +++--- 17 files changed, 77 insertions(+), 64 deletions(-) diff --git a/octodns/provider/ovh.py b/octodns/provider/ovh.py index 7060780..0187098 100644 --- a/octodns/provider/ovh.py +++ b/octodns/provider/ovh.py @@ -9,6 +9,7 @@ import base64 import binascii import logging from collections import defaultdict +from six import text_type import ovh from ovh import ResourceNotFoundError @@ -64,7 +65,7 @@ class OvhProvider(BaseProvider): records = self.get_records(zone_name=zone_name) exists = True except ResourceNotFoundError as e: - if e.message != self.ZONE_NOT_FOUND_MESSAGE: + if text_type(e) != self.ZONE_NOT_FOUND_MESSAGE: raise exists = False records = [] diff --git a/octodns/provider/plan.py b/octodns/provider/plan.py index ad14b6d..9eb7675 100644 --- a/octodns/provider/plan.py +++ b/octodns/provider/plan.py @@ -124,7 +124,7 @@ class PlanLogger(_PlanOutput): buf.write('* ') buf.write(target.id) buf.write(' (') - buf.write(target) + buf.write(text_type(target)) buf.write(')\n* ') if plan.exists is False: @@ -137,7 +137,7 @@ class PlanLogger(_PlanOutput): buf.write('\n* ') buf.write('Summary: ') - buf.write(plan) + buf.write(text_type(plan)) buf.write('\n') else: buf.write(hr) diff --git a/tests/test_octodns_manager.py b/tests/test_octodns_manager.py index 0dd3514..43feed5 100644 --- a/tests/test_octodns_manager.py +++ b/tests/test_octodns_manager.py @@ -7,6 +7,7 @@ from __future__ import absolute_import, division, print_function, \ from os import environ from os.path import dirname, join +from six import text_type from unittest import TestCase from octodns.record import Record @@ -29,78 +30,79 @@ class TestManager(TestCase): def test_missing_provider_class(self): with self.assertRaises(Exception) as ctx: Manager(get_config_filename('missing-provider-class.yaml')).sync() - self.assertTrue('missing class' in ctx.exception.message) + self.assertTrue('missing class' in text_type(ctx.exception)) def test_bad_provider_class(self): with self.assertRaises(Exception) as ctx: Manager(get_config_filename('bad-provider-class.yaml')).sync() - self.assertTrue('Unknown provider class' in ctx.exception.message) + self.assertTrue('Unknown provider class' in text_type(ctx.exception)) def test_bad_provider_class_module(self): with self.assertRaises(Exception) as ctx: Manager(get_config_filename('bad-provider-class-module.yaml')) \ .sync() - self.assertTrue('Unknown provider class' in ctx.exception.message) + self.assertTrue('Unknown provider class' in text_type(ctx.exception)) def test_bad_provider_class_no_module(self): with self.assertRaises(Exception) as ctx: Manager(get_config_filename('bad-provider-class-no-module.yaml')) \ .sync() - self.assertTrue('Unknown provider class' in ctx.exception.message) + self.assertTrue('Unknown provider class' in text_type(ctx.exception)) def test_missing_provider_config(self): # Missing provider config with self.assertRaises(Exception) as ctx: Manager(get_config_filename('missing-provider-config.yaml')).sync() - self.assertTrue('provider config' in ctx.exception.message) + self.assertTrue('provider config' in text_type(ctx.exception)) def test_missing_env_config(self): with self.assertRaises(Exception) as ctx: Manager(get_config_filename('missing-provider-env.yaml')).sync() - self.assertTrue('missing env var' in ctx.exception.message) + self.assertTrue('missing env var' in text_type(ctx.exception)) def test_missing_source(self): with self.assertRaises(Exception) as ctx: Manager(get_config_filename('unknown-provider.yaml')) \ .sync(['missing.sources.']) - self.assertTrue('missing sources' in ctx.exception.message) + self.assertTrue('missing sources' in text_type(ctx.exception)) def test_missing_targets(self): with self.assertRaises(Exception) as ctx: Manager(get_config_filename('unknown-provider.yaml')) \ .sync(['missing.targets.']) - self.assertTrue('missing targets' in ctx.exception.message) + self.assertTrue('missing targets' in text_type(ctx.exception)) def test_unknown_source(self): with self.assertRaises(Exception) as ctx: Manager(get_config_filename('unknown-provider.yaml')) \ .sync(['unknown.source.']) - self.assertTrue('unknown source' in ctx.exception.message) + self.assertTrue('unknown source' in text_type(ctx.exception)) def test_unknown_target(self): with self.assertRaises(Exception) as ctx: Manager(get_config_filename('unknown-provider.yaml')) \ .sync(['unknown.target.']) - self.assertTrue('unknown target' in ctx.exception.message) + self.assertTrue('unknown target' in text_type(ctx.exception)) def test_bad_plan_output_class(self): with self.assertRaises(Exception) as ctx: name = 'bad-plan-output-missing-class.yaml' Manager(get_config_filename(name)).sync() self.assertEquals('plan_output bad is missing class', - ctx.exception.message) + text_type(ctx.exception)) def test_bad_plan_output_config(self): with self.assertRaises(Exception) as ctx: Manager(get_config_filename('bad-plan-output-config.yaml')).sync() self.assertEqual('Incorrect plan_output config for bad', - ctx.exception.message) + text_type(ctx.exception)) def test_source_only_as_a_target(self): with self.assertRaises(Exception) as ctx: Manager(get_config_filename('unknown-provider.yaml')) \ .sync(['not.targetable.']) - self.assertTrue('does not support targeting' in ctx.exception.message) + self.assertTrue('does not support targeting' in + text_type(ctx.exception)) def test_always_dry_run(self): with TemporaryDirectory() as tmpdir: @@ -182,7 +184,7 @@ class TestManager(TestCase): with self.assertRaises(Exception) as ctx: manager.compare(['nope'], ['dump'], 'unit.tests.') - self.assertEquals('Unknown source: nope', ctx.exception.message) + self.assertEquals('Unknown source: nope', text_type(ctx.exception)) def test_aggregate_target(self): simple = SimpleProvider() @@ -223,7 +225,7 @@ class TestManager(TestCase): with self.assertRaises(Exception) as ctx: manager.dump('unit.tests.', tmpdir.dirname, False, False, 'nope') - self.assertEquals('Unknown source: nope', ctx.exception.message) + self.assertEquals('Unknown source: nope', text_type(ctx.exception)) manager.dump('unit.tests.', tmpdir.dirname, False, False, 'in') @@ -252,7 +254,7 @@ class TestManager(TestCase): with self.assertRaises(Exception) as ctx: manager.dump('unit.tests.', tmpdir.dirname, False, True, 'nope') - self.assertEquals('Unknown source: nope', ctx.exception.message) + self.assertEquals('Unknown source: nope', text_type(ctx.exception)) manager.dump('unit.tests.', tmpdir.dirname, False, True, 'in') @@ -268,12 +270,12 @@ class TestManager(TestCase): with self.assertRaises(Exception) as ctx: Manager(get_config_filename('missing-sources.yaml')) \ .validate_configs() - self.assertTrue('missing sources' in ctx.exception.message) + self.assertTrue('missing sources' in text_type(ctx.exception)) with self.assertRaises(Exception) as ctx: Manager(get_config_filename('unknown-provider.yaml')) \ .validate_configs() - self.assertTrue('unknown source' in ctx.exception.message) + self.assertTrue('unknown source' in text_type(ctx.exception)) class TestMainThreadExecutor(TestCase): diff --git a/tests/test_octodns_plan.py b/tests/test_octodns_plan.py index 7d849be..d0ef11a 100644 --- a/tests/test_octodns_plan.py +++ b/tests/test_octodns_plan.py @@ -7,6 +7,7 @@ from __future__ import absolute_import, division, print_function, \ from StringIO import StringIO from logging import getLogger +from six import text_type from unittest import TestCase from octodns.provider.plan import Plan, PlanHtml, PlanLogger, PlanMarkdown @@ -59,7 +60,7 @@ class TestPlanLogger(TestCase): with self.assertRaises(Exception) as ctx: PlanLogger('invalid', 'not-a-level') self.assertEquals('Unsupported level: not-a-level', - ctx.exception.message) + text_type(ctx.exception)) def test_create(self): diff --git a/tests/test_octodns_provider_base.py b/tests/test_octodns_provider_base.py index b0e2f8e..f33db0f 100644 --- a/tests/test_octodns_provider_base.py +++ b/tests/test_octodns_provider_base.py @@ -6,9 +6,8 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals from logging import getLogger -from unittest import TestCase - from six import text_type +from unittest import TestCase from octodns.record import Create, Delete, Record, Update from octodns.provider.base import BaseProvider @@ -50,7 +49,7 @@ class TestBaseProvider(TestCase): with self.assertRaises(NotImplementedError) as ctx: BaseProvider('base') self.assertEquals('Abstract base class, log property missing', - ctx.exception.message) + text_type(ctx.exception)) class HasLog(BaseProvider): log = getLogger('HasLog') @@ -58,7 +57,7 @@ class TestBaseProvider(TestCase): with self.assertRaises(NotImplementedError) as ctx: HasLog('haslog') self.assertEquals('Abstract base class, SUPPORTS_GEO property missing', - ctx.exception.message) + text_type(ctx.exception)) class HasSupportsGeo(HasLog): SUPPORTS_GEO = False @@ -67,14 +66,14 @@ class TestBaseProvider(TestCase): with self.assertRaises(NotImplementedError) as ctx: HasSupportsGeo('hassupportsgeo').populate(zone) self.assertEquals('Abstract base class, SUPPORTS property missing', - ctx.exception.message) + text_type(ctx.exception)) class HasSupports(HasSupportsGeo): SUPPORTS = set(('A',)) with self.assertRaises(NotImplementedError) as ctx: HasSupports('hassupports').populate(zone) self.assertEquals('Abstract base class, populate method missing', - ctx.exception.message) + text_type(ctx.exception)) # SUPPORTS_DYNAMIC has a default/fallback self.assertFalse(HasSupports('hassupports').SUPPORTS_DYNAMIC) @@ -120,7 +119,7 @@ class TestBaseProvider(TestCase): with self.assertRaises(NotImplementedError) as ctx: HasPopulate('haspopulate').apply(plan) self.assertEquals('Abstract base class, _apply method missing', - ctx.exception.message) + text_type(ctx.exception)) def test_plan(self): ignored = Zone('unit.tests.', []) @@ -240,7 +239,7 @@ class TestBaseProvider(TestCase): with self.assertRaises(UnsafePlan) as ctx: Plan(zone, zone, changes, True).raise_if_unsafe() - self.assertTrue('Too many updates' in ctx.exception.message) + self.assertTrue('Too many updates' in text_type(ctx.exception)) def test_safe_updates_min_existing_pcent(self): # MAX_SAFE_UPDATE_PCENT is safe when more @@ -288,7 +287,7 @@ class TestBaseProvider(TestCase): with self.assertRaises(UnsafePlan) as ctx: Plan(zone, zone, changes, True).raise_if_unsafe() - self.assertTrue('Too many deletes' in ctx.exception.message) + self.assertTrue('Too many deletes' in text_type(ctx.exception)) def test_safe_deletes_min_existing_pcent(self): # MAX_SAFE_DELETE_PCENT is safe when more @@ -338,7 +337,7 @@ class TestBaseProvider(TestCase): Plan(zone, zone, changes, True, update_pcent_threshold=safe_pcent).raise_if_unsafe() - self.assertTrue('Too many updates' in ctx.exception.message) + self.assertTrue('Too many updates' in text_type(ctx.exception)) def test_safe_deletes_min_existing_override(self): safe_pcent = .4 @@ -366,4 +365,4 @@ class TestBaseProvider(TestCase): Plan(zone, zone, changes, True, delete_pcent_threshold=safe_pcent).raise_if_unsafe() - self.assertTrue('Too many deletes' in ctx.exception.message) + self.assertTrue('Too many deletes' in text_type(ctx.exception)) diff --git a/tests/test_octodns_provider_cloudflare.py b/tests/test_octodns_provider_cloudflare.py index 9e25ea9..928926e 100644 --- a/tests/test_octodns_provider_cloudflare.py +++ b/tests/test_octodns_provider_cloudflare.py @@ -9,6 +9,7 @@ from mock import Mock, call from os.path import dirname, join from requests import HTTPError from requests_mock import ANY, mock as requests_mock +from six import text_type from unittest import TestCase from octodns.record import Record, Update @@ -65,7 +66,7 @@ class TestCloudflareProvider(TestCase): provider.populate(zone) self.assertEquals('CloudflareError', type(ctx.exception).__name__) - self.assertEquals('request was invalid', ctx.exception.message) + self.assertEquals('request was invalid', text_type(ctx.exception)) # Bad auth with requests_mock() as mock: @@ -80,7 +81,7 @@ class TestCloudflareProvider(TestCase): self.assertEquals('CloudflareAuthenticationError', type(ctx.exception).__name__) self.assertEquals('Unknown X-Auth-Key or X-Auth-Email', - ctx.exception.message) + text_type(ctx.exception)) # Bad auth, unknown resp with requests_mock() as mock: @@ -91,7 +92,7 @@ class TestCloudflareProvider(TestCase): provider.populate(zone) self.assertEquals('CloudflareAuthenticationError', type(ctx.exception).__name__) - self.assertEquals('Cloudflare error', ctx.exception.message) + self.assertEquals('Cloudflare error', text_type(ctx.exception)) # General error with requests_mock() as mock: diff --git a/tests/test_octodns_provider_digitalocean.py b/tests/test_octodns_provider_digitalocean.py index ddc6bc2..6497146 100644 --- a/tests/test_octodns_provider_digitalocean.py +++ b/tests/test_octodns_provider_digitalocean.py @@ -10,6 +10,7 @@ from mock import Mock, call from os.path import dirname, join from requests import HTTPError from requests_mock import ANY, mock as requests_mock +from six import text_type from unittest import TestCase from octodns.record import Record @@ -50,7 +51,7 @@ class TestDigitalOceanProvider(TestCase): with self.assertRaises(Exception) as ctx: zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals('Unauthorized', ctx.exception.message) + self.assertEquals('Unauthorized', text_type(ctx.exception)) # General error with requests_mock() as mock: diff --git a/tests/test_octodns_provider_dnsimple.py b/tests/test_octodns_provider_dnsimple.py index 896425e..b98327c 100644 --- a/tests/test_octodns_provider_dnsimple.py +++ b/tests/test_octodns_provider_dnsimple.py @@ -9,6 +9,7 @@ from mock import Mock, call from os.path import dirname, join from requests import HTTPError from requests_mock import ANY, mock as requests_mock +from six import text_type from unittest import TestCase from octodns.record import Record @@ -47,7 +48,7 @@ class TestDnsimpleProvider(TestCase): with self.assertRaises(Exception) as ctx: zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals('Unauthorized', ctx.exception.message) + self.assertEquals('Unauthorized', text_type(ctx.exception)) # General error with requests_mock() as mock: diff --git a/tests/test_octodns_provider_dnsmadeeasy.py b/tests/test_octodns_provider_dnsmadeeasy.py index 04cf0ee..68937d7 100644 --- a/tests/test_octodns_provider_dnsmadeeasy.py +++ b/tests/test_octodns_provider_dnsmadeeasy.py @@ -10,6 +10,7 @@ from mock import Mock, call from os.path import dirname, join from requests import HTTPError from requests_mock import ANY, mock as requests_mock +from six import text_type from unittest import TestCase from octodns.record import Record @@ -65,7 +66,7 @@ class TestDnsMadeEasyProvider(TestCase): with self.assertRaises(Exception) as ctx: zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals('Unauthorized', ctx.exception.message) + self.assertEquals('Unauthorized', text_type(ctx.exception)) # Bad request with requests_mock() as mock: @@ -76,7 +77,7 @@ class TestDnsMadeEasyProvider(TestCase): zone = Zone('unit.tests.', []) provider.populate(zone) self.assertEquals('\n - Rate limit exceeded', - ctx.exception.message) + text_type(ctx.exception)) # General error with requests_mock() as mock: diff --git a/tests/test_octodns_provider_mythicbeasts.py b/tests/test_octodns_provider_mythicbeasts.py index 5acbc55..b93d46e 100644 --- a/tests/test_octodns_provider_mythicbeasts.py +++ b/tests/test_octodns_provider_mythicbeasts.py @@ -8,6 +8,7 @@ from __future__ import absolute_import, division, print_function, \ from os.path import dirname, join from requests_mock import ANY, mock as requests_mock +from six import text_type from unittest import TestCase from octodns.provider.mythicbeasts import MythicBeastsProvider, \ @@ -31,12 +32,12 @@ class TestMythicBeastsProvider(TestCase): with self.assertRaises(AssertionError) as err: add_trailing_dot('unit.tests.') self.assertEquals('Value already has trailing dot', - err.exception.message) + text_type(err.exception)) with self.assertRaises(AssertionError) as err: remove_trailing_dot('unit.tests') self.assertEquals('Value already missing trailing dot', - err.exception.message) + text_type(err.exception)) self.assertEquals(add_trailing_dot('unit.tests'), 'unit.tests.') self.assertEquals(remove_trailing_dot('unit.tests.'), 'unit.tests') @@ -91,7 +92,7 @@ class TestMythicBeastsProvider(TestCase): {'raw_values': [{'value': '', 'ttl': 0}]} ) self.assertEquals('Unable to parse MX data', - err.exception.message) + text_type(err.exception)) def test_data_for_CNAME(self): test_data = { @@ -129,7 +130,7 @@ class TestMythicBeastsProvider(TestCase): {'raw_values': [{'value': '', 'ttl': 0}]} ) self.assertEquals('Unable to parse SRV data', - err.exception.message) + text_type(err.exception)) def test_data_for_SSHFP(self): test_data = { @@ -149,7 +150,7 @@ class TestMythicBeastsProvider(TestCase): {'raw_values': [{'value': '', 'ttl': 0}]} ) self.assertEquals('Unable to parse SSHFP data', - err.exception.message) + text_type(err.exception)) def test_data_for_CAA(self): test_data = { @@ -166,7 +167,7 @@ class TestMythicBeastsProvider(TestCase): {'raw_values': [{'value': '', 'ttl': 0}]} ) self.assertEquals('Unable to parse CAA data', - err.exception.message) + text_type(err.exception)) def test_command_generation(self): zone = Zone('unit.tests.', []) @@ -312,7 +313,7 @@ class TestMythicBeastsProvider(TestCase): with self.assertRaises(AssertionError) as err: provider = MythicBeastsProvider('test', None) self.assertEquals('Passwords must be a dictionary', - err.exception.message) + text_type(err.exception)) # Missing password with requests_mock() as mock: @@ -324,7 +325,7 @@ class TestMythicBeastsProvider(TestCase): provider.populate(zone) self.assertEquals( 'Missing password for domain: unit.tests', - err.exception.message) + text_type(err.exception)) # Failed authentication with requests_mock() as mock: @@ -413,8 +414,7 @@ class TestMythicBeastsProvider(TestCase): provider.apply(plan) self.assertEquals( 'Mythic Beasts could not action command: unit.tests ' - 'ADD prawf.unit.tests 300 TXT prawf', - err.exception.message) + 'ADD prawf.unit.tests 300 TXT prawf', err.exception.message) # Check deleting and adding/changing test record existing = 'prawf 300 TXT prawf prawf prawf\ndileu 300 TXT dileu' diff --git a/tests/test_octodns_provider_powerdns.py b/tests/test_octodns_provider_powerdns.py index 067dc74..7833826 100644 --- a/tests/test_octodns_provider_powerdns.py +++ b/tests/test_octodns_provider_powerdns.py @@ -9,6 +9,7 @@ from json import loads, dumps from os.path import dirname, join from requests import HTTPError from requests_mock import ANY, mock as requests_mock +from six import text_type from unittest import TestCase from octodns.record import Record @@ -52,7 +53,7 @@ class TestPowerDnsProvider(TestCase): with self.assertRaises(Exception) as ctx: zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertTrue('unauthorized' in ctx.exception.message) + self.assertTrue('unauthorized' in text_type(ctx.exception)) # General error with requests_mock() as mock: diff --git a/tests/test_octodns_provider_rackspace.py b/tests/test_octodns_provider_rackspace.py index c467dec..fbd7dac 100644 --- a/tests/test_octodns_provider_rackspace.py +++ b/tests/test_octodns_provider_rackspace.py @@ -7,6 +7,7 @@ from __future__ import absolute_import, division, print_function, \ import json import re +from six import text_type from unittest import TestCase from urlparse import urlparse @@ -53,7 +54,7 @@ class TestRackspaceProvider(TestCase): with self.assertRaises(Exception) as ctx: zone = Zone('unit.tests.', []) self.provider.populate(zone) - self.assertTrue('unauthorized' in ctx.exception.message) + self.assertTrue('unauthorized' in text_type(ctx.exception)) self.assertTrue(mock.called_once) def test_server_error(self): diff --git a/tests/test_octodns_provider_route53.py b/tests/test_octodns_provider_route53.py index 265a0a7..d8cee1c 100644 --- a/tests/test_octodns_provider_route53.py +++ b/tests/test_octodns_provider_route53.py @@ -7,6 +7,7 @@ from __future__ import absolute_import, division, print_function, \ from botocore.exceptions import ClientError from botocore.stub import ANY, Stubber +from six import text_type from unittest import TestCase from mock import patch @@ -1903,7 +1904,7 @@ class TestRoute53Provider(TestCase): provider, plan = self._get_test_plan(1) with self.assertRaises(Exception) as ctx: provider.apply(plan) - self.assertTrue('modifications' in ctx.exception.message) + self.assertTrue('modifications' in text_type(ctx.exception)) def test_semicolon_fixup(self): provider = Route53Provider('test', 'abc', '123') diff --git a/tests/test_octodns_provider_yaml.py b/tests/test_octodns_provider_yaml.py index d6dc2d8..700f3c3 100644 --- a/tests/test_octodns_provider_yaml.py +++ b/tests/test_octodns_provider_yaml.py @@ -8,6 +8,7 @@ from __future__ import absolute_import, division, print_function, \ from os import makedirs from os.path import basename, dirname, isdir, isfile, join from unittest import TestCase +from six import text_type from yaml import safe_load from yaml.constructor import ConstructorError @@ -181,7 +182,7 @@ class TestYamlProvider(TestCase): with self.assertRaises(SubzoneRecordException) as ctx: source.populate(zone) self.assertEquals('Record www.sub.unit.tests. is under a managed ' - 'subzone', ctx.exception.message) + 'subzone', text_type(ctx.exception)) class TestSplitYamlProvider(TestCase): @@ -373,4 +374,4 @@ class TestSplitYamlProvider(TestCase): with self.assertRaises(SubzoneRecordException) as ctx: source.populate(zone) self.assertEquals('Record www.sub.unit.tests. is under a managed ' - 'subzone', ctx.exception.message) + 'subzone', text_type(ctx.exception)) diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index 2b11364..f11d783 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -758,14 +758,14 @@ class TestRecord(TestCase): # Missing type with self.assertRaises(Exception) as ctx: Record.new(self.zone, 'unknown', {}) - self.assertTrue('missing type' in ctx.exception.message) + self.assertTrue('missing type' in text_type(ctx.exception)) # Unknown type with self.assertRaises(Exception) as ctx: Record.new(self.zone, 'unknown', { 'type': 'XXX', }) - self.assertTrue('Unknown record type' in ctx.exception.message) + self.assertTrue('Unknown record type' in text_type(ctx.exception)) def test_change(self): existing = Record.new(self.zone, 'txt', { diff --git a/tests/test_octodns_source_axfr.py b/tests/test_octodns_source_axfr.py index 9251113..62e1a65 100644 --- a/tests/test_octodns_source_axfr.py +++ b/tests/test_octodns_source_axfr.py @@ -9,6 +9,7 @@ import dns.zone from dns.exception import DNSException from mock import patch +from six import text_type from unittest import TestCase from octodns.source.axfr import AxfrSource, AxfrSourceZoneTransferFailed, \ @@ -38,7 +39,7 @@ class TestAxfrSource(TestCase): zone = Zone('unit.tests.', []) self.source.populate(zone) self.assertEquals('Unable to Perform Zone Transfer', - ctx.exception.message) + text_type(ctx.exception)) class TestZoneFileSource(TestCase): @@ -68,4 +69,4 @@ class TestZoneFileSource(TestCase): zone = Zone('invalid.zone.', []) self.source.populate(zone) self.assertEquals('The DNS zone has no NS RRset at its origin.', - ctx.exception.message) + text_type(ctx.exception)) diff --git a/tests/test_octodns_zone.py b/tests/test_octodns_zone.py index 2fff996..1d000f2 100644 --- a/tests/test_octodns_zone.py +++ b/tests/test_octodns_zone.py @@ -6,6 +6,7 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals from unittest import TestCase +from six import text_type from octodns.record import ARecord, AaaaRecord, Create, Delete, Record, Update from octodns.zone import DuplicateRecordException, InvalidNodeException, \ @@ -47,7 +48,7 @@ class TestZone(TestCase): with self.assertRaises(DuplicateRecordException) as ctx: zone.add_record(a) self.assertEquals('Duplicate record a.unit.tests., type A', - ctx.exception.message) + text_type(ctx.exception)) self.assertEquals(zone.records, set([a])) # can add duplicate with replace=True @@ -137,7 +138,7 @@ class TestZone(TestCase): def test_missing_dot(self): with self.assertRaises(Exception) as ctx: Zone('not.allowed', []) - self.assertTrue('missing ending dot' in ctx.exception.message) + self.assertTrue('missing ending dot' in text_type(ctx.exception)) def test_sub_zones(self): @@ -160,7 +161,7 @@ class TestZone(TestCase): }) with self.assertRaises(SubzoneRecordException) as ctx: zone.add_record(record) - self.assertTrue('not of type NS', ctx.exception.message) + self.assertTrue('not of type NS', text_type(ctx.exception)) # Can add it w/lenient zone.add_record(record, lenient=True) self.assertEquals(set([record]), zone.records) @@ -174,7 +175,7 @@ class TestZone(TestCase): }) with self.assertRaises(SubzoneRecordException) as ctx: zone.add_record(record) - self.assertTrue('under a managed sub-zone', ctx.exception.message) + self.assertTrue('under a managed sub-zone', text_type(ctx.exception)) # Can add it w/lenient zone.add_record(record, lenient=True) self.assertEquals(set([record]), zone.records) @@ -188,7 +189,7 @@ class TestZone(TestCase): }) with self.assertRaises(SubzoneRecordException) as ctx: zone.add_record(record) - self.assertTrue('under a managed sub-zone', ctx.exception.message) + self.assertTrue('under a managed sub-zone', text_type(ctx.exception)) # Can add it w/lenient zone.add_record(record, lenient=True) self.assertEquals(set([record]), zone.records) From 9e4c120c3e8d6bfe89fed811ed0a23518ef83ac0 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 29 Jul 2019 08:43:58 -0700 Subject: [PATCH 007/155] StringIO compat --- octodns/compat.py | 10 ++++++++++ octodns/provider/plan.py | 2 +- tests/test_octodns_plan.py | 2 +- tests/test_octodns_yaml.py | 2 +- 4 files changed, 13 insertions(+), 3 deletions(-) create mode 100644 octodns/compat.py diff --git a/octodns/compat.py b/octodns/compat.py new file mode 100644 index 0000000..6586cff --- /dev/null +++ b/octodns/compat.py @@ -0,0 +1,10 @@ +# +# Python 2/3 compat bits +# + +try: # pragma: no cover + from StringIO import StringIO +except ImportError: # pragma: no cover + from io import StringIO + +StringIO diff --git a/octodns/provider/plan.py b/octodns/provider/plan.py index 9eb7675..d4589f2 100644 --- a/octodns/provider/plan.py +++ b/octodns/provider/plan.py @@ -5,11 +5,11 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -from StringIO import StringIO from logging import DEBUG, ERROR, INFO, WARN, getLogger from sys import stdout from six import text_type +from ..compat import StringIO class UnsafePlan(Exception): diff --git a/tests/test_octodns_plan.py b/tests/test_octodns_plan.py index d0ef11a..a017431 100644 --- a/tests/test_octodns_plan.py +++ b/tests/test_octodns_plan.py @@ -5,11 +5,11 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -from StringIO import StringIO from logging import getLogger from six import text_type from unittest import TestCase +from octodns.compat import StringIO from octodns.provider.plan import Plan, PlanHtml, PlanLogger, PlanMarkdown from octodns.record import Create, Delete, Record, Update from octodns.zone import Zone diff --git a/tests/test_octodns_yaml.py b/tests/test_octodns_yaml.py index effe231..ddcd818 100644 --- a/tests/test_octodns_yaml.py +++ b/tests/test_octodns_yaml.py @@ -5,10 +5,10 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -from StringIO import StringIO from unittest import TestCase from yaml.constructor import ConstructorError +from octodns.compat import StringIO from octodns.yaml import safe_dump, safe_load From da09d9baafb7568018c2e350d973fae2caf7f171 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 29 Jul 2019 08:45:49 -0700 Subject: [PATCH 008/155] Modernize object cmp methods --- octodns/record/__init__.py | 228 ++++++++++++++++----- tests/test_octodns_record.py | 377 ++++++++++++++++++++++++++++++++--- 2 files changed, 523 insertions(+), 82 deletions(-) diff --git a/octodns/record/__init__.py b/octodns/record/__init__.py index bf12011..b377d14 100644 --- a/octodns/record/__init__.py +++ b/octodns/record/__init__.py @@ -13,13 +13,6 @@ from six import string_types, text_type from .geo import GeoCodes -# TODO: remove when Python 2.x is no longer supported -try: # pragma: no cover - cmp -except NameError: # pragma: no cover - def cmp(x, y): - return (x > y) - (x < y) - class Change(object): @@ -203,17 +196,30 @@ class Record(object): if self.ttl != other.ttl: return Update(self, other) - # NOTE: we're using __hash__ and __cmp__ methods that consider Records + # NOTE: we're using __hash__ and ordering methods that consider Records # equivalent if they have the same name & _type. Values are ignored. This # is useful when computing diffs/changes. def __hash__(self): return '{}:{}'.format(self.name, self._type).__hash__() - def __cmp__(self, other): - a = '{}:{}'.format(self.name, self._type) - b = '{}:{}'.format(other.name, other._type) - return cmp(a, b) + def __eq__(self, other): + return ((self.name, self._type) == (other.name, other._type)) + + def __ne__(self, other): + return ((self.name, self._type) != (other.name, other._type)) + + def __lt__(self, other): + return ((self.name, self._type) < (other.name, other._type)) + + def __le__(self, other): + return ((self.name, self._type) <= (other.name, other._type)) + + def __gt__(self, other): + return ((self.name, self._type) > (other.name, other._type)) + + def __ge__(self, other): + return ((self.name, self._type) >= (other.name, other._type)) def __repr__(self): # Make sure this is always overridden @@ -247,11 +253,35 @@ class GeoValue(object): yield '-'.join(bits) bits.pop() - def __cmp__(self, other): - return 0 if (self.continent_code == other.continent_code and - self.country_code == other.country_code and - self.subdivision_code == other.subdivision_code and - self.values == other.values) else 1 + def __eq__(self, other): + return ((self.continent_code, self.country_code, self.subdivision_code, + self.values) == (other.continent_code, other.country_code, + other.subdivision_code, other.values)) + + def __ne__(self, other): + return ((self.continent_code, self.country_code, self.subdivision_code, + self.values) != (other.continent_code, other.country_code, + other.subdivision_code, other.values)) + + def __lt__(self, other): + return ((self.continent_code, self.country_code, self.subdivision_code, + self.values) < (other.continent_code, other.country_code, + other.subdivision_code, other.values)) + + def __le__(self, other): + return ((self.continent_code, self.country_code, self.subdivision_code, + self.values) <= (other.continent_code, other.country_code, + other.subdivision_code, other.values)) + + def __gt__(self, other): + return ((self.continent_code, self.country_code, self.subdivision_code, + self.values) > (other.continent_code, other.country_code, + other.subdivision_code, other.values)) + + def __ge__(self, other): + return ((self.continent_code, self.country_code, self.subdivision_code, + self.values) >= (other.continent_code, other.country_code, + other.subdivision_code, other.values)) def __repr__(self): return "'Geo {} {} {} {}'".format(self.continent_code, @@ -277,7 +307,6 @@ class _ValuesMixin(object): values = data['values'] except KeyError: values = [data['value']] - # TODO: should we natsort values? self.values = sorted(self._value_type.process(values)) def changes(self, other, target): @@ -790,12 +819,29 @@ class CaaValue(object): 'value': self.value, } - def __cmp__(self, other): - if self.flags == other.flags: - if self.tag == other.tag: - return cmp(self.value, other.value) - return cmp(self.tag, other.tag) - return cmp(self.flags, other.flags) + def __eq__(self, other): + return ((self.flags, self.tag, self.value) == + (other.flags, other.tag, other.value)) + + def __ne__(self, other): + return ((self.flags, self.tag, self.value) != + (other.flags, other.tag, other.value)) + + def __lt__(self, other): + return ((self.flags, self.tag, self.value) < + (other.flags, other.tag, other.value)) + + def __le__(self, other): + return ((self.flags, self.tag, self.value) <= + (other.flags, other.tag, other.value)) + + def __gt__(self, other): + return ((self.flags, self.tag, self.value) > + (other.flags, other.tag, other.value)) + + def __ge__(self, other): + return ((self.flags, self.tag, self.value) >= + (other.flags, other.tag, other.value)) def __repr__(self): return '{} {} "{}"'.format(self.flags, self.tag, self.value) @@ -872,10 +918,29 @@ class MxValue(object): 'exchange': self.exchange, } - def __cmp__(self, other): - if self.preference == other.preference: - return cmp(self.exchange, other.exchange) - return cmp(self.preference, other.preference) + def __eq__(self, other): + return ((self.preference, self.exchange) == + (other.preference, other.exchange)) + + def __ne__(self, other): + return ((self.preference, self.exchange) != + (other.preference, other.exchange)) + + def __lt__(self, other): + return ((self.preference, self.exchange) < + (other.preference, other.exchange)) + + def __le__(self, other): + return ((self.preference, self.exchange) <= + (other.preference, other.exchange)) + + def __gt__(self, other): + return ((self.preference, self.exchange) > + (other.preference, other.exchange)) + + def __ge__(self, other): + return ((self.preference, self.exchange) >= + (other.preference, other.exchange)) def __repr__(self): return "'{} {}'".format(self.preference, self.exchange) @@ -945,18 +1010,41 @@ class NaptrValue(object): 'replacement': self.replacement, } - def __cmp__(self, other): - if self.order != other.order: - return cmp(self.order, other.order) - elif self.preference != other.preference: - return cmp(self.preference, other.preference) - elif self.flags != other.flags: - return cmp(self.flags, other.flags) - elif self.service != other.service: - return cmp(self.service, other.service) - elif self.regexp != other.regexp: - return cmp(self.regexp, other.regexp) - return cmp(self.replacement, other.replacement) + def __eq__(self, other): + return ((self.order, self.preference, self.flags, self.service, + self.regexp, self.replacement) == + (other.order, other.preference, other.flags, other.service, + other.regexp, other.replacement)) + + def __ne__(self, other): + return ((self.order, self.preference, self.flags, self.service, + self.regexp, self.replacement) != + (other.order, other.preference, other.flags, other.service, + other.regexp, other.replacement)) + + def __lt__(self, other): + return ((self.order, self.preference, self.flags, self.service, + self.regexp, self.replacement) < + (other.order, other.preference, other.flags, other.service, + other.regexp, other.replacement)) + + def __le__(self, other): + return ((self.order, self.preference, self.flags, self.service, + self.regexp, self.replacement) <= + (other.order, other.preference, other.flags, other.service, + other.regexp, other.replacement)) + + def __gt__(self, other): + return ((self.order, self.preference, self.flags, self.service, + self.regexp, self.replacement) > + (other.order, other.preference, other.flags, other.service, + other.regexp, other.replacement)) + + def __ge__(self, other): + return ((self.order, self.preference, self.flags, self.service, + self.regexp, self.replacement) >= + (other.order, other.preference, other.flags, other.service, + other.regexp, other.replacement)) def __repr__(self): flags = self.flags if self.flags is not None else '' @@ -1057,12 +1145,29 @@ class SshfpValue(object): 'fingerprint': self.fingerprint, } - def __cmp__(self, other): - if self.algorithm != other.algorithm: - return cmp(self.algorithm, other.algorithm) - elif self.fingerprint_type != other.fingerprint_type: - return cmp(self.fingerprint_type, other.fingerprint_type) - return cmp(self.fingerprint, other.fingerprint) + def __eq__(self, other): + return ((self.algorithm, self.fingerprint_type, self.fingerprint) == + (other.algorithm, other.fingerprint_type, other.fingerprint)) + + def __ne__(self, other): + return ((self.algorithm, self.fingerprint_type, self.fingerprint) != + (other.algorithm, other.fingerprint_type, other.fingerprint)) + + def __lt__(self, other): + return ((self.algorithm, self.fingerprint_type, self.fingerprint) < + (other.algorithm, other.fingerprint_type, other.fingerprint)) + + def __le__(self, other): + return ((self.algorithm, self.fingerprint_type, self.fingerprint) <= + (other.algorithm, other.fingerprint_type, other.fingerprint)) + + def __gt__(self, other): + return ((self.algorithm, self.fingerprint_type, self.fingerprint) > + (other.algorithm, other.fingerprint_type, other.fingerprint)) + + def __ge__(self, other): + return ((self.algorithm, self.fingerprint_type, self.fingerprint) >= + (other.algorithm, other.fingerprint_type, other.fingerprint)) def __repr__(self): return "'{} {} {}'".format(self.algorithm, self.fingerprint_type, @@ -1178,14 +1283,29 @@ class SrvValue(object): 'target': self.target, } - def __cmp__(self, other): - if self.priority != other.priority: - return cmp(self.priority, other.priority) - elif self.weight != other.weight: - return cmp(self.weight, other.weight) - elif self.port != other.port: - return cmp(self.port, other.port) - return cmp(self.target, other.target) + def __eq__(self, other): + return ((self.priority, self.weight, self.port, self.target) == + (other.priority, other.weight, other.port, other.target)) + + def __ne__(self, other): + return ((self.priority, self.weight, self.port, self.target) != + (other.priority, other.weight, other.port, other.target)) + + def __lt__(self, other): + return ((self.priority, self.weight, self.port, self.target) < + (other.priority, other.weight, other.port, other.target)) + + def __le__(self, other): + return ((self.priority, self.weight, self.port, self.target) <= + (other.priority, other.weight, other.port, other.target)) + + def __gt__(self, other): + return ((self.priority, self.weight, self.port, self.target) > + (other.priority, other.weight, other.port, other.target)) + + def __ge__(self, other): + return ((self.priority, self.weight, self.port, self.target) >= + (other.priority, other.weight, other.port, other.target)) def __repr__(self): return "'{} {} {} {}'".format(self.priority, self.weight, self.port, diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index f11d783..0b96fd7 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -5,12 +5,14 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals +from six import text_type from unittest import TestCase from octodns.record import ARecord, AaaaRecord, AliasRecord, CaaRecord, \ - CnameRecord, Create, Delete, GeoValue, MxRecord, NaptrRecord, NaptrValue, \ - NsRecord, PtrRecord, Record, SshfpRecord, SpfRecord, SrvRecord, \ - TxtRecord, Update, ValidationError, _Dynamic, _DynamicPool, _DynamicRule + CaaValue, CnameRecord, Create, Delete, GeoValue, MxRecord, MxValue, \ + NaptrRecord, NaptrValue, NsRecord, PtrRecord, Record, SshfpRecord, \ + SshfpValue, SpfRecord, SrvRecord, SrvValue, TxtRecord, Update, \ + ValidationError, _Dynamic, _DynamicPool, _DynamicRule from octodns.zone import Zone from helpers import DynamicProvider, GeoProvider, SimpleProvider @@ -482,109 +484,112 @@ class TestRecord(TestCase): # full sorting # equivalent b_naptr_value = b.values[0] - self.assertEquals(0, b_naptr_value.__cmp__(b_naptr_value)) + self.assertTrue(b_naptr_value == b_naptr_value) + self.assertFalse(b_naptr_value != b_naptr_value) + self.assertTrue(b_naptr_value <= b_naptr_value) + self.assertTrue(b_naptr_value >= b_naptr_value) # by order - self.assertEquals(1, b_naptr_value.__cmp__(NaptrValue({ + self.assertTrue(b_naptr_value > NaptrValue({ 'order': 10, 'preference': 31, 'flags': 'M', 'service': 'N', 'regexp': 'O', 'replacement': 'x', - }))) - self.assertEquals(-1, b_naptr_value.__cmp__(NaptrValue({ + })) + self.assertTrue(b_naptr_value < NaptrValue({ 'order': 40, 'preference': 31, 'flags': 'M', 'service': 'N', 'regexp': 'O', 'replacement': 'x', - }))) + })) # by preference - self.assertEquals(1, b_naptr_value.__cmp__(NaptrValue({ + self.assertTrue(b_naptr_value > NaptrValue({ 'order': 30, 'preference': 10, 'flags': 'M', 'service': 'N', 'regexp': 'O', 'replacement': 'x', - }))) - self.assertEquals(-1, b_naptr_value.__cmp__(NaptrValue({ + })) + self.assertTrue(b_naptr_value < NaptrValue({ 'order': 30, 'preference': 40, 'flags': 'M', 'service': 'N', 'regexp': 'O', 'replacement': 'x', - }))) + })) # by flags - self.assertEquals(1, b_naptr_value.__cmp__(NaptrValue({ + self.assertTrue(b_naptr_value > NaptrValue({ 'order': 30, 'preference': 31, 'flags': 'A', 'service': 'N', 'regexp': 'O', 'replacement': 'x', - }))) - self.assertEquals(-1, b_naptr_value.__cmp__(NaptrValue({ + })) + self.assertTrue(b_naptr_value < NaptrValue({ 'order': 30, 'preference': 31, 'flags': 'Z', 'service': 'N', 'regexp': 'O', 'replacement': 'x', - }))) + })) # by service - self.assertEquals(1, b_naptr_value.__cmp__(NaptrValue({ + self.assertTrue(b_naptr_value > NaptrValue({ 'order': 30, 'preference': 31, 'flags': 'M', 'service': 'A', 'regexp': 'O', 'replacement': 'x', - }))) - self.assertEquals(-1, b_naptr_value.__cmp__(NaptrValue({ + })) + self.assertTrue(b_naptr_value < NaptrValue({ 'order': 30, 'preference': 31, 'flags': 'M', 'service': 'Z', 'regexp': 'O', 'replacement': 'x', - }))) + })) # by regexp - self.assertEquals(1, b_naptr_value.__cmp__(NaptrValue({ + self.assertTrue(b_naptr_value > NaptrValue({ 'order': 30, 'preference': 31, 'flags': 'M', 'service': 'N', 'regexp': 'A', 'replacement': 'x', - }))) - self.assertEquals(-1, b_naptr_value.__cmp__(NaptrValue({ + })) + self.assertTrue(b_naptr_value < NaptrValue({ 'order': 30, 'preference': 31, 'flags': 'M', 'service': 'N', 'regexp': 'Z', 'replacement': 'x', - }))) + })) # by replacement - self.assertEquals(1, b_naptr_value.__cmp__(NaptrValue({ + self.assertTrue(b_naptr_value > NaptrValue({ 'order': 30, 'preference': 31, 'flags': 'M', 'service': 'N', 'regexp': 'O', 'replacement': 'a', - }))) - self.assertEquals(-1, b_naptr_value.__cmp__(NaptrValue({ + })) + self.assertTrue(b_naptr_value < NaptrValue({ 'order': 30, 'preference': 31, 'flags': 'M', 'service': 'N', 'regexp': 'O', 'replacement': 'z', - }))) + })) # __repr__ doesn't blow up a.__repr__() @@ -796,6 +801,38 @@ class TestRecord(TestCase): self.assertEquals(values, geo.values) self.assertEquals(['NA-US', 'NA'], list(geo.parents)) + a = GeoValue('NA-US-CA', values) + b = GeoValue('AP-JP', values) + c = GeoValue('NA-US-CA', ['2.3.4.5']) + + self.assertEqual(a, a) + self.assertEqual(b, b) + self.assertEqual(c, c) + + self.assertNotEqual(a, b) + self.assertNotEqual(a, c) + self.assertNotEqual(b, a) + self.assertNotEqual(b, c) + self.assertNotEqual(c, a) + self.assertNotEqual(c, b) + + self.assertTrue(a > b) + self.assertTrue(a < c) + self.assertTrue(b < a) + self.assertTrue(b < c) + self.assertTrue(c > a) + self.assertTrue(c > b) + + self.assertTrue(a >= a) + self.assertTrue(a >= b) + self.assertTrue(a <= c) + self.assertTrue(b <= a) + self.assertTrue(b <= b) + self.assertTrue(b <= c) + self.assertTrue(c > a) + self.assertTrue(c > b) + self.assertTrue(c >= b) + def test_healthcheck(self): new = Record.new(self.zone, 'a', { 'ttl': 44, @@ -851,6 +888,290 @@ class TestRecord(TestCase): }) self.assertFalse(new.ignored) + def test_ordering_functions(self): + a = Record.new(self.zone, 'a', { + 'ttl': 44, + 'type': 'A', + 'value': '1.2.3.4', + }) + b = Record.new(self.zone, 'b', { + 'ttl': 44, + 'type': 'A', + 'value': '1.2.3.4', + }) + c = Record.new(self.zone, 'c', { + 'ttl': 44, + 'type': 'A', + 'value': '1.2.3.4', + }) + aaaa = Record.new(self.zone, 'a', { + 'ttl': 44, + 'type': 'AAAA', + 'value': '2601:644:500:e210:62f8:1dff:feb8:947a', + }) + + self.assertEquals(a, a) + self.assertEquals(b, b) + self.assertEquals(c, c) + self.assertEquals(aaaa, aaaa) + + self.assertNotEqual(a, b) + self.assertNotEqual(a, c) + self.assertNotEqual(a, aaaa) + self.assertNotEqual(b, a) + self.assertNotEqual(b, c) + self.assertNotEqual(b, aaaa) + self.assertNotEqual(c, a) + self.assertNotEqual(c, b) + self.assertNotEqual(c, aaaa) + self.assertNotEqual(aaaa, a) + self.assertNotEqual(aaaa, b) + self.assertNotEqual(aaaa, c) + + self.assertTrue(a < b) + self.assertTrue(a < c) + self.assertTrue(a < aaaa) + self.assertTrue(b > a) + self.assertTrue(b < c) + self.assertTrue(b > aaaa) + self.assertTrue(c > a) + self.assertTrue(c > b) + self.assertTrue(c > aaaa) + self.assertTrue(aaaa > a) + self.assertTrue(aaaa < b) + self.assertTrue(aaaa < c) + + self.assertTrue(a <= a) + self.assertTrue(a <= b) + self.assertTrue(a <= c) + self.assertTrue(a <= aaaa) + self.assertTrue(b >= a) + self.assertTrue(b >= b) + self.assertTrue(b <= c) + self.assertTrue(b >= aaaa) + self.assertTrue(c >= a) + self.assertTrue(c >= b) + self.assertTrue(c >= c) + self.assertTrue(c >= aaaa) + self.assertTrue(aaaa >= a) + self.assertTrue(aaaa <= b) + self.assertTrue(aaaa <= c) + self.assertTrue(aaaa <= aaaa) + + def test_caa_value(self): + a = CaaValue({'flags': 0, 'tag': 'a', 'value': 'v'}) + b = CaaValue({'flags': 1, 'tag': 'a', 'value': 'v'}) + c = CaaValue({'flags': 0, 'tag': 'c', 'value': 'v'}) + d = CaaValue({'flags': 0, 'tag': 'a', 'value': 'z'}) + + self.assertEqual(a, a) + self.assertEqual(b, b) + self.assertEqual(c, c) + self.assertEqual(d, d) + + self.assertNotEqual(a, b) + self.assertNotEqual(a, c) + self.assertNotEqual(a, d) + self.assertNotEqual(b, a) + self.assertNotEqual(b, c) + self.assertNotEqual(b, d) + self.assertNotEqual(c, a) + self.assertNotEqual(c, b) + self.assertNotEqual(c, d) + + self.assertTrue(a < b) + self.assertTrue(a < c) + self.assertTrue(a < d) + + self.assertTrue(b > a) + self.assertTrue(b > c) + self.assertTrue(b > d) + + self.assertTrue(c > a) + self.assertTrue(c < b) + self.assertTrue(c > d) + + self.assertTrue(d > a) + self.assertTrue(d < b) + self.assertTrue(d < c) + + self.assertTrue(a <= b) + self.assertTrue(a <= c) + self.assertTrue(a <= d) + self.assertTrue(a <= a) + self.assertTrue(a >= a) + + self.assertTrue(b >= a) + self.assertTrue(b >= c) + self.assertTrue(b >= d) + self.assertTrue(b >= b) + self.assertTrue(b <= b) + + self.assertTrue(c >= a) + self.assertTrue(c <= b) + self.assertTrue(c >= d) + self.assertTrue(c >= c) + self.assertTrue(c <= c) + + self.assertTrue(d >= a) + self.assertTrue(d <= b) + self.assertTrue(d <= c) + self.assertTrue(d >= d) + self.assertTrue(d <= d) + + def test_mx_value(self): + a = MxValue({'preference': 0, 'priority': 'a', 'exchange': 'v', + 'value': '1'}) + b = MxValue({'preference': 10, 'priority': 'a', 'exchange': 'v', + 'value': '2'}) + c = MxValue({'preference': 0, 'priority': 'b', 'exchange': 'z', + 'value': '3'}) + + self.assertEqual(a, a) + self.assertEqual(b, b) + self.assertEqual(c, c) + + self.assertNotEqual(a, b) + self.assertNotEqual(a, c) + self.assertNotEqual(b, a) + self.assertNotEqual(b, c) + self.assertNotEqual(c, a) + self.assertNotEqual(c, b) + + self.assertTrue(a < b) + self.assertTrue(a < c) + + self.assertTrue(b > a) + self.assertTrue(b > c) + + self.assertTrue(c > a) + self.assertTrue(c < b) + + self.assertTrue(a <= b) + self.assertTrue(a <= c) + self.assertTrue(a <= a) + self.assertTrue(a >= a) + + self.assertTrue(b >= a) + self.assertTrue(b >= c) + self.assertTrue(b >= b) + self.assertTrue(b <= b) + + self.assertTrue(c >= a) + self.assertTrue(c <= b) + self.assertTrue(c >= c) + self.assertTrue(c <= c) + + def test_sshfp_value(self): + a = SshfpValue({'algorithm': 0, 'fingerprint_type': 0, + 'fingerprint': 'abcd'}) + b = SshfpValue({'algorithm': 1, 'fingerprint_type': 0, + 'fingerprint': 'abcd'}) + c = SshfpValue({'algorithm': 0, 'fingerprint_type': 1, + 'fingerprint': 'abcd'}) + d = SshfpValue({'algorithm': 0, 'fingerprint_type': 0, + 'fingerprint': 'bcde'}) + + self.assertEqual(a, a) + self.assertEqual(b, b) + self.assertEqual(c, c) + self.assertEqual(d, d) + + self.assertNotEqual(a, b) + self.assertNotEqual(a, c) + self.assertNotEqual(a, d) + self.assertNotEqual(b, a) + self.assertNotEqual(b, c) + self.assertNotEqual(b, d) + self.assertNotEqual(c, a) + self.assertNotEqual(c, b) + self.assertNotEqual(c, d) + self.assertNotEqual(d, a) + self.assertNotEqual(d, b) + self.assertNotEqual(d, c) + + self.assertTrue(a < b) + self.assertTrue(a < c) + + self.assertTrue(b > a) + self.assertTrue(b > c) + + self.assertTrue(c > a) + self.assertTrue(c < b) + + self.assertTrue(a <= b) + self.assertTrue(a <= c) + self.assertTrue(a <= a) + self.assertTrue(a >= a) + + self.assertTrue(b >= a) + self.assertTrue(b >= c) + self.assertTrue(b >= b) + self.assertTrue(b <= b) + + self.assertTrue(c >= a) + self.assertTrue(c <= b) + self.assertTrue(c >= c) + self.assertTrue(c <= c) + + def test_srv_value(self): + a = SrvValue({'priority': 0, 'weight': 0, 'port': 0, 'target': 'foo.'}) + b = SrvValue({'priority': 1, 'weight': 0, 'port': 0, 'target': 'foo.'}) + c = SrvValue({'priority': 0, 'weight': 2, 'port': 0, 'target': 'foo.'}) + d = SrvValue({'priority': 0, 'weight': 0, 'port': 3, 'target': 'foo.'}) + e = SrvValue({'priority': 0, 'weight': 0, 'port': 0, 'target': 'mmm.'}) + + self.assertEqual(a, a) + self.assertEqual(b, b) + self.assertEqual(c, c) + self.assertEqual(d, d) + self.assertEqual(e, e) + + self.assertNotEqual(a, b) + self.assertNotEqual(a, c) + self.assertNotEqual(a, d) + self.assertNotEqual(a, e) + self.assertNotEqual(b, a) + self.assertNotEqual(b, c) + self.assertNotEqual(b, d) + self.assertNotEqual(b, e) + self.assertNotEqual(c, a) + self.assertNotEqual(c, b) + self.assertNotEqual(c, d) + self.assertNotEqual(c, e) + self.assertNotEqual(d, a) + self.assertNotEqual(d, b) + self.assertNotEqual(d, c) + self.assertNotEqual(d, e) + self.assertNotEqual(e, a) + self.assertNotEqual(e, b) + self.assertNotEqual(e, c) + self.assertNotEqual(e, d) + + self.assertTrue(a < b) + self.assertTrue(a < c) + + self.assertTrue(b > a) + self.assertTrue(b > c) + + self.assertTrue(c > a) + self.assertTrue(c < b) + + self.assertTrue(a <= b) + self.assertTrue(a <= c) + self.assertTrue(a <= a) + self.assertTrue(a >= a) + + self.assertTrue(b >= a) + self.assertTrue(b >= c) + self.assertTrue(b >= b) + self.assertTrue(b <= b) + + self.assertTrue(c >= a) + self.assertTrue(c <= b) + self.assertTrue(c >= c) + self.assertTrue(c <= c) + class TestRecordValidation(TestCase): zone = Zone('unit.tests.', []) From ce67824015ae25914e185309931c0346dcf7c1bd Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 29 Jul 2019 08:48:17 -0700 Subject: [PATCH 009/155] Handle python3 sourcing of urlparse --- tests/test_octodns_provider_rackspace.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_octodns_provider_rackspace.py b/tests/test_octodns_provider_rackspace.py index fbd7dac..b257166 100644 --- a/tests/test_octodns_provider_rackspace.py +++ b/tests/test_octodns_provider_rackspace.py @@ -9,7 +9,10 @@ import json import re from six import text_type from unittest import TestCase -from urlparse import urlparse +try: + from urlparse import urlparse +except ImportError: + from urllib.parse import urlparse from requests import HTTPError from requests_mock import ANY, mock as requests_mock From 7d8f181367d540ddc3f99a5a1340e8bd7e5db6ac Mon Sep 17 00:00:00 2001 From: Arunothia Marappan Date: Thu, 19 Sep 2019 10:48:18 -0700 Subject: [PATCH 010/155] Adding Akamai entry to supported providers --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 75a006c..be0a4ab 100644 --- a/README.md +++ b/README.md @@ -172,6 +172,7 @@ The above command pulled the existing data out of Route53 and placed the results | Provider | Requirements | Record Support | Dynamic/Geo Support | Notes | |--|--|--|--|--| | [AzureProvider](/octodns/provider/azuredns.py) | azure-mgmt-dns | A, AAAA, CAA, CNAME, MX, NS, PTR, SRV, TXT | No | | +| [Akamai](/octodns/provider/fastdns.py) | edgegrid-python | A, AAAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, SSHFP, TXT | No | | | [CloudflareProvider](/octodns/provider/cloudflare.py) | | A, AAAA, ALIAS, CAA, CNAME, MX, NS, SPF, SRV, TXT | No | CAA tags restricted | | [ConstellixProvider](/octodns/provider/constellix.py) | | A, AAAA, ALIAS (ANAME), CAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | CAA tags restricted | | [DigitalOceanProvider](/octodns/provider/digitalocean.py) | | A, AAAA, CAA, CNAME, MX, NS, TXT, SRV | No | CAA tags restricted | From 736d588e86942a215a01607ce3983c3ea1165395 Mon Sep 17 00:00:00 2001 From: Maikel Poot Date: Fri, 20 Sep 2019 10:18:51 +0200 Subject: [PATCH 011/155] Changed requirements to version 2.22.0 Fixes: ERROR: requests 2.20.0 has requirement urllib3<1.25,>=1.21.1, but you'll have urllib3 1.25.5 which is incompatible. --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 75dc1df..f67f2ee 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,7 +18,7 @@ natsort==5.5.0 nsone==0.9.100 ovh==0.4.8 python-dateutil==2.6.1 -requests==2.20.0 +requests==2.22.0 s3transfer==0.1.13 six==1.11.0 setuptools==38.5.2 From bb3f0c0b4a9c0705aad39cfd5aac3cbca8bfab47 Mon Sep 17 00:00:00 2001 From: Maikel Poot Date: Wed, 25 Sep 2019 07:01:06 +0200 Subject: [PATCH 012/155] Added TransIP provider and tests --- octodns/provider/transip.py | 328 +++++++++++++++++++++++++ tests/test_octodns_provider_transip.py | 234 ++++++++++++++++++ 2 files changed, 562 insertions(+) create mode 100644 octodns/provider/transip.py create mode 100644 tests/test_octodns_provider_transip.py diff --git a/octodns/provider/transip.py b/octodns/provider/transip.py new file mode 100644 index 0000000..2ce9180 --- /dev/null +++ b/octodns/provider/transip.py @@ -0,0 +1,328 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from suds import WebFault + +from collections import defaultdict +from .base import BaseProvider +from logging import getLogger +from ..record import Record +from transip.service.domain import DomainService +from transip.service.objects import DnsEntry + + +class TransipProvider(BaseProvider): + ''' + Transip DNS provider + + transip: + class: octodns.provider.transip.TransipProvider + # Your Transip account name (required) + account: yourname + # The api key (required) + key: | + \''' + -----BEGIN PRIVATE KEY----- + ... + -----END PRIVATE KEY----- + \''' + + ''' + SUPPORTS_GEO = False + SUPPORTS_DYNAMIC = False + SUPPORTS = set( + ('A', 'AAAA', 'CNAME', 'MX', 'SRV', 'SPF', 'TXT', 'SSHFP', 'CAA')) + # unsupported by OctoDNS: 'TLSA', 'CAA' + MIN_TTL = 120 + TIMEOUT = 15 + ROOT_RECORD = '@' + + def __init__(self, id, account, key, *args, **kwargs): + self.log = getLogger('TransipProvider[{}]'.format(id)) + self.log.debug('__init__: id=%s, account=%s, token=***', id, + account) + super(TransipProvider, self).__init__(id, *args, **kwargs) + + self._client = DomainService(account, key) + + self.account = account + self.key = key + + self._zones = None + self._zone_records = {} + + self._currentZone = {} + + def populate(self, zone, target=False, lenient=False): + + exists = False + self._currentZone = zone + self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, + target, lenient) + + before = len(zone.records) + try: + zoneInfo = self._client.get_info(zone.name[:-1]) + except WebFault as e: + if e.fault.faultcode == '102' and target == False: + self.log.warning( + 'populate: (%s) Zone %s not found in account ', + e.fault.faultcode, zone.name) + exists = False + return exists + elif e.fault.faultcode == '102' and target == True: + self.log.warning('populate: Transip can\'t create new zones') + raise Exception( + ('populate: ({}) Transip used ' + + 'as target for non-existing zone: {}').format( + e.fault.faultcode, zone.name)) + else: + self.log.error('populate: (%s) %s ', e.fault.faultcode, + e.fault.faultstring) + raise e + + self.log.debug('populate: found %s records for zone %s', + len(zoneInfo.dnsEntries), zone.name) + exists = True + if zoneInfo.dnsEntries: + values = defaultdict(lambda: defaultdict(list)) + for record in zoneInfo.dnsEntries: + name = zone.hostname_from_fqdn(record['name']) + if name == self.ROOT_RECORD: + name = '' + + if record['type'] in self.SUPPORTS: + values[name][record['type']].append(record) + + for name, types in values.items(): + for _type, records in types.items(): + data_for = getattr(self, '_data_for_{}'.format(_type)) + record = Record.new(zone, name, data_for(_type, records), + source=self, lenient=lenient) + zone.add_record(record, lenient=lenient) + self.log.info('populate: found %s records, exists = %s', + len(zone.records) - before, exists) + + self._currentZone = {} + return exists + + def _apply(self, plan): + desired = plan.desired + changes = plan.changes + self.log.debug('apply: zone=%s, changes=%d', desired.name, + len(changes)) + # for change in changes: + # class_name = change.__class__.__name__ + # getattr(self, '_apply_{}'.format(class_name))(change) + + self._currentZone = plan.desired + try: + self._client.get_info(plan.desired.name[:-1]) + except WebFault as e: + self.log.warning('_apply: %s ', e.message) + raise e + + _dns_entries = [] + for record in plan.desired.records: + if record._type in self.SUPPORTS: + entries_for = getattr(self, '_entries_for_{}'.format(record._type)) + + # Root records have '@' as name + name = record.name + if name == '': + name = self.ROOT_RECORD + + _dns_entries.extend(entries_for(name, record)) + + try: + self._client.set_dns_entries(plan.desired.name[:-1], _dns_entries) + except WebFault as e: + self.log.warning(('_apply: Set DNS returned ' + + 'one or more errors: {}').format( + e.fault.faultstring)) + raise Exception(200, e.fault.faultstring) + + self._currentZone = {} + + def _entries_for_multiple(self, name, record): + _entries = [] + + for value in record.values: + _entries.append(DnsEntry(name, record.ttl, record._type, value)) + + return _entries + + def _entries_for_single(self, name, record): + + return [DnsEntry(name, record.ttl, record._type, record.value)] + + _entries_for_A = _entries_for_multiple + _entries_for_AAAA = _entries_for_multiple + _entries_for_NS = _entries_for_multiple + _entries_for_SPF = _entries_for_multiple + _entries_for_CNAME = _entries_for_single + + def _entries_for_MX(self, name, record): + _entries = [] + + for value in record.values: + content = "{} {}".format(value.preference, value.exchange) + _entries.append(DnsEntry(name, record.ttl, record._type, content)) + + return _entries + + def _entries_for_SRV(self, name, record): + _entries = [] + + for value in record.values: + content = "{} {} {} {}".format(value.priority, value.weight, + value.port, value.target) + _entries.append(DnsEntry(name, record.ttl, record._type, content)) + + return _entries + + def _entries_for_SSHFP(self, name, record): + _entries = [] + + for value in record.values: + content = "{} {} {}".format(value.algorithm, value.fingerprint_type, + value.fingerprint) + _entries.append(DnsEntry(name, record.ttl, record._type, content)) + + return _entries + + def _entries_for_CAA(self, name, record): + _entries = [] + + for value in record.values: + content = "{} {} {}".format(value.flags, value.tag, + value.value) + _entries.append(DnsEntry(name, record.ttl, record._type, content)) + + return _entries + + def _entries_for_TXT(self, name, record): + _entries = [] + + for value in record.values: + value = value.replace('\\;', ';') + _entries.append(DnsEntry(name, record.ttl, record._type, value)) + + return _entries + + def _parse_to_fqdn(self, value): + + if (value[-1] != '.'): + self.log.debug('parseToFQDN: changed %s to %s', value, + '{}.{}'.format(value, self._currentZone.name)) + value = '{}.{}'.format(value, self._currentZone.name) + + return value + + def _get_lowest_ttl(self, records): + _ttl = 100000 + for record in records: + _ttl = min(_ttl, record['expire']) + return _ttl + + def _data_for_multiple(self, _type, records): + + _values = [] + for record in records: + _values.append(record['content']) + + return { + 'ttl': self._get_lowest_ttl(records), + 'type': _type, + 'values': _values + } + + _data_for_A = _data_for_multiple + _data_for_AAAA = _data_for_multiple + _data_for_NS = _data_for_multiple + _data_for_SPF = _data_for_multiple + + def _data_for_CNAME(self, _type, records): + return { + 'ttl': records[0]['expire'], + 'type': _type, + 'value': self._parse_to_fqdn(records[0]['content']) + } + + def _data_for_MX(self, _type, records): + values = [] + for record in records: + preference, exchange = record['content'].split(" ", 1) + values.append({ + 'preference': preference, + 'exchange': self._parse_to_fqdn(exchange) + }) + return { + 'ttl': self._get_lowest_ttl(records), + 'type': _type, + 'values': values + } + + def _data_for_SRV(self, _type, records): + values = [] + for record in records: + priority, weight, port, target = record['content'].split(' ', 3) + values.append({ + 'port': port, + 'priority': priority, + 'target': self._parse_to_fqdn(target), + 'weight': weight + }) + + return { + 'type': _type, + 'ttl': self._get_lowest_ttl(records), + 'values': values + } + + def _data_for_SSHFP(self, _type, records): + values = [] + for record in records: + algorithm, fp_type, fingerprint = record['content'].split(' ', 2) + values.append({ + 'algorithm': algorithm, + 'fingerprint': fingerprint.lower(), + 'fingerprint_type': fp_type + }) + + return { + 'type': _type, + 'ttl': self._get_lowest_ttl(records), + 'values': values + } + + def _data_for_CAA(self, _type, records): + values = [] + for record in records: + flags, tag, value = record['content'].split(' ', 2) + values.append({ + 'flags': flags, + 'tag': tag, + 'value': value + }) + + return { + 'type': _type, + 'ttl': self._get_lowest_ttl(records), + 'values': values + } + + def _data_for_TXT(self, _type, records): + values = [] + for record in records: + values.append(record['content'].replace(';', '\\;')) + + return { + 'type': _type, + 'ttl': self._get_lowest_ttl(records), + 'values': values + } diff --git a/tests/test_octodns_provider_transip.py b/tests/test_octodns_provider_transip.py new file mode 100644 index 0000000..dbf7eab --- /dev/null +++ b/tests/test_octodns_provider_transip.py @@ -0,0 +1,234 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +# from mock import Mock, call +from os.path import dirname, join + +from suds import WebFault + +from requests_mock import ANY, mock as requests_mock +from unittest import TestCase + +from octodns.record import Record +from octodns.provider.transip import TransipProvider +from octodns.provider.yaml import YamlProvider +from octodns.zone import Zone +from transip.service.domain import DomainService +from transip.service.objects import DnsEntry + + + + +class MockDomainService(DomainService): + + def __init__(self, *args, **kwargs): + super(MockDomainService, self).__init__('MockDomainService', *args, **kwargs) + self.mockupEntries = [] + + def mockup(self, records): + + provider = TransipProvider('', '', ''); + + _dns_entries = [] + for record in records: + if record._type in provider.SUPPORTS: + entries_for = getattr(provider, '_entries_for_{}'.format(record._type)) + + # Root records have '@' as name + name = record.name + if name == '': + name = provider.ROOT_RECORD + + _dns_entries.extend(entries_for(name, record)) + + _dns_entries.append(DnsEntry('@', '3600', 'NS', 'ns01.transip.nl.')) + + + self.mockupEntries = _dns_entries + + # Skips authentication layer and returns the entries loaded by "Mockup" + def get_info(self, domain_name): + + if str(domain_name) == str('notfound.unit.tests'): + self.raiseZoneNotFound() + + result = lambda: None + setattr(result, "dnsEntries", self.mockupEntries) + return result + + def set_dns_entries(self, domain_name, dns_entries): + if str(domain_name) == str('failsetdns.unit.tests'): + self.raiseSaveError() + + return True + + def raiseZoneNotFound(self): + fault = lambda: None + setattr(fault, "faultstring", '102 is zone not found') + setattr(fault, "faultcode", str('102')) + document = {} + raise WebFault(fault, document) + + def raiseInvalidAuth(self): + fault = lambda: None + setattr(fault, "faultstring", '200 is invalid auth') + setattr(fault, "faultcode", str('200')) + document = {} + raise WebFault(fault, document) + + def raiseSaveError(self): + fault = lambda: None + setattr(fault, "faultstring", '202 error while saving') + setattr(fault, "faultcode", str('202')) + document = {} + raise WebFault(fault, document) + + + +class TestTransipProvider(TestCase): + bogus_key = str("""-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEA0U5HGCkLrz423IyUf3u4cKN2WrNz1x5KNr6PvH2M/zxas+zB +elbxkdT3AQ+wmfcIvOuTmFRTHv35q2um1aBrPxVw+2s+lWo28VwIRttwIB1vIeWu +lSBnkEZQRLyPI2tH0i5QoMX4CVPf9rvij3Uslimi84jdzDfPFIh6jZ6C8nLipOTG +0IMhge1ofVfB0oSy5H+7PYS2858QLAf5ruYbzbAxZRivS402wGmQ0d0Lc1KxraAj +kiMM5yj/CkH/Vm2w9I6+tLFeASE4ub5HCP5G/ig4dbYtqZMQMpqyAbGxd5SOVtyn +UHagAJUxf8DT3I8PyjEHjxdOPUsxNyRtepO/7QIDAQABAoIBAQC7fiZ7gxE/ezjD +2n6PsHFpHVTBLS2gzzZl0dCKZeFvJk6ODJDImaeuHhrh7X8ifMNsEI9XjnojMhl8 +MGPzy88mZHugDNK0H8B19x5G8v1/Fz7dG5WHas660/HFkS+b59cfdXOugYiOOn9O +08HBBpLZNRUOmVUuQfQTjapSwGLG8PocgpyRD4zx0LnldnJcqYCxwCdev+AAsPnq +ibNtOd/MYD37w9MEGcaxLE8wGgkv8yd97aTjkgE+tp4zsM4QE4Rag133tsLLNznT +4Qr/of15M3NW/DXq/fgctyRcJjZpU66eCXLCz2iRTnLyyxxDC2nwlxKbubV+lcS0 +S4hbfd/BAoGBAO8jXxEaiybR0aIhhSR5esEc3ymo8R8vBN3ZMJ+vr5jEPXr/ZuFj +/R4cZ2XV3VoQJG0pvIOYVPZ5DpJM7W+zSXtJ/7bLXy4Bnmh/rc+YYgC+AXQoLSil +iD2OuB2xAzRAK71DVSO0kv8gEEXCersPT2i6+vC2GIlJvLcYbOdRKWGxAoGBAOAQ +aJbRLtKujH+kMdoMI7tRlL8XwI+SZf0FcieEu//nFyerTePUhVgEtcE+7eQ7hyhG +fIXUFx/wALySoqFzdJDLc8U8pTLhbUaoLOTjkwnCTKQVprhnISqQqqh/0U5u47IE +RWzWKN6OHb0CezNTq80Dr6HoxmPCnJHBHn5LinT9AoGAQSpvZpbIIqz8pmTiBl2A +QQ2gFpcuFeRXPClKYcmbXVLkuhbNL1BzEniFCLAt4LQTaRf9ghLJ3FyCxwVlkpHV +zV4N6/8hkcTpKOraL38D/dXJSaEFJVVuee/hZl3tVJjEEpA9rDwx7ooLRSdJEJ6M +ciq55UyKBSdt4KssSiDI2RECgYBL3mJ7xuLy5bWfNsrGiVvD/rC+L928/5ZXIXPw +26oI0Yfun7ulDH4GOroMcDF/GYT/Zzac3h7iapLlR0WYI47xxGI0A//wBZLJ3QIu +krxkDo2C9e3Y/NqnHgsbOQR3aWbiDT4wxydZjIeXS3LKA2fl6Hyc90PN3cTEOb8I +hq2gRQKBgEt0SxhhtyB93SjgTzmUZZ7PiEf0YJatfM6cevmjWHexrZH+x31PB72s +fH2BQyTKKzoCLB1k/6HRaMnZdrWyWSZ7JKz3AHJ8+58d0Hr8LTrzDM1L6BbjeDct +N4OiVz1I3rbZGYa396lpxO6ku8yCglisL1yrSP6DdEUp66ntpKVd +-----END RSA PRIVATE KEY-----""") + + + def make_expected(self): + expected = Zone('unit.tests.', []) + source = YamlProvider('test', join(dirname(__file__), 'config')) + source.populate(expected) + return expected + + def test_populate(self): + + _expected = self.make_expected() + + with self.assertRaises(WebFault) as ctx: + provider = TransipProvider('test', 'unittest', self.bogus_key) + zone = Zone('unit.tests.', []) + provider.populate(zone, True) + + self.assertEquals(str('WebFault'), + str(ctx.exception.__class__.__name__)) + + self.assertEquals(str('200'), ctx.exception.fault.faultcode) + + with self.assertRaises(Exception) as ctx: + provider = TransipProvider('test', 'unittest', self.bogus_key) + provider._client = MockDomainService('unittest', self.bogus_key) + zone = Zone('notfound.unit.tests.', []) + provider.populate(zone, True) + + self.assertEquals(str('Exception'), + str(ctx.exception.__class__.__name__)) + + self.assertEquals('populate: (102) Transip used as target for non-existing zone: notfound.unit.tests.', ctx.exception.message) + + provider = TransipProvider('test', 'unittest', self.bogus_key) + provider._client = MockDomainService('unittest', self.bogus_key) + zone = Zone('notfound.unit.tests.', []) + provider.populate(zone, False) + + provider = TransipProvider('test', 'unittest', self.bogus_key) + provider._client = MockDomainService('unittest', self.bogus_key) + provider._client.mockup(_expected.records) + zone = Zone('unit.tests.', []) + provider.populate(zone, False) + + provider._currentZone = zone + self.assertEquals("www.unit.tests.", provider._parse_to_fqdn("www")) + + + provider = TransipProvider('test', 'unittest', self.bogus_key) + provider._client = MockDomainService('unittest', self.bogus_key) + zone = Zone('unit.tests.', []) + exists = provider.populate(zone, True) + self.assertTrue(exists, 'populate should return true') + + + return + + def test_plan(self): + + _expected = self.make_expected() + + print(_expected.name) + + provider = TransipProvider('test', 'unittest', self.bogus_key) + provider._client = MockDomainService('unittest', self.bogus_key) + plan = provider.plan(_expected) + + self.assertEqual(12, plan.change_counts['Create']) + self.assertEqual(0, plan.change_counts['Update']) + self.assertEqual(0, plan.change_counts['Delete']) + + return + + def test_apply(self): + + _expected = self.make_expected() + + provider = TransipProvider('test', 'unittest', self.bogus_key) + provider._client = MockDomainService('unittest', self.bogus_key) + plan = provider.plan(_expected) + #self.assertEqual(11, plan.changes) + changes = provider.apply(plan) + + + + with self.assertRaises(Exception) as ctx: + provider = TransipProvider('test', 'unittest', self.bogus_key) + provider._client = MockDomainService('unittest', self.bogus_key) + plan = provider.plan(_expected) + plan.desired.name = 'notfound.unit.tests.' + changes = provider.apply(plan) + # self.assertEqual(11, changes) + + self.assertEquals(str('WebFault'), + str(ctx.exception.__class__.__name__)) + + _expected = self.make_expected() + + with self.assertRaises(Exception) as ctx: + provider = TransipProvider('test', 'unittest', self.bogus_key) + provider._client = MockDomainService('unittest', self.bogus_key) + plan = provider.plan(_expected) + plan.desired.name = 'failsetdns.unit.tests.' + changes = provider.apply(plan) + # self.assertEqual(11, changes) + + + #provider = TransipProvider('test', 'unittest', self.bogus_key) + + #plan = provider.plan(_expected) + +# changes = provider.apply(plan) +# self.assertEquals(29, changes) + + From 30c8c4d313d150c2c81324f0b0a7de06f634bdc0 Mon Sep 17 00:00:00 2001 From: Maikel Poot Date: Wed, 25 Sep 2019 08:18:17 +0200 Subject: [PATCH 013/155] Add transip requirement and add provider to readme --- README.md | 1 + requirements.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/README.md b/README.md index be0a4ab..7124b20 100644 --- a/README.md +++ b/README.md @@ -188,6 +188,7 @@ The above command pulled the existing data out of Route53 and placed the results | [Rackspace](/octodns/provider/rackspace.py) | | A, AAAA, ALIAS, CNAME, MX, NS, PTR, SPF, TXT | No | | | [Route53](/octodns/provider/route53.py) | boto3 | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | Both | CNAME health checks don't support a Host header | | [Selectel](/octodns/provider/selectel.py) | | A, AAAA, CNAME, MX, NS, SPF, SRV, TXT | No | | +| [Transip](/octodns/provider/transip.py) | transip | A, AAAA, CNAME, MX, SRV, SPF, TXT, SSHFP, CAA | No | | | [AxfrSource](/octodns/source/axfr.py) | | A, AAAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | read-only | | [ZoneFileSource](/octodns/source/axfr.py) | | A, AAAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | read-only | | [TinyDnsFileSource](/octodns/source/tinydns.py) | | A, CNAME, MX, NS, PTR | No | read-only | diff --git a/requirements.txt b/requirements.txt index f67f2ee..bb373d0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,3 +22,4 @@ requests==2.22.0 s3transfer==0.1.13 six==1.11.0 setuptools==38.5.2 +transip==2.0.0 From 7056d299072f3c03c446ef8b5435f4339b1bb4b4 Mon Sep 17 00:00:00 2001 From: Maikel Poot Date: Wed, 25 Sep 2019 08:18:59 +0200 Subject: [PATCH 014/155] fixes lint warning. --- octodns/provider/transip.py | 20 +++-- tests/test_octodns_provider_transip.py | 110 +++++++++++++++---------- 2 files changed, 79 insertions(+), 51 deletions(-) diff --git a/octodns/provider/transip.py b/octodns/provider/transip.py index 2ce9180..8014310 100644 --- a/octodns/provider/transip.py +++ b/octodns/provider/transip.py @@ -36,7 +36,7 @@ class TransipProvider(BaseProvider): SUPPORTS_DYNAMIC = False SUPPORTS = set( ('A', 'AAAA', 'CNAME', 'MX', 'SRV', 'SPF', 'TXT', 'SSHFP', 'CAA')) - # unsupported by OctoDNS: 'TLSA', 'CAA' + # unsupported by OctoDNS: 'TLSA' MIN_TTL = 120 TIMEOUT = 15 ROOT_RECORD = '@' @@ -68,17 +68,17 @@ class TransipProvider(BaseProvider): try: zoneInfo = self._client.get_info(zone.name[:-1]) except WebFault as e: - if e.fault.faultcode == '102' and target == False: + if e.fault.faultcode == '102' and target is False: self.log.warning( 'populate: (%s) Zone %s not found in account ', e.fault.faultcode, zone.name) exists = False return exists - elif e.fault.faultcode == '102' and target == True: + elif e.fault.faultcode == '102' and target is True: self.log.warning('populate: Transip can\'t create new zones') raise Exception( ('populate: ({}) Transip used ' + - 'as target for non-existing zone: {}').format( + 'as target for non-existing zone: {}').format( e.fault.faultcode, zone.name)) else: self.log.error('populate: (%s) %s ', e.fault.faultcode, @@ -129,7 +129,8 @@ class TransipProvider(BaseProvider): _dns_entries = [] for record in plan.desired.records: if record._type in self.SUPPORTS: - entries_for = getattr(self, '_entries_for_{}'.format(record._type)) + entries_for = getattr(self, + '_entries_for_{}'.format(record._type)) # Root records have '@' as name name = record.name @@ -143,7 +144,7 @@ class TransipProvider(BaseProvider): except WebFault as e: self.log.warning(('_apply: Set DNS returned ' + 'one or more errors: {}').format( - e.fault.faultstring)) + e.fault.faultstring)) raise Exception(200, e.fault.faultstring) self._currentZone = {} @@ -189,8 +190,9 @@ class TransipProvider(BaseProvider): _entries = [] for value in record.values: - content = "{} {} {}".format(value.algorithm, value.fingerprint_type, - value.fingerprint) + content = "{} {} {}".format(value.algorithm, + value.fingerprint_type, + value.fingerprint) _entries.append(DnsEntry(name, record.ttl, record._type, content)) return _entries @@ -200,7 +202,7 @@ class TransipProvider(BaseProvider): for value in record.values: content = "{} {} {}".format(value.flags, value.tag, - value.value) + value.value) _entries.append(DnsEntry(name, record.ttl, record._type, content)) return _entries diff --git a/tests/test_octodns_provider_transip.py b/tests/test_octodns_provider_transip.py index dbf7eab..811c1e2 100644 --- a/tests/test_octodns_provider_transip.py +++ b/tests/test_octodns_provider_transip.py @@ -10,10 +10,8 @@ from os.path import dirname, join from suds import WebFault -from requests_mock import ANY, mock as requests_mock from unittest import TestCase -from octodns.record import Record from octodns.provider.transip import TransipProvider from octodns.provider.yaml import YamlProvider from octodns.zone import Zone @@ -21,22 +19,35 @@ from transip.service.domain import DomainService from transip.service.objects import DnsEntry +class MockFault(object): + faultstring = "" + faultcode = "" + + def __init__(self, code, string, *args, **kwargs): + self.faultstring = string + self.faultcode = code + + +class MockResponse(object): + dnsEntries = [] class MockDomainService(DomainService): def __init__(self, *args, **kwargs): - super(MockDomainService, self).__init__('MockDomainService', *args, **kwargs) + super(MockDomainService, self).__init__('MockDomainService', *args, + **kwargs) self.mockupEntries = [] def mockup(self, records): - provider = TransipProvider('', '', ''); + provider = TransipProvider('', '', '') _dns_entries = [] for record in records: if record._type in provider.SUPPORTS: - entries_for = getattr(provider, '_entries_for_{}'.format(record._type)) + entries_for = getattr(provider, + '_entries_for_{}'.format(record._type)) # Root records have '@' as name name = record.name @@ -45,50 +56,48 @@ class MockDomainService(DomainService): _dns_entries.extend(entries_for(name, record)) - _dns_entries.append(DnsEntry('@', '3600', 'NS', 'ns01.transip.nl.')) - + # NS is not supported as a DNS Entry, + # so it should cover the if statement + _dns_entries.append( + DnsEntry('@', '3600', 'NS', 'ns01.transip.nl.')) self.mockupEntries = _dns_entries # Skips authentication layer and returns the entries loaded by "Mockup" def get_info(self, domain_name): + # Special 'domain' to trigger error if str(domain_name) == str('notfound.unit.tests'): self.raiseZoneNotFound() - result = lambda: None - setattr(result, "dnsEntries", self.mockupEntries) + result = MockResponse() + result.dnsEntries = self.mockupEntries return result def set_dns_entries(self, domain_name, dns_entries): + + # Special 'domain' to trigger error if str(domain_name) == str('failsetdns.unit.tests'): self.raiseSaveError() return True def raiseZoneNotFound(self): - fault = lambda: None - setattr(fault, "faultstring", '102 is zone not found') - setattr(fault, "faultcode", str('102')) + fault = MockFault(str('102'), '102 is zone not found') document = {} raise WebFault(fault, document) def raiseInvalidAuth(self): - fault = lambda: None - setattr(fault, "faultstring", '200 is invalid auth') - setattr(fault, "faultcode", str('200')) + fault = MockFault(str('200'), '200 is invalid auth') document = {} raise WebFault(fault, document) def raiseSaveError(self): - fault = lambda: None - setattr(fault, "faultstring", '202 error while saving') - setattr(fault, "faultcode", str('202')) + fault = MockFault(str('200'), '202 random error') document = {} raise WebFault(fault, document) - class TestTransipProvider(TestCase): bogus_key = str("""-----BEGIN RSA PRIVATE KEY----- MIIEowIBAAKCAQEA0U5HGCkLrz423IyUf3u4cKN2WrNz1x5KNr6PvH2M/zxas+zB @@ -118,7 +127,6 @@ fH2BQyTKKzoCLB1k/6HRaMnZdrWyWSZ7JKz3AHJ8+58d0Hr8LTrzDM1L6BbjeDct N4OiVz1I3rbZGYa396lpxO6ku8yCglisL1yrSP6DdEUp66ntpKVd -----END RSA PRIVATE KEY-----""") - def make_expected(self): expected = Zone('unit.tests.', []) source = YamlProvider('test', join(dirname(__file__), 'config')) @@ -126,9 +134,10 @@ N4OiVz1I3rbZGYa396lpxO6ku8yCglisL1yrSP6DdEUp66ntpKVd return expected def test_populate(self): - _expected = self.make_expected() + # Unhappy Plan - Not authenticated + # Live test against API, will fail in an unauthorized error with self.assertRaises(WebFault) as ctx: provider = TransipProvider('test', 'unittest', self.bogus_key) zone = Zone('unit.tests.', []) @@ -139,6 +148,9 @@ N4OiVz1I3rbZGYa396lpxO6ku8yCglisL1yrSP6DdEUp66ntpKVd self.assertEquals(str('200'), ctx.exception.fault.faultcode) + # Unhappy Plan - Zone does not exists + # Will trigger an exception if provider is used as a target for a + # non-existing zone with self.assertRaises(Exception) as ctx: provider = TransipProvider('test', 'unittest', self.bogus_key) provider._client = MockDomainService('unittest', self.bogus_key) @@ -148,38 +160,48 @@ N4OiVz1I3rbZGYa396lpxO6ku8yCglisL1yrSP6DdEUp66ntpKVd self.assertEquals(str('Exception'), str(ctx.exception.__class__.__name__)) - self.assertEquals('populate: (102) Transip used as target for non-existing zone: notfound.unit.tests.', ctx.exception.message) + self.assertEquals( + 'populate: (102) Transip used as target' + + ' for non-existing zone: notfound.unit.tests.', + ctx.exception.message) + # Happy Plan - Zone does not exists + # Won't trigger an exception if provider is NOT used as a target for a + # non-existing zone. provider = TransipProvider('test', 'unittest', self.bogus_key) provider._client = MockDomainService('unittest', self.bogus_key) zone = Zone('notfound.unit.tests.', []) provider.populate(zone, False) + # Happy Plan - Populate with mockup records provider = TransipProvider('test', 'unittest', self.bogus_key) provider._client = MockDomainService('unittest', self.bogus_key) provider._client.mockup(_expected.records) zone = Zone('unit.tests.', []) provider.populate(zone, False) + # Transip allows relative values for types like cname, mx. + # Test is these are correctly appended with the domain provider._currentZone = zone self.assertEquals("www.unit.tests.", provider._parse_to_fqdn("www")) + self.assertEquals("www.unit.tests.", + provider._parse_to_fqdn("www.unit.tests.")) + self.assertEquals("www.sub.sub.sub.unit.tests.", + provider._parse_to_fqdn("www.sub.sub.sub")) - + # Happy Plan - Even if the zone has no records the zone should exist provider = TransipProvider('test', 'unittest', self.bogus_key) provider._client = MockDomainService('unittest', self.bogus_key) zone = Zone('unit.tests.', []) exists = provider.populate(zone, True) self.assertTrue(exists, 'populate should return true') - return def test_plan(self): - _expected = self.make_expected() - print(_expected.name) - + # Test Happy plan, only create provider = TransipProvider('test', 'unittest', self.bogus_key) provider._client = MockDomainService('unittest', self.bogus_key) plan = provider.plan(_expected) @@ -191,29 +213,38 @@ N4OiVz1I3rbZGYa396lpxO6ku8yCglisL1yrSP6DdEUp66ntpKVd return def test_apply(self): - _expected = self.make_expected() + # Test happy flow. Create all supoorted records provider = TransipProvider('test', 'unittest', self.bogus_key) provider._client = MockDomainService('unittest', self.bogus_key) plan = provider.plan(_expected) - #self.assertEqual(11, plan.changes) + self.assertEqual(12, len(plan.changes)) changes = provider.apply(plan) + self.assertEqual(changes, len(plan.changes)) - - + # Test unhappy flow. Trigger 'not found error' in apply stage + # This should normally not happen as populate will capture it first + # but just in case. + changes = [] # reset changes with self.assertRaises(Exception) as ctx: provider = TransipProvider('test', 'unittest', self.bogus_key) provider._client = MockDomainService('unittest', self.bogus_key) plan = provider.plan(_expected) plan.desired.name = 'notfound.unit.tests.' changes = provider.apply(plan) - # self.assertEqual(11, changes) + + # Changes should not be set due to an Exception + self.assertEqual([], changes) self.assertEquals(str('WebFault'), str(ctx.exception.__class__.__name__)) - _expected = self.make_expected() + self.assertEquals(str('102'), ctx.exception.fault.faultcode) + + # Test unhappy flow. Trigger a unrecoverable error while saving + _expected = self.make_expected() # reset expected + changes = [] # reset changes with self.assertRaises(Exception) as ctx: provider = TransipProvider('test', 'unittest', self.bogus_key) @@ -221,14 +252,9 @@ N4OiVz1I3rbZGYa396lpxO6ku8yCglisL1yrSP6DdEUp66ntpKVd plan = provider.plan(_expected) plan.desired.name = 'failsetdns.unit.tests.' changes = provider.apply(plan) - # self.assertEqual(11, changes) - - - #provider = TransipProvider('test', 'unittest', self.bogus_key) - - #plan = provider.plan(_expected) - -# changes = provider.apply(plan) -# self.assertEquals(29, changes) + # Changes should not be set due to an Exception + self.assertEqual([], changes) + self.assertEquals(str('Exception'), + str(ctx.exception.__class__.__name__)) From 59e44b865cfa5e7d3a1f40d34f41d5b2dd285aac Mon Sep 17 00:00:00 2001 From: Maikel Poot Date: Wed, 25 Sep 2019 11:24:13 +0200 Subject: [PATCH 015/155] Added detection for edge case that could happen with existing records where the value is '@' TransIP allows '@' as value to alias the root record. '@' was on populate appended with the zone, which trigger an unneeded update. '@' => '@.example.com.' -> 'example.com' This fix will stop the unneeded update --- octodns/provider/transip.py | 7 ++++++- tests/test_octodns_provider_transip.py | 2 ++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/octodns/provider/transip.py b/octodns/provider/transip.py index 8014310..adde617 100644 --- a/octodns/provider/transip.py +++ b/octodns/provider/transip.py @@ -218,7 +218,12 @@ class TransipProvider(BaseProvider): def _parse_to_fqdn(self, value): - if (value[-1] != '.'): + # TransIP allows '@' as value to alias the root record. + # this provider won't set an '@' value, but can be an existing record + if value == self.ROOT_RECORD: + value = self._currentZone.name + + if value[-1] != '.': self.log.debug('parseToFQDN: changed %s to %s', value, '{}.{}'.format(value, self._currentZone.name)) value = '{}.{}'.format(value, self._currentZone.name) diff --git a/tests/test_octodns_provider_transip.py b/tests/test_octodns_provider_transip.py index 811c1e2..d6bcaa7 100644 --- a/tests/test_octodns_provider_transip.py +++ b/tests/test_octodns_provider_transip.py @@ -188,6 +188,8 @@ N4OiVz1I3rbZGYa396lpxO6ku8yCglisL1yrSP6DdEUp66ntpKVd provider._parse_to_fqdn("www.unit.tests.")) self.assertEquals("www.sub.sub.sub.unit.tests.", provider._parse_to_fqdn("www.sub.sub.sub")) + self.assertEquals("unit.tests.", + provider._parse_to_fqdn("@")) # Happy Plan - Even if the zone has no records the zone should exist provider = TransipProvider('test', 'unittest', self.bogus_key) From cebc629a06295a9cae3c5821b448e11737eddbbe Mon Sep 17 00:00:00 2001 From: Maikel Poot Date: Wed, 25 Sep 2019 14:33:49 +0200 Subject: [PATCH 016/155] Enforce values as basic string to fix yaml export error Fixes an exception in combination with the yamlProvider as a target The unmodified value object isn't represented as string while building the yaml output The Exception: yaml.representer.RepresenterError: ('cannot represent an object', 1.1.1.1) yaml/representer.py@249, represent_undefined() --- octodns/provider/transip.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/octodns/provider/transip.py b/octodns/provider/transip.py index adde617..050398a 100644 --- a/octodns/provider/transip.py +++ b/octodns/provider/transip.py @@ -218,6 +218,9 @@ class TransipProvider(BaseProvider): def _parse_to_fqdn(self, value): + # Enforce switch from suds.sax.text.Text to string + value = ''+value + # TransIP allows '@' as value to alias the root record. # this provider won't set an '@' value, but can be an existing record if value == self.ROOT_RECORD: @@ -240,7 +243,8 @@ class TransipProvider(BaseProvider): _values = [] for record in records: - _values.append(record['content']) + # Enforce switch from suds.sax.text.Text to string + _values.append(''+record['content']) return { 'ttl': self._get_lowest_ttl(records), From 9cab94a83a4621334880059b8007fe7bdd2bd8ba Mon Sep 17 00:00:00 2001 From: Maikel Poot Date: Wed, 25 Sep 2019 14:45:41 +0200 Subject: [PATCH 017/155] Some codestyle review changes. --- octodns/provider/transip.py | 37 +++++++++++++++---------------------- 1 file changed, 15 insertions(+), 22 deletions(-) diff --git a/octodns/provider/transip.py b/octodns/provider/transip.py index 050398a..aa5b5b3 100644 --- a/octodns/provider/transip.py +++ b/octodns/provider/transip.py @@ -52,9 +52,6 @@ class TransipProvider(BaseProvider): self.account = account self.key = key - self._zones = None - self._zone_records = {} - self._currentZone = {} def populate(self, zone, target=False, lenient=False): @@ -72,7 +69,6 @@ class TransipProvider(BaseProvider): self.log.warning( 'populate: (%s) Zone %s not found in account ', e.fault.faultcode, zone.name) - exists = False return exists elif e.fault.faultcode == '102' and target is True: self.log.warning('populate: Transip can\'t create new zones') @@ -115,9 +111,6 @@ class TransipProvider(BaseProvider): changes = plan.changes self.log.debug('apply: zone=%s, changes=%d', desired.name, len(changes)) - # for change in changes: - # class_name = change.__class__.__name__ - # getattr(self, '_apply_{}'.format(class_name))(change) self._currentZone = plan.desired try: @@ -265,24 +258,24 @@ class TransipProvider(BaseProvider): } def _data_for_MX(self, _type, records): - values = [] + _values = [] for record in records: preference, exchange = record['content'].split(" ", 1) - values.append({ + _values.append({ 'preference': preference, 'exchange': self._parse_to_fqdn(exchange) }) return { 'ttl': self._get_lowest_ttl(records), 'type': _type, - 'values': values + 'values': _values } def _data_for_SRV(self, _type, records): - values = [] + _values = [] for record in records: priority, weight, port, target = record['content'].split(' ', 3) - values.append({ + _values.append({ 'port': port, 'priority': priority, 'target': self._parse_to_fqdn(target), @@ -292,14 +285,14 @@ class TransipProvider(BaseProvider): return { 'type': _type, 'ttl': self._get_lowest_ttl(records), - 'values': values + 'values': _values } def _data_for_SSHFP(self, _type, records): - values = [] + _values = [] for record in records: algorithm, fp_type, fingerprint = record['content'].split(' ', 2) - values.append({ + _values.append({ 'algorithm': algorithm, 'fingerprint': fingerprint.lower(), 'fingerprint_type': fp_type @@ -308,14 +301,14 @@ class TransipProvider(BaseProvider): return { 'type': _type, 'ttl': self._get_lowest_ttl(records), - 'values': values + 'values': _values } def _data_for_CAA(self, _type, records): - values = [] + _values = [] for record in records: flags, tag, value = record['content'].split(' ', 2) - values.append({ + _values.append({ 'flags': flags, 'tag': tag, 'value': value @@ -324,16 +317,16 @@ class TransipProvider(BaseProvider): return { 'type': _type, 'ttl': self._get_lowest_ttl(records), - 'values': values + 'values': _values } def _data_for_TXT(self, _type, records): - values = [] + _values = [] for record in records: - values.append(record['content'].replace(';', '\\;')) + _values.append(record['content'].replace(';', '\\;')) return { 'type': _type, 'ttl': self._get_lowest_ttl(records), - 'values': values + 'values': _values } From 71f215932d94b323cddba6ed6e0b8f35ba3b9a4d Mon Sep 17 00:00:00 2001 From: Maikel Poot Date: Wed, 25 Sep 2019 14:51:53 +0200 Subject: [PATCH 018/155] whitespaces around operators to make /script/lint happy again --- octodns/provider/transip.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/octodns/provider/transip.py b/octodns/provider/transip.py index aa5b5b3..64692ee 100644 --- a/octodns/provider/transip.py +++ b/octodns/provider/transip.py @@ -212,7 +212,7 @@ class TransipProvider(BaseProvider): def _parse_to_fqdn(self, value): # Enforce switch from suds.sax.text.Text to string - value = ''+value + value = '' + value # TransIP allows '@' as value to alias the root record. # this provider won't set an '@' value, but can be an existing record @@ -237,7 +237,7 @@ class TransipProvider(BaseProvider): _values = [] for record in records: # Enforce switch from suds.sax.text.Text to string - _values.append(''+record['content']) + _values.append('' + record['content']) return { 'ttl': self._get_lowest_ttl(records), From a035ee8c84955d2dcdf40c0ea15f1901224ae000 Mon Sep 17 00:00:00 2001 From: Maikel Poot Date: Thu, 26 Sep 2019 14:49:14 +0200 Subject: [PATCH 019/155] Give the option to use a private_key_file. Transip sdk also supports a private_key_file, so forwarding that option to the provider. Could be handy in combination with k8s secrets. --- octodns/provider/transip.py | 14 +++++++++++--- tests/test_octodns_provider_transip.py | 13 +++++++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/octodns/provider/transip.py b/octodns/provider/transip.py index 64692ee..92d607d 100644 --- a/octodns/provider/transip.py +++ b/octodns/provider/transip.py @@ -23,13 +23,16 @@ class TransipProvider(BaseProvider): class: octodns.provider.transip.TransipProvider # Your Transip account name (required) account: yourname - # The api key (required) + # Path to a private key file (required if key is not used) + key_file: /path/to/file + # The api key as string (required if key_file is not used) key: | \''' -----BEGIN PRIVATE KEY----- ... -----END PRIVATE KEY----- \''' + # if both `key_file` and `key` are presented `key_file` is used ''' SUPPORTS_GEO = False @@ -41,13 +44,18 @@ class TransipProvider(BaseProvider): TIMEOUT = 15 ROOT_RECORD = '@' - def __init__(self, id, account, key, *args, **kwargs): + def __init__(self, id, account, key=None, key_file=None, *args, **kwargs): self.log = getLogger('TransipProvider[{}]'.format(id)) self.log.debug('__init__: id=%s, account=%s, token=***', id, account) super(TransipProvider, self).__init__(id, *args, **kwargs) - self._client = DomainService(account, key) + if key_file is not None: + self._client = DomainService(account, private_key_file=key_file) + elif key is not None: + self._client = DomainService(account, private_key=key) + else: + raise Exception('Missing `key` of `key_file` parameter in config') self.account = account self.key = key diff --git a/tests/test_octodns_provider_transip.py b/tests/test_octodns_provider_transip.py index d6bcaa7..8d85e1f 100644 --- a/tests/test_octodns_provider_transip.py +++ b/tests/test_octodns_provider_transip.py @@ -133,6 +133,19 @@ N4OiVz1I3rbZGYa396lpxO6ku8yCglisL1yrSP6DdEUp66ntpKVd source.populate(expected) return expected + def test_init(self): + with self.assertRaises(Exception) as ctx: + TransipProvider('test', 'unittest') + + self.assertEquals( + str('Missing `key` of `key_file` parameter in config'), + str(ctx.exception)) + + TransipProvider('test', 'unittest', key=self.bogus_key) + + # Existence and content of the key is tested in the SDK on client call + TransipProvider('test', 'unittest', key_file='/fake/path') + def test_populate(self): _expected = self.make_expected() From 637c2547782b15e21473f51960eb1ab4b3a94c1d Mon Sep 17 00:00:00 2001 From: Maikel Poot Date: Mon, 30 Sep 2019 13:18:57 +0200 Subject: [PATCH 020/155] Handling PR Review comments. - Added Specific exceptions - str() instead of concatenation - removed zone not found warning --- octodns/provider/transip.py | 29 +++++++++++++++++++------- tests/test_octodns_provider_transip.py | 4 ++-- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/octodns/provider/transip.py b/octodns/provider/transip.py index 92d607d..09920a9 100644 --- a/octodns/provider/transip.py +++ b/octodns/provider/transip.py @@ -15,6 +15,18 @@ from transip.service.domain import DomainService from transip.service.objects import DnsEntry +class TransipException(Exception): + pass + + +class TransipConfigException(TransipException): + pass + + +class TransipNewZoneException(TransipException): + pass + + class TransipProvider(BaseProvider): ''' Transip DNS provider @@ -55,7 +67,9 @@ class TransipProvider(BaseProvider): elif key is not None: self._client = DomainService(account, private_key=key) else: - raise Exception('Missing `key` of `key_file` parameter in config') + raise TransipConfigException( + 'Missing `key` of `key_file` parameter in config' + ) self.account = account self.key = key @@ -74,13 +88,12 @@ class TransipProvider(BaseProvider): zoneInfo = self._client.get_info(zone.name[:-1]) except WebFault as e: if e.fault.faultcode == '102' and target is False: - self.log.warning( - 'populate: (%s) Zone %s not found in account ', - e.fault.faultcode, zone.name) + # Zone not found in account, and not a target so just + # leave an empty zone. return exists elif e.fault.faultcode == '102' and target is True: self.log.warning('populate: Transip can\'t create new zones') - raise Exception( + raise TransipNewZoneException( ('populate: ({}) Transip used ' + 'as target for non-existing zone: {}').format( e.fault.faultcode, zone.name)) @@ -146,7 +159,7 @@ class TransipProvider(BaseProvider): self.log.warning(('_apply: Set DNS returned ' + 'one or more errors: {}').format( e.fault.faultstring)) - raise Exception(200, e.fault.faultstring) + raise TransipException(200, e.fault.faultstring) self._currentZone = {} @@ -220,7 +233,7 @@ class TransipProvider(BaseProvider): def _parse_to_fqdn(self, value): # Enforce switch from suds.sax.text.Text to string - value = '' + value + value = str(value) # TransIP allows '@' as value to alias the root record. # this provider won't set an '@' value, but can be an existing record @@ -245,7 +258,7 @@ class TransipProvider(BaseProvider): _values = [] for record in records: # Enforce switch from suds.sax.text.Text to string - _values.append('' + record['content']) + _values.append(str(record['content'])) return { 'ttl': self._get_lowest_ttl(records), diff --git a/tests/test_octodns_provider_transip.py b/tests/test_octodns_provider_transip.py index 8d85e1f..c56509a 100644 --- a/tests/test_octodns_provider_transip.py +++ b/tests/test_octodns_provider_transip.py @@ -170,7 +170,7 @@ N4OiVz1I3rbZGYa396lpxO6ku8yCglisL1yrSP6DdEUp66ntpKVd zone = Zone('notfound.unit.tests.', []) provider.populate(zone, True) - self.assertEquals(str('Exception'), + self.assertEquals(str('TransipNewZoneException'), str(ctx.exception.__class__.__name__)) self.assertEquals( @@ -271,5 +271,5 @@ N4OiVz1I3rbZGYa396lpxO6ku8yCglisL1yrSP6DdEUp66ntpKVd # Changes should not be set due to an Exception self.assertEqual([], changes) - self.assertEquals(str('Exception'), + self.assertEquals(str('TransipException'), str(ctx.exception.__class__.__name__)) From 97608b382399b798f7b865f082b943685b3e2fd9 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 30 Sep 2019 09:29:30 -0700 Subject: [PATCH 021/155] v0.9.7 version bump and CHANGELOG updates --- .gitignore | 4 ++++ CHANGELOG.md | 8 ++++++++ octodns/__init__.py | 2 +- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 1efa084..715b687 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,7 @@ +# +# Do not add editor or OS specific ignores here. Have a look at adding +# `excludesfile` to your `~/.gitconfig` to globally ignore such things. +# *.pyc .coverage .env diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f283ec..d80a48f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +## v0.9.7 - 2019-09-30 - It's about time + +* AkamaiProvider, ConstellixProvider, MythicBeastsProvider, SelectelProvider, + & TransipPovider providers added +* Route53Provider seperator fix +* YamlProvider export error around stringification +* PyPi markdown rendering fix + ## v0.9.6 - 2019-07-16 - The little one that fixes stuff from the big one * Reduced dynamic record value weight range to 0-15 so that Dyn and Route53 diff --git a/octodns/__init__.py b/octodns/__init__.py index 6422577..57300de 100644 --- a/octodns/__init__.py +++ b/octodns/__init__.py @@ -3,4 +3,4 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -__VERSION__ = '0.9.6' +__VERSION__ = '0.9.7' From a1d2217604fad7ec8c3fccc73f7d6e892071eeb3 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 30 Sep 2019 10:17:50 -0700 Subject: [PATCH 022/155] Fix/hack README rendering so that pypi's markdown handling libs are happy --- README.md | 4 ++-- setup.py | 36 +++++++++++++++++++++++++++++++++++- 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 7124b20..83f0bd1 100644 --- a/README.md +++ b/README.md @@ -90,8 +90,8 @@ Now that we have something to tell OctoDNS about our providers & zones we need t ttl: 60 type: A values: - - 1.2.3.4 - - 1.2.3.5 + - 1.2.3.4 + - 1.2.3.5 ``` Further information can be found in [Records Documentation](/docs/records.md). diff --git a/setup.py b/setup.py index 75a39d7..5cb741b 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,6 @@ #!/usr/bin/env python +from StringIO import StringIO from os.path import dirname, join import octodns @@ -21,6 +22,39 @@ console_scripts = { for name in cmds } + +def long_description(): + buf = StringIO() + yaml_block = False + supported_providers = False + with open('README.md') as fh: + for line in fh: + if line == '```yaml\n': + yaml_block = True + continue + elif yaml_block and line == '---\n': + # skip the line + continue + elif yaml_block and line == '```\n': + yaml_block = False + continue + elif supported_providers: + if line.startswith('## '): + supported_providers = False + # write this line out, no continue + else: + # We're ignoring this one + continue + elif line == '## Supported providers\n': + supported_providers = True + continue + buf.write(line) + buf = buf.getvalue() + with open('/tmp/mod', 'w') as fh: + fh.write(buf) + return buf + + setup( author='Ross McFarland', author_email='rwmcfa1@gmail.com', @@ -40,7 +74,7 @@ setup( 'requests>=2.20.0' ], license='MIT', - long_description=open('README.md').read(), + long_description=long_description(), long_description_content_type='text/markdown', name='octodns', packages=find_packages(), From e3ad57d15b055db9a441d0a5d7a7b924ff8600d5 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 30 Sep 2019 10:18:35 -0700 Subject: [PATCH 023/155] twine check before upload --- script/release | 1 + 1 file changed, 1 insertion(+) diff --git a/script/release b/script/release index dd3e1b1..f2c90bf 100755 --- a/script/release +++ b/script/release @@ -22,5 +22,6 @@ git tag -s "v$VERSION" -m "Release $VERSION" git push origin "v$VERSION" echo "Tagged and pushed v$VERSION" python setup.py sdist +twine check dist/*$VERSION.tar.gz twine upload dist/*$VERSION.tar.gz echo "Uploaded $VERSION" From 659a60de46ad0bb75811266691e57b44e58b46ff Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 30 Sep 2019 10:18:53 -0700 Subject: [PATCH 024/155] v0.9.8 version bump and CHANGELOG updates --- CHANGELOG.md | 4 ++++ octodns/__init__.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d80a48f..76ff8b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## v0.9.8 - 2019-09-30 - One with no changes b/c PyPi description problems + +* No material changes + ## v0.9.7 - 2019-09-30 - It's about time * AkamaiProvider, ConstellixProvider, MythicBeastsProvider, SelectelProvider, diff --git a/octodns/__init__.py b/octodns/__init__.py index 57300de..71b5b1a 100644 --- a/octodns/__init__.py +++ b/octodns/__init__.py @@ -3,4 +3,4 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -__VERSION__ = '0.9.7' +__VERSION__ = '0.9.8' From 3296b55aa62ccbe558dff22fd465d95cd0b7cfb1 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 30 Sep 2019 10:22:05 -0700 Subject: [PATCH 025/155] Include readme_renderer[md] in dev requirements --- requirements-dev.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements-dev.txt b/requirements-dev.txt index 77dd50c..a2833ae 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,5 +5,6 @@ pycodestyle==2.4.0 pycountry>=18.12.8 pycountry_convert>=0.7.2 pyflakes==1.6.0 +readme_renderer[md]==24.0 requests_mock twine==1.13.0 From db77d5b3bbd89adf76a80e37c7d5d65c8cc3056e Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 5 Oct 2019 07:58:38 -0700 Subject: [PATCH 026/155] python3 compat for azure provider --- octodns/provider/azuredns.py | 10 +++++++--- requirements.txt | 6 +++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 0bca46d..3d8122a 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -175,6 +175,10 @@ class _AzureRecord(object): :type return: bool ''' + + def key_dict(d): + return sum([hash('{}:{}'.format(k, v)) for k, v in d.items()]) + def parse_dict(params): vals = [] for char in params: @@ -185,7 +189,7 @@ class _AzureRecord(object): vals.append(record.__dict__) except: vals.append(list_records.__dict__) - vals.sort() + vals.sort(key=key_dict) return vals return (self.resource_group == b.resource_group) & \ @@ -373,13 +377,13 @@ class AzureProvider(BaseProvider): self._populate_zones() self._check_zone(zone_name) - _records = set() + _records = [] records = self._dns_client.record_sets.list_by_dns_zone if self._check_zone(zone_name): exists = True for azrecord in records(self._resource_group, zone_name): if _parse_azure_type(azrecord.type) in self.SUPPORTS: - _records.add(azrecord) + _records.append(azrecord) for azrecord in _records: record_name = azrecord.name if azrecord.name != '@' else '' typ = _parse_azure_type(azrecord.type) diff --git a/requirements.txt b/requirements.txt index 765b65c..19d45a8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ PyYaml==4.2b1 -azure-common==1.1.18 -azure-mgmt-dns==2.1.0 +azure-common==1.1.23 +azure-mgmt-dns==3.0.0 boto3==1.7.5 botocore==1.10.5 dnspython==1.15.0 @@ -13,7 +13,7 @@ google-cloud-dns==0.29.0 incf.countryutils==1.0 ipaddress==1.0.22 jmespath==0.9.3 -msrestazure==0.6.0 +msrestazure==0.6.2 natsort==5.5.0 nsone==0.9.100 ovh==0.4.8 From 470dd822026655eba24d213ab9d54946b1a7f936 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 5 Oct 2019 08:03:43 -0700 Subject: [PATCH 027/155] python 3 support for constellix provider --- octodns/provider/constellix.py | 3 ++- tests/test_octodns_provider_constellix.py | 10 ++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/octodns/provider/constellix.py b/octodns/provider/constellix.py index 939284d..050f120 100644 --- a/octodns/provider/constellix.py +++ b/octodns/provider/constellix.py @@ -9,6 +9,7 @@ from collections import defaultdict from requests import Session from base64 import b64encode from ipaddress import ip_address +from six import string_types import hashlib import hmac import logging @@ -122,7 +123,7 @@ class ConstellixClient(object): # change relative values to absolute value = record['value'] if record['type'] in ['ALIAS', 'CNAME', 'MX', 'NS', 'SRV']: - if isinstance(value, unicode): + if isinstance(value, string_types): record['value'] = self._absolutize_value(value, zone_name) if isinstance(value, list): diff --git a/tests/test_octodns_provider_constellix.py b/tests/test_octodns_provider_constellix.py index 346bb17..7914c53 100644 --- a/tests/test_octodns_provider_constellix.py +++ b/tests/test_octodns_provider_constellix.py @@ -10,6 +10,7 @@ from mock import Mock, call from os.path import dirname, join from requests import HTTPError from requests_mock import ANY, mock as requests_mock +from six import text_type from unittest import TestCase from octodns.record import Record @@ -65,7 +66,7 @@ class TestConstellixProvider(TestCase): with self.assertRaises(Exception) as ctx: zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals('Unauthorized', ctx.exception.message) + self.assertEquals('Unauthorized', text_type(ctx.exception)) # Bad request with requests_mock() as mock: @@ -77,7 +78,7 @@ class TestConstellixProvider(TestCase): zone = Zone('unit.tests.', []) provider.populate(zone) self.assertEquals('\n - "unittests" is not a valid domain name', - ctx.exception.message) + text_type(ctx.exception)) # General error with requests_mock() as mock: @@ -148,6 +149,11 @@ class TestConstellixProvider(TestCase): call('POST', '/', data={'names': ['unit.tests']}), # get all domains to build the cache call('GET', '/'), + ]) + # These two checks are broken up so that ordering doesn't break things. + # Python3 doesn't make the calls in a consistent order so different + # things follow the GET / on different runs + provider._client._request.assert_has_calls([ call('POST', '/123123/records/SRV', data={ 'roundRobin': [{ 'priority': 10, From 742305c20b62b97d354dc0336b0f87dca4a1de1d Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 5 Oct 2019 08:47:12 -0700 Subject: [PATCH 028/155] six.moves.urllib.parse --- octodns/provider/fastdns.py | 2 +- tests/test_octodns_provider_rackspace.py | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/octodns/provider/fastdns.py b/octodns/provider/fastdns.py index f851303..8f651f0 100644 --- a/octodns/provider/fastdns.py +++ b/octodns/provider/fastdns.py @@ -7,7 +7,7 @@ from __future__ import absolute_import, division, print_function, \ from requests import Session from akamai.edgegrid import EdgeGridAuth -from urlparse import urljoin +from six.moves.urllib.parse import urljoin from collections import defaultdict from logging import getLogger diff --git a/tests/test_octodns_provider_rackspace.py b/tests/test_octodns_provider_rackspace.py index b257166..b0dcad7 100644 --- a/tests/test_octodns_provider_rackspace.py +++ b/tests/test_octodns_provider_rackspace.py @@ -8,11 +8,8 @@ from __future__ import absolute_import, division, print_function, \ import json import re from six import text_type +from six.moves.urllib.parse import urlparse from unittest import TestCase -try: - from urlparse import urlparse -except ImportError: - from urllib.parse import urlparse from requests import HTTPError from requests_mock import ANY, mock as requests_mock From 47199fdfab404dc762d04c9733ccdd9dbb342933 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 5 Oct 2019 08:49:19 -0700 Subject: [PATCH 029/155] FastDNS python3 --- tests/test_octodns_provider_fastdns.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_octodns_provider_fastdns.py b/tests/test_octodns_provider_fastdns.py index 5f503c7..a8bed74 100644 --- a/tests/test_octodns_provider_fastdns.py +++ b/tests/test_octodns_provider_fastdns.py @@ -9,6 +9,7 @@ from __future__ import absolute_import, division, print_function, \ from os.path import dirname, join from requests import HTTPError from requests_mock import ANY, mock as requests_mock +from six import text_type from unittest import TestCase from octodns.record import Record @@ -147,4 +148,4 @@ class TestFastdnsProvider(TestCase): changes = provider.apply(plan) except NameError as e: expected = "contractId not specified to create zone" - self.assertEquals(e.message, expected) + self.assertEquals(text_type(e), expected) From 00781985008b2e1b122160e568b2818deb960cb8 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 5 Oct 2019 08:57:14 -0700 Subject: [PATCH 030/155] GoogleCloud python3 --- tests/test_octodns_provider_googlecloud.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_octodns_provider_googlecloud.py b/tests/test_octodns_provider_googlecloud.py index d7f0e0c..e642668 100644 --- a/tests/test_octodns_provider_googlecloud.py +++ b/tests/test_octodns_provider_googlecloud.py @@ -193,9 +193,14 @@ class DummyIterator: def __iter__(self): return self + # python2 def next(self): return next(self.iterable) + # python3 + def __next__(self): + return next(self.iterable) + class TestGoogleCloudProvider(TestCase): @patch('octodns.provider.googlecloud.dns') @@ -247,7 +252,7 @@ class TestGoogleCloudProvider(TestCase): return_values_for_status = iter( ["pending"] * 11 + ['done', 'done']) type(status_mock).status = PropertyMock( - side_effect=return_values_for_status.next) + side_effect=lambda: next(return_values_for_status)) gcloud_zone_mock.changes = Mock(return_value=status_mock) provider = self._get_provider() From 484a5118f438f8e00ce29027e6b2af167aae1eb5 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 5 Oct 2019 09:10:59 -0700 Subject: [PATCH 031/155] MythicBeastsProvider python3 --- octodns/provider/mythicbeasts.py | 2 +- tests/test_octodns_provider_mythicbeasts.py | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/octodns/provider/mythicbeasts.py b/octodns/provider/mythicbeasts.py index 17029db..b255a74 100644 --- a/octodns/provider/mythicbeasts.py +++ b/octodns/provider/mythicbeasts.py @@ -328,7 +328,7 @@ class MythicBeastsProvider(BaseProvider): exists = True for line in resp.content.splitlines(): - match = MythicBeastsProvider.RE_POPLINE.match(line) + match = MythicBeastsProvider.RE_POPLINE.match(line.decode("utf-8")) if match is None: self.log.debug('failed to match line: %s', line) diff --git a/tests/test_octodns_provider_mythicbeasts.py b/tests/test_octodns_provider_mythicbeasts.py index b93d46e..498b408 100644 --- a/tests/test_octodns_provider_mythicbeasts.py +++ b/tests/test_octodns_provider_mythicbeasts.py @@ -441,11 +441,11 @@ class TestMythicBeastsProvider(TestCase): plan = provider.plan(wanted) # Octo ignores NS records (15-1) - self.assertEquals(1, len(filter(lambda u: isinstance(u, Update), - plan.changes))) - self.assertEquals(1, len(filter(lambda d: isinstance(d, Delete), - plan.changes))) - self.assertEquals(14, len(filter(lambda c: isinstance(c, Create), - plan.changes))) + self.assertEquals(1, len(list(filter( + lambda u: isinstance(u, Update), plan.changes)))) + self.assertEquals(1, len(list(filter( + lambda d: isinstance(d, Delete), plan.changes)))) + self.assertEquals(14, len(list(filter( + lambda c: isinstance(c, Create), plan.changes)))) self.assertEquals(16, provider.apply(plan)) self.assertTrue(plan.exists) From 37543e6a762f94687a219134c6c9f75b900c93a8 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 5 Oct 2019 09:18:42 -0700 Subject: [PATCH 032/155] OvhProvider python3 --- octodns/provider/ovh.py | 2 +- tests/test_octodns_provider_ovh.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/octodns/provider/ovh.py b/octodns/provider/ovh.py index 0187098..a0e47f8 100644 --- a/octodns/provider/ovh.py +++ b/octodns/provider/ovh.py @@ -345,7 +345,7 @@ class OvhProvider(BaseProvider): @staticmethod def _is_valid_dkim_key(key): try: - base64.decodestring(key) + base64.decodestring(bytearray(key, 'utf-8')) except binascii.Error: return False return True diff --git a/tests/test_octodns_provider_ovh.py b/tests/test_octodns_provider_ovh.py index d3f468d..1f41abf 100644 --- a/tests/test_octodns_provider_ovh.py +++ b/tests/test_octodns_provider_ovh.py @@ -428,7 +428,7 @@ class TestOvhProvider(TestCase): ), call(u'/domain/zone/unit.tests/refresh')] - post_mock.assert_has_calls(wanted_calls) + post_mock.assert_has_calls(wanted_calls, any_order=True) # Get for delete calls wanted_get_calls = [ @@ -440,7 +440,7 @@ class TestOvhProvider(TestCase): subDomain=u''), call(u'/domain/zone/unit.tests/record', fieldType=u'A', subDomain='fake')] - get_mock.assert_has_calls(wanted_get_calls) + get_mock.assert_has_calls(wanted_get_calls, any_order=True) # 4 delete calls for update and delete delete_mock.assert_has_calls( [call(u'/domain/zone/unit.tests/record/100'), From 0acff67faa4c38ca36a13f164ed6903c504a64fc Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 5 Oct 2019 14:38:58 -0700 Subject: [PATCH 033/155] Ns1Provider python3 --- octodns/provider/ns1.py | 5 ++--- tests/test_octodns_provider_ns1.py | 13 +++++++++---- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index d3faf21..b4fd850 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -10,7 +10,7 @@ from itertools import chain from collections import OrderedDict, defaultdict from nsone import NSONE from nsone.rest.errors import RateLimitException, ResourceException -from incf.countryutils import transformations +from pycountry_convert import country_alpha2_to_continent_code from time import sleep from six import text_type @@ -62,8 +62,7 @@ class Ns1Provider(BaseProvider): us_state = meta.get('us_state', []) ca_province = meta.get('ca_province', []) for cntry in country: - cn = transformations.cc_to_cn(cntry) - con = transformations.cn_to_ctca2(cn) + con = country_alpha2_to_continent_code(cntry) key = '{}-{}'.format(con, cntry) geo[key].extend(answer['answer']) for state in us_state: diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index 178ce53..91d1a3f 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -5,6 +5,7 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals +from collections import defaultdict from mock import Mock, call, patch from nsone.rest.errors import AuthException, RateLimitException, \ ResourceException @@ -373,8 +374,12 @@ class TestNs1Provider(TestCase): load_mock.side_effect = [nsone_zone, nsone_zone] plan = provider.plan(desired) self.assertEquals(3, len(plan.changes)) - self.assertIsInstance(plan.changes[0], Update) - self.assertIsInstance(plan.changes[2], Delete) + # Shouldn't rely on order so just count classes + classes = defaultdict(lambda: 0) + for change in plan.changes: + classes[change.__class__] += 1 + self.assertEquals(1, classes[Delete]) + self.assertEquals(2, classes[Update]) # ugh, we need a mock record that can be returned from loadRecord for # the update and delete targets, we can add our side effects to that to # trigger rate limit handling @@ -397,7 +402,7 @@ class TestNs1Provider(TestCase): call('unit.tests', u'A'), call('geo', u'A'), call('delete-me', u'A'), - ]) + ], any_order=True) mock_record.assert_has_calls([ call.update(answers=[{'answer': [u'1.2.3.4'], 'meta': {}}], filters=[], @@ -424,7 +429,7 @@ class TestNs1Provider(TestCase): ttl=34), call.delete(), call.delete() - ]) + ], any_order=True) def test_escaping(self): provider = Ns1Provider('test', 'api-key') From aeb70b24888cd448e8dcf073b391d7c649ab1b46 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 5 Oct 2019 20:01:53 -0700 Subject: [PATCH 034/155] Route53Provider python 3, rm incf.countryutils, lots of cmp removal and ordering fixes --- CHANGELOG.md | 6 + octodns/provider/route53.py | 106 ++++++--- requirements.txt | 1 - tests/test_octodns_provider_rackspace.py | 1 - tests/test_octodns_provider_route53.py | 268 +++++++++++++---------- 5 files changed, 241 insertions(+), 141 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 76ff8b0..e30d0df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## v0.9.9 - 2019-??-?? - Python 3.7 Support + +* Route53 _mod_keyer ordering wasn't complete/reliable and in python 3 this + resulted in randomness. This has been addressed and may result in value + reordering on next plan, no actual changes in behavior should occur. + ## v0.9.8 - 2019-09-30 - One with no changes b/c PyPi description problems * No material changes diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index 0bc7a99..6d2cfeb 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -8,8 +8,8 @@ from __future__ import absolute_import, division, print_function, \ from boto3 import client from botocore.config import Config from collections import defaultdict -from incf.countryutils.transformations import cca_to_ctca2 from ipaddress import AddressValueError, ip_address +from pycountry_convert import country_alpha2_to_continent_code from uuid import uuid4 import logging import re @@ -20,13 +20,6 @@ from ..record import Record, Update from ..record.geo import GeoCodes from .base import BaseProvider -# TODO: remove when Python 2.x is no longer supported -try: # pragma: no cover - cmp -except NameError: # pragma: no cover - def cmp(x, y): - return (x > y) - (x < y) - octal_re = re.compile(r'\\(\d\d\d)') @@ -155,7 +148,7 @@ class _Route53Record(object): } } - # NOTE: we're using __hash__ and __cmp__ methods that consider + # NOTE: we're using __hash__ and ordering methods that consider # _Route53Records equivalent if they have the same class, fqdn, and _type. # Values are ignored. This is useful when computing diffs/changes. @@ -163,17 +156,34 @@ class _Route53Record(object): 'sub-classes should never use this method' return '{}:{}'.format(self.fqdn, self._type).__hash__() - def __cmp__(self, other): - '''sub-classes should call up to this and return its value if non-zero. - When it's zero they should compute their own __cmp__''' - if self.__class__ != other.__class__: - return cmp(self.__class__, other.__class__) - elif self.fqdn != other.fqdn: - return cmp(self.fqdn, other.fqdn) - elif self._type != other._type: - return cmp(self._type, other._type) - # We're ignoring ttl, it's not an actual differentiator - return 0 + def __eq__(self, other): + '''Sub-classes should call up to this and return its value if true. + When it's false they should compute their own __eq__, same for other + ordering methods.''' + return self.__class__.__name__ == other.__class__.__name__ and \ + self.fqdn == other.fqdn and \ + self._type == other._type + + def __ne__(self, other): + return self.__class__.__name__ != other.__class__.__name__ or \ + self.fqdn != other.fqdn or \ + self._type != other._type + + def __lt__(self, other): + return (((self.__class__.__name__, self.fqdn, self._type)) < + ((other.__class__.__name__, other.fqdn, other._type))) + + def __le__(self, other): + return (((self.__class__.__name__, self.fqdn, self._type)) <= + ((other.__class__.__name__, other.fqdn, other._type))) + + def __gt__(self, other): + return (((self.__class__.__name__, self.fqdn, self._type)) > + ((other.__class__.__name__, other.fqdn, other._type))) + + def __ge__(self, other): + return (((self.__class__.__name__, self.fqdn, self._type)) >= + ((other.__class__.__name__, other.fqdn, other._type))) def __repr__(self): return '_Route53Record<{} {} {} {}>'.format(self.fqdn, self._type, @@ -514,11 +524,50 @@ class _Route53GeoRecord(_Route53Record): return '{}:{}:{}'.format(self.fqdn, self._type, self.geo.code).__hash__() - def __cmp__(self, other): - ret = super(_Route53GeoRecord, self).__cmp__(other) - if ret != 0: - return ret - return cmp(self.geo.code, other.geo.code) + def __eq__(self, other): + return super(_Route53GeoRecord, self).__eq__(other) and \ + self.geo.code == other.geo.code + + def __ne__(self, other): + # super will handle class != class, so if it's true we have 2 geo + # objects with the same name and type, so just need to compare codes + return super(_Route53GeoRecord, self).__ne__(other) or \ + self.geo.code != other.geo.code + + def __lt__(self, other): + # super eq will check class, name, and type + if super(_Route53GeoRecord, self).__eq__(other): + # if it's True we're dealing with two geo's with the same name and + # type, so we just need to compare codes + return self.geo.code < other.geo.code + # Super is not equal so we'll just let it decide lt + return super(_Route53GeoRecord, self).__lt__(other) + + def __le__(self, other): + # super eq will check class, name, and type + if super(_Route53GeoRecord, self).__eq__(other): + # Just need to compare codes, everything else is equal + return self.geo.code <= other.geo.code + # Super is not equal so geo.code doesn't matter, let it decide with lt, + # can't be eq + return super(_Route53GeoRecord, self).__lt__(other) + + def __gt__(self, other): + # super eq will check class, name, and type + if super(_Route53GeoRecord, self).__eq__(other): + # Just need to compare codes, everything else is equal + return self.geo.code > other.geo.code + # Super is not equal so we'll just let it decide gt + return super(_Route53GeoRecord, self).__gt__(other) + + def __ge__(self, other): + # super eq will check class, name, and type + if super(_Route53GeoRecord, self).__eq__(other): + # Just need to compare codes, everything else is equal + return self.geo.code >= other.geo.code + # Super is not equal so geo.code doesn't matter, let it decide with gt, + # can't be eq + return super(_Route53GeoRecord, self).__gt__(other) def __repr__(self): return '_Route53GeoRecord<{} {} {} {} {}>'.format(self.fqdn, @@ -561,7 +610,10 @@ def _mod_keyer(mod): if rrset.get('GeoLocation', False): unique_id = rrset['SetIdentifier'] else: - unique_id = rrset['Name'] + try: + unique_id = '{}-{}'.format(rrset['Name'], rrset['SetIdentifier']) + except KeyError: + unique_id = rrset['Name'] # Prioritise within the action_priority, ensuring targets come first. if rrset.get('GeoLocation', False): @@ -708,7 +760,7 @@ class Route53Provider(BaseProvider): if cc == '*': # This is the default return - cn = cca_to_ctca2(cc) + cn = country_alpha2_to_continent_code(cc) try: return '{}-{}-{}'.format(cn, cc, loc['SubdivisionCode']) except KeyError: diff --git a/requirements.txt b/requirements.txt index 19d45a8..6b6af09 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,6 @@ futures==3.2.0; python_version < '3.0' edgegrid-python==1.1.1 google-cloud-core==0.28.1 google-cloud-dns==0.29.0 -incf.countryutils==1.0 ipaddress==1.0.22 jmespath==0.9.3 msrestazure==0.6.2 diff --git a/tests/test_octodns_provider_rackspace.py b/tests/test_octodns_provider_rackspace.py index b0dcad7..3dfdd5f 100644 --- a/tests/test_octodns_provider_rackspace.py +++ b/tests/test_octodns_provider_rackspace.py @@ -40,7 +40,6 @@ with open('./tests/fixtures/rackspace-sample-recordset-page2.json') as fh: class TestRackspaceProvider(TestCase): def setUp(self): - self.maxDiff = 1000 with requests_mock() as mock: mock.post(ANY, status_code=200, text=AUTH_RESPONSE) self.provider = RackspaceProvider('identity', 'test', 'api-key', diff --git a/tests/test_octodns_provider_route53.py b/tests/test_octodns_provider_route53.py index 87dfd09..b0ee342 100644 --- a/tests/test_octodns_provider_route53.py +++ b/tests/test_octodns_provider_route53.py @@ -2091,6 +2091,58 @@ class TestRoute53Records(TestCase): e.__repr__() f.__repr__() + def test_route53_record_ordering(self): + # Matches + a = _Route53Record(None, self.record_a, False) + b = _Route53Record(None, self.record_a, False) + self.assertTrue(a == b) + self.assertFalse(a != b) + self.assertFalse(a < b) + self.assertTrue(a <= b) + self.assertFalse(a > b) + self.assertTrue(a >= b) + + # Change the fqdn is greater + fqdn = _Route53Record(None, self.record_a, False, + fqdn_override='other') + self.assertFalse(a == fqdn) + self.assertTrue(a != fqdn) + self.assertFalse(a < fqdn) + self.assertFalse(a <= fqdn) + self.assertTrue(a > fqdn) + self.assertTrue(a >= fqdn) + + provider = DummyProvider() + geo_a = _Route53GeoRecord(provider, self.record_a, 'NA-US', + self.record_a.geo['NA-US'], False) + geo_b = _Route53GeoRecord(provider, self.record_a, 'NA-US', + self.record_a.geo['NA-US'], False) + self.assertTrue(geo_a == geo_b) + self.assertFalse(geo_a != geo_b) + self.assertFalse(geo_a < geo_b) + self.assertTrue(geo_a <= geo_b) + self.assertFalse(geo_a > geo_b) + self.assertTrue(geo_a >= geo_b) + + # Other base + geo_fqdn = _Route53GeoRecord(provider, self.record_a, 'NA-US', + self.record_a.geo['NA-US'], False) + geo_fqdn.fqdn = 'other' + self.assertFalse(geo_a == geo_fqdn) + self.assertTrue(geo_a != geo_fqdn) + self.assertFalse(geo_a < geo_fqdn) + self.assertFalse(geo_a <= geo_fqdn) + self.assertTrue(geo_a > geo_fqdn) + self.assertTrue(geo_a >= geo_fqdn) + + # Other class + self.assertFalse(a == geo_a) + self.assertTrue(a != geo_a) + self.assertFalse(a < geo_a) + self.assertFalse(a <= geo_a) + self.assertTrue(a > geo_a) + self.assertTrue(a >= geo_a) + def test_dynamic_value_delete(self): provider = DummyProvider() geo = _Route53DynamicValue(provider, self.record_a, 'iad', '2.2.2.2', @@ -2207,70 +2259,112 @@ class TestRoute53Records(TestCase): creating=True) self.assertEquals(18, len(route53_records)) + expected_mods = [r.mod('CREATE', []) for r in route53_records] + # Sort so that we get a consistent order and don't rely on set ordering + expected_mods.sort(key=_mod_keyer) + # Convert the route53_records into mods self.assertEquals([{ 'Action': 'CREATE', 'ResourceRecordSet': { 'HealthCheckId': 'hc42', 'Name': '_octodns-ap-southeast-1-value.unit.tests.', - 'ResourceRecords': [{ - 'Value': '1.4.1.2'}], - 'SetIdentifier': 'ap-southeast-1-001', + 'ResourceRecords': [{'Value': '1.4.1.1'}], + 'SetIdentifier': 'ap-southeast-1-000', 'TTL': 60, 'Type': 'A', - 'Weight': 2 - } + 'Weight': 2} }, { 'Action': 'CREATE', 'ResourceRecordSet': { 'HealthCheckId': 'hc42', 'Name': '_octodns-ap-southeast-1-value.unit.tests.', - 'ResourceRecords': [{ - 'Value': '1.4.1.1'}], - 'SetIdentifier': 'ap-southeast-1-000', + 'ResourceRecords': [{'Value': '1.4.1.2'}], + 'SetIdentifier': 'ap-southeast-1-001', 'TTL': 60, 'Type': 'A', - 'Weight': 2 - } + 'Weight': 2} + }, { + 'Action': 'CREATE', + 'ResourceRecordSet': { + 'Name': '_octodns-default-pool.unit.tests.', + 'ResourceRecords': [ + {'Value': '1.1.2.1'}, + {'Value': '1.1.2.2'}], + 'TTL': 60, + 'Type': 'A'} + }, { + 'Action': 'CREATE', + 'ResourceRecordSet': { + 'HealthCheckId': 'hc42', + 'Name': '_octodns-eu-central-1-value.unit.tests.', + 'ResourceRecords': [{'Value': '1.3.1.1'}], + 'SetIdentifier': 'eu-central-1-000', + 'TTL': 60, + 'Type': 'A', + 'Weight': 1} + }, { + 'Action': 'CREATE', + 'ResourceRecordSet': { + 'HealthCheckId': 'hc42', + 'Name': '_octodns-eu-central-1-value.unit.tests.', + 'ResourceRecords': [{'Value': '1.3.1.2'}], + 'SetIdentifier': 'eu-central-1-001', + 'TTL': 60, + 'Type': 'A', + 'Weight': 1} + }, { + 'Action': 'CREATE', + 'ResourceRecordSet': { + 'HealthCheckId': 'hc42', + 'Name': '_octodns-us-east-1-value.unit.tests.', + 'ResourceRecords': [{'Value': '1.5.1.1'}], + 'SetIdentifier': 'us-east-1-000', + 'TTL': 60, + 'Type': 'A', + 'Weight': 1} + }, { + 'Action': 'CREATE', + 'ResourceRecordSet': { + 'HealthCheckId': 'hc42', + 'Name': '_octodns-us-east-1-value.unit.tests.', + 'ResourceRecords': [{'Value': '1.5.1.2'}], + 'SetIdentifier': 'us-east-1-001', + 'TTL': 60, + 'Type': 'A', + 'Weight': 1} }, { 'Action': 'CREATE', 'ResourceRecordSet': { 'AliasTarget': { - 'DNSName': '_octodns-ap-southeast-1-pool.unit.tests.', + 'DNSName': '_octodns-ap-southeast-1-value.unit.tests.', 'EvaluateTargetHealth': True, - 'HostedZoneId': 'z45' - }, - 'GeoLocation': { - 'CountryCode': 'JP'}, - 'Name': 'unit.tests.', - 'SetIdentifier': '0-ap-southeast-1-AS-JP', - 'Type': 'A' - } + 'HostedZoneId': 'z45'}, + 'Failover': 'PRIMARY', + 'Name': '_octodns-ap-southeast-1-pool.unit.tests.', + 'SetIdentifier': 'ap-southeast-1-Primary', + 'Type': 'A'} }, { 'Action': 'CREATE', 'ResourceRecordSet': { 'AliasTarget': { - 'DNSName': '_octodns-eu-central-1-pool.unit.tests.', + 'DNSName': '_octodns-eu-central-1-value.unit.tests.', 'EvaluateTargetHealth': True, 'HostedZoneId': 'z45'}, - 'GeoLocation': { - 'CountryCode': 'US', - 'SubdivisionCode': 'FL', - }, - 'Name': 'unit.tests.', - 'SetIdentifier': '1-eu-central-1-NA-US-FL', + 'Failover': 'PRIMARY', + 'Name': '_octodns-eu-central-1-pool.unit.tests.', + 'SetIdentifier': 'eu-central-1-Primary', 'Type': 'A'} }, { 'Action': 'CREATE', 'ResourceRecordSet': { 'AliasTarget': { - 'DNSName': '_octodns-us-east-1-pool.unit.tests.', + 'DNSName': '_octodns-us-east-1-value.unit.tests.', 'EvaluateTargetHealth': True, 'HostedZoneId': 'z45'}, - 'GeoLocation': { - 'CountryCode': '*'}, - 'Name': 'unit.tests.', - 'SetIdentifier': '2-us-east-1-None', + 'Failover': 'PRIMARY', + 'Name': '_octodns-us-east-1-pool.unit.tests.', + 'SetIdentifier': 'us-east-1-Primary', 'Type': 'A'} }, { 'Action': 'CREATE', @@ -2287,123 +2381,72 @@ class TestRoute53Records(TestCase): 'Action': 'CREATE', 'ResourceRecordSet': { 'AliasTarget': { - 'DNSName': '_octodns-ap-southeast-1-pool.unit.tests.', + 'DNSName': '_octodns-us-east-1-pool.unit.tests.', 'EvaluateTargetHealth': True, 'HostedZoneId': 'z45'}, - 'GeoLocation': { - 'CountryCode': 'CN'}, - 'Name': 'unit.tests.', - 'SetIdentifier': '0-ap-southeast-1-AS-CN', + 'Failover': 'SECONDARY', + 'Name': '_octodns-eu-central-1-pool.unit.tests.', + 'SetIdentifier': 'eu-central-1-Secondary-us-east-1', 'Type': 'A'} }, { 'Action': 'CREATE', 'ResourceRecordSet': { 'AliasTarget': { - 'DNSName': '_octodns-us-east-1-value.unit.tests.', + 'DNSName': '_octodns-default-pool.unit.tests.', 'EvaluateTargetHealth': True, 'HostedZoneId': 'z45'}, - 'Failover': 'PRIMARY', + 'Failover': 'SECONDARY', 'Name': '_octodns-us-east-1-pool.unit.tests.', - 'SetIdentifier': 'us-east-1-Primary', + 'SetIdentifier': 'us-east-1-Secondary-default', 'Type': 'A'} }, { 'Action': 'CREATE', 'ResourceRecordSet': { 'AliasTarget': { - 'DNSName': '_octodns-eu-central-1-pool.unit.tests.', + 'DNSName': '_octodns-ap-southeast-1-pool.unit.tests.', 'EvaluateTargetHealth': True, 'HostedZoneId': 'z45'}, 'GeoLocation': { - 'ContinentCode': 'EU'}, + 'CountryCode': 'CN'}, 'Name': 'unit.tests.', - 'SetIdentifier': '1-eu-central-1-EU', + 'SetIdentifier': '0-ap-southeast-1-AS-CN', 'Type': 'A'} }, { 'Action': 'CREATE', 'ResourceRecordSet': { 'AliasTarget': { - 'DNSName': '_octodns-eu-central-1-value.unit.tests.', + 'DNSName': '_octodns-ap-southeast-1-pool.unit.tests.', 'EvaluateTargetHealth': True, 'HostedZoneId': 'z45'}, - 'Failover': 'PRIMARY', - 'Name': '_octodns-eu-central-1-pool.unit.tests.', - 'SetIdentifier': 'eu-central-1-Primary', - 'Type': 'A'} - }, { - 'Action': 'CREATE', - 'ResourceRecordSet': { - 'Name': '_octodns-default-pool.unit.tests.', - 'ResourceRecords': [{ - 'Value': '1.1.2.1'}, - { - 'Value': '1.1.2.2'}], - 'TTL': 60, + 'GeoLocation': { + 'CountryCode': 'JP'}, + 'Name': 'unit.tests.', + 'SetIdentifier': '0-ap-southeast-1-AS-JP', 'Type': 'A'} - }, { - 'Action': 'CREATE', - 'ResourceRecordSet': { - 'HealthCheckId': 'hc42', - 'Name': '_octodns-eu-central-1-value.unit.tests.', - 'ResourceRecords': [{ - 'Value': '1.3.1.2'}], - 'SetIdentifier': 'eu-central-1-001', - 'TTL': 60, - 'Type': 'A', - 'Weight': 1} - }, { - 'Action': 'CREATE', - 'ResourceRecordSet': { - 'HealthCheckId': 'hc42', - 'Name': '_octodns-eu-central-1-value.unit.tests.', - 'ResourceRecords': [{ - 'Value': '1.3.1.1'}], - 'SetIdentifier': 'eu-central-1-000', - 'TTL': 60, - 'Type': 'A', - 'Weight': 1} }, { 'Action': 'CREATE', 'ResourceRecordSet': { 'AliasTarget': { - 'DNSName': '_octodns-default-pool.unit.tests.', + 'DNSName': '_octodns-eu-central-1-pool.unit.tests.', 'EvaluateTargetHealth': True, 'HostedZoneId': 'z45'}, - 'Failover': 'SECONDARY', - 'Name': '_octodns-us-east-1-pool.unit.tests.', - 'SetIdentifier': 'us-east-1-Secondary-default', + 'GeoLocation': { + 'ContinentCode': 'EU'}, + 'Name': 'unit.tests.', + 'SetIdentifier': '1-eu-central-1-EU', 'Type': 'A'} - }, { - 'Action': 'CREATE', - 'ResourceRecordSet': { - 'HealthCheckId': 'hc42', - 'Name': '_octodns-us-east-1-value.unit.tests.', - 'ResourceRecords': [{ - 'Value': '1.5.1.2'}], - 'SetIdentifier': 'us-east-1-001', - 'TTL': 60, - 'Type': 'A', - 'Weight': 1} - }, { - 'Action': 'CREATE', - 'ResourceRecordSet': { - 'HealthCheckId': 'hc42', - 'Name': '_octodns-us-east-1-value.unit.tests.', - 'ResourceRecords': [{ - 'Value': '1.5.1.1'}], - 'SetIdentifier': 'us-east-1-000', - 'TTL': 60, - 'Type': 'A', - 'Weight': 1} }, { 'Action': 'CREATE', 'ResourceRecordSet': { 'AliasTarget': { - 'DNSName': '_octodns-ap-southeast-1-value.unit.tests.', + 'DNSName': '_octodns-eu-central-1-pool.unit.tests.', 'EvaluateTargetHealth': True, 'HostedZoneId': 'z45'}, - 'Failover': 'PRIMARY', - 'Name': '_octodns-ap-southeast-1-pool.unit.tests.', - 'SetIdentifier': 'ap-southeast-1-Primary', + 'GeoLocation': { + 'CountryCode': 'US', + 'SubdivisionCode': 'FL'}, + 'Name': 'unit.tests.', + 'SetIdentifier': '1-eu-central-1-NA-US-FL', 'Type': 'A'} }, { 'Action': 'CREATE', @@ -2412,11 +2455,12 @@ class TestRoute53Records(TestCase): 'DNSName': '_octodns-us-east-1-pool.unit.tests.', 'EvaluateTargetHealth': True, 'HostedZoneId': 'z45'}, - 'Failover': 'SECONDARY', - 'Name': '_octodns-eu-central-1-pool.unit.tests.', - 'SetIdentifier': 'eu-central-1-Secondary-us-east-1', + 'GeoLocation': { + 'CountryCode': '*'}, + 'Name': 'unit.tests.', + 'SetIdentifier': '2-us-east-1-None', 'Type': 'A'} - }], [r.mod('CREATE', []) for r in route53_records]) + }], expected_mods) for route53_record in route53_records: # Smoke test stringification From c82e94792e37e2d99df0a0d774a8615871bb84d1 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 7 Oct 2019 07:47:43 -0700 Subject: [PATCH 035/155] RackspaceProvider python3, value types hashing --- octodns/provider/rackspace.py | 15 ++++----- octodns/record/__init__.py | 15 ++++++++- tests/test_octodns_provider_rackspace.py | 8 ++--- tests/test_octodns_record.py | 40 ++++++++++++++++++++++++ 4 files changed, 66 insertions(+), 12 deletions(-) diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py index 5038929..28b7f05 100644 --- a/octodns/provider/rackspace.py +++ b/octodns/provider/rackspace.py @@ -7,13 +7,16 @@ from __future__ import absolute_import, division, print_function, \ from requests import HTTPError, Session, post from collections import defaultdict import logging -import string import time from ..record import Record from .base import BaseProvider +def _value_keyer(v): + return '{}-{}-{}'.format(v.get('type', ''), v['name'], v.get('data', '')) + + def add_trailing_dot(s): assert s assert s[-1] != '.' @@ -28,12 +31,12 @@ def remove_trailing_dot(s): def escape_semicolon(s): assert s - return string.replace(s, ';', '\\;') + return s.replace(';', '\\;') def unescape_semicolon(s): assert s - return string.replace(s, '\\;', ';') + return s.replace('\\;', ';') class RackspaceProvider(BaseProvider): @@ -367,11 +370,9 @@ class RackspaceProvider(BaseProvider): self._delete('domains/{}/records?{}'.format(domain_id, params)) if updates: - data = {"records": sorted(updates, key=lambda v: v['name'])} + data = {"records": sorted(updates, key=_value_keyer)} self._put('domains/{}/records'.format(domain_id), data=data) if creates: - data = {"records": sorted(creates, key=lambda v: v['type'] + - v['name'] + - v.get('data', ''))} + data = {"records": sorted(creates, key=_value_keyer)} self._post('domains/{}/records'.format(domain_id), data=data) diff --git a/octodns/record/__init__.py b/octodns/record/__init__.py index b377d14..a782018 100644 --- a/octodns/record/__init__.py +++ b/octodns/record/__init__.py @@ -723,7 +723,8 @@ class _IpList(object): @classmethod def process(cls, values): - return values + # Translating None into '' so that the list will be sortable in python3 + return [v if v is not None else '' for v in values] class Ipv4List(_IpList): @@ -918,6 +919,9 @@ class MxValue(object): 'exchange': self.exchange, } + def __hash__(self): + return hash('{} {}'.format(self.preference, self.exchange)) + def __eq__(self, other): return ((self.preference, self.exchange) == (other.preference, other.exchange)) @@ -1010,6 +1014,9 @@ class NaptrValue(object): 'replacement': self.replacement, } + def __hash__(self): + return hash(self.__repr__()) + def __eq__(self, other): return ((self.order, self.preference, self.flags, self.service, self.regexp, self.replacement) == @@ -1145,6 +1152,9 @@ class SshfpValue(object): 'fingerprint': self.fingerprint, } + def __hash__(self): + return hash(self.__repr__()) + def __eq__(self, other): return ((self.algorithm, self.fingerprint_type, self.fingerprint) == (other.algorithm, other.fingerprint_type, other.fingerprint)) @@ -1283,6 +1293,9 @@ class SrvValue(object): 'target': self.target, } + def __hash__(self): + return hash(self.__repr__()) + def __eq__(self, other): return ((self.priority, self.weight, self.port, self.target) == (other.priority, other.weight, other.port, other.target)) diff --git a/tests/test_octodns_provider_rackspace.py b/tests/test_octodns_provider_rackspace.py index 3dfdd5f..0a6564d 100644 --- a/tests/test_octodns_provider_rackspace.py +++ b/tests/test_octodns_provider_rackspace.py @@ -792,13 +792,13 @@ class TestRackspaceProvider(TestCase): ExpectedUpdates = { "records": [{ "name": "unit.tests", - "id": "A-222222", - "data": "1.2.3.5", + "id": "A-111111", + "data": "1.2.3.4", "ttl": 3600 }, { "name": "unit.tests", - "id": "A-111111", - "data": "1.2.3.4", + "id": "A-222222", + "data": "1.2.3.5", "ttl": 3600 }, { "name": "unit.tests", diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index c26b301..0845ddf 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -594,6 +594,30 @@ class TestRecord(TestCase): # __repr__ doesn't blow up a.__repr__() + # Hash + v = NaptrValue({ + 'order': 30, + 'preference': 31, + 'flags': 'M', + 'service': 'N', + 'regexp': 'O', + 'replacement': 'z', + }) + o = NaptrValue({ + 'order': 30, + 'preference': 32, + 'flags': 'M', + 'service': 'N', + 'regexp': 'O', + 'replacement': 'z', + }) + values = set() + values.add(v) + self.assertTrue(v in values) + self.assertFalse(o in values) + values.add(o) + self.assertTrue(o in values) + def test_ns(self): a_values = ['5.6.7.8.', '6.7.8.9.', '7.8.9.0.'] a_data = {'ttl': 30, 'values': a_values} @@ -1114,6 +1138,14 @@ class TestRecord(TestCase): self.assertTrue(c >= c) self.assertTrue(c <= c) + # Hash + values = set() + values.add(a) + self.assertTrue(a in values) + self.assertFalse(b in values) + values.add(b) + self.assertTrue(b in values) + def test_srv_value(self): a = SrvValue({'priority': 0, 'weight': 0, 'port': 0, 'target': 'foo.'}) b = SrvValue({'priority': 1, 'weight': 0, 'port': 0, 'target': 'foo.'}) @@ -1172,6 +1204,14 @@ class TestRecord(TestCase): self.assertTrue(c >= c) self.assertTrue(c <= c) + # Hash + values = set() + values.add(a) + self.assertTrue(a in values) + self.assertFalse(b in values) + values.add(b) + self.assertTrue(b in values) + class TestRecordValidation(TestCase): zone = Zone('unit.tests.', []) From 25768c476fee352148a8af583709410426469679 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 7 Oct 2019 07:48:55 -0700 Subject: [PATCH 036/155] SelectelProvider python3 (tests) --- tests/test_octodns_provider_selectel.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_octodns_provider_selectel.py b/tests/test_octodns_provider_selectel.py index a2ba39e..7ad1e6b 100644 --- a/tests/test_octodns_provider_selectel.py +++ b/tests/test_octodns_provider_selectel.py @@ -6,6 +6,7 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals from unittest import TestCase +from six import text_type import requests_mock @@ -288,7 +289,7 @@ class TestSelectelProvider(TestCase): with self.assertRaises(Exception) as ctx: SelectelProvider(123, 'fail_token') - self.assertEquals(ctx.exception.message, + self.assertEquals(text_type(ctx.exception), 'Authorization failed. Invalid or empty token.') @requests_mock.Mocker() From 90a60d3dbdd453fb691b698bc2fe8f7936e37584 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 7 Oct 2019 07:53:08 -0700 Subject: [PATCH 037/155] TransipProvider python3 --- octodns/provider/transip.py | 2 +- tests/test_octodns_provider_transip.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/octodns/provider/transip.py b/octodns/provider/transip.py index 09920a9..7458e36 100644 --- a/octodns/provider/transip.py +++ b/octodns/provider/transip.py @@ -137,7 +137,7 @@ class TransipProvider(BaseProvider): try: self._client.get_info(plan.desired.name[:-1]) except WebFault as e: - self.log.warning('_apply: %s ', e.message) + self.log.exception('_apply: get_info failed') raise e _dns_entries = [] diff --git a/tests/test_octodns_provider_transip.py b/tests/test_octodns_provider_transip.py index c56509a..3fbfc44 100644 --- a/tests/test_octodns_provider_transip.py +++ b/tests/test_octodns_provider_transip.py @@ -5,8 +5,8 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -# from mock import Mock, call from os.path import dirname, join +from six import text_type from suds import WebFault @@ -176,7 +176,7 @@ N4OiVz1I3rbZGYa396lpxO6ku8yCglisL1yrSP6DdEUp66ntpKVd self.assertEquals( 'populate: (102) Transip used as target' + ' for non-existing zone: notfound.unit.tests.', - ctx.exception.message) + text_type(ctx.exception)) # Happy Plan - Zone does not exists # Won't trigger an exception if provider is NOT used as a target for a From 14063186f3e9bc419fa6eb50f30e9a7dc4feaa9c Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 7 Oct 2019 07:56:10 -0700 Subject: [PATCH 038/155] YamlProvider python3, tests --- tests/test_octodns_provider_yaml.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/test_octodns_provider_yaml.py b/tests/test_octodns_provider_yaml.py index 700f3c3..e16aaa5 100644 --- a/tests/test_octodns_provider_yaml.py +++ b/tests/test_octodns_provider_yaml.py @@ -82,9 +82,9 @@ class TestYamlProvider(TestCase): target.populate(reloaded) self.assertDictEqual( {'included': ['test']}, - filter( + list(filter( lambda x: x.name == 'included', reloaded.records - )[0]._octodns) + ))[0]._octodns) self.assertFalse(zone.changes(reloaded, target=source)) @@ -120,7 +120,7 @@ class TestYamlProvider(TestCase): self.assertTrue('value' in data.pop('www.sub')) # make sure nothing is left - self.assertEquals([], data.keys()) + self.assertEquals([], list(data.keys())) with open(dynamic_yaml_file) as fh: data = safe_load(fh.read()) @@ -149,7 +149,7 @@ class TestYamlProvider(TestCase): # self.assertTrue('dynamic' in dyna) # make sure nothing is left - self.assertEquals([], data.keys()) + self.assertEquals([], list(data.keys())) def test_empty(self): source = YamlProvider('test', join(dirname(__file__), 'config')) @@ -255,8 +255,8 @@ class TestSplitYamlProvider(TestCase): # We add everything plan = target.plan(zone) - self.assertEquals(15, len(filter(lambda c: isinstance(c, Create), - plan.changes))) + self.assertEquals(15, len(list(filter( + lambda c: isinstance(c, Create), plan.changes)))) self.assertFalse(isdir(zone_dir)) # Now actually do it @@ -264,8 +264,8 @@ class TestSplitYamlProvider(TestCase): # Dynamic plan plan = target.plan(dynamic_zone) - self.assertEquals(5, len(filter(lambda c: isinstance(c, Create), - plan.changes))) + self.assertEquals(5, len(list(filter( + lambda c: isinstance(c, Create), plan.changes)))) self.assertFalse(isdir(dynamic_zone_dir)) # Apply it self.assertEquals(5, target.apply(plan)) @@ -276,16 +276,16 @@ class TestSplitYamlProvider(TestCase): target.populate(reloaded) self.assertDictEqual( {'included': ['test']}, - filter( + list(filter( lambda x: x.name == 'included', reloaded.records - )[0]._octodns) + ))[0]._octodns) self.assertFalse(zone.changes(reloaded, target=source)) # A 2nd sync should still create everything plan = target.plan(zone) - self.assertEquals(15, len(filter(lambda c: isinstance(c, Create), - plan.changes))) + self.assertEquals(15, len(list(filter( + lambda c: isinstance(c, Create), plan.changes)))) yaml_file = join(zone_dir, '$unit.tests.yaml') self.assertTrue(isfile(yaml_file)) From e0c5962d79df12ace77d763833010b0193b6dce4 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 7 Oct 2019 07:58:44 -0700 Subject: [PATCH 039/155] AxfrSource python3 --- octodns/source/axfr.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/octodns/source/axfr.py b/octodns/source/axfr.py index f35c4b3..be2acf5 100644 --- a/octodns/source/axfr.py +++ b/octodns/source/axfr.py @@ -15,6 +15,7 @@ from dns.exception import DNSException from collections import defaultdict from os import listdir from os.path import join +from six import text_type import logging from ..record import Record @@ -179,8 +180,7 @@ class ZoneFileSourceNotFound(ZoneFileSourceException): class ZoneFileSourceLoadFailure(ZoneFileSourceException): def __init__(self, error): - super(ZoneFileSourceLoadFailure, self).__init__( - error.message) + super(ZoneFileSourceLoadFailure, self).__init__(text_type(error)) class ZoneFileSource(AxfrBaseSource): From 0708b797da2b1208bd66b10881deca40585ce6d4 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 7 Oct 2019 08:29:50 -0700 Subject: [PATCH 040/155] TinyDnsSource python3 --- octodns/source/tinydns.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/octodns/source/tinydns.py b/octodns/source/tinydns.py index dc2bc1b..0ee100a 100755 --- a/octodns/source/tinydns.py +++ b/octodns/source/tinydns.py @@ -67,7 +67,8 @@ class TinyDnsBaseSource(BaseSource): values = [] for record in records: - new_value = record[0].decode('unicode-escape').replace(";", "\\;") + new_value = record[0].encode('latin1').decode('unicode-escape') \ + .replace(";", "\\;") values.append(new_value) try: From db8de8acb888713db8d88784de090c0e45c122ad Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 7 Oct 2019 08:41:28 -0700 Subject: [PATCH 041/155] Fix Manager ordering assumptions --- tests/config/provider-problems.yaml | 28 ++++++++++++++++++++++++++++ tests/config/unknown-provider.yaml | 16 ---------------- tests/test_octodns_manager.py | 10 +++++----- 3 files changed, 33 insertions(+), 21 deletions(-) create mode 100644 tests/config/provider-problems.yaml diff --git a/tests/config/provider-problems.yaml b/tests/config/provider-problems.yaml new file mode 100644 index 0000000..9071046 --- /dev/null +++ b/tests/config/provider-problems.yaml @@ -0,0 +1,28 @@ +providers: + yaml: + class: octodns.provider.yaml.YamlProvider + directory: ./config + simple_source: + class: helpers.SimpleSource +zones: + missing.sources.: + targets: + - yaml + missing.targets.: + sources: + - yaml + unknown.source.: + sources: + - not-there + targets: + - yaml + unknown.target.: + sources: + - yaml + targets: + - not-there-either + not.targetable.: + sources: + - yaml + targets: + - simple_source diff --git a/tests/config/unknown-provider.yaml b/tests/config/unknown-provider.yaml index 9071046..a0e9f55 100644 --- a/tests/config/unknown-provider.yaml +++ b/tests/config/unknown-provider.yaml @@ -5,24 +5,8 @@ providers: simple_source: class: helpers.SimpleSource zones: - missing.sources.: - targets: - - yaml - missing.targets.: - sources: - - yaml unknown.source.: sources: - not-there targets: - yaml - unknown.target.: - sources: - - yaml - targets: - - not-there-either - not.targetable.: - sources: - - yaml - targets: - - simple_source diff --git a/tests/test_octodns_manager.py b/tests/test_octodns_manager.py index 43feed5..60a1922 100644 --- a/tests/test_octodns_manager.py +++ b/tests/test_octodns_manager.py @@ -62,25 +62,25 @@ class TestManager(TestCase): def test_missing_source(self): with self.assertRaises(Exception) as ctx: - Manager(get_config_filename('unknown-provider.yaml')) \ + Manager(get_config_filename('provider-problems.yaml')) \ .sync(['missing.sources.']) self.assertTrue('missing sources' in text_type(ctx.exception)) def test_missing_targets(self): with self.assertRaises(Exception) as ctx: - Manager(get_config_filename('unknown-provider.yaml')) \ + Manager(get_config_filename('provider-problems.yaml')) \ .sync(['missing.targets.']) self.assertTrue('missing targets' in text_type(ctx.exception)) def test_unknown_source(self): with self.assertRaises(Exception) as ctx: - Manager(get_config_filename('unknown-provider.yaml')) \ + Manager(get_config_filename('provider-problems.yaml')) \ .sync(['unknown.source.']) self.assertTrue('unknown source' in text_type(ctx.exception)) def test_unknown_target(self): with self.assertRaises(Exception) as ctx: - Manager(get_config_filename('unknown-provider.yaml')) \ + Manager(get_config_filename('provider-problems.yaml')) \ .sync(['unknown.target.']) self.assertTrue('unknown target' in text_type(ctx.exception)) @@ -99,7 +99,7 @@ class TestManager(TestCase): def test_source_only_as_a_target(self): with self.assertRaises(Exception) as ctx: - Manager(get_config_filename('unknown-provider.yaml')) \ + Manager(get_config_filename('provider-problems.yaml')) \ .sync(['not.targetable.']) self.assertTrue('does not support targeting' in text_type(ctx.exception)) From bfa1fadde97f7a3567103e2bcb500d7e484f0967 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 7 Oct 2019 08:44:25 -0700 Subject: [PATCH 042/155] Fix CloudflareProvider test ordering assumptions --- tests/test_octodns_provider_cloudflare.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/tests/test_octodns_provider_cloudflare.py b/tests/test_octodns_provider_cloudflare.py index c656cab..013c5f7 100644 --- a/tests/test_octodns_provider_cloudflare.py +++ b/tests/test_octodns_provider_cloudflare.py @@ -313,7 +313,7 @@ class TestCloudflareProvider(TestCase): 'dns_records/fc12ab34cd5611334422ab3322997653'), call('DELETE', '/zones/ff12ab34cd5611334422ab3322997650/' 'dns_records/fc12ab34cd5611334422ab3322997654') - ]) + ], any_order=True) def test_update_add_swap(self): provider = CloudflareProvider('test', 'email', 'token') @@ -743,23 +743,25 @@ class TestCloudflareProvider(TestCase): # the CDN. self.assertEquals(3, len(zone.records)) - record = list(zone.records)[0] - self.assertEquals('multi', record.name) - self.assertEquals('multi.unit.tests.', record.fqdn) + ordered = sorted(zone.records, key=lambda r: r.name) + + record = ordered[0] + self.assertEquals('a', record.name) + self.assertEquals('a.unit.tests.', record.fqdn) self.assertEquals('CNAME', record._type) - self.assertEquals('multi.unit.tests.cdn.cloudflare.net.', record.value) + self.assertEquals('a.unit.tests.cdn.cloudflare.net.', record.value) - record = list(zone.records)[1] + record = ordered[1] self.assertEquals('cname', record.name) self.assertEquals('cname.unit.tests.', record.fqdn) self.assertEquals('CNAME', record._type) self.assertEquals('cname.unit.tests.cdn.cloudflare.net.', record.value) - record = list(zone.records)[2] - self.assertEquals('a', record.name) - self.assertEquals('a.unit.tests.', record.fqdn) + record = ordered[2] + self.assertEquals('multi', record.name) + self.assertEquals('multi.unit.tests.', record.fqdn) self.assertEquals('CNAME', record._type) - self.assertEquals('a.unit.tests.cdn.cloudflare.net.', record.value) + self.assertEquals('multi.unit.tests.cdn.cloudflare.net.', record.value) # CDN enabled records can't be updated, we don't know the real values # never point a Cloudflare record to itself. From 7958233fccf9ea22d95e2fd06c48d7d0a4529e26 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 7 Oct 2019 09:17:48 -0700 Subject: [PATCH 043/155] Consistently order changes :-/ Many providers make their modifications in the order that changes comes. In python3 this causes things to be inconsistently ordered. That mostly works, but could result in hidenbugs (e.g. Route53Provider's batching could be completely different based on the order it sees changes.) Sorting changes consistently is a good thing and it shouldn't hurt situations where providers are already doing their own ordering. All-in-all more consistent is better and we have to be explicit with python 3. --- octodns/provider/plan.py | 6 +- octodns/record/__init__.py | 6 ++ tests/test_octodns_provider_cloudflare.py | 2 +- tests/test_octodns_provider_digitalocean.py | 15 ++- tests/test_octodns_provider_dnsimple.py | 27 +++++- tests/test_octodns_provider_dnsmadeeasy.py | 22 ++++- tests/test_octodns_provider_ns1.py | 10 +- tests/test_octodns_provider_ovh.py | 101 ++++++++++---------- tests/test_octodns_provider_route53.py | 4 +- 9 files changed, 130 insertions(+), 63 deletions(-) diff --git a/octodns/provider/plan.py b/octodns/provider/plan.py index d4589f2..5b8be68 100644 --- a/octodns/provider/plan.py +++ b/octodns/provider/plan.py @@ -28,7 +28,11 @@ class Plan(object): delete_pcent_threshold=MAX_SAFE_DELETE_PCENT): self.existing = existing self.desired = desired - self.changes = changes + # Sort changes to ensure we always have a consistent ordering for + # things that make assumptions about that. Many providers will do their + # own ordering to ensure things happen in a way that makes sense to + # them and/or is as safe as possible. + self.changes = sorted(changes) self.exists = exists self.update_pcent_threshold = update_pcent_threshold self.delete_pcent_threshold = delete_pcent_threshold diff --git a/octodns/record/__init__.py b/octodns/record/__init__.py index a782018..47d2e9b 100644 --- a/octodns/record/__init__.py +++ b/octodns/record/__init__.py @@ -25,6 +25,12 @@ class Change(object): 'Returns new if we have one, existing otherwise' return self.new or self.existing + def __lt__(self, other): + self_record = self.record + other_record = other.record + return ((self_record.name, self_record._type) < + (other_record.name, other_record._type)) + class Create(Change): diff --git a/tests/test_octodns_provider_cloudflare.py b/tests/test_octodns_provider_cloudflare.py index 013c5f7..3581033 100644 --- a/tests/test_octodns_provider_cloudflare.py +++ b/tests/test_octodns_provider_cloudflare.py @@ -313,7 +313,7 @@ class TestCloudflareProvider(TestCase): 'dns_records/fc12ab34cd5611334422ab3322997653'), call('DELETE', '/zones/ff12ab34cd5611334422ab3322997650/' 'dns_records/fc12ab34cd5611334422ab3322997654') - ], any_order=True) + ]) def test_update_add_swap(self): provider = CloudflareProvider('test', 'email', 'token') diff --git a/tests/test_octodns_provider_digitalocean.py b/tests/test_octodns_provider_digitalocean.py index 448b26e..ebb5319 100644 --- a/tests/test_octodns_provider_digitalocean.py +++ b/tests/test_octodns_provider_digitalocean.py @@ -176,7 +176,20 @@ class TestDigitalOceanProvider(TestCase): call('GET', '/domains/unit.tests/records', {'page': 1}), # delete the initial A record call('DELETE', '/domains/unit.tests/records/11189877'), - # created at least one of the record with expected data + # created at least some of the record with expected data + call('POST', '/domains/unit.tests/records', data={ + 'data': '1.2.3.4', + 'name': '@', + 'ttl': 300, 'type': 'A'}), + call('POST', '/domains/unit.tests/records', data={ + 'data': '1.2.3.5', + 'name': '@', + 'ttl': 300, 'type': 'A'}), + call('POST', '/domains/unit.tests/records', data={ + 'data': 'ca.unit.tests.', + 'flags': 0, 'name': '@', + 'tag': 'issue', + 'ttl': 3600, 'type': 'CAA'}), call('POST', '/domains/unit.tests/records', data={ 'name': '_srv._tcp', 'weight': 20, diff --git a/tests/test_octodns_provider_dnsimple.py b/tests/test_octodns_provider_dnsimple.py index 9e8586c..e3a9b8d 100644 --- a/tests/test_octodns_provider_dnsimple.py +++ b/tests/test_octodns_provider_dnsimple.py @@ -139,7 +139,32 @@ class TestDnsimpleProvider(TestCase): provider._client._request.assert_has_calls([ # created the domain call('POST', '/domains', data={'name': 'unit.tests'}), - # created at least one of the record with expected data + # created at least some of the record with expected data + call('POST', '/zones/unit.tests/records', data={ + 'content': '1.2.3.4', + 'type': 'A', + 'name': '', + 'ttl': 300}), + call('POST', '/zones/unit.tests/records', data={ + 'content': '1.2.3.5', + 'type': 'A', + 'name': '', + 'ttl': 300}), + call('POST', '/zones/unit.tests/records', data={ + 'content': '0 issue "ca.unit.tests"', + 'type': 'CAA', + 'name': '', + 'ttl': 3600}), + call('POST', '/zones/unit.tests/records', data={ + 'content': '1 1 7491973e5f8b39d5327cd4e08bc81b05f7710b49', + 'type': 'SSHFP', + 'name': '', + 'ttl': 3600}), + call('POST', '/zones/unit.tests/records', data={ + 'content': '1 1 bf6b6825d2977c511a475bbefb88aad54a92ac73', + 'type': 'SSHFP', + 'name': '', + 'ttl': 3600}), call('POST', '/zones/unit.tests/records', data={ 'content': '20 30 foo-1.unit.tests.', 'priority': 10, diff --git a/tests/test_octodns_provider_dnsmadeeasy.py b/tests/test_octodns_provider_dnsmadeeasy.py index ea14376..ba61b94 100644 --- a/tests/test_octodns_provider_dnsmadeeasy.py +++ b/tests/test_octodns_provider_dnsmadeeasy.py @@ -149,7 +149,27 @@ class TestDnsMadeEasyProvider(TestCase): call('POST', '/', data={'name': 'unit.tests'}), # get all domains to build the cache call('GET', '/'), - # created at least one of the record with expected data + # created at least some of the record with expected data + call('POST', '/123123/records', data={ + 'type': 'A', + 'name': '', + 'value': '1.2.3.4', + 'ttl': 300}), + call('POST', '/123123/records', data={ + 'type': 'A', + 'name': '', + 'value': '1.2.3.5', + 'ttl': 300}), + call('POST', '/123123/records', data={ + 'type': 'ANAME', + 'name': '', + 'value': 'aname.unit.tests.', + 'ttl': 1800}), + call('POST', '/123123/records', data={ + 'name': '', + 'value': 'ca.unit.tests', + 'issuerCritical': 0, 'caaType': 'issue', + 'ttl': 3600, 'type': 'CAA'}), call('POST', '/123123/records', data={ 'name': '_srv._tcp', 'weight': 20, diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index 91d1a3f..9d23806 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -400,9 +400,9 @@ class TestNs1Provider(TestCase): self.assertEquals(3, got_n) nsone_zone.loadRecord.assert_has_calls([ call('unit.tests', u'A'), - call('geo', u'A'), call('delete-me', u'A'), - ], any_order=True) + call('geo', u'A'), + ]) mock_record.assert_has_calls([ call.update(answers=[{'answer': [u'1.2.3.4'], 'meta': {}}], filters=[], @@ -410,6 +410,8 @@ class TestNs1Provider(TestCase): call.update(answers=[{u'answer': [u'1.2.3.4'], u'meta': {}}], filters=[], ttl=32), + call.delete(), + call.delete(), call.update( answers=[ {u'answer': [u'101.102.103.104'], u'meta': {}}, @@ -427,9 +429,7 @@ class TestNs1Provider(TestCase): {u'filter': u'select_first_n', u'config': {u'N': 1}}, ], ttl=34), - call.delete(), - call.delete() - ], any_order=True) + ]) def test_escaping(self): provider = Ns1Provider('test', 'api-key') diff --git a/tests/test_octodns_provider_ovh.py b/tests/test_octodns_provider_ovh.py index 1f41abf..924591f 100644 --- a/tests/test_octodns_provider_ovh.py +++ b/tests/test_octodns_provider_ovh.py @@ -382,65 +382,64 @@ class TestOvhProvider(TestCase): get_mock.side_effect = [[100], [101], [102], [103]] provider.apply(plan) wanted_calls = [ - call(u'/domain/zone/unit.tests/record', fieldType=u'TXT', - subDomain='txt', target=u'TXT text', ttl=1400), - call(u'/domain/zone/unit.tests/record', fieldType=u'DKIM', - subDomain='dkim', target=self.valid_dkim_key, - ttl=1300), - call(u'/domain/zone/unit.tests/record', fieldType=u'A', - subDomain=u'', target=u'1.2.3.4', ttl=100), - call(u'/domain/zone/unit.tests/record', fieldType=u'SRV', + call('/domain/zone/unit.tests/record', fieldType='A', + subDomain='', target='1.2.3.4', ttl=100), + call('/domain/zone/unit.tests/record', fieldType='AAAA', + subDomain='', target='1:1ec:1::1', ttl=200), + call('/domain/zone/unit.tests/record', fieldType='MX', + subDomain='', target='10 mx1.unit.tests.', ttl=400), + call('/domain/zone/unit.tests/record', fieldType='SPF', + subDomain='', + target='v=spf1 include:unit.texts.redirect ~all', + ttl=1000), + call('/domain/zone/unit.tests/record', fieldType='SSHFP', + subDomain='', + target='1 1 bf6b6825d2977c511a475bbefb88aad54a92ac73', + ttl=1100), + call('/domain/zone/unit.tests/record', fieldType='PTR', + subDomain='4', target='unit.tests.', ttl=900), + call('/domain/zone/unit.tests/record', fieldType='SRV', subDomain='_srv._tcp', - target=u'10 20 30 foo-1.unit.tests.', ttl=800), - call(u'/domain/zone/unit.tests/record', fieldType=u'SRV', + target='10 20 30 foo-1.unit.tests.', ttl=800), + call('/domain/zone/unit.tests/record', fieldType='SRV', subDomain='_srv._tcp', - target=u'40 50 60 foo-2.unit.tests.', ttl=800), - call(u'/domain/zone/unit.tests/record', fieldType=u'PTR', - subDomain='4', target=u'unit.tests.', ttl=900), - call(u'/domain/zone/unit.tests/record', fieldType=u'NS', - subDomain='www3', target=u'ns3.unit.tests.', ttl=700), - call(u'/domain/zone/unit.tests/record', fieldType=u'NS', - subDomain='www3', target=u'ns4.unit.tests.', ttl=700), - call(u'/domain/zone/unit.tests/record', - fieldType=u'SSHFP', subDomain=u'', ttl=1100, - target=u'1 1 bf6b6825d2977c511a475bbefb88a' - u'ad54' - u'a92ac73', - ), - call(u'/domain/zone/unit.tests/record', fieldType=u'AAAA', - subDomain=u'', target=u'1:1ec:1::1', ttl=200), - call(u'/domain/zone/unit.tests/record', fieldType=u'MX', - subDomain=u'', target=u'10 mx1.unit.tests.', ttl=400), - call(u'/domain/zone/unit.tests/record', fieldType=u'CNAME', - subDomain='www2', target=u'unit.tests.', ttl=300), - call(u'/domain/zone/unit.tests/record', fieldType=u'SPF', - subDomain=u'', ttl=1000, - target=u'v=spf1 include:unit.texts.' - u'redirect ~all', - ), - call(u'/domain/zone/unit.tests/record', fieldType=u'A', - subDomain='sub', target=u'1.2.3.4', ttl=200), - call(u'/domain/zone/unit.tests/record', fieldType=u'NAPTR', - subDomain='naptr', ttl=500, - target=u'10 100 "S" "SIP+D2U" "!^.*$!sip:' - u'info@bar' - u'.example.com!" .' - ), - call(u'/domain/zone/unit.tests/refresh')] - - post_mock.assert_has_calls(wanted_calls, any_order=True) + target='40 50 60 foo-2.unit.tests.', ttl=800), + call('/domain/zone/unit.tests/record', fieldType='DKIM', + subDomain='dkim', + target='p=MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCxLaG' + '16G4SaEcXVdiIxTg7gKSGbHKQLm30CHib1h9FzS9nkcyvQSyQj1r' + 'MFyqC//tft3ohx3nvJl+bGCWxdtLYDSmir9PW54e5CTdxEh8MWRk' + 'BO3StF6QG/tAh3aTGDmkqhIJGLb87iHvpmVKqURmEUzJPv5KPJfW' + 'LofADI+q9lQIDAQAB', ttl=1300), + call('/domain/zone/unit.tests/record', fieldType='NAPTR', + subDomain='naptr', + target='10 100 "S" "SIP+D2U" "!^.*$!sip:info@bar.exam' + 'ple.com!" .', ttl=500), + call('/domain/zone/unit.tests/record', fieldType='A', + subDomain='sub', target='1.2.3.4', ttl=200), + call('/domain/zone/unit.tests/record', fieldType='TXT', + subDomain='txt', target='TXT text', ttl=1400), + call('/domain/zone/unit.tests/record', fieldType='CNAME', + subDomain='www2', target='unit.tests.', ttl=300), + call('/domain/zone/unit.tests/record', fieldType='NS', + subDomain='www3', target='ns3.unit.tests.', ttl=700), + call('/domain/zone/unit.tests/record', fieldType='NS', + subDomain='www3', target='ns4.unit.tests.', ttl=700), + call('/domain/zone/unit.tests/refresh')] + + post_mock.assert_has_calls(wanted_calls) # Get for delete calls wanted_get_calls = [ - call(u'/domain/zone/unit.tests/record', fieldType=u'TXT', - subDomain='txt'), - call(u'/domain/zone/unit.tests/record', fieldType=u'DKIM', - subDomain='dkim'), call(u'/domain/zone/unit.tests/record', fieldType=u'A', subDomain=u''), + call(u'/domain/zone/unit.tests/record', fieldType=u'DKIM', + subDomain='dkim'), call(u'/domain/zone/unit.tests/record', fieldType=u'A', - subDomain='fake')] - get_mock.assert_has_calls(wanted_get_calls, any_order=True) + subDomain='fake'), + call(u'/domain/zone/unit.tests/record', fieldType=u'TXT', + subDomain='txt')] + get_mock.assert_has_calls(wanted_get_calls) # 4 delete calls for update and delete delete_mock.assert_has_calls( [call(u'/domain/zone/unit.tests/record/100'), diff --git a/tests/test_octodns_provider_route53.py b/tests/test_octodns_provider_route53.py index b0ee342..7691804 100644 --- a/tests/test_octodns_provider_route53.py +++ b/tests/test_octodns_provider_route53.py @@ -1882,10 +1882,10 @@ class TestRoute53Provider(TestCase): @patch('octodns.provider.route53.Route53Provider._really_apply') def test_apply_3(self, really_apply_mock, _): - # with a max of seven modifications, four calls + # with a max of seven modifications, three calls provider, plan = self._get_test_plan(7) provider.apply(plan) - self.assertEquals(4, really_apply_mock.call_count) + self.assertEquals(3, really_apply_mock.call_count) @patch('octodns.provider.route53.Route53Provider._load_records') @patch('octodns.provider.route53.Route53Provider._really_apply') From be06a5da94fe3ffdbac2d44dec9e426e7ba320e3 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 7 Oct 2019 09:31:15 -0700 Subject: [PATCH 044/155] Make sure map and keys are lists when needed --- octodns/provider/dyn.py | 4 ++-- tests/test_octodns_provider_dyn.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/octodns/provider/dyn.py b/octodns/provider/dyn.py index c306238..30ba8bd 100644 --- a/octodns/provider/dyn.py +++ b/octodns/provider/dyn.py @@ -903,10 +903,10 @@ class DynProvider(BaseProvider): # Sort the values for consistent ordering so that we can compare values = sorted(values, key=_dynamic_value_sort_key) # Ensure that weight is included and if not use the default - values = map(lambda v: { + values = list(map(lambda v: { 'value': v['value'], 'weight': v.get('weight', 1), - }, values) + }, values)) # Walk through our existing pools looking for a match we can use for pool in pools: diff --git a/tests/test_octodns_provider_dyn.py b/tests/test_octodns_provider_dyn.py index 4f224fc..7c023fd 100644 --- a/tests/test_octodns_provider_dyn.py +++ b/tests/test_octodns_provider_dyn.py @@ -670,8 +670,8 @@ class TestDynProviderGeo(TestCase): tds = provider.traffic_directors self.assertEquals(set(['unit.tests.', 'geo.unit.tests.']), set(tds.keys())) - self.assertEquals(['A'], tds['unit.tests.'].keys()) - self.assertEquals(['A'], tds['geo.unit.tests.'].keys()) + self.assertEquals(['A'], list(tds['unit.tests.'].keys())) + self.assertEquals(['A'], list(tds['geo.unit.tests.'].keys())) provider.log.warn.assert_called_with("Unsupported TrafficDirector " "'%s'", 'something else') From 25cc4f42db439327747bb8101a6d4d7381f0ebb8 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 7 Oct 2019 09:33:14 -0700 Subject: [PATCH 045/155] Explicit list on filter when checking for non-existant targets --- octodns/manager.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/octodns/manager.py b/octodns/manager.py index 3984a5a..1c6a401 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -263,7 +263,8 @@ class Manager(object): except KeyError: raise Exception('Zone {} is missing targets'.format(zone_name)) if eligible_targets: - targets = filter(lambda d: d in eligible_targets, targets) + targets = list(filter(lambda d: d in eligible_targets, + targets)) if not targets: # Don't bother planning (and more importantly populating) zones From 3c75dc81f890c707895af25e22941050062c2ea4 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 7 Oct 2019 09:34:10 -0700 Subject: [PATCH 046/155] Require python 3.7 to pass --- .travis.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index b1a8204..eb609ff 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,9 +3,6 @@ matrix: include: - python: 2.7 - python: 3.7 - dist: xenial # required for Python >= 3.7 on Travis CI - allow_failures: - - python: 3.7 before_install: pip install --upgrade pip script: ./script/cibuild notifications: From 7867ad2093f775fbb4d83564600c299e9e51f31f Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 7 Oct 2019 09:40:23 -0700 Subject: [PATCH 047/155] Use six's StringIO, remove compat.py --- octodns/compat.py | 10 ---------- octodns/provider/plan.py | 3 +-- tests/test_octodns_plan.py | 3 +-- tests/test_octodns_yaml.py | 2 +- 4 files changed, 3 insertions(+), 15 deletions(-) delete mode 100644 octodns/compat.py diff --git a/octodns/compat.py b/octodns/compat.py deleted file mode 100644 index 6586cff..0000000 --- a/octodns/compat.py +++ /dev/null @@ -1,10 +0,0 @@ -# -# Python 2/3 compat bits -# - -try: # pragma: no cover - from StringIO import StringIO -except ImportError: # pragma: no cover - from io import StringIO - -StringIO diff --git a/octodns/provider/plan.py b/octodns/provider/plan.py index 5b8be68..af6863a 100644 --- a/octodns/provider/plan.py +++ b/octodns/provider/plan.py @@ -8,8 +8,7 @@ from __future__ import absolute_import, division, print_function, \ from logging import DEBUG, ERROR, INFO, WARN, getLogger from sys import stdout -from six import text_type -from ..compat import StringIO +from six import StringIO, text_type class UnsafePlan(Exception): diff --git a/tests/test_octodns_plan.py b/tests/test_octodns_plan.py index a017431..9cf812d 100644 --- a/tests/test_octodns_plan.py +++ b/tests/test_octodns_plan.py @@ -6,10 +6,9 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals from logging import getLogger -from six import text_type +from six import StringIO, text_type from unittest import TestCase -from octodns.compat import StringIO from octodns.provider.plan import Plan, PlanHtml, PlanLogger, PlanMarkdown from octodns.record import Create, Delete, Record, Update from octodns.zone import Zone diff --git a/tests/test_octodns_yaml.py b/tests/test_octodns_yaml.py index ddcd818..f211854 100644 --- a/tests/test_octodns_yaml.py +++ b/tests/test_octodns_yaml.py @@ -5,10 +5,10 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals +from six import StringIO from unittest import TestCase from yaml.constructor import ConstructorError -from octodns.compat import StringIO from octodns.yaml import safe_dump, safe_load From 7c34247e3ba81352ab0730b7b341050da11b7412 Mon Sep 17 00:00:00 2001 From: Josef Vogt Date: Wed, 9 Oct 2019 15:56:28 +0200 Subject: [PATCH 048/155] Fix 'server error: zone not found' for NS1 provider --- README.md | 2 +- octodns/provider/ns1.py | 6 +++--- requirements.txt | 2 +- tests/test_octodns_provider_ns1.py | 8 ++++---- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 83f0bd1..c3ddb0b 100644 --- a/README.md +++ b/README.md @@ -182,7 +182,7 @@ The above command pulled the existing data out of Route53 and placed the results | [EtcHostsProvider](/octodns/provider/etc_hosts.py) | | A, AAAA, ALIAS, CNAME | No | | | [GoogleCloudProvider](/octodns/provider/googlecloud.py) | google-cloud-dns | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | No | | | [MythicBeastsProvider](/octodns/provider/mythicbeasts.py) | Mythic Beasts | A, AAAA, ALIAS, CNAME, MX, NS, SRV, SSHFP, CAA, TXT | No | | -| [Ns1Provider](/octodns/provider/ns1.py) | nsone | All | Partial Geo | No health checking for GeoDNS | +| [Ns1Provider](/octodns/provider/ns1.py) | ns1-python | All | Partial Geo | No health checking for GeoDNS | | [OVH](/octodns/provider/ovh.py) | ovh | A, AAAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, SSHFP, TXT, DKIM | No | | | [PowerDnsProvider](/octodns/provider/powerdns.py) | | All | No | | | [Rackspace](/octodns/provider/rackspace.py) | | A, AAAA, ALIAS, CNAME, MX, NS, PTR, SPF, TXT | No | | diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 5fdf5b0..626b59e 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -8,8 +8,8 @@ from __future__ import absolute_import, division, print_function, \ from logging import getLogger from itertools import chain from collections import OrderedDict, defaultdict -from nsone import NSONE -from nsone.rest.errors import RateLimitException, ResourceException +from ns1 import NS1 +from ns1.rest.errors import RateLimitException, ResourceException from incf.countryutils import transformations from time import sleep @@ -36,7 +36,7 @@ class Ns1Provider(BaseProvider): self.log = getLogger('Ns1Provider[{}]'.format(id)) self.log.debug('__init__: id=%s, api_key=***', id) super(Ns1Provider, self).__init__(id, *args, **kwargs) - self._client = NSONE(apiKey=api_key) + self._client = NS1(apiKey=api_key) def _data_for_A(self, _type, record): # record meta (which would include geo information is only diff --git a/requirements.txt b/requirements.txt index bb373d0..1960ffe 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,7 +15,7 @@ ipaddress==1.0.22 jmespath==0.9.3 msrestazure==0.6.0 natsort==5.5.0 -nsone==0.9.100 +ns1-python==0.12.0 ovh==0.4.8 python-dateutil==2.6.1 requests==2.22.0 diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index 178ce53..22449f6 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -6,7 +6,7 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals from mock import Mock, call, patch -from nsone.rest.errors import AuthException, RateLimitException, \ +from ns1.rest.errors import AuthException, RateLimitException, \ ResourceException from unittest import TestCase @@ -171,7 +171,7 @@ class TestNs1Provider(TestCase): 'domain': 'unit.tests.', }] - @patch('nsone.NSONE.loadZone') + @patch('ns1.NS1.loadZone') def test_populate(self, load_mock): provider = Ns1Provider('test', 'api-key') @@ -290,8 +290,8 @@ class TestNs1Provider(TestCase): self.assertEquals(self.expected, zone.records) self.assertEquals(('unit.tests',), load_mock.call_args[0]) - @patch('nsone.NSONE.createZone') - @patch('nsone.NSONE.loadZone') + @patch('ns1.NS1.createZone') + @patch('ns1.NS1.loadZone') def test_sync(self, load_mock, create_mock): provider = Ns1Provider('test', 'api-key') From 0a7d63ef06e59dab5f0cd6b110c39b30b0114800 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Wed, 9 Oct 2019 14:35:53 -0700 Subject: [PATCH 049/155] Show line numbers missing coverage --- script/coverage | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/coverage b/script/coverage index 8552eba..ad8189e 100755 --- a/script/coverage +++ b/script/coverage @@ -29,7 +29,7 @@ export GOOGLE_APPLICATION_CREDENTIALS= coverage run --branch --source=octodns --omit=octodns/cmds/* "$(command -v nosetests)" --with-xunit "$@" coverage html coverage xml -coverage report +coverage report --show-missing coverage report | grep ^TOTAL | grep -qv 100% && { echo "Incomplete code coverage" >&2 exit 1 From 3f487197df7e6dacdd703b8e9888d29b67f74ba1 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Wed, 9 Oct 2019 15:09:37 -0700 Subject: [PATCH 050/155] Manager throws ManagerException rather than Exception, more robust tests --- octodns/manager.py | 71 ++++++++++++++++++----------------- tests/test_octodns_manager.py | 39 +++++++++---------- 2 files changed, 57 insertions(+), 53 deletions(-) diff --git a/octodns/manager.py b/octodns/manager.py index 1c6a401..85de76b 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -69,6 +69,10 @@ class MainThreadExecutor(object): return MakeThreadFuture(func, args, kwargs) +class ManagerException(Exception): + pass + + class Manager(object): log = logging.getLogger('Manager') @@ -105,16 +109,16 @@ class Manager(object): _class = provider_config.pop('class') except KeyError: self.log.exception('Invalid provider class') - raise Exception('Provider {} is missing class' - .format(provider_name)) + raise ManagerException('Provider {} is missing class' + .format(provider_name)) _class = self._get_named_class('provider', _class) kwargs = self._build_kwargs(provider_config) try: self.providers[provider_name] = _class(provider_name, **kwargs) except TypeError: self.log.exception('Invalid provider config') - raise Exception('Incorrect provider config for {}' - .format(provider_name)) + raise ManagerException('Incorrect provider config for {}' + .format(provider_name)) zone_tree = {} # sort by reversed strings so that parent zones always come first @@ -148,8 +152,8 @@ class Manager(object): _class = plan_output_config.pop('class') except KeyError: self.log.exception('Invalid plan_output class') - raise Exception('plan_output {} is missing class' - .format(plan_output_name)) + raise ManagerException('plan_output {} is missing class' + .format(plan_output_name)) _class = self._get_named_class('plan_output', _class) kwargs = self._build_kwargs(plan_output_config) try: @@ -157,8 +161,8 @@ class Manager(object): _class(plan_output_name, **kwargs) except TypeError: self.log.exception('Invalid plan_output config') - raise Exception('Incorrect plan_output config for {}' - .format(plan_output_name)) + raise ManagerException('Incorrect plan_output config for {}' + .format(plan_output_name)) def _get_named_class(self, _type, _class): try: @@ -167,13 +171,15 @@ class Manager(object): except (ImportError, ValueError): self.log.exception('_get_{}_class: Unable to import ' 'module %s', _class) - raise Exception('Unknown {} class: {}'.format(_type, _class)) + raise ManagerException('Unknown {} class: {}' + .format(_type, _class)) try: return getattr(module, class_name) except AttributeError: self.log.exception('_get_{}_class: Unable to get class %s ' 'from module %s', class_name, module) - raise Exception('Unknown {} class: {}'.format(_type, _class)) + raise ManagerException('Unknown {} class: {}' + .format(_type, _class)) def _build_kwargs(self, source): # Build up the arguments we need to pass to the provider @@ -186,9 +192,9 @@ class Manager(object): v = environ[env_var] except KeyError: self.log.exception('Invalid provider config') - raise Exception('Incorrect provider config, ' - 'missing env var {}' - .format(env_var)) + raise ManagerException('Incorrect provider config, ' + 'missing env var {}' + .format(env_var)) except AttributeError: pass kwargs[k] = v @@ -256,12 +262,14 @@ class Manager(object): try: sources = config['sources'] except KeyError: - raise Exception('Zone {} is missing sources'.format(zone_name)) + raise ManagerException('Zone {} is missing sources' + .format(zone_name)) try: targets = config['targets'] except KeyError: - raise Exception('Zone {} is missing targets'.format(zone_name)) + raise ManagerException('Zone {} is missing targets' + .format(zone_name)) if eligible_targets: targets = list(filter(lambda d: d in eligible_targets, targets)) @@ -276,26 +284,23 @@ class Manager(object): self.log.info('sync: sources=%s -> targets=%s', sources, targets) try: - collected = [] - for source in sources: - collected.append(self.providers[source]) - sources = collected + sources = [self.providers[source] for source in sources] except KeyError: - raise Exception('Zone {}, unknown source: {}'.format(zone_name, - source)) + raise ManagerException('Zone {}, unknown source: {}' + .format(zone_name, source)) try: trgs = [] for target in targets: trg = self.providers[target] if not isinstance(trg, BaseProvider): - raise Exception('{} - "{}" does not support targeting' - .format(trg, target)) + raise ManagerException('{} - "{}" does not support ' + 'targeting'.format(trg, target)) trgs.append(trg) targets = trgs except KeyError: - raise Exception('Zone {}, unknown target: {}'.format(zone_name, - target)) + raise ManagerException('Zone {}, unknown target: {}' + .format(zone_name, target)) futures.append(self._executor.submit(self._populate_and_plan, zone_name, sources, targets)) @@ -348,7 +353,7 @@ class Manager(object): a = [self.providers[source] for source in a] b = [self.providers[source] for source in b] except KeyError as e: - raise Exception('Unknown source: {}'.format(e.args[0])) + raise ManagerException('Unknown source: {}'.format(e.args[0])) sub_zones = self.configured_sub_zones(zone) za = Zone(zone, sub_zones) @@ -374,7 +379,7 @@ class Manager(object): try: sources = [self.providers[s] for s in sources] except KeyError as e: - raise Exception('Unknown source: {}'.format(e.args[0])) + raise ManagerException('Unknown source: {}'.format(e.args[0])) clz = YamlProvider if split: @@ -397,16 +402,14 @@ class Manager(object): try: sources = config['sources'] except KeyError: - raise Exception('Zone {} is missing sources'.format(zone_name)) + raise ManagerException('Zone {} is missing sources' + .format(zone_name)) try: - collected = [] - for source in sources: - collected.append(self.providers[source]) - sources = collected + sources = [self.providers[source] for source in sources] except KeyError: - raise Exception('Zone {}, unknown source: {}'.format(zone_name, - source)) + raise ManagerException('Zone {}, unknown source: {}' + .format(zone_name, source)) for source in sources: if isinstance(source, YamlProvider): diff --git a/tests/test_octodns_manager.py b/tests/test_octodns_manager.py index 60a1922..13eea95 100644 --- a/tests/test_octodns_manager.py +++ b/tests/test_octodns_manager.py @@ -11,7 +11,8 @@ from six import text_type from unittest import TestCase from octodns.record import Record -from octodns.manager import _AggregateTarget, MainThreadExecutor, Manager +from octodns.manager import _AggregateTarget, MainThreadExecutor, Manager, \ + ManagerException from octodns.yaml import safe_load from octodns.zone import Zone @@ -28,77 +29,77 @@ def get_config_filename(which): class TestManager(TestCase): def test_missing_provider_class(self): - with self.assertRaises(Exception) as ctx: + with self.assertRaises(ManagerException) as ctx: Manager(get_config_filename('missing-provider-class.yaml')).sync() self.assertTrue('missing class' in text_type(ctx.exception)) def test_bad_provider_class(self): - with self.assertRaises(Exception) as ctx: + with self.assertRaises(ManagerException) as ctx: Manager(get_config_filename('bad-provider-class.yaml')).sync() self.assertTrue('Unknown provider class' in text_type(ctx.exception)) def test_bad_provider_class_module(self): - with self.assertRaises(Exception) as ctx: + with self.assertRaises(ManagerException) as ctx: Manager(get_config_filename('bad-provider-class-module.yaml')) \ .sync() self.assertTrue('Unknown provider class' in text_type(ctx.exception)) def test_bad_provider_class_no_module(self): - with self.assertRaises(Exception) as ctx: + with self.assertRaises(ManagerException) as ctx: Manager(get_config_filename('bad-provider-class-no-module.yaml')) \ .sync() self.assertTrue('Unknown provider class' in text_type(ctx.exception)) def test_missing_provider_config(self): # Missing provider config - with self.assertRaises(Exception) as ctx: + with self.assertRaises(ManagerException) as ctx: Manager(get_config_filename('missing-provider-config.yaml')).sync() self.assertTrue('provider config' in text_type(ctx.exception)) def test_missing_env_config(self): - with self.assertRaises(Exception) as ctx: + with self.assertRaises(ManagerException) as ctx: Manager(get_config_filename('missing-provider-env.yaml')).sync() self.assertTrue('missing env var' in text_type(ctx.exception)) def test_missing_source(self): - with self.assertRaises(Exception) as ctx: + with self.assertRaises(ManagerException) as ctx: Manager(get_config_filename('provider-problems.yaml')) \ .sync(['missing.sources.']) self.assertTrue('missing sources' in text_type(ctx.exception)) def test_missing_targets(self): - with self.assertRaises(Exception) as ctx: + with self.assertRaises(ManagerException) as ctx: Manager(get_config_filename('provider-problems.yaml')) \ .sync(['missing.targets.']) self.assertTrue('missing targets' in text_type(ctx.exception)) def test_unknown_source(self): - with self.assertRaises(Exception) as ctx: + with self.assertRaises(ManagerException) as ctx: Manager(get_config_filename('provider-problems.yaml')) \ .sync(['unknown.source.']) self.assertTrue('unknown source' in text_type(ctx.exception)) def test_unknown_target(self): - with self.assertRaises(Exception) as ctx: + with self.assertRaises(ManagerException) as ctx: Manager(get_config_filename('provider-problems.yaml')) \ .sync(['unknown.target.']) self.assertTrue('unknown target' in text_type(ctx.exception)) def test_bad_plan_output_class(self): - with self.assertRaises(Exception) as ctx: + with self.assertRaises(ManagerException) as ctx: name = 'bad-plan-output-missing-class.yaml' Manager(get_config_filename(name)).sync() self.assertEquals('plan_output bad is missing class', text_type(ctx.exception)) def test_bad_plan_output_config(self): - with self.assertRaises(Exception) as ctx: + with self.assertRaises(ManagerException) as ctx: Manager(get_config_filename('bad-plan-output-config.yaml')).sync() self.assertEqual('Incorrect plan_output config for bad', text_type(ctx.exception)) def test_source_only_as_a_target(self): - with self.assertRaises(Exception) as ctx: + with self.assertRaises(ManagerException) as ctx: Manager(get_config_filename('provider-problems.yaml')) \ .sync(['not.targetable.']) self.assertTrue('does not support targeting' in @@ -182,7 +183,7 @@ class TestManager(TestCase): 'unit.tests.') self.assertEquals(14, len(changes)) - with self.assertRaises(Exception) as ctx: + with self.assertRaises(ManagerException) as ctx: manager.compare(['nope'], ['dump'], 'unit.tests.') self.assertEquals('Unknown source: nope', text_type(ctx.exception)) @@ -222,7 +223,7 @@ class TestManager(TestCase): environ['YAML_TMP_DIR'] = tmpdir.dirname manager = Manager(get_config_filename('simple.yaml')) - with self.assertRaises(Exception) as ctx: + with self.assertRaises(ManagerException) as ctx: manager.dump('unit.tests.', tmpdir.dirname, False, False, 'nope') self.assertEquals('Unknown source: nope', text_type(ctx.exception)) @@ -251,7 +252,7 @@ class TestManager(TestCase): environ['YAML_TMP_DIR'] = tmpdir.dirname manager = Manager(get_config_filename('simple-split.yaml')) - with self.assertRaises(Exception) as ctx: + with self.assertRaises(ManagerException) as ctx: manager.dump('unit.tests.', tmpdir.dirname, False, True, 'nope') self.assertEquals('Unknown source: nope', text_type(ctx.exception)) @@ -267,12 +268,12 @@ class TestManager(TestCase): def test_validate_configs(self): Manager(get_config_filename('simple-validate.yaml')).validate_configs() - with self.assertRaises(Exception) as ctx: + with self.assertRaises(ManagerException) as ctx: Manager(get_config_filename('missing-sources.yaml')) \ .validate_configs() self.assertTrue('missing sources' in text_type(ctx.exception)) - with self.assertRaises(Exception) as ctx: + with self.assertRaises(ManagerException) as ctx: Manager(get_config_filename('unknown-provider.yaml')) \ .validate_configs() self.assertTrue('unknown source' in text_type(ctx.exception)) From 00fa158c59b243a80901e1802a437eece6a1264c Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Wed, 9 Oct 2019 15:31:27 -0700 Subject: [PATCH 051/155] filter -> [... if]s --- octodns/manager.py | 3 +- octodns/provider/base.py | 2 +- tests/test_octodns_provider_mythicbeasts.py | 12 +++---- tests/test_octodns_provider_yaml.py | 37 +++++++++------------ 4 files changed, 24 insertions(+), 30 deletions(-) diff --git a/octodns/manager.py b/octodns/manager.py index 85de76b..9a1b598 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -271,8 +271,7 @@ class Manager(object): raise ManagerException('Zone {} is missing targets' .format(zone_name)) if eligible_targets: - targets = list(filter(lambda d: d in eligible_targets, - targets)) + targets = [t for t in targets if t in eligible_targets] if not targets: # Don't bother planning (and more importantly populating) zones diff --git a/octodns/provider/base.py b/octodns/provider/base.py index 9f03e78..ae87844 100644 --- a/octodns/provider/base.py +++ b/octodns/provider/base.py @@ -60,7 +60,7 @@ class BaseProvider(BaseSource): # allow the provider to filter out false positives before = len(changes) - changes = list(filter(self._include_change, changes)) + changes = [c for c in changes if self._include_change(c)] after = len(changes) if before != after: self.log.info('plan: filtered out %s changes', before - after) diff --git a/tests/test_octodns_provider_mythicbeasts.py b/tests/test_octodns_provider_mythicbeasts.py index 498b408..960bd65 100644 --- a/tests/test_octodns_provider_mythicbeasts.py +++ b/tests/test_octodns_provider_mythicbeasts.py @@ -441,11 +441,11 @@ class TestMythicBeastsProvider(TestCase): plan = provider.plan(wanted) # Octo ignores NS records (15-1) - self.assertEquals(1, len(list(filter( - lambda u: isinstance(u, Update), plan.changes)))) - self.assertEquals(1, len(list(filter( - lambda d: isinstance(d, Delete), plan.changes)))) - self.assertEquals(14, len(list(filter( - lambda c: isinstance(c, Create), plan.changes)))) + self.assertEquals(1, len([c for c in plan.changes + if isinstance(c, Update)])) + self.assertEquals(1, len([c for c in plan.changes + if isinstance(c, Delete)])) + self.assertEquals(14, len([c for c in plan.changes + if isinstance(c, Create)])) self.assertEquals(16, provider.apply(plan)) self.assertTrue(plan.exists) diff --git a/tests/test_octodns_provider_yaml.py b/tests/test_octodns_provider_yaml.py index e16aaa5..a47dca1 100644 --- a/tests/test_octodns_provider_yaml.py +++ b/tests/test_octodns_provider_yaml.py @@ -58,9 +58,8 @@ class TestYamlProvider(TestCase): # We add everything plan = target.plan(zone) - self.assertEquals(15, len(list(filter(lambda c: - isinstance(c, Create), - plan.changes)))) + self.assertEquals(15, len([c for c in plan.changes + if isinstance(c, Create)])) self.assertFalse(isfile(yaml_file)) # Now actually do it @@ -69,9 +68,8 @@ class TestYamlProvider(TestCase): # Dynamic plan plan = target.plan(dynamic_zone) - self.assertEquals(5, len(list(filter(lambda c: - isinstance(c, Create), - plan.changes)))) + self.assertEquals(5, len([c for c in plan.changes + if isinstance(c, Create)])) self.assertFalse(isfile(dynamic_yaml_file)) # Apply it self.assertEquals(5, target.apply(plan)) @@ -82,17 +80,15 @@ class TestYamlProvider(TestCase): target.populate(reloaded) self.assertDictEqual( {'included': ['test']}, - list(filter( - lambda x: x.name == 'included', reloaded.records - ))[0]._octodns) + [x for x in reloaded.records + if x.name == 'included'][0]._octodns) self.assertFalse(zone.changes(reloaded, target=source)) # A 2nd sync should still create everything plan = target.plan(zone) - self.assertEquals(15, len(list(filter(lambda c: - isinstance(c, Create), - plan.changes)))) + self.assertEquals(15, len([c for c in plan.changes + if isinstance(c, Create)])) with open(yaml_file) as fh: data = safe_load(fh.read()) @@ -255,8 +251,8 @@ class TestSplitYamlProvider(TestCase): # We add everything plan = target.plan(zone) - self.assertEquals(15, len(list(filter( - lambda c: isinstance(c, Create), plan.changes)))) + self.assertEquals(15, len([c for c in plan.changes + if isinstance(c, Create)])) self.assertFalse(isdir(zone_dir)) # Now actually do it @@ -264,8 +260,8 @@ class TestSplitYamlProvider(TestCase): # Dynamic plan plan = target.plan(dynamic_zone) - self.assertEquals(5, len(list(filter( - lambda c: isinstance(c, Create), plan.changes)))) + self.assertEquals(5, len([c for c in plan.changes + if isinstance(c, Create)])) self.assertFalse(isdir(dynamic_zone_dir)) # Apply it self.assertEquals(5, target.apply(plan)) @@ -276,16 +272,15 @@ class TestSplitYamlProvider(TestCase): target.populate(reloaded) self.assertDictEqual( {'included': ['test']}, - list(filter( - lambda x: x.name == 'included', reloaded.records - ))[0]._octodns) + [x for x in reloaded.records + if x.name == 'included'][0]._octodns) self.assertFalse(zone.changes(reloaded, target=source)) # A 2nd sync should still create everything plan = target.plan(zone) - self.assertEquals(15, len(list(filter( - lambda c: isinstance(c, Create), plan.changes)))) + self.assertEquals(15, len([c for c in plan.changes + if isinstance(c, Create)])) yaml_file = join(zone_dir, '$unit.tests.yaml') self.assertTrue(isfile(yaml_file)) From 4d0bc29acc5286e382edf3af72c7c5b8f3656b21 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Wed, 9 Oct 2019 15:36:25 -0700 Subject: [PATCH 052/155] Remove a couple more filters( --- octodns/manager.py | 2 +- octodns/source/tinydns.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/octodns/manager.py b/octodns/manager.py index 9a1b598..1a86336 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -254,7 +254,7 @@ class Manager(object): zones = self.config['zones'].items() if eligible_zones: - zones = filter(lambda d: d[0] in eligible_zones, zones) + zones = [z for z in zones if z[0] in eligible_zones] futures = [] for zone_name, config in zones: diff --git a/octodns/source/tinydns.py b/octodns/source/tinydns.py index 0ee100a..9c44ed8 100755 --- a/octodns/source/tinydns.py +++ b/octodns/source/tinydns.py @@ -253,7 +253,7 @@ class TinyDnsFileSource(TinyDnsBaseSource): # Ignore hidden files continue with open(join(self.directory, filename), 'r') as fh: - lines += filter(lambda l: l, fh.read().split('\n')) + lines += [l for l in fh.read().split('\n') if l] self._cache = lines From 10ad30e7ea4a463838989258428245e0d7c55b1e Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Wed, 9 Oct 2019 15:41:18 -0700 Subject: [PATCH 053/155] map( to [...] --- octodns/provider/dyn.py | 4 ++-- octodns/provider/ovh.py | 3 +-- octodns/zone.py | 4 ++-- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/octodns/provider/dyn.py b/octodns/provider/dyn.py index 30ba8bd..f4f8e53 100644 --- a/octodns/provider/dyn.py +++ b/octodns/provider/dyn.py @@ -903,10 +903,10 @@ class DynProvider(BaseProvider): # Sort the values for consistent ordering so that we can compare values = sorted(values, key=_dynamic_value_sort_key) # Ensure that weight is included and if not use the default - values = list(map(lambda v: { + values = [{ 'value': v['value'], 'weight': v.get('weight', 1), - }, values)) + } for v in values] # Walk through our existing pools looking for a match we can use for pool in pools: diff --git a/octodns/provider/ovh.py b/octodns/provider/ovh.py index a0e47f8..17aff8d 100644 --- a/octodns/provider/ovh.py +++ b/octodns/provider/ovh.py @@ -326,8 +326,7 @@ class OvhProvider(BaseProvider): splitted = value.split('\\;') found_key = False for splitted_value in splitted: - sub_split = list(map(lambda x: x.strip(), - splitted_value.split("=", 1))) + sub_split = [x.strip() for x in splitted_value.split("=", 1)] if len(sub_split) < 2: return False key, value = sub_split[0], sub_split[1] diff --git a/octodns/zone.py b/octodns/zone.py index 1191b6f..5f099ac 100644 --- a/octodns/zone.py +++ b/octodns/zone.py @@ -84,8 +84,8 @@ class Zone(object): raise DuplicateRecordException('Duplicate record {}, type {}' .format(record.fqdn, record._type)) - elif not lenient and (((record._type == 'CNAME' and len(node) > 0) or - ('CNAME' in map(lambda r: r._type, node)))): + elif not lenient and ((record._type == 'CNAME' and len(node) > 0) or + ('CNAME' in [r._type for r in node])): # We're adding a CNAME to existing records or adding to an existing # CNAME raise InvalidNodeException('Invalid state, CNAME at {} cannot ' From b5c75d189c7d9b0fe4f30abe8f8e074c1a1e36dd Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Wed, 9 Oct 2019 16:01:39 -0700 Subject: [PATCH 054/155] Convert sources building back out to for x in y from list comprehension --- octodns/manager.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/octodns/manager.py b/octodns/manager.py index 1a86336..14a8f07 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -283,7 +283,10 @@ class Manager(object): self.log.info('sync: sources=%s -> targets=%s', sources, targets) try: - sources = [self.providers[source] for source in sources] + collected = [] + for source in sources: + collected.append(self.providers[source]) + sources = collected except KeyError: raise ManagerException('Zone {}, unknown source: {}' .format(zone_name, source)) @@ -405,7 +408,10 @@ class Manager(object): .format(zone_name)) try: - sources = [self.providers[source] for source in sources] + collected = [] + for source in sources: + collected.append(self.providers[source]) + sources = collected except KeyError: raise ManagerException('Zone {}, unknown source: {}' .format(zone_name, source)) From 6959b58b7533a94de78d49e0db706ee182009339 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Wed, 9 Oct 2019 16:03:06 -0700 Subject: [PATCH 055/155] Update requirements and setup.py, remove incf.countryutils, promote pycountry-convert --- requirements-dev.txt | 2 -- requirements.txt | 6 ++++-- setup.py | 3 ++- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index a2833ae..d9888b8 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,8 +2,6 @@ coverage mock nose pycodestyle==2.4.0 -pycountry>=18.12.8 -pycountry_convert>=0.7.2 pyflakes==1.6.0 readme_renderer[md]==24.0 requests_mock diff --git a/requirements.txt b/requirements.txt index 4d98736..204fd63 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,8 +6,8 @@ botocore==1.10.5 dnspython==1.15.0 docutils==0.14 dyn==1.8.1 -futures==3.2.0; python_version < '3.0' edgegrid-python==1.1.1 +futures==3.2.0; python_version < '3.0' google-cloud-core==0.28.1 google-cloud-dns==0.29.0 ipaddress==1.0.22 @@ -16,9 +16,11 @@ msrestazure==0.6.2 natsort==5.5.0 ns1-python==0.12.0 ovh==0.4.8 +pycountry-convert==0.7.2 +pycountry==19.8.18 python-dateutil==2.6.1 requests==2.22.0 s3transfer==0.1.13 -six==1.12.0 setuptools==38.5.2 +six==1.12.0 transip==2.0.0 diff --git a/setup.py b/setup.py index 5cb741b..d58ed2c 100644 --- a/setup.py +++ b/setup.py @@ -66,9 +66,10 @@ setup( 'PyYaml>=4.2b1', 'dnspython>=1.15.0', 'futures>=3.2.0', - 'incf.countryutils>=1.0', 'ipaddress>=1.0.22', 'natsort>=5.5.0', + 'pycountry>=19.8.18', + 'pycountry-convert>=0.7.2', # botocore doesn't like >=2.7.0 for some reason 'python-dateutil>=2.6.0,<2.7.0', 'requests>=2.20.0' From 294c2cc9b3f8a1820437a16436ace7c4508f65e1 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Wed, 9 Oct 2019 16:07:03 -0700 Subject: [PATCH 056/155] Update CHANGELOG for python 3 work --- CHANGELOG.md | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e30d0df..7e5e6ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,28 @@ ## v0.9.9 - 2019-??-?? - Python 3.7 Support -* Route53 _mod_keyer ordering wasn't complete/reliable and in python 3 this - resulted in randomness. This has been addressed and may result in value - reordering on next plan, no actual changes in behavior should occur. +* Extensive pass through the whole codebase to support Python 3 + * Tons of updates to replace `def __cmp__` with `__eq__` and friends to + preserve custom equality and ordering behaviors that are essential to + octoDNS's processes. + * Quite a few objects required the addition of `__eq__` and friends so that + they're sortable in Python 3 now that those things are more strict. A few + places this required jumping through hoops of sorts. Thankfully our tests + are pretty thorough and caught a lot of issues and hopefully the whole + plan, review, apply process will backstop that. + * Explicit ordering of changes by (name, type) to address inconsistent + ordering for a number of providers that just convert changes into API + calls as they come. Python 2 sets ordered consistently, Python 3 they do + not. https://github.com/github/octodns/pull/384/commits/7958233fccf9ea22d95e2fd06c48d7d0a4529e26 + * Route53 _mod_keyer ordering wasn't 100% complete and thus unreliable and + random in Python 3. This has been addressed and may result in value + reordering on next plan, no actual changes in behavior should occur. + * `incf.countryutils` (in pypi) was last released in 2009 is not python 3 + compatible (it's country data is also pretty stale.) `pycountry_convert` + appears to have the functionality required to replace its usage so it has + been removed as a dependency/requirement. + * Bunch of additional unit tests and supporting config to exercise new code + and verify things that were run into during the Python 3 work + * lots of `six`ing of things ## v0.9.8 - 2019-09-30 - One with no changes b/c PyPi description problems From baa1f7472f0127aab0111ef18486d290f7f7714e Mon Sep 17 00:00:00 2001 From: Adam Smith Date: Sun, 13 Oct 2019 21:07:30 -0700 Subject: [PATCH 057/155] ConstellixProvider: change ALIAS to CNAME to allow record deletion --- octodns/provider/constellix.py | 4 ++++ tests/test_octodns_provider_constellix.py | 15 ++++++++++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/octodns/provider/constellix.py b/octodns/provider/constellix.py index 939284d..5bd506d 100644 --- a/octodns/provider/constellix.py +++ b/octodns/provider/constellix.py @@ -148,6 +148,10 @@ class ConstellixClient(object): self._request('POST', path, data=params) def record_delete(self, zone_name, record_type, record_id): + # change ALIAS records to ANAME + if record_type == 'ALIAS': + record_type = 'ANAME' + zone_id = self.domains.get(zone_name, False) path = '/{}/records/{}/{}'.format(zone_id, record_type, record_id) self._request('DELETE', path) diff --git a/tests/test_octodns_provider_constellix.py b/tests/test_octodns_provider_constellix.py index 346bb17..80489c8 100644 --- a/tests/test_octodns_provider_constellix.py +++ b/tests/test_octodns_provider_constellix.py @@ -187,6 +187,14 @@ class TestConstellixProvider(TestCase): 'value': [ '3.2.3.4' ] + }, { + 'id': 11189899, + 'type': 'ALIAS', + 'name': 'alias', + 'ttl': 600, + 'value': [{ + 'value': 'aname.unit.tests.' + }] } ]) @@ -201,8 +209,8 @@ class TestConstellixProvider(TestCase): })) plan = provider.plan(wanted) - self.assertEquals(2, len(plan.changes)) - self.assertEquals(2, provider.apply(plan)) + self.assertEquals(3, len(plan.changes)) + self.assertEquals(3, provider.apply(plan)) # recreate for update, and deletes for the 2 parts of the other provider._client._request.assert_has_calls([ @@ -214,5 +222,6 @@ class TestConstellixProvider(TestCase): 'ttl': 300 }), call('DELETE', '/123123/records/A/11189897'), - call('DELETE', '/123123/records/A/11189898') + call('DELETE', '/123123/records/A/11189898'), + call('DELETE', '/123123/records/ANAME/11189899') ], any_order=True) From b3bd4382cc6c32c45bf874320a99c241b314c26e Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 14 Oct 2019 07:32:09 -0700 Subject: [PATCH 058/155] Apply suggestions from code review Co-Authored-By: Theo Julienne --- octodns/manager.py | 4 ++++ octodns/provider/rackspace.py | 2 +- octodns/record/__init__.py | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/octodns/manager.py b/octodns/manager.py index 14a8f07..bc78f5b 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -283,6 +283,8 @@ class Manager(object): self.log.info('sync: sources=%s -> targets=%s', sources, targets) try: + # rather than using a list comprehension, we break this loop out + # so that the `except` block below can reference the `source` collected = [] for source in sources: collected.append(self.providers[source]) @@ -408,6 +410,8 @@ class Manager(object): .format(zone_name)) try: + # rather than using a list comprehension, we break this loop out + # so that the `except` block below can reference the `source` collected = [] for source in sources: collected.append(self.providers[source]) diff --git a/octodns/provider/rackspace.py b/octodns/provider/rackspace.py index 28b7f05..7fed05b 100644 --- a/octodns/provider/rackspace.py +++ b/octodns/provider/rackspace.py @@ -14,7 +14,7 @@ from .base import BaseProvider def _value_keyer(v): - return '{}-{}-{}'.format(v.get('type', ''), v['name'], v.get('data', '')) + return (v.get('type', ''), v['name'], v.get('data', '')) def add_trailing_dot(s): diff --git a/octodns/record/__init__.py b/octodns/record/__init__.py index 47d2e9b..c0f6482 100644 --- a/octodns/record/__init__.py +++ b/octodns/record/__init__.py @@ -926,7 +926,7 @@ class MxValue(object): } def __hash__(self): - return hash('{} {}'.format(self.preference, self.exchange)) + return hash((self.preference, self.exchange)) def __eq__(self, other): return ((self.preference, self.exchange) == From 25b41a4a92cfde89f157c0e116908b509c63a266 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 14 Oct 2019 07:47:36 -0700 Subject: [PATCH 059/155] Adopt Route53Provider _equality_tuple suggestion --- octodns/provider/route53.py | 70 +++++++------------------------------ 1 file changed, 12 insertions(+), 58 deletions(-) diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index 6d2cfeb..decbd4d 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -156,34 +156,29 @@ class _Route53Record(object): 'sub-classes should never use this method' return '{}:{}'.format(self.fqdn, self._type).__hash__() + def _equality_tuple(self): + return (self.__class__.__name__, self.fqdn, self._type) + def __eq__(self, other): '''Sub-classes should call up to this and return its value if true. When it's false they should compute their own __eq__, same for other ordering methods.''' - return self.__class__.__name__ == other.__class__.__name__ and \ - self.fqdn == other.fqdn and \ - self._type == other._type + return self._equality_tuple() == other._equality_tuple() def __ne__(self, other): - return self.__class__.__name__ != other.__class__.__name__ or \ - self.fqdn != other.fqdn or \ - self._type != other._type + return self._equality_tuple() != other._equality_tuple() def __lt__(self, other): - return (((self.__class__.__name__, self.fqdn, self._type)) < - ((other.__class__.__name__, other.fqdn, other._type))) + return self._equality_tuple() < other._equality_tuple() def __le__(self, other): - return (((self.__class__.__name__, self.fqdn, self._type)) <= - ((other.__class__.__name__, other.fqdn, other._type))) + return self._equality_tuple() <= other._equality_tuple() def __gt__(self, other): - return (((self.__class__.__name__, self.fqdn, self._type)) > - ((other.__class__.__name__, other.fqdn, other._type))) + return self._equality_tuple() > other._equality_tuple() def __ge__(self, other): - return (((self.__class__.__name__, self.fqdn, self._type)) >= - ((other.__class__.__name__, other.fqdn, other._type))) + return self._equality_tuple() >= other._equality_tuple() def __repr__(self): return '_Route53Record<{} {} {} {}>'.format(self.fqdn, self._type, @@ -524,50 +519,9 @@ class _Route53GeoRecord(_Route53Record): return '{}:{}:{}'.format(self.fqdn, self._type, self.geo.code).__hash__() - def __eq__(self, other): - return super(_Route53GeoRecord, self).__eq__(other) and \ - self.geo.code == other.geo.code - - def __ne__(self, other): - # super will handle class != class, so if it's true we have 2 geo - # objects with the same name and type, so just need to compare codes - return super(_Route53GeoRecord, self).__ne__(other) or \ - self.geo.code != other.geo.code - - def __lt__(self, other): - # super eq will check class, name, and type - if super(_Route53GeoRecord, self).__eq__(other): - # if it's True we're dealing with two geo's with the same name and - # type, so we just need to compare codes - return self.geo.code < other.geo.code - # Super is not equal so we'll just let it decide lt - return super(_Route53GeoRecord, self).__lt__(other) - - def __le__(self, other): - # super eq will check class, name, and type - if super(_Route53GeoRecord, self).__eq__(other): - # Just need to compare codes, everything else is equal - return self.geo.code <= other.geo.code - # Super is not equal so geo.code doesn't matter, let it decide with lt, - # can't be eq - return super(_Route53GeoRecord, self).__lt__(other) - - def __gt__(self, other): - # super eq will check class, name, and type - if super(_Route53GeoRecord, self).__eq__(other): - # Just need to compare codes, everything else is equal - return self.geo.code > other.geo.code - # Super is not equal so we'll just let it decide gt - return super(_Route53GeoRecord, self).__gt__(other) - - def __ge__(self, other): - # super eq will check class, name, and type - if super(_Route53GeoRecord, self).__eq__(other): - # Just need to compare codes, everything else is equal - return self.geo.code >= other.geo.code - # Super is not equal so geo.code doesn't matter, let it decide with gt, - # can't be eq - return super(_Route53GeoRecord, self).__gt__(other) + def _equality_tuple(self): + return super(_Route53GeoRecord, self)._equality_tuple() + \ + (self.geo.code,) def __repr__(self): return '_Route53GeoRecord<{} {} {} {} {}>'.format(self.fqdn, From b8e2ec124b1783f0e57ea435da0e93ef694eb750 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 14 Oct 2019 07:48:17 -0700 Subject: [PATCH 060/155] Fix Manager comment wrapping --- octodns/manager.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/octodns/manager.py b/octodns/manager.py index bc78f5b..3c91aa9 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -283,8 +283,9 @@ class Manager(object): self.log.info('sync: sources=%s -> targets=%s', sources, targets) try: - # rather than using a list comprehension, we break this loop out - # so that the `except` block below can reference the `source` + # rather than using a list comprehension, we break this loop + # out so that the `except` block below can reference the + # `source` collected = [] for source in sources: collected.append(self.providers[source]) @@ -410,8 +411,9 @@ class Manager(object): .format(zone_name)) try: - # rather than using a list comprehension, we break this loop out - # so that the `except` block below can reference the `source` + # rather than using a list comprehension, we break this loop + # out so that the `except` block below can reference the + # `source` collected = [] for source in sources: collected.append(self.providers[source]) From 74048bf974a81d0b85802b6bb1da617ecf07cbe7 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 14 Oct 2019 07:48:47 -0700 Subject: [PATCH 061/155] Use if, else rather than try, except KeyError --- octodns/provider/route53.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index decbd4d..968d8b8 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -564,9 +564,9 @@ def _mod_keyer(mod): if rrset.get('GeoLocation', False): unique_id = rrset['SetIdentifier'] else: - try: + if 'SetIdentifier' in rrset: unique_id = '{}-{}'.format(rrset['Name'], rrset['SetIdentifier']) - except KeyError: + else: unique_id = rrset['Name'] # Prioritise within the action_priority, ensuring targets come first. From 2b33f95c1759e61698797963c04727cf19f1893e Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 14 Oct 2019 08:13:07 -0700 Subject: [PATCH 062/155] EqualityTupleMixin impl, use everywhere we were doing tuple compares --- octodns/equality.py | 30 +++++ octodns/provider/route53.py | 26 +---- octodns/record/__init__.py | 204 ++++----------------------------- tests/test_octodns_equality.py | 68 +++++++++++ 4 files changed, 126 insertions(+), 202 deletions(-) create mode 100644 octodns/equality.py create mode 100644 tests/test_octodns_equality.py diff --git a/octodns/equality.py b/octodns/equality.py new file mode 100644 index 0000000..965b8a0 --- /dev/null +++ b/octodns/equality.py @@ -0,0 +1,30 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + + +class EqualityTupleMixin: + + def _equality_tuple(self): + raise NotImplementedError('_equality_tuple method not implemented') + + def __eq__(self, other): + return self._equality_tuple() == other._equality_tuple() + + def __ne__(self, other): + return self._equality_tuple() != other._equality_tuple() + + def __lt__(self, other): + return self._equality_tuple() < other._equality_tuple() + + def __le__(self, other): + return self._equality_tuple() <= other._equality_tuple() + + def __gt__(self, other): + return self._equality_tuple() > other._equality_tuple() + + def __ge__(self, other): + return self._equality_tuple() >= other._equality_tuple() diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index 968d8b8..66da6b5 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -16,6 +16,7 @@ import re from six import text_type +from ..equality import EqualityTupleMixin from ..record import Record, Update from ..record.geo import GeoCodes from .base import BaseProvider @@ -29,7 +30,7 @@ def _octal_replace(s): return octal_re.sub(lambda m: chr(int(m.group(1), 8)), s) -class _Route53Record(object): +class _Route53Record(EqualityTupleMixin): @classmethod def _new_dynamic(cls, provider, record, hosted_zone_id, creating): @@ -157,29 +158,10 @@ class _Route53Record(object): return '{}:{}'.format(self.fqdn, self._type).__hash__() def _equality_tuple(self): + '''Sub-classes should call up to this and return its value and add + any additional fields they need to hav considered.''' return (self.__class__.__name__, self.fqdn, self._type) - def __eq__(self, other): - '''Sub-classes should call up to this and return its value if true. - When it's false they should compute their own __eq__, same for other - ordering methods.''' - return self._equality_tuple() == other._equality_tuple() - - def __ne__(self, other): - return self._equality_tuple() != other._equality_tuple() - - def __lt__(self, other): - return self._equality_tuple() < other._equality_tuple() - - def __le__(self, other): - return self._equality_tuple() <= other._equality_tuple() - - def __gt__(self, other): - return self._equality_tuple() > other._equality_tuple() - - def __ge__(self, other): - return self._equality_tuple() >= other._equality_tuple() - def __repr__(self): return '_Route53Record<{} {} {} {}>'.format(self.fqdn, self._type, self.ttl, self.values) diff --git a/octodns/record/__init__.py b/octodns/record/__init__.py index c0f6482..0847018 100644 --- a/octodns/record/__init__.py +++ b/octodns/record/__init__.py @@ -11,6 +11,7 @@ import re from six import string_types, text_type +from ..equality import EqualityTupleMixin from .geo import GeoCodes @@ -76,7 +77,7 @@ class ValidationError(Exception): self.reasons = reasons -class Record(object): +class Record(EqualityTupleMixin): log = getLogger('Record') @classmethod @@ -209,30 +210,15 @@ class Record(object): def __hash__(self): return '{}:{}'.format(self.name, self._type).__hash__() - def __eq__(self, other): - return ((self.name, self._type) == (other.name, other._type)) - - def __ne__(self, other): - return ((self.name, self._type) != (other.name, other._type)) - - def __lt__(self, other): - return ((self.name, self._type) < (other.name, other._type)) - - def __le__(self, other): - return ((self.name, self._type) <= (other.name, other._type)) - - def __gt__(self, other): - return ((self.name, self._type) > (other.name, other._type)) - - def __ge__(self, other): - return ((self.name, self._type) >= (other.name, other._type)) + def _equality_tuple(self): + return (self.name, self._type) def __repr__(self): # Make sure this is always overridden raise NotImplementedError('Abstract base class, __repr__ required') -class GeoValue(object): +class GeoValue(EqualityTupleMixin): geo_re = re.compile(r'^(?P\w\w)(-(?P\w\w)' r'(-(?P\w\w))?)?$') @@ -259,35 +245,9 @@ class GeoValue(object): yield '-'.join(bits) bits.pop() - def __eq__(self, other): - return ((self.continent_code, self.country_code, self.subdivision_code, - self.values) == (other.continent_code, other.country_code, - other.subdivision_code, other.values)) - - def __ne__(self, other): - return ((self.continent_code, self.country_code, self.subdivision_code, - self.values) != (other.continent_code, other.country_code, - other.subdivision_code, other.values)) - - def __lt__(self, other): - return ((self.continent_code, self.country_code, self.subdivision_code, - self.values) < (other.continent_code, other.country_code, - other.subdivision_code, other.values)) - - def __le__(self, other): - return ((self.continent_code, self.country_code, self.subdivision_code, - self.values) <= (other.continent_code, other.country_code, - other.subdivision_code, other.values)) - - def __gt__(self, other): - return ((self.continent_code, self.country_code, self.subdivision_code, - self.values) > (other.continent_code, other.country_code, - other.subdivision_code, other.values)) - - def __ge__(self, other): - return ((self.continent_code, self.country_code, self.subdivision_code, - self.values) >= (other.continent_code, other.country_code, - other.subdivision_code, other.values)) + def _equality_tuple(self): + return (self.continent_code, self.country_code, self.subdivision_code, + self.values) def __repr__(self): return "'Geo {} {} {} {}'".format(self.continent_code, @@ -787,7 +747,7 @@ class AliasRecord(_ValueMixin, Record): _value_type = AliasValue -class CaaValue(object): +class CaaValue(EqualityTupleMixin): # https://tools.ietf.org/html/rfc6844#page-5 @classmethod @@ -826,29 +786,8 @@ class CaaValue(object): 'value': self.value, } - def __eq__(self, other): - return ((self.flags, self.tag, self.value) == - (other.flags, other.tag, other.value)) - - def __ne__(self, other): - return ((self.flags, self.tag, self.value) != - (other.flags, other.tag, other.value)) - - def __lt__(self, other): - return ((self.flags, self.tag, self.value) < - (other.flags, other.tag, other.value)) - - def __le__(self, other): - return ((self.flags, self.tag, self.value) <= - (other.flags, other.tag, other.value)) - - def __gt__(self, other): - return ((self.flags, self.tag, self.value) > - (other.flags, other.tag, other.value)) - - def __ge__(self, other): - return ((self.flags, self.tag, self.value) >= - (other.flags, other.tag, other.value)) + def _equality_tuple(self): + return (self.flags, self.tag, self.value) def __repr__(self): return '{} {} "{}"'.format(self.flags, self.tag, self.value) @@ -872,7 +811,7 @@ class CnameRecord(_DynamicMixin, _ValueMixin, Record): return reasons -class MxValue(object): +class MxValue(EqualityTupleMixin): @classmethod def validate(cls, data, _type): @@ -928,29 +867,8 @@ class MxValue(object): def __hash__(self): return hash((self.preference, self.exchange)) - def __eq__(self, other): - return ((self.preference, self.exchange) == - (other.preference, other.exchange)) - - def __ne__(self, other): - return ((self.preference, self.exchange) != - (other.preference, other.exchange)) - - def __lt__(self, other): - return ((self.preference, self.exchange) < - (other.preference, other.exchange)) - - def __le__(self, other): - return ((self.preference, self.exchange) <= - (other.preference, other.exchange)) - - def __gt__(self, other): - return ((self.preference, self.exchange) > - (other.preference, other.exchange)) - - def __ge__(self, other): - return ((self.preference, self.exchange) >= - (other.preference, other.exchange)) + def _equality_tuple(self): + return (self.preference, self.exchange) def __repr__(self): return "'{} {}'".format(self.preference, self.exchange) @@ -961,7 +879,7 @@ class MxRecord(_ValuesMixin, Record): _value_type = MxValue -class NaptrValue(object): +class NaptrValue(EqualityTupleMixin): VALID_FLAGS = ('S', 'A', 'U', 'P') @classmethod @@ -1023,41 +941,9 @@ class NaptrValue(object): def __hash__(self): return hash(self.__repr__()) - def __eq__(self, other): - return ((self.order, self.preference, self.flags, self.service, - self.regexp, self.replacement) == - (other.order, other.preference, other.flags, other.service, - other.regexp, other.replacement)) - - def __ne__(self, other): - return ((self.order, self.preference, self.flags, self.service, - self.regexp, self.replacement) != - (other.order, other.preference, other.flags, other.service, - other.regexp, other.replacement)) - - def __lt__(self, other): - return ((self.order, self.preference, self.flags, self.service, - self.regexp, self.replacement) < - (other.order, other.preference, other.flags, other.service, - other.regexp, other.replacement)) - - def __le__(self, other): - return ((self.order, self.preference, self.flags, self.service, - self.regexp, self.replacement) <= - (other.order, other.preference, other.flags, other.service, - other.regexp, other.replacement)) - - def __gt__(self, other): - return ((self.order, self.preference, self.flags, self.service, - self.regexp, self.replacement) > - (other.order, other.preference, other.flags, other.service, - other.regexp, other.replacement)) - - def __ge__(self, other): - return ((self.order, self.preference, self.flags, self.service, - self.regexp, self.replacement) >= - (other.order, other.preference, other.flags, other.service, - other.regexp, other.replacement)) + def _equality_tuple(self): + return (self.order, self.preference, self.flags, self.service, + self.regexp, self.replacement) def __repr__(self): flags = self.flags if self.flags is not None else '' @@ -1107,7 +993,7 @@ class PtrRecord(_ValueMixin, Record): _value_type = PtrValue -class SshfpValue(object): +class SshfpValue(EqualityTupleMixin): VALID_ALGORITHMS = (1, 2, 3, 4) VALID_FINGERPRINT_TYPES = (1, 2) @@ -1161,29 +1047,8 @@ class SshfpValue(object): def __hash__(self): return hash(self.__repr__()) - def __eq__(self, other): - return ((self.algorithm, self.fingerprint_type, self.fingerprint) == - (other.algorithm, other.fingerprint_type, other.fingerprint)) - - def __ne__(self, other): - return ((self.algorithm, self.fingerprint_type, self.fingerprint) != - (other.algorithm, other.fingerprint_type, other.fingerprint)) - - def __lt__(self, other): - return ((self.algorithm, self.fingerprint_type, self.fingerprint) < - (other.algorithm, other.fingerprint_type, other.fingerprint)) - - def __le__(self, other): - return ((self.algorithm, self.fingerprint_type, self.fingerprint) <= - (other.algorithm, other.fingerprint_type, other.fingerprint)) - - def __gt__(self, other): - return ((self.algorithm, self.fingerprint_type, self.fingerprint) > - (other.algorithm, other.fingerprint_type, other.fingerprint)) - - def __ge__(self, other): - return ((self.algorithm, self.fingerprint_type, self.fingerprint) >= - (other.algorithm, other.fingerprint_type, other.fingerprint)) + def _equality_tuple(self): + return (self.algorithm, self.fingerprint_type, self.fingerprint) def __repr__(self): return "'{} {} {}'".format(self.algorithm, self.fingerprint_type, @@ -1244,7 +1109,7 @@ class SpfRecord(_ChunkedValuesMixin, Record): _value_type = _ChunkedValue -class SrvValue(object): +class SrvValue(EqualityTupleMixin): @classmethod def validate(cls, data, _type): @@ -1302,29 +1167,8 @@ class SrvValue(object): def __hash__(self): return hash(self.__repr__()) - def __eq__(self, other): - return ((self.priority, self.weight, self.port, self.target) == - (other.priority, other.weight, other.port, other.target)) - - def __ne__(self, other): - return ((self.priority, self.weight, self.port, self.target) != - (other.priority, other.weight, other.port, other.target)) - - def __lt__(self, other): - return ((self.priority, self.weight, self.port, self.target) < - (other.priority, other.weight, other.port, other.target)) - - def __le__(self, other): - return ((self.priority, self.weight, self.port, self.target) <= - (other.priority, other.weight, other.port, other.target)) - - def __gt__(self, other): - return ((self.priority, self.weight, self.port, self.target) > - (other.priority, other.weight, other.port, other.target)) - - def __ge__(self, other): - return ((self.priority, self.weight, self.port, self.target) >= - (other.priority, other.weight, other.port, other.target)) + def _equality_tuple(self): + return (self.priority, self.weight, self.port, self.target) def __repr__(self): return "'{} {} {} {}'".format(self.priority, self.weight, self.port, diff --git a/tests/test_octodns_equality.py b/tests/test_octodns_equality.py new file mode 100644 index 0000000..dcdc460 --- /dev/null +++ b/tests/test_octodns_equality.py @@ -0,0 +1,68 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from unittest import TestCase + +from octodns.equality import EqualityTupleMixin + + +class TestEqualityTupleMixin(TestCase): + + def test_basics(self): + + class Simple(EqualityTupleMixin): + + def __init__(self, a, b, c): + self.a = a + self.b = b + self.c = c + + def _equality_tuple(self): + return (self.a, self.b) + + one = Simple(1, 2, 3) + same = Simple(1, 2, 3) + matches = Simple(1, 2, 'ignored') + doesnt = Simple(2, 3, 4) + + # equality + self.assertEquals(one, one) + self.assertEquals(one, same) + self.assertEquals(same, one) + # only a & c are considered + self.assertEquals(one, matches) + self.assertEquals(matches, one) + self.assertNotEquals(one, doesnt) + self.assertNotEquals(doesnt, one) + + # lt + self.assertTrue(one < doesnt) + self.assertFalse(doesnt < one) + self.assertFalse(one < same) + + # le + self.assertTrue(one <= doesnt) + self.assertFalse(doesnt <= one) + self.assertTrue(one <= same) + + # gt + self.assertFalse(one > doesnt) + self.assertTrue(doesnt > one) + self.assertFalse(one > same) + + # ge + self.assertFalse(one >= doesnt) + self.assertTrue(doesnt >= one) + self.assertTrue(one >= same) + + def test_not_implemented(self): + + class MissingMethod(EqualityTupleMixin): + pass + + with self.assertRaises(NotImplementedError): + MissingMethod() == MissingMethod() From e1daccc0b762181f96fbc86cd0b300d2f36655ac Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 14 Oct 2019 08:14:30 -0700 Subject: [PATCH 063/155] Update CHANGELOG.md Co-Authored-By: Theo Julienne --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e5e6ac..503e53c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,7 @@ ordering for a number of providers that just convert changes into API calls as they come. Python 2 sets ordered consistently, Python 3 they do not. https://github.com/github/octodns/pull/384/commits/7958233fccf9ea22d95e2fd06c48d7d0a4529e26 - * Route53 _mod_keyer ordering wasn't 100% complete and thus unreliable and + * Route53 `_mod_keyer` ordering wasn't 100% complete and thus unreliable and random in Python 3. This has been addressed and may result in value reordering on next plan, no actual changes in behavior should occur. * `incf.countryutils` (in pypi) was last released in 2009 is not python 3 From 759c44f35be46f50e87ed9dc7022c177504a4ecc Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 14 Oct 2019 08:39:45 -0700 Subject: [PATCH 064/155] EqualityTupleMixin needs an explicit inhert from object to make 2.7 happy --- octodns/equality.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/equality.py b/octodns/equality.py index 965b8a0..bd22c7d 100644 --- a/octodns/equality.py +++ b/octodns/equality.py @@ -6,7 +6,7 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -class EqualityTupleMixin: +class EqualityTupleMixin(object): def _equality_tuple(self): raise NotImplementedError('_equality_tuple method not implemented') From 4e09f8a838e767aaefa3886ec8208dce9345ab37 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 14 Oct 2019 19:07:12 -0700 Subject: [PATCH 065/155] Use six.StringIO in setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d58ed2c..362db58 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -from StringIO import StringIO +from six import StringIO from os.path import dirname, join import octodns From 2f45cbc086d11f74c91e4b50da9224c6079849a5 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 14 Oct 2019 19:10:47 -0700 Subject: [PATCH 066/155] No six for setup.py, try/except both options --- setup.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 362db58..6ae8fd5 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,9 @@ #!/usr/bin/env python -from six import StringIO +try: + from io import StringIO +except ImportError: + from StringIO import StringIO from os.path import dirname, join import octodns From 4a41c98c1681ebad6c8b9630301284fd4df25d01 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 14 Oct 2019 19:14:38 -0700 Subject: [PATCH 067/155] setup.py install_requires futures only on 2.7 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6ae8fd5..b6c2cc4 100644 --- a/setup.py +++ b/setup.py @@ -68,7 +68,7 @@ setup( install_requires=[ 'PyYaml>=4.2b1', 'dnspython>=1.15.0', - 'futures>=3.2.0', + 'futures>=3.2.0; python_version<"3.2"', 'ipaddress>=1.0.22', 'natsort>=5.5.0', 'pycountry>=19.8.18', From 6f9842301ed4e21b80bd36dfb1ac96dbb98c7d4d Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 14 Oct 2019 19:31:04 -0700 Subject: [PATCH 068/155] Prefer StringIO.StringIO over io. --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index b6c2cc4..4f28232 100644 --- a/setup.py +++ b/setup.py @@ -1,9 +1,9 @@ #!/usr/bin/env python try: - from io import StringIO -except ImportError: from StringIO import StringIO +except ImportError: + from io import StringIO from os.path import dirname, join import octodns From 9e948aa4c88e1e679659673d1a259994c094ef17 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 18 Oct 2019 10:36:14 -0700 Subject: [PATCH 069/155] Validate Record name & fqdn length --- octodns/record/__init__.py | 37 ++++++++++++++++++++++-------------- tests/test_octodns_record.py | 28 +++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 14 deletions(-) diff --git a/octodns/record/__init__.py b/octodns/record/__init__.py index 0847018..98c1836 100644 --- a/octodns/record/__init__.py +++ b/octodns/record/__init__.py @@ -82,6 +82,7 @@ class Record(EqualityTupleMixin): @classmethod def new(cls, zone, name, data, source=None, lenient=False): + name = text_type(name) fqdn = '{}.{}'.format(name, zone.name) if name else zone.name try: _type = data['type'] @@ -105,7 +106,7 @@ class Record(EqualityTupleMixin): }[_type] except KeyError: raise Exception('Unknown record type: "{}"'.format(_type)) - reasons = _class.validate(name, data) + reasons = _class.validate(name, fqdn, data) try: lenient |= data['octodns']['lenient'] except KeyError: @@ -118,8 +119,16 @@ class Record(EqualityTupleMixin): return _class(zone, name, data, source=source) @classmethod - def validate(cls, name, data): + def validate(cls, name, fqdn, data): reasons = [] + n = len(fqdn) + if n > 253: + reasons.append('invalid fqdn, "{}" is too long at {} chars, max ' + 'is 253'.format(fqdn, n)) + n = len(name) + if n > 63: + reasons.append('invalid name, "{}" is too long at {} chars, max ' + 'is 63'.format(name, n)) try: ttl = int(data['ttl']) if ttl < 0: @@ -258,8 +267,8 @@ class GeoValue(EqualityTupleMixin): class _ValuesMixin(object): @classmethod - def validate(cls, name, data): - reasons = super(_ValuesMixin, cls).validate(name, data) + def validate(cls, name, fqdn, data): + reasons = super(_ValuesMixin, cls).validate(name, fqdn, data) values = data.get('values', data.get('value', [])) @@ -311,8 +320,8 @@ class _GeoMixin(_ValuesMixin): ''' @classmethod - def validate(cls, name, data): - reasons = super(_GeoMixin, cls).validate(name, data) + def validate(cls, name, fqdn, data): + reasons = super(_GeoMixin, cls).validate(name, fqdn, data) try: geo = dict(data['geo']) for code, values in geo.items(): @@ -358,8 +367,8 @@ class _GeoMixin(_ValuesMixin): class _ValueMixin(object): @classmethod - def validate(cls, name, data): - reasons = super(_ValueMixin, cls).validate(name, data) + def validate(cls, name, fqdn, data): + reasons = super(_ValueMixin, cls).validate(name, fqdn, data) reasons.extend(cls._value_type.validate(data.get('value', None), cls._type)) return reasons @@ -485,8 +494,8 @@ class _DynamicMixin(object): r'(-(?P\w\w))?)?$') @classmethod - def validate(cls, name, data): - reasons = super(_DynamicMixin, cls).validate(name, data) + def validate(cls, name, fqdn, data): + reasons = super(_DynamicMixin, cls).validate(name, fqdn, data) if 'dynamic' not in data: return reasons @@ -803,11 +812,11 @@ class CnameRecord(_DynamicMixin, _ValueMixin, Record): _value_type = CnameValue @classmethod - def validate(cls, name, data): + def validate(cls, name, fqdn, data): reasons = [] if name == '': reasons.append('root CNAME not allowed') - reasons.extend(super(CnameRecord, cls).validate(name, data)) + reasons.extend(super(CnameRecord, cls).validate(name, fqdn, data)) return reasons @@ -1181,11 +1190,11 @@ class SrvRecord(_ValuesMixin, Record): _name_re = re.compile(r'^_[^\.]+\.[^\.]+') @classmethod - def validate(cls, name, data): + def validate(cls, name, fqdn, data): reasons = [] if not cls._name_re.match(name): reasons.append('invalid name') - reasons.extend(super(SrvRecord, cls).validate(name, data)) + reasons.extend(super(SrvRecord, cls).validate(name, fqdn, data)) return reasons diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index 0845ddf..f313342 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -1217,6 +1217,34 @@ class TestRecordValidation(TestCase): zone = Zone('unit.tests.', []) def test_base(self): + # fqdn length, DNS defins max as 253 + with self.assertRaises(ValidationError) as ctx: + # The . will put this over the edge + name = 'x' * (253 - len(self.zone.name)) + Record.new(self.zone, name, { + 'ttl': 300, + 'type': 'A', + 'value': '1.2.3.4', + }) + reason = ctx.exception.reasons[0] + self.assertTrue(reason.startswith('invalid fqdn, "xxxx')) + self.assertTrue(reason.endswith('.unit.tests." is too long at 254' + ' chars, max is 253')) + + # label length, DNS defins max as 63 + with self.assertRaises(ValidationError) as ctx: + # The . will put this over the edge + name = 'x' * 64 + Record.new(self.zone, name, { + 'ttl': 300, + 'type': 'A', + 'value': '1.2.3.4', + }) + reason = ctx.exception.reasons[0] + self.assertTrue(reason.startswith('invalid name, "xxxx')) + self.assertTrue(reason.endswith('xxx" is too long at 64' + ' chars, max is 63')) + # no ttl with self.assertRaises(ValidationError) as ctx: Record.new(self.zone, '', { From 193d2da4ddc53dde846005fc48a0103932b497e4 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 4 Nov 2019 14:56:36 -0800 Subject: [PATCH 070/155] v0.9.9 version bump and CHANGELOG update --- CHANGELOG.md | 3 ++- octodns/__init__.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 503e53c..1cb7b1b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## v0.9.9 - 2019-??-?? - Python 3.7 Support +## v0.9.9 - 2019-11-04 - Python 3.7 Support * Extensive pass through the whole codebase to support Python 3 * Tons of updates to replace `def __cmp__` with `__eq__` and friends to @@ -23,6 +23,7 @@ * Bunch of additional unit tests and supporting config to exercise new code and verify things that were run into during the Python 3 work * lots of `six`ing of things +* Validate Record name & fqdn length ## v0.9.8 - 2019-09-30 - One with no changes b/c PyPi description problems diff --git a/octodns/__init__.py b/octodns/__init__.py index 71b5b1a..404d688 100644 --- a/octodns/__init__.py +++ b/octodns/__init__.py @@ -3,4 +3,4 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -__VERSION__ = '0.9.8' +__VERSION__ = '0.9.9' From 7a623f167f01d8590afb9286c315839ab43b299f Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 4 Nov 2019 14:57:28 -0800 Subject: [PATCH 071/155] Test python setup build in CI --- script/cibuild | 2 ++ 1 file changed, 2 insertions(+) diff --git a/script/cibuild b/script/cibuild index d048e8e..a2dc527 100755 --- a/script/cibuild +++ b/script/cibuild @@ -27,4 +27,6 @@ echo "## lint ################################################################## script/lint echo "## tests/coverage ##############################################################" script/coverage +echo "## validate setup.py build #####################################################" +python setup.py build echo "## complete ####################################################################" From 6428d3090d73b801ceda5a64e1d6483e01bdac35 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 22 Nov 2019 17:09:29 -0800 Subject: [PATCH 072/155] Initial actions PR Per @yzguy's issue https://github.com/github/octodns/issues/425 --- .github/workflows/main.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 .github/workflows/main.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..726e2e8 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,24 @@ +name: OctoDNS +on: [pull_request] + +jobs: + ci: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [2.7, 3.7] + steps: + - uses: actions/checkout@master + - name: Setup python + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + architecture: x64 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install virtualenv + - name: CI Build + run: | + ./script/cibuild From e600575e47b56766520fe9d98348f340a06531cd Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 22 Nov 2019 17:29:41 -0800 Subject: [PATCH 073/155] Delete .travis.yml --- .travis.yml | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index eb609ff..0000000 --- a/.travis.yml +++ /dev/null @@ -1,10 +0,0 @@ -language: python -matrix: - include: - - python: 2.7 - - python: 3.7 -before_install: pip install --upgrade pip -script: ./script/cibuild -notifications: - email: - - ross@github.com From 0c05ff075c60488858a658ec17b66f22f6d0616b Mon Sep 17 00:00:00 2001 From: Brian E Clow Date: Tue, 26 Nov 2019 10:00:29 -0800 Subject: [PATCH 074/155] Bumping setuptools from 38.5.2 to 40.3.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 204fd63..66fa130 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,6 +21,6 @@ pycountry==19.8.18 python-dateutil==2.6.1 requests==2.22.0 s3transfer==0.1.13 -setuptools==38.5.2 +setuptools==40.3.0 six==1.12.0 transip==2.0.0 From 1e74d59a76791f6d5f9c610a10166f7cac400fca Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 9 Dec 2019 06:35:31 -0800 Subject: [PATCH 075/155] Bump ns1-python to 0.13.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 66fa130..6a26ad3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,7 @@ ipaddress==1.0.22 jmespath==0.9.3 msrestazure==0.6.2 natsort==5.5.0 -ns1-python==0.12.0 +ns1-python==0.13.0 ovh==0.4.8 pycountry-convert==0.7.2 pycountry==19.8.18 From f599d91902ab3f8e5b09638efd6872e772657d93 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 9 Dec 2019 08:09:41 -0800 Subject: [PATCH 076/155] Update ns1 to 0.13.0 and use newer API methods Results in lots of churn in the tests, but actually for the better as it doesn't have to jump through nearly as many hoops to mock things now. --- octodns/provider/ns1.py | 57 +++--- tests/test_octodns_provider_ns1.py | 285 +++++++++++++++-------------- 2 files changed, 177 insertions(+), 165 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index b737a19..19b33af 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -38,7 +38,9 @@ class Ns1Provider(BaseProvider): self.log = getLogger('Ns1Provider[{}]'.format(id)) self.log.debug('__init__: id=%s, api_key=***', id) super(Ns1Provider, self).__init__(id, *args, **kwargs) - self._client = NS1(apiKey=api_key) + client = NS1(apiKey=api_key) + self._records = client.records() + self._zones = client.zones() def _data_for_A(self, _type, record): # record meta (which would include geo information is only @@ -189,18 +191,29 @@ class Ns1Provider(BaseProvider): target, lenient) try: - nsone_zone = self._client.loadZone(zone.name[:-1]) - records = nsone_zone.data['records'] + nsone_zone_name = zone.name[:-1] + nsone_zone = self._zones.retrieve(nsone_zone_name) + + records = [] + geo_records = [] # change answers for certain types to always be absolute - for record in records: + for record in nsone_zone['records']: if record['type'] in ['ALIAS', 'CNAME', 'MX', 'NS', 'PTR', 'SRV']: for i, a in enumerate(record['short_answers']): if not a.endswith('.'): record['short_answers'][i] = '{}.'.format(a) - geo_records = nsone_zone.search(has_geo=True) + if record.get('tier', 1) > 1: + # Need to get the full record data for geo records + record = self._records.retrieve(nsone_zone_name, + record['domain'], + record['type']) + geo_records.append(record) + else: + records.append(record) + exists = True except ResourceException as e: if e.message != self.ZONE_NOT_FOUND_MESSAGE: @@ -299,53 +312,49 @@ class Ns1Provider(BaseProvider): for v in record.values] return {'answers': values, 'ttl': record.ttl} - def _get_name(self, record): - return record.fqdn[:-1] if record.name == '' else record.name - def _apply_Create(self, nsone_zone, change): new = change.new - name = self._get_name(new) + zone = new.zone.name[:-1] + domain = new.fqdn[:-1] _type = new._type params = getattr(self, '_params_for_{}'.format(_type))(new) - meth = getattr(nsone_zone, 'add_{}'.format(_type)) try: - meth(name, **params) + self._records.create(zone, domain, _type, **params) except RateLimitException as e: period = float(e.period) self.log.warn('_apply_Create: rate limit encountered, pausing ' 'for %ds and trying again', period) sleep(period) - meth(name, **params) + self._records.create(zone, domain, _type, **params) def _apply_Update(self, nsone_zone, change): - existing = change.existing - name = self._get_name(existing) - _type = existing._type - record = nsone_zone.loadRecord(name, _type) new = change.new + zone = new.zone.name[:-1] + domain = new.fqdn[:-1] + _type = new._type params = getattr(self, '_params_for_{}'.format(_type))(new) try: - record.update(**params) + self._records.update(zone, domain, _type, **params) except RateLimitException as e: period = float(e.period) self.log.warn('_apply_Update: rate limit encountered, pausing ' 'for %ds and trying again', period) sleep(period) - record.update(**params) + self._records.update(zone, domain, _type, **params) def _apply_Delete(self, nsone_zone, change): existing = change.existing - name = self._get_name(existing) + zone = existing.zone.name[:-1] + domain = existing.fqdn[:-1] _type = existing._type - record = nsone_zone.loadRecord(name, _type) try: - record.delete() + self._records.delete(zone, domain, _type) except RateLimitException as e: period = float(e.period) self.log.warn('_apply_Delete: rate limit encountered, pausing ' 'for %ds and trying again', period) sleep(period) - record.delete() + self._records.delete(zone, domain, _type) def _apply(self, plan): desired = plan.desired @@ -355,12 +364,12 @@ class Ns1Provider(BaseProvider): domain_name = desired.name[:-1] try: - nsone_zone = self._client.loadZone(domain_name) + nsone_zone = self._zones.retrieve(domain_name) except ResourceException as e: if e.message != self.ZONE_NOT_FOUND_MESSAGE: raise self.log.debug('_apply: no matching zone, creating') - nsone_zone = self._client.createZone(domain_name) + nsone_zone = self._zones.create(domain_name) for change in changes: class_name = change.__class__.__name__ diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index 7ef182c..9034552 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -6,7 +6,7 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals from collections import defaultdict -from mock import Mock, call, patch +from mock import call, patch from ns1.rest.errors import AuthException, RateLimitException, \ ResourceException from unittest import TestCase @@ -16,14 +16,6 @@ from octodns.provider.ns1 import Ns1Provider from octodns.zone import Zone -class DummyZone(object): - - def __init__(self, records): - self.data = { - 'records': records - } - - class TestNs1Provider(TestCase): zone = Zone('unit.tests.', []) expected = set() @@ -172,43 +164,40 @@ class TestNs1Provider(TestCase): 'domain': 'unit.tests.', }] - @patch('ns1.NS1.loadZone') - def test_populate(self, load_mock): + @patch('ns1.rest.zones.Zones.retrieve') + def test_populate(self, zone_retrieve_mock): provider = Ns1Provider('test', 'api-key') # Bad auth - load_mock.side_effect = AuthException('unauthorized') + zone_retrieve_mock.side_effect = AuthException('unauthorized') zone = Zone('unit.tests.', []) with self.assertRaises(AuthException) as ctx: provider.populate(zone) - self.assertEquals(load_mock.side_effect, ctx.exception) + self.assertEquals(zone_retrieve_mock.side_effect, ctx.exception) # General error - load_mock.reset_mock() - load_mock.side_effect = ResourceException('boom') + zone_retrieve_mock.reset_mock() + zone_retrieve_mock.side_effect = ResourceException('boom') zone = Zone('unit.tests.', []) with self.assertRaises(ResourceException) as ctx: provider.populate(zone) - self.assertEquals(load_mock.side_effect, ctx.exception) - self.assertEquals(('unit.tests',), load_mock.call_args[0]) + self.assertEquals(zone_retrieve_mock.side_effect, ctx.exception) + self.assertEquals(('unit.tests',), zone_retrieve_mock.call_args[0]) # Non-existent zone doesn't populate anything - load_mock.reset_mock() - load_mock.side_effect = \ + zone_retrieve_mock.reset_mock() + zone_retrieve_mock.side_effect = \ ResourceException('server error: zone not found') zone = Zone('unit.tests.', []) exists = provider.populate(zone) self.assertEquals(set(), zone.records) - self.assertEquals(('unit.tests',), load_mock.call_args[0]) + self.assertEquals(('unit.tests',), zone_retrieve_mock.call_args[0]) self.assertFalse(exists) # Existing zone w/o records - load_mock.reset_mock() - nsone_zone = DummyZone([]) - load_mock.side_effect = [nsone_zone] - zone_search = Mock() - zone_search.return_value = [ - { + zone_retrieve_mock.reset_mock() + nsone_zone = { + 'records': [{ "domain": "geo.unit.tests", "zone": "unit.tests", "type": "A", @@ -222,21 +211,18 @@ class TestNs1Provider(TestCase): 'meta': {'iso_region_code': ['NA-US-WA']}}, ], 'ttl': 34, - }, - ] - nsone_zone.search = zone_search + }], + } + zone_retrieve_mock.side_effect = [nsone_zone] zone = Zone('unit.tests.', []) provider.populate(zone) self.assertEquals(1, len(zone.records)) - self.assertEquals(('unit.tests',), load_mock.call_args[0]) + self.assertEquals(('unit.tests',), zone_retrieve_mock.call_args[0]) # Existing zone w/records - load_mock.reset_mock() - nsone_zone = DummyZone(self.nsone_records) - load_mock.side_effect = [nsone_zone] - zone_search = Mock() - zone_search.return_value = [ - { + zone_retrieve_mock.reset_mock() + nsone_zone = { + 'records': self.nsone_records + [{ "domain": "geo.unit.tests", "zone": "unit.tests", "type": "A", @@ -250,26 +236,23 @@ class TestNs1Provider(TestCase): 'meta': {'iso_region_code': ['NA-US-WA']}}, ], 'ttl': 34, - }, - ] - nsone_zone.search = zone_search + }], + } + zone_retrieve_mock.side_effect = [nsone_zone] zone = Zone('unit.tests.', []) provider.populate(zone) self.assertEquals(self.expected, zone.records) - self.assertEquals(('unit.tests',), load_mock.call_args[0]) + self.assertEquals(('unit.tests',), zone_retrieve_mock.call_args[0]) # Test skipping unsupported record type - load_mock.reset_mock() - nsone_zone = DummyZone(self.nsone_records + [{ - 'type': 'UNSUPPORTED', - 'ttl': 42, - 'short_answers': ['unsupported'], - 'domain': 'unsupported.unit.tests.', - }]) - load_mock.side_effect = [nsone_zone] - zone_search = Mock() - zone_search.return_value = [ - { + zone_retrieve_mock.reset_mock() + nsone_zone = { + 'records': self.nsone_records + [{ + 'type': 'UNSUPPORTED', + 'ttl': 42, + 'short_answers': ['unsupported'], + 'domain': 'unsupported.unit.tests.', + }, { "domain": "geo.unit.tests", "zone": "unit.tests", "type": "A", @@ -283,17 +266,23 @@ class TestNs1Provider(TestCase): 'meta': {'iso_region_code': ['NA-US-WA']}}, ], 'ttl': 34, - }, - ] - nsone_zone.search = zone_search + }], + } + zone_retrieve_mock.side_effect = [nsone_zone] zone = Zone('unit.tests.', []) provider.populate(zone) self.assertEquals(self.expected, zone.records) - self.assertEquals(('unit.tests',), load_mock.call_args[0]) + self.assertEquals(('unit.tests',), zone_retrieve_mock.call_args[0]) - @patch('ns1.NS1.createZone') - @patch('ns1.NS1.loadZone') - def test_sync(self, load_mock, create_mock): + @patch('ns1.rest.records.Records.delete') + @patch('ns1.rest.records.Records.update') + @patch('ns1.rest.records.Records.create') + @patch('ns1.rest.records.Records.retrieve') + @patch('ns1.rest.zones.Zones.create') + @patch('ns1.rest.zones.Zones.retrieve') + def test_sync(self, zone_retrieve_mock, zone_create_mock, + record_retrieve_mock, record_create_mock, + record_update_mock, record_delete_mock): provider = Ns1Provider('test', 'api-key') desired = Zone('unit.tests.', []) @@ -307,71 +296,98 @@ class TestNs1Provider(TestCase): self.assertTrue(plan.exists) # Fails, general error - load_mock.reset_mock() - create_mock.reset_mock() - load_mock.side_effect = ResourceException('boom') + zone_retrieve_mock.reset_mock() + zone_create_mock.reset_mock() + zone_retrieve_mock.side_effect = ResourceException('boom') with self.assertRaises(ResourceException) as ctx: provider.apply(plan) - self.assertEquals(load_mock.side_effect, ctx.exception) + self.assertEquals(zone_retrieve_mock.side_effect, ctx.exception) # Fails, bad auth - load_mock.reset_mock() - create_mock.reset_mock() - load_mock.side_effect = \ + zone_retrieve_mock.reset_mock() + zone_create_mock.reset_mock() + zone_retrieve_mock.side_effect = \ ResourceException('server error: zone not found') - create_mock.side_effect = AuthException('unauthorized') + zone_create_mock.side_effect = AuthException('unauthorized') with self.assertRaises(AuthException) as ctx: provider.apply(plan) - self.assertEquals(create_mock.side_effect, ctx.exception) + self.assertEquals(zone_create_mock.side_effect, ctx.exception) # non-existent zone, create - load_mock.reset_mock() - create_mock.reset_mock() - load_mock.side_effect = \ + zone_retrieve_mock.reset_mock() + zone_create_mock.reset_mock() + zone_retrieve_mock.side_effect = \ ResourceException('server error: zone not found') - # ugh, need a mock zone with a mock prop since we're using getattr, we - # can actually control side effects on `meth` with that. - mock_zone = Mock() - mock_zone.add_SRV = Mock() - mock_zone.add_SRV.side_effect = [ + + zone_create_mock.side_effect = ['foo'] + # Test out the create rate-limit handling, then 9 successes + record_create_mock.side_effect = [ RateLimitException('boo', period=0), - None, - ] - create_mock.side_effect = [mock_zone] + ] + ([None] * 9) + got_n = provider.apply(plan) self.assertEquals(expected_n, got_n) + # Zone was created + zone_create_mock.assert_has_calls([call('unit.tests')]) + # Checking that we got some of the expected records too + record_create_mock.assert_has_calls([ + call('unit.tests', 'unit.tests', 'A', answers=[ + {'answer': ['1.2.3.4'], 'meta': {}} + ], filters=[], ttl=32), + call('unit.tests', 'unit.tests', 'CAA', answers=[ + (0, 'issue', 'ca.unit.tests') + ], ttl=40), + call('unit.tests', 'unit.tests', 'MX', answers=[ + (10, 'mx1.unit.tests.'), (20, 'mx2.unit.tests.') + ], ttl=35), + ]) + # Update & delete - load_mock.reset_mock() - create_mock.reset_mock() - nsone_zone = DummyZone(self.nsone_records + [{ - 'type': 'A', - 'ttl': 42, - 'short_answers': ['9.9.9.9'], - 'domain': 'delete-me.unit.tests.', - }]) - nsone_zone.data['records'][0]['short_answers'][0] = '2.2.2.2' - nsone_zone.loadRecord = Mock() - zone_search = Mock() - zone_search.return_value = [ - { + zone_retrieve_mock.reset_mock() + zone_create_mock.reset_mock() + + nsone_zone = { + 'records': self.nsone_records + [{ + 'type': 'A', + 'ttl': 42, + 'short_answers': ['9.9.9.9'], + 'domain': 'delete-me.unit.tests.', + }, { "domain": "geo.unit.tests", "zone": "unit.tests", "type": "A", - "answers": [ - {'answer': ['1.1.1.1'], 'meta': {}}, - {'answer': ['1.2.3.4'], - 'meta': {'ca_province': ['ON']}}, - {'answer': ['2.3.4.5'], 'meta': {'us_state': ['NY']}}, - {'answer': ['3.4.5.6'], 'meta': {'country': ['US']}}, - {'answer': ['4.5.6.7'], - 'meta': {'iso_region_code': ['NA-US-WA']}}, + "short_answers": [ + '1.1.1.1', + '1.2.3.4', + '2.3.4.5', + '3.4.5.6', + '4.5.6.7', ], + 'tier': 3, # This flags it as advacned, full load required 'ttl': 34, - }, - ] - nsone_zone.search = zone_search - load_mock.side_effect = [nsone_zone, nsone_zone] + }], + } + nsone_zone['records'][0]['short_answers'][0] = '2.2.2.2' + + record_retrieve_mock.side_effect = [{ + "domain": "geo.unit.tests", + "zone": "unit.tests", + "type": "A", + "answers": [ + {'answer': ['1.1.1.1'], 'meta': {}}, + {'answer': ['1.2.3.4'], + 'meta': {'ca_province': ['ON']}}, + {'answer': ['2.3.4.5'], 'meta': {'us_state': ['NY']}}, + {'answer': ['3.4.5.6'], 'meta': {'country': ['US']}}, + {'answer': ['4.5.6.7'], + 'meta': {'iso_region_code': ['NA-US-WA']}}, + ], + 'tier': 3, + 'ttl': 34, + }] + + zone_retrieve_mock.side_effect = [nsone_zone, nsone_zone] plan = provider.plan(desired) self.assertEquals(3, len(plan.changes)) # Shouldn't rely on order so just count classes @@ -380,55 +396,42 @@ class TestNs1Provider(TestCase): classes[change.__class__] += 1 self.assertEquals(1, classes[Delete]) self.assertEquals(2, classes[Update]) - # ugh, we need a mock record that can be returned from loadRecord for - # the update and delete targets, we can add our side effects to that to - # trigger rate limit handling - mock_record = Mock() - mock_record.update.side_effect = [ + + record_update_mock.side_effect = [ RateLimitException('one', period=0), None, None, ] - mock_record.delete.side_effect = [ + record_delete_mock.side_effect = [ RateLimitException('two', period=0), None, None, ] - nsone_zone.loadRecord.side_effect = [mock_record, mock_record, - mock_record] + got_n = provider.apply(plan) self.assertEquals(3, got_n) - nsone_zone.loadRecord.assert_has_calls([ - call('unit.tests', u'A'), - call('delete-me', u'A'), - call('geo', u'A'), - ]) - mock_record.assert_has_calls([ - call.update(answers=[{'answer': [u'1.2.3.4'], 'meta': {}}], - filters=[], - ttl=32), - call.update(answers=[{u'answer': [u'1.2.3.4'], u'meta': {}}], - filters=[], - ttl=32), - call.delete(), - call.delete(), - call.update( - answers=[ - {u'answer': [u'101.102.103.104'], u'meta': {}}, - {u'answer': [u'101.102.103.105'], u'meta': {}}, - { - u'answer': [u'201.202.203.204'], - u'meta': { - u'iso_region_code': [u'NA-US-NY'] - }, - }, - ], + + record_update_mock.assert_has_calls([ + call('unit.tests', 'unit.tests', 'A', answers=[ + {'answer': ['1.2.3.4'], 'meta': {}}], + filters=[], + ttl=32), + call('unit.tests', 'unit.tests', 'A', answers=[ + {'answer': ['1.2.3.4'], 'meta': {}}], + filters=[], + ttl=32), + call('unit.tests', 'geo.unit.tests', 'A', answers=[ + {'answer': ['101.102.103.104'], 'meta': {}}, + {'answer': ['101.102.103.105'], 'meta': {}}, + { + 'answer': ['201.202.203.204'], + 'meta': {'iso_region_code': ['NA-US-NY']} + }], filters=[ - {u'filter': u'shuffle', u'config': {}}, - {u'filter': u'geotarget_country', u'config': {}}, - {u'filter': u'select_first_n', u'config': {u'N': 1}}, - ], - ttl=34), + {'filter': 'shuffle', 'config': {}}, + {'filter': 'geotarget_country', 'config': {}}, + {'filter': 'select_first_n', 'config': {'N': 1}}], + ttl=34) ]) def test_escaping(self): From c4987f1a09048b6b557ce19bc194ee6598a175fe Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 9 Dec 2019 08:13:09 -0800 Subject: [PATCH 077/155] s/nsone/ns1/g --- octodns/provider/ns1.py | 22 +++++++++++----------- tests/test_octodns_provider_ns1.py | 30 +++++++++++++++--------------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 19b33af..cf78241 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -23,7 +23,7 @@ class Ns1Provider(BaseProvider): ''' Ns1 provider - nsone: + ns1: class: octodns.provider.ns1.Ns1Provider api_key: env/NS1_API_KEY ''' @@ -191,14 +191,14 @@ class Ns1Provider(BaseProvider): target, lenient) try: - nsone_zone_name = zone.name[:-1] - nsone_zone = self._zones.retrieve(nsone_zone_name) + ns1_zone_name = zone.name[:-1] + ns1_zone = self._zones.retrieve(ns1_zone_name) records = [] geo_records = [] # change answers for certain types to always be absolute - for record in nsone_zone['records']: + for record in ns1_zone['records']: if record['type'] in ['ALIAS', 'CNAME', 'MX', 'NS', 'PTR', 'SRV']: for i, a in enumerate(record['short_answers']): @@ -207,7 +207,7 @@ class Ns1Provider(BaseProvider): if record.get('tier', 1) > 1: # Need to get the full record data for geo records - record = self._records.retrieve(nsone_zone_name, + record = self._records.retrieve(ns1_zone_name, record['domain'], record['type']) geo_records.append(record) @@ -312,7 +312,7 @@ class Ns1Provider(BaseProvider): for v in record.values] return {'answers': values, 'ttl': record.ttl} - def _apply_Create(self, nsone_zone, change): + def _apply_Create(self, ns1_zone, change): new = change.new zone = new.zone.name[:-1] domain = new.fqdn[:-1] @@ -327,7 +327,7 @@ class Ns1Provider(BaseProvider): sleep(period) self._records.create(zone, domain, _type, **params) - def _apply_Update(self, nsone_zone, change): + def _apply_Update(self, ns1_zone, change): new = change.new zone = new.zone.name[:-1] domain = new.fqdn[:-1] @@ -342,7 +342,7 @@ class Ns1Provider(BaseProvider): sleep(period) self._records.update(zone, domain, _type, **params) - def _apply_Delete(self, nsone_zone, change): + def _apply_Delete(self, ns1_zone, change): existing = change.existing zone = existing.zone.name[:-1] domain = existing.fqdn[:-1] @@ -364,14 +364,14 @@ class Ns1Provider(BaseProvider): domain_name = desired.name[:-1] try: - nsone_zone = self._zones.retrieve(domain_name) + ns1_zone = self._zones.retrieve(domain_name) except ResourceException as e: if e.message != self.ZONE_NOT_FOUND_MESSAGE: raise self.log.debug('_apply: no matching zone, creating') - nsone_zone = self._zones.create(domain_name) + ns1_zone = self._zones.create(domain_name) for change in changes: class_name = change.__class__.__name__ - getattr(self, '_apply_{}'.format(class_name))(nsone_zone, + getattr(self, '_apply_{}'.format(class_name))(ns1_zone, change) diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index 9034552..0f23222 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -108,7 +108,7 @@ class TestNs1Provider(TestCase): }, })) - nsone_records = [{ + ns1_records = [{ 'type': 'A', 'ttl': 32, 'short_answers': ['1.2.3.4'], @@ -196,7 +196,7 @@ class TestNs1Provider(TestCase): # Existing zone w/o records zone_retrieve_mock.reset_mock() - nsone_zone = { + ns1_zone = { 'records': [{ "domain": "geo.unit.tests", "zone": "unit.tests", @@ -213,7 +213,7 @@ class TestNs1Provider(TestCase): 'ttl': 34, }], } - zone_retrieve_mock.side_effect = [nsone_zone] + zone_retrieve_mock.side_effect = [ns1_zone] zone = Zone('unit.tests.', []) provider.populate(zone) self.assertEquals(1, len(zone.records)) @@ -221,8 +221,8 @@ class TestNs1Provider(TestCase): # Existing zone w/records zone_retrieve_mock.reset_mock() - nsone_zone = { - 'records': self.nsone_records + [{ + ns1_zone = { + 'records': self.ns1_records + [{ "domain": "geo.unit.tests", "zone": "unit.tests", "type": "A", @@ -238,7 +238,7 @@ class TestNs1Provider(TestCase): 'ttl': 34, }], } - zone_retrieve_mock.side_effect = [nsone_zone] + zone_retrieve_mock.side_effect = [ns1_zone] zone = Zone('unit.tests.', []) provider.populate(zone) self.assertEquals(self.expected, zone.records) @@ -246,8 +246,8 @@ class TestNs1Provider(TestCase): # Test skipping unsupported record type zone_retrieve_mock.reset_mock() - nsone_zone = { - 'records': self.nsone_records + [{ + ns1_zone = { + 'records': self.ns1_records + [{ 'type': 'UNSUPPORTED', 'ttl': 42, 'short_answers': ['unsupported'], @@ -268,7 +268,7 @@ class TestNs1Provider(TestCase): 'ttl': 34, }], } - zone_retrieve_mock.side_effect = [nsone_zone] + zone_retrieve_mock.side_effect = [ns1_zone] zone = Zone('unit.tests.', []) provider.populate(zone) self.assertEquals(self.expected, zone.records) @@ -347,8 +347,8 @@ class TestNs1Provider(TestCase): zone_retrieve_mock.reset_mock() zone_create_mock.reset_mock() - nsone_zone = { - 'records': self.nsone_records + [{ + ns1_zone = { + 'records': self.ns1_records + [{ 'type': 'A', 'ttl': 42, 'short_answers': ['9.9.9.9'], @@ -368,7 +368,7 @@ class TestNs1Provider(TestCase): 'ttl': 34, }], } - nsone_zone['records'][0]['short_answers'][0] = '2.2.2.2' + ns1_zone['records'][0]['short_answers'][0] = '2.2.2.2' record_retrieve_mock.side_effect = [{ "domain": "geo.unit.tests", @@ -387,7 +387,7 @@ class TestNs1Provider(TestCase): 'ttl': 34, }] - zone_retrieve_mock.side_effect = [nsone_zone, nsone_zone] + zone_retrieve_mock.side_effect = [ns1_zone, ns1_zone] plan = provider.plan(desired) self.assertEquals(3, len(plan.changes)) # Shouldn't rely on order so just count classes @@ -470,7 +470,7 @@ class TestNs1Provider(TestCase): def test_data_for_CNAME(self): provider = Ns1Provider('test', 'api-key') - # answers from nsone + # answers from ns1 a_record = { 'ttl': 31, 'type': 'CNAME', @@ -484,7 +484,7 @@ class TestNs1Provider(TestCase): self.assertEqual(a_expected, provider._data_for_CNAME(a_record['type'], a_record)) - # no answers from nsone + # no answers from ns1 b_record = { 'ttl': 32, 'type': 'CNAME', From 9e8a417c356d8f00c5e5096f0864f70bf50d5ab5 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 9 Dec 2019 08:26:20 -0800 Subject: [PATCH 078/155] Refactor thin Ns1Client wrapper out of provider --- octodns/provider/ns1.py | 98 ++++++++++++++++++++++++++--------------- 1 file changed, 63 insertions(+), 35 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index cf78241..8acbd53 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -19,6 +19,56 @@ from ..record import Record from .base import BaseProvider +class Ns1Client(object): + log = getLogger('NS1Client') + + def __init__(self, api_key, retry_delay=1): + self.retry_delay = retry_delay + + client = NS1(apiKey=api_key) + self._records = client.records() + self._zones = client.zones() + + def zones_retrieve(self, name): + return self._zones.retrieve(name) + + def zones_create(self, name): + return self._zones.create(name) + + def records_retrieve(self, zone, domain, _type): + return self._records.retrieve(zone, domain, _type) + + def records_create(self, zone, domain, _type, **params): + try: + return self._records.create(zone, domain, _type, **params) + except RateLimitException as e: + period = float(e.period) + self.log.warn('_apply_Create: rate limit encountered, pausing ' + 'for %ds and trying again', period) + sleep(period) + return self._records.create(zone, domain, _type, **params) + + def records_update(self, zone, domain, _type, **params): + try: + return self._records.update(zone, domain, _type, **params) + except RateLimitException as e: + period = float(e.period) + self.log.warn('_apply_Update: rate limit encountered, pausing ' + 'for %ds and trying again', period) + sleep(period) + return self._records.update(zone, domain, _type, **params) + + def records_delete(self, zone, domain, _type): + try: + return self._records.delete(zone, domain, _type) + except RateLimitException as e: + period = float(e.period) + self.log.warn('_apply_Delete: rate limit encountered, pausing ' + 'for %ds and trying again', period) + sleep(period) + return self._records.delete(zone, domain, _type) + + class Ns1Provider(BaseProvider): ''' Ns1 provider @@ -34,13 +84,12 @@ class Ns1Provider(BaseProvider): ZONE_NOT_FOUND_MESSAGE = 'server error: zone not found' - def __init__(self, id, api_key, *args, **kwargs): + def __init__(self, id, api_key, retry_delay=1, *args, **kwargs): self.log = getLogger('Ns1Provider[{}]'.format(id)) - self.log.debug('__init__: id=%s, api_key=***', id) + self.log.debug('__init__: id=%s, api_key=***, retry_delay=%d', id, + retry_delay) super(Ns1Provider, self).__init__(id, *args, **kwargs) - client = NS1(apiKey=api_key) - self._records = client.records() - self._zones = client.zones() + self._client = Ns1Client(api_key, retry_delay) def _data_for_A(self, _type, record): # record meta (which would include geo information is only @@ -192,7 +241,7 @@ class Ns1Provider(BaseProvider): try: ns1_zone_name = zone.name[:-1] - ns1_zone = self._zones.retrieve(ns1_zone_name) + ns1_zone = self._client.zones_retrieve(ns1_zone_name) records = [] geo_records = [] @@ -207,9 +256,9 @@ class Ns1Provider(BaseProvider): if record.get('tier', 1) > 1: # Need to get the full record data for geo records - record = self._records.retrieve(ns1_zone_name, - record['domain'], - record['type']) + record = self._client.records_retrieve(ns1_zone_name, + record['domain'], + record['type']) geo_records.append(record) else: records.append(record) @@ -318,14 +367,7 @@ class Ns1Provider(BaseProvider): domain = new.fqdn[:-1] _type = new._type params = getattr(self, '_params_for_{}'.format(_type))(new) - try: - self._records.create(zone, domain, _type, **params) - except RateLimitException as e: - period = float(e.period) - self.log.warn('_apply_Create: rate limit encountered, pausing ' - 'for %ds and trying again', period) - sleep(period) - self._records.create(zone, domain, _type, **params) + self._client.records_create(zone, domain, _type, **params) def _apply_Update(self, ns1_zone, change): new = change.new @@ -333,28 +375,14 @@ class Ns1Provider(BaseProvider): domain = new.fqdn[:-1] _type = new._type params = getattr(self, '_params_for_{}'.format(_type))(new) - try: - self._records.update(zone, domain, _type, **params) - except RateLimitException as e: - period = float(e.period) - self.log.warn('_apply_Update: rate limit encountered, pausing ' - 'for %ds and trying again', period) - sleep(period) - self._records.update(zone, domain, _type, **params) + self._client.records_update(zone, domain, _type, **params) def _apply_Delete(self, ns1_zone, change): existing = change.existing zone = existing.zone.name[:-1] domain = existing.fqdn[:-1] _type = existing._type - try: - self._records.delete(zone, domain, _type) - except RateLimitException as e: - period = float(e.period) - self.log.warn('_apply_Delete: rate limit encountered, pausing ' - 'for %ds and trying again', period) - sleep(period) - self._records.delete(zone, domain, _type) + self._client.records_delete(zone, domain, _type) def _apply(self, plan): desired = plan.desired @@ -364,12 +392,12 @@ class Ns1Provider(BaseProvider): domain_name = desired.name[:-1] try: - ns1_zone = self._zones.retrieve(domain_name) + ns1_zone = self._client.zones_retrieve(domain_name) except ResourceException as e: if e.message != self.ZONE_NOT_FOUND_MESSAGE: raise self.log.debug('_apply: no matching zone, creating') - ns1_zone = self._zones.create(domain_name) + ns1_zone = self._client.zones_create(domain_name) for change in changes: class_name = change.__class__.__name__ From 4fd2daa8a990521ae89d3be1a183f75ddee0fcd5 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 9 Dec 2019 08:56:55 -0800 Subject: [PATCH 079/155] Implement reworked NS1 retry mechinism --- octodns/provider/ns1.py | 59 +++++++++++++----------------- tests/test_octodns_provider_ns1.py | 46 ++++++++++++++++++++++- 2 files changed, 71 insertions(+), 34 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 8acbd53..da2d64a 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -22,51 +22,44 @@ from .base import BaseProvider class Ns1Client(object): log = getLogger('NS1Client') - def __init__(self, api_key, retry_delay=1): - self.retry_delay = retry_delay + def __init__(self, api_key, retry_count=4): + self.retry_count = retry_count client = NS1(apiKey=api_key) self._records = client.records() self._zones = client.zones() + def _try(self, method, *args, **kwargs): + tries = self.retry_count + while tries: + try: + return method(*args, **kwargs) + except RateLimitException as e: + period = float(e.period) + self.log.warn('rate limit encountered, pausing ' + 'for %ds and trying again, %d remaining', + period, tries) + sleep(period) + tries -= 1 + raise + def zones_retrieve(self, name): - return self._zones.retrieve(name) + return self._try(self._zones.retrieve, name) def zones_create(self, name): - return self._zones.create(name) + return self._try(self._zones.create, name) def records_retrieve(self, zone, domain, _type): - return self._records.retrieve(zone, domain, _type) + return self._try(self._records.retrieve, zone, domain, _type) def records_create(self, zone, domain, _type, **params): - try: - return self._records.create(zone, domain, _type, **params) - except RateLimitException as e: - period = float(e.period) - self.log.warn('_apply_Create: rate limit encountered, pausing ' - 'for %ds and trying again', period) - sleep(period) - return self._records.create(zone, domain, _type, **params) + return self._try(self._records.create, zone, domain, _type, **params) def records_update(self, zone, domain, _type, **params): - try: - return self._records.update(zone, domain, _type, **params) - except RateLimitException as e: - period = float(e.period) - self.log.warn('_apply_Update: rate limit encountered, pausing ' - 'for %ds and trying again', period) - sleep(period) - return self._records.update(zone, domain, _type, **params) + return self._try(self._records.update, zone, domain, _type, **params) def records_delete(self, zone, domain, _type): - try: - return self._records.delete(zone, domain, _type) - except RateLimitException as e: - period = float(e.period) - self.log.warn('_apply_Delete: rate limit encountered, pausing ' - 'for %ds and trying again', period) - sleep(period) - return self._records.delete(zone, domain, _type) + return self._try(self._records.delete, zone, domain, _type) class Ns1Provider(BaseProvider): @@ -84,12 +77,12 @@ class Ns1Provider(BaseProvider): ZONE_NOT_FOUND_MESSAGE = 'server error: zone not found' - def __init__(self, id, api_key, retry_delay=1, *args, **kwargs): + def __init__(self, id, api_key, retry_count=4, *args, **kwargs): self.log = getLogger('Ns1Provider[{}]'.format(id)) - self.log.debug('__init__: id=%s, api_key=***, retry_delay=%d', id, - retry_delay) + self.log.debug('__init__: id=%s, api_key=***, retry_count=%d', id, + retry_count) super(Ns1Provider, self).__init__(id, *args, **kwargs) - self._client = Ns1Client(api_key, retry_delay) + self._client = Ns1Client(api_key, retry_count) def _data_for_A(self, _type, record): # record meta (which would include geo information is only diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index 0f23222..0743943 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -9,10 +9,11 @@ from collections import defaultdict from mock import call, patch from ns1.rest.errors import AuthException, RateLimitException, \ ResourceException +from six import text_type from unittest import TestCase from octodns.record import Delete, Record, Update -from octodns.provider.ns1 import Ns1Provider +from octodns.provider.ns1 import Ns1Client, Ns1Provider from octodns.zone import Zone @@ -497,3 +498,46 @@ class TestNs1Provider(TestCase): } self.assertEqual(b_expected, provider._data_for_CNAME(b_record['type'], b_record)) + + +class TestNs1Client(TestCase): + + @patch('ns1.rest.zones.Zones.retrieve') + def test_retry_behavior(self, zone_retrieve_mock): + client = Ns1Client('dummy-key') + + # No retry required, just calls and is returned + zone_retrieve_mock.reset_mock() + zone_retrieve_mock.side_effect = ['foo'] + self.assertEquals('foo', client.zones_retrieve('unit.tests')) + zone_retrieve_mock.assert_has_calls([call('unit.tests')]) + + # One retry required + zone_retrieve_mock.reset_mock() + zone_retrieve_mock.side_effect = [ + RateLimitException('boo', period=0), + 'foo' + ] + self.assertEquals('foo', client.zones_retrieve('unit.tests')) + zone_retrieve_mock.assert_has_calls([call('unit.tests')]) + + # Two retries required + zone_retrieve_mock.reset_mock() + zone_retrieve_mock.side_effect = [ + RateLimitException('boo', period=0), + 'foo' + ] + self.assertEquals('foo', client.zones_retrieve('unit.tests')) + zone_retrieve_mock.assert_has_calls([call('unit.tests')]) + + # Exhaust our retries + zone_retrieve_mock.reset_mock() + zone_retrieve_mock.side_effect = [ + RateLimitException('first', period=0), + RateLimitException('boo', period=0), + RateLimitException('boo', period=0), + RateLimitException('last', period=0), + ] + with self.assertRaises(RateLimitException) as ctx: + client.zones_retrieve('unit.tests') + self.assertEquals('last', text_type(ctx.exception)) From f0bc9add22bd29b814e61d0740ac14ad6e6906ab Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 9 Dec 2019 14:30:02 -0800 Subject: [PATCH 080/155] Rough draft/expirimentation on dynamic creation --- octodns/provider/ns1.py | 167 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 163 insertions(+), 4 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index da2d64a..445e612 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -70,13 +70,54 @@ class Ns1Provider(BaseProvider): class: octodns.provider.ns1.Ns1Provider api_key: env/NS1_API_KEY ''' - SUPPORTS_GEO = True - SUPPORTS_DYNAMIC = False + SUPPORTS_GEO = False + SUPPORTS_DYNAMIC = True SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CAA', 'CNAME', 'MX', 'NAPTR', 'NS', 'PTR', 'SPF', 'SRV', 'TXT')) ZONE_NOT_FOUND_MESSAGE = 'server error: zone not found' + _DYNAMIC_FILTERS = [{ + 'config': {}, + 'filter': 'up' + }, { + 'config': {}, + 'filter': u'geotarget_regional' + }, { + 'config': {}, + 'filter': u'select_first_region' + }, { + 'config': { + 'eliminate': u'1' + }, + 'filter': 'priority' + }, { + 'config': {}, + 'filter': u'weighted_shuffle' + }, { + 'config': { + 'N': u'1' + }, + 'filter': u'select_first_n' + }] + _REGION_TO_CONTINENT = { + 'AFRICA': 'AF', + 'ASIAPAC': 'AS', + 'EUROPE': 'EU', + 'SOUTH-AMERICA': 'SA', + 'US-CENTRAL': 'NA', + 'US-EAST': 'NA', + 'US-WEST': 'NA', + } + _CONTINENT_TO_REGIONS = { + 'AF': ('AFRICA',), + 'AS': ('ASIAPAC',), + 'EU': ('EUROPE',), + 'SA': ('SOUTH-AMERICA',), + # TODO: what about CA, MX, and all the other NA countries? + 'NA': ('US-CENTRAL', 'US-EAST', 'US-WEST'), + } + def __init__(self, id, api_key, retry_count=4, *args, **kwargs): self.log = getLogger('Ns1Provider[{}]'.format(id)) self.log.debug('__init__: id=%s, api_key=***, retry_count=%d', id, @@ -282,9 +323,124 @@ class Ns1Provider(BaseProvider): len(zone.records) - before, exists) return exists + def _encode_notes(self, data): + return ' '.join(['{}:{}'.format(k, v) + for k, v in sorted(data.items())]) + def _params_for_A(self, record): - params = {'answers': record.values, 'ttl': record.ttl} - if hasattr(record, 'geo'): + params = {'ttl': record.ttl} + + if hasattr(record, 'dynamic'): + + pools = record.dynamic.pools + + # Convert rules to regions + regions = {} + for i, rule in enumerate(record.dynamic.rules): + pool_name = rule.data['pool'] + + notes = { + 'rule-order': i, + } + + fallback = pools[pool_name].data.get('fallback', None) + if fallback: + notes['fallback'] = fallback + + country = set() + georegion = set() + us_state = set() + + for geo in rule.data.get('geos', []): + n = len(geo) + if n == 8: + # US state + us_state.add(geo[-2:]) + elif n == 5: + # Country + country.add(geo[-2:]) + else: + # Continent + georegion.update(self._CONTINENT_TO_REGIONS[geo]) + + meta = { + 'note': self._encode_notes(notes), + } + if georegion: + meta['georegion'] = sorted(georegion) + if country: + meta['country'] = sorted(country) + if us_state: + meta['us_state'] = sorted(us_state) + + regions[pool_name] = { + 'meta': meta, + } + + # Build a list of primary values for each pool + pool_answers = defaultdict(list) + for pool_name, pool in sorted(pools.items()): + for value in pool.data['values']: + pool_answers[pool_name].append({ + 'answer': [value['value']], + 'weight': value['weight'], + }) + + default_answers = [{ + 'answer': [v], + 'weight': 1, + } for v in record.values] + + # Build our list of answers + answers = [] + for pool_name in sorted(pools.keys()): + priority = 1 + + # Dynamic/health checked + current_pool_name = pool_name + while current_pool_name: + pool = pools[current_pool_name] + for answer in pool_answers[current_pool_name]: + answer = { + 'answer': answer['answer'], + 'meta': { + 'priority': priority, + 'note': self._encode_notes({ + 'from': current_pool_name, + }), + 'weight': answer['weight'], + }, + 'region': pool_name, # the one we're answering + } + answers.append(answer) + + current_pool_name = pool.data.get('fallback', None) + priority += 1 + + # Static/default + for answer in default_answers: + answer = { + 'answer': answer['answer'], + 'meta': { + 'priority': priority, + 'note': self._encode_notes({ + 'from': '--default--', + }), + 'weight': 1, + }, + 'region': pool_name, # the one we're answering + } + answers.append(answer) + + params.update({ + 'answers': answers, + 'filters': self._DYNAMIC_FILTERS, + 'regions': regions, + }) + + return params + + elif hasattr(record, 'geo'): # purposefully set non-geo answers to have an empty meta, # so that we know we did this on purpose if/when troubleshooting params['answers'] = [{"answer": [x], "meta": {}} @@ -315,6 +471,9 @@ class Ns1Provider(BaseProvider): {"filter": "select_first_n", "config": {"N": 1}} ) + else: + params['answers'] = record.values + self.log.debug("params for A: %s", params) return params From bec8c086da738d86ee0ab0ffd3bc483c6abc12b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20H=C3=BCgli?= Date: Tue, 10 Dec 2019 15:33:41 +0100 Subject: [PATCH 081/155] add new value delegation_set_id to aws provider --- octodns/provider/route53.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index 66da6b5..e4e4b50 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -618,8 +618,9 @@ class Route53Provider(BaseProvider): def __init__(self, id, access_key_id=None, secret_access_key=None, max_changes=1000, client_max_attempts=None, - session_token=None, *args, **kwargs): + session_token=None, delegation_set_id=None, *args, **kwargs): self.max_changes = max_changes + self.delegation_set_id = delegation_set_id _msg = 'access_key_id={}, secret_access_key=***, ' \ 'session_token=***'.format(access_key_id) use_fallback_auth = access_key_id is None and \ @@ -676,7 +677,12 @@ class Route53Provider(BaseProvider): ref = uuid4().hex self.log.debug('_get_zone_id: no matching zone, creating, ' 'ref=%s', ref) + if self.delegation_set_id: resp = self._conn.create_hosted_zone(Name=name, + CallerReference=ref, + DelegationSetId=self.delegation_set_id) + else: + resp = self._conn.create_hosted_zone(Name=name, CallerReference=ref) self.r53_zones[name] = id = resp['HostedZone']['Id'] return id From 65922bbe5cd29702ea7a927e058372079474c00f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20H=C3=BCgli?= Date: Tue, 10 Dec 2019 15:45:16 +0100 Subject: [PATCH 082/155] fix lint --- octodns/provider/route53.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index e4e4b50..9677ff0 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -678,9 +678,11 @@ class Route53Provider(BaseProvider): self.log.debug('_get_zone_id: no matching zone, creating, ' 'ref=%s', ref) if self.delegation_set_id: - resp = self._conn.create_hosted_zone(Name=name, + resp = self._conn.create_hosted_zone( + Name=name, CallerReference=ref, - DelegationSetId=self.delegation_set_id) + DelegationSetId=self.delegation_set_id + ) else: resp = self._conn.create_hosted_zone(Name=name, CallerReference=ref) From fcf32e2d445e39c1963e4d63a1023f62fca53d8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20H=C3=BCgli?= Date: Tue, 10 Dec 2019 16:00:03 +0100 Subject: [PATCH 083/155] final fix lint --- octodns/provider/route53.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index 9677ff0..3d658d0 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -675,17 +675,16 @@ class Route53Provider(BaseProvider): return id if create: ref = uuid4().hex + del_set = self.delegation_set_id self.log.debug('_get_zone_id: no matching zone, creating, ' 'ref=%s', ref) if self.delegation_set_id: - resp = self._conn.create_hosted_zone( - Name=name, + resp = self._conn.create_hosted_zone(Name=name, CallerReference=ref, - DelegationSetId=self.delegation_set_id - ) + DelegationSetId=del_set) else: resp = self._conn.create_hosted_zone(Name=name, - CallerReference=ref) + CallerReference=ref) self.r53_zones[name] = id = resp['HostedZone']['Id'] return id return None From b9d0586c7f5043e8fd922a5b408518ccdcfbdf4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20H=C3=BCgli?= Date: Tue, 10 Dec 2019 17:03:31 +0100 Subject: [PATCH 084/155] Update octodns/provider/route53.py Co-Authored-By: Ross McFarland --- octodns/provider/route53.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index 3d658d0..89fb7a8 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -678,7 +678,7 @@ class Route53Provider(BaseProvider): del_set = self.delegation_set_id self.log.debug('_get_zone_id: no matching zone, creating, ' 'ref=%s', ref) - if self.delegation_set_id: + if del_set: resp = self._conn.create_hosted_zone(Name=name, CallerReference=ref, DelegationSetId=del_set) From 334e64c8f5148c1f25d9227b8793a4feff38a2b8 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Tue, 10 Dec 2019 12:20:25 -0800 Subject: [PATCH 085/155] Python 3 friendly way to re-raise when tries expire --- octodns/provider/ns1.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index da2d64a..0383dbf 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -31,17 +31,18 @@ class Ns1Client(object): def _try(self, method, *args, **kwargs): tries = self.retry_count - while tries: + while True: # We'll raise to break after our tries expire try: return method(*args, **kwargs) except RateLimitException as e: + if tries <= 1: + raise period = float(e.period) self.log.warn('rate limit encountered, pausing ' 'for %ds and trying again, %d remaining', period, tries) sleep(period) tries -= 1 - raise def zones_retrieve(self, name): return self._try(self._zones.retrieve, name) From ea2a52c307d5e03d7b1a7550273c1d3e607a2026 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Tue, 10 Dec 2019 12:20:25 -0800 Subject: [PATCH 086/155] Python 3 friendly way to re-raise when tries expire --- octodns/provider/ns1.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 445e612..7244c3a 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -31,17 +31,18 @@ class Ns1Client(object): def _try(self, method, *args, **kwargs): tries = self.retry_count - while tries: + while True: # We'll raise to break after our tries expire try: return method(*args, **kwargs) except RateLimitException as e: + if tries <= 1: + raise period = float(e.period) self.log.warn('rate limit encountered, pausing ' 'for %ds and trying again, %d remaining', period, tries) sleep(period) tries -= 1 - raise def zones_retrieve(self, name): return self._try(self._zones.retrieve, name) From 7a472506ccdadef44bc164f17829b32c2961c55e Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Tue, 10 Dec 2019 13:50:11 -0800 Subject: [PATCH 087/155] Implement _data_for_dynamic_A w/some related refactoring --- octodns/provider/ns1.py | 138 +++++++++++++++++++++++++++-- tests/test_octodns_provider_ns1.py | 19 +++- 2 files changed, 147 insertions(+), 10 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 7244c3a..0b6e16a 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -19,6 +19,10 @@ from ..record import Record from .base import BaseProvider +class Ns1Exception(Exception): + pass + + class Ns1Client(object): log = getLogger('NS1Client') @@ -126,7 +130,18 @@ class Ns1Provider(BaseProvider): super(Ns1Provider, self).__init__(id, *args, **kwargs) self._client = Ns1Client(api_key, retry_count) - def _data_for_A(self, _type, record): + def _encode_notes(self, data): + return ' '.join(['{}:{}'.format(k, v) + for k, v in sorted(data.items())]) + + def _parse_notes(self, note): + data = {} + for piece in note.split(' '): + k, v = piece.split(':', 1) + data[k] = v + return data + + def _data_for_geo_A(self, _type, record): # record meta (which would include geo information is only # returned when getting a record's detail, not from zone detail geo = defaultdict(list) @@ -171,6 +186,116 @@ class Ns1Provider(BaseProvider): data['geo'] = geo return data + def _data_for_dynamic_A(self, _type, record): + # First make sure we have the expected filters config + if self._DYNAMIC_FILTERS != record['filters']: + self.log.error('_data_for_dynamic_A: %s %s has unsupported ' + 'filters', record['domain'], _type) + raise Ns1Exception('Unrecognized advanced record') + + # All regions (pools) will include the list of default values + # (eventually) at higher priorities, we'll just add them to this set to + # we'll have the complete collection. + default = set() + # Fill out the pools by walking the answers and looking at their + # region. + pools = defaultdict(lambda: {'fallback': None, 'values': []}) + for answer in record['answers']: + # region (group name in the UI) is the pool name + pool_name = answer['region'] + pool = pools[answer['region']] + + meta = answer['meta'] + value = text_type(answer['answer'][0]) + if meta['priority'] == 1: + # priority 1 means this answer is part of the pools own values + pool['values'].append({ + 'value': value, + 'weight': int(meta.get('weight', 1)), + }) + else: + # It's a fallback, we only care about it if it's a + # final/default + notes = self._parse_notes(meta.get('note', '')) + if notes.get('from', False) == '--default--': + default.add(value) + + # The regions objects map to rules, but it's a bit fuzzy since they're + # tied to pools on the NS1 side, e.g. we can only have 1 rule per pool, + # that may eventually run into problems, but I don't have any use-cases + # examples currently where it would + rules = [] + for pool_name, region in sorted(record['regions'].items()): + meta = region['meta'] + notes = self._parse_notes(meta.get('note', '')) + + # The group notes field in the UI is a `note` on the region here, + # that's where we can find our pool's fallback. + if 'fallback' in notes: + # set the fallback pool name + pools[pool_name]['fallback'] = notes['fallback'] + + geos = set() + + # continents are mapped (imperfectly) to regions, but what about + # Canada/North America + for georegion in meta.get('georegion', []): + geos.add(self._REGION_TO_CONTINENT[georegion]) + + # Countries are easy enough to map, we just have ot find their + # continent + for country in meta.get('country', []): + con = country_alpha2_to_continent_code(country) + geos.add('{}-{}'.format(con, country)) + + # States are easy too, just assume NA-US (CA providences aren't + # supported by octoDNS currently) + for state in meta.get('us_state', []): + geos.add('NA-US-{}'.format(state)) + + rule = { + 'pool': pool_name, + '_order': notes['rule-order'], + } + if geos: + rule['geos'] = geos + rules.append(rule) + + # Order and convert to a list + default = sorted(default) + # Order + rules.sort(key=lambda r: (r['_order'], r['pool'])) + + return { + 'dynamic': { + 'pools': pools, + 'rules': rules, + }, + 'ttl': record['ttl'], + 'type': _type, + 'values': sorted(default), + } + + def _data_for_A(self, _type, record): + if record.get('tier', 1) > 1: + # Advanced record, see if it's first answer has a note + try: + first_answer_note = record['answers'][0]['meta']['note'] + except (IndexError, KeyError): + first_answer_note = '' + # If that note includes a `from` (pool name) it's a dynamic record + if 'from:' in first_answer_note: + return self._data_for_dynamic_A(_type, record) + # If not it's an old geo record + return self._data_for_geo_A(_type, record) + + # This is a basic record, just convert it + return { + 'ttl': record['ttl'], + 'type': _type, + 'values': [text_type(x) for x in record['short_answers']] + } + _data_for_AAAA = _data_for_A def _data_for_SPF(self, _type, record): @@ -316,23 +441,18 @@ class Ns1Provider(BaseProvider): continue data_for = getattr(self, '_data_for_{}'.format(_type)) name = zone.hostname_from_fqdn(record['domain']) - record = Record.new(zone, name, data_for(_type, record), - source=self, lenient=lenient) + data = data_for(_type, record) + record = Record.new(zone, name, data, source=self, lenient=lenient) zone_hash[(_type, name)] = record [zone.add_record(r, lenient=lenient) for r in zone_hash.values()] self.log.info('populate: found %s records, exists=%s', len(zone.records) - before, exists) return exists - def _encode_notes(self, data): - return ' '.join(['{}:{}'.format(k, v) - for k, v in sorted(data.items())]) - def _params_for_A(self, record): params = {'ttl': record.ttl} - if hasattr(record, 'dynamic'): - + if hasattr(record, 'dynamic') and record.dynamic: pools = record.dynamic.pools # Convert rules to regions diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index 0743943..fedcc2e 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -165,8 +165,9 @@ class TestNs1Provider(TestCase): 'domain': 'unit.tests.', }] + @patch('ns1.rest.records.Records.retrieve') @patch('ns1.rest.zones.Zones.retrieve') - def test_populate(self, zone_retrieve_mock): + def test_populate(self, zone_retrieve_mock, record_retrieve_mock): provider = Ns1Provider('test', 'api-key') # Bad auth @@ -197,6 +198,7 @@ class TestNs1Provider(TestCase): # Existing zone w/o records zone_retrieve_mock.reset_mock() + record_retrieve_mock.reset_mock() ns1_zone = { 'records': [{ "domain": "geo.unit.tests", @@ -211,17 +213,23 @@ class TestNs1Provider(TestCase): {'answer': ['4.5.6.7'], 'meta': {'iso_region_code': ['NA-US-WA']}}, ], + 'tier': 3, 'ttl': 34, }], } zone_retrieve_mock.side_effect = [ns1_zone] + # Its tier 3 so we'll do a full lookup + record_retrieve_mock.side_effect = ns1_zone['records'] zone = Zone('unit.tests.', []) provider.populate(zone) self.assertEquals(1, len(zone.records)) self.assertEquals(('unit.tests',), zone_retrieve_mock.call_args[0]) + record_retrieve_mock.assert_has_calls([call('unit.tests', + 'geo.unit.tests', 'A')]) # Existing zone w/records zone_retrieve_mock.reset_mock() + record_retrieve_mock.reset_mock() ns1_zone = { 'records': self.ns1_records + [{ "domain": "geo.unit.tests", @@ -236,17 +244,23 @@ class TestNs1Provider(TestCase): {'answer': ['4.5.6.7'], 'meta': {'iso_region_code': ['NA-US-WA']}}, ], + 'tier': 3, 'ttl': 34, }], } zone_retrieve_mock.side_effect = [ns1_zone] + # Its tier 3 so we'll do a full lookup + record_retrieve_mock.side_effect = ns1_zone['records'] zone = Zone('unit.tests.', []) provider.populate(zone) self.assertEquals(self.expected, zone.records) self.assertEquals(('unit.tests',), zone_retrieve_mock.call_args[0]) + record_retrieve_mock.assert_has_calls([call('unit.tests', + 'geo.unit.tests', 'A')]) # Test skipping unsupported record type zone_retrieve_mock.reset_mock() + record_retrieve_mock.reset_mock() ns1_zone = { 'records': self.ns1_records + [{ 'type': 'UNSUPPORTED', @@ -266,6 +280,7 @@ class TestNs1Provider(TestCase): {'answer': ['4.5.6.7'], 'meta': {'iso_region_code': ['NA-US-WA']}}, ], + 'tier': 3, 'ttl': 34, }], } @@ -274,6 +289,8 @@ class TestNs1Provider(TestCase): provider.populate(zone) self.assertEquals(self.expected, zone.records) self.assertEquals(('unit.tests',), zone_retrieve_mock.call_args[0]) + record_retrieve_mock.assert_has_calls([call('unit.tests', + 'geo.unit.tests', 'A')]) @patch('ns1.rest.records.Records.delete') @patch('ns1.rest.records.Records.update') From f6c60b69b72e9538d164ccfa41ceefe82bebc011 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Wed, 11 Dec 2019 15:05:52 -0800 Subject: [PATCH 088/155] WIP monitors management --- octodns/provider/ns1.py | 110 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 103 insertions(+), 7 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 0b6e16a..8d230cf 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -32,6 +32,7 @@ class Ns1Client(object): client = NS1(apiKey=api_key) self._records = client.records() self._zones = client.zones() + self._monitors = client.monitors() def _try(self, method, *args, **kwargs): tries = self.retry_count @@ -66,6 +67,17 @@ class Ns1Client(object): def records_delete(self, zone, domain, _type): return self._try(self._records.delete, zone, domain, _type) + def monitors_list(self): + return self._try(self._monitors.list) + + def monitors_create(self, **params): + body = {} # TODO: not clear what this is supposed to be + return self._try(self._monitors.create, body, **params) + + def monitors_update(self, job_id, **params): + body = {} # TODO: not clear what this is supposed to be + return self._try(self._monitors.update, job_id, body, **params) + class Ns1Provider(BaseProvider): ''' @@ -136,9 +148,10 @@ class Ns1Provider(BaseProvider): def _parse_notes(self, note): data = {} - for piece in note.split(' '): - k, v = piece.split(':', 1) - data[k] = v + if note: + for piece in note.split(' '): + k, v = piece.split(':', 1) + data[k] = v return data def _data_for_geo_A(self, _type, record): @@ -258,7 +271,7 @@ class Ns1Provider(BaseProvider): '_order': notes['rule-order'], } if geos: - rule['geos'] = geos + rule['geos'] = sorted(geos) rules.append(rule) # Order and convert to a list @@ -449,6 +462,79 @@ class Ns1Provider(BaseProvider): len(zone.records) - before, exists) return exists + def _extra_changes(self, desired, changes, **kwargs): + # TODO: check monitors to see if they need updated + return [] + + def _monitors_for(self, record): + # TODO: should this just be a global cache by fqdn, type, and value? + expected_host = record.fqdn[:-1] + expected_type = record._type + + monitors = {} + + # TODO: cache here or in Ns1Client + for monitor in self._client.monitors_list(): + data = self._parse_notes(monitor['notes']) + if expected_host == data['host'] or \ + expected_type == data['type']: + # This monitor does not belong to this record + config = monitor['config'] + value = config['host'] + monitors[value] = monitor + + return monitors + + def _sync_monitor(self, record, value, existing): + host = record.fqdn[:-1] + _type = record._type + + request = 'GET {path} HTTP/1.0\\r\\nHost: {host}\\r\\n' \ + 'User-agent: NS1\\r\\n\\r\\n'.format(path=record.healthcheck_path, + host=host) + + expected = { + 'active': True, + 'config': { + 'connect_timeout': 2000, + 'host': value, + 'port': record.healthcheck_port, + 'response_timeout': 10000, + 'send': request, + 'ssl': record.healthcheck_protocol == 'HTTPS', + }, + 'frequency': 60, + 'job_type': 'tcp', + 'name': '{} - {} - {}'.format(host, _type, value), + 'notes': self._encode_notes({ + 'host': host, + 'type': _type, + }), + 'policy': 'quorum', + 'rapid_recheck': False, + 'region_scope': 'fixed', + # TODO: what should we do here dal, sjc, lga, sin, ams + 'regions': ['lga'], + 'rules': [{ + 'comparison': 'contains', + 'key': 'output', + 'value': '200 OK', + }], + } + + if existing: + monitor_id = existing['id'] + # See if the monitor needs updating + for k, v in expected.items(): + if existing.get(k, '--missing--') != v: + self._client.monitors_update(monitor_id, **expected) + break + else: + return self._client.monitors_create(**expected)['id'] + + # TODO: this needs to return the feed + return None + def _params_for_A(self, record): params = {'ttl': record.ttl} @@ -498,13 +584,21 @@ class Ns1Provider(BaseProvider): 'meta': meta, } - # Build a list of primary values for each pool + existing_monitors = self._monitors_for(record) + + # Build a list of primary values for each pool, including their + # monitor pool_answers = defaultdict(list) for pool_name, pool in sorted(pools.items()): for value in pool.data['values']: + weight = value['weight'] + value = value['value'] + existing = existing_monitors.get(value) + monitor_id = self._sync_monitor(record, value, existing) pool_answers[pool_name].append({ - 'answer': [value['value']], - 'weight': value['weight'], + 'answer': [value], + 'weight': weight, + 'monitor_id': monitor_id, }) default_answers = [{ @@ -531,6 +625,7 @@ class Ns1Provider(BaseProvider): }), 'weight': answer['weight'], }, + 'up': True, # TODO: this should be a monitor/feed 'region': pool_name, # the one we're answering } answers.append(answer) @@ -547,6 +642,7 @@ class Ns1Provider(BaseProvider): 'note': self._encode_notes({ 'from': '--default--', }), + 'up': True, 'weight': 1, }, 'region': pool_name, # the one we're answering From 55f4194daf8ece7fb63c3b9a6bf2d2fdf9a8d38a Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Thu, 12 Dec 2019 13:23:35 -0800 Subject: [PATCH 089/155] Functionally complement and untested ns1 dynamic support --- octodns/provider/ns1.py | 559 +++++++++++++++++++---------- tests/test_octodns_provider_ns1.py | 8 +- 2 files changed, 374 insertions(+), 193 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 8d230cf..e5cb1ed 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -12,10 +12,11 @@ from ns1 import NS1 from ns1.rest.errors import RateLimitException, ResourceException from pycountry_convert import country_alpha2_to_continent_code from time import sleep +from uuid import uuid4 from six import text_type -from ..record import Record +from ..record import Record, Update from .base import BaseProvider @@ -33,6 +34,9 @@ class Ns1Client(object): self._records = client.records() self._zones = client.zones() self._monitors = client.monitors() + self._notifylists = client.notifylists() + self._datasource = client.datasource() + self._datafeed = client.datafeed() def _try(self, method, *args, **kwargs): tries = self.retry_count @@ -49,35 +53,62 @@ class Ns1Client(object): sleep(period) tries -= 1 - def zones_retrieve(self, name): - return self._try(self._zones.retrieve, name) + def datafeed_create(self, sourceid, name, config): + return self._try(self._datafeed.create, sourceid, name, config) - def zones_create(self, name): - return self._try(self._zones.create, name) + def datafeed_delete(self, sourceid, feedid): + return self._try(self._datafeed.delete, sourceid, feedid) - def records_retrieve(self, zone, domain, _type): - return self._try(self._records.retrieve, zone, domain, _type) + def datafeed_list(self, sourceid): + return self._try(self._datafeed.list, sourceid) - def records_create(self, zone, domain, _type, **params): - return self._try(self._records.create, zone, domain, _type, **params) + def datasource_create(self, **body): + return self._try(self._datasource.create, **body) - def records_update(self, zone, domain, _type, **params): - return self._try(self._records.update, zone, domain, _type, **params) + def datasource_list(self): + return self._try(self._datasource.list) - def records_delete(self, zone, domain, _type): - return self._try(self._records.delete, zone, domain, _type) + def monitors_create(self, **params): + body = {} + return self._try(self._monitors.create, body, **params) + + def monitors_delete(self, jobid): + return self._try(self._monitors.delete, jobid) def monitors_list(self): return self._try(self._monitors.list) - def monitors_create(self, **params): - body = {} # TODO: not clear what this is supposed to be - return self._try(self._monitors.create, body, **params) - def monitors_update(self, job_id, **params): - body = {} # TODO: not clear what this is supposed to be + body = {} return self._try(self._monitors.update, job_id, body, **params) + def notifylists_delete(self, nlid): + return self._try(self._notifylists.delete, nlid) + + def notifylists_create(self, **body): + return self._try(self._notifylists.create, body) + + def notifylists_list(self): + return self._try(self._notifylists.list) + + def records_create(self, zone, domain, _type, **params): + return self._try(self._records.create, zone, domain, _type, **params) + + def records_delete(self, zone, domain, _type): + return self._try(self._records.delete, zone, domain, _type) + + def records_retrieve(self, zone, domain, _type): + return self._try(self._records.retrieve, zone, domain, _type) + + def records_update(self, zone, domain, _type, **params): + return self._try(self._records.update, zone, domain, _type, **params) + + def zones_create(self, name): + return self._try(self._zones.create, name) + + def zones_retrieve(self, name): + return self._try(self._zones.retrieve, name) + class Ns1Provider(BaseProvider): ''' @@ -140,7 +171,11 @@ class Ns1Provider(BaseProvider): self.log.debug('__init__: id=%s, api_key=***, retry_count=%d', id, retry_count) super(Ns1Provider, self).__init__(id, *args, **kwargs) + self._client = Ns1Client(api_key, retry_count) + self.__monitors = None + self.__datasource_id = None + self.__feeds_for_monitors = None def _encode_notes(self, data): return ' '.join(['{}:{}'.format(k, v) @@ -462,38 +497,139 @@ class Ns1Provider(BaseProvider): len(zone.records) - before, exists) return exists - def _extra_changes(self, desired, changes, **kwargs): - # TODO: check monitors to see if they need updated - return [] + def _params_for_geo_A(self, record): + # purposefully set non-geo answers to have an empty meta, + # so that we know we did this on purpose if/when troubleshooting + params = { + 'answers': [{"answer": [x], "meta": {}} for x in record.values], + 'ttl': record.ttl, + } - def _monitors_for(self, record): - # TODO: should this just be a global cache by fqdn, type, and value? - expected_host = record.fqdn[:-1] - expected_type = record._type + has_country = False + for iso_region, target in record.geo.items(): + key = 'iso_region_code' + value = iso_region + if not has_country and \ + len(value.split('-')) > 1: # pragma: nocover + has_country = True + for answer in target.values: + params['answers'].append( + { + 'answer': [answer], + 'meta': {key: [value]}, + }, + ) + + params['filters'] = [] + if has_country: + params['filters'].append( + {"filter": "shuffle", "config": {}} + ) + params['filters'].append( + {"filter": "geotarget_country", "config": {}} + ) + params['filters'].append( + {"filter": "select_first_n", + "config": {"N": 1}} + ) + + return params, None + + @property + def _monitors(self): + # TODO: cache in sync, here and for others + if self.__monitors is None: + self.__monitors = {m['id']: m + for m in self._client.monitors_list()} + return self.__monitors + def _monitors_for(self, record): monitors = {} - # TODO: cache here or in Ns1Client - for monitor in self._client.monitors_list(): - data = self._parse_notes(monitor['notes']) - if expected_host == data['host'] or \ - expected_type == data['type']: - # This monitor does not belong to this record - config = monitor['config'] - value = config['host'] - monitors[value] = monitor + if getattr(record, 'dynamic', False): + # TODO: should this just be a global cache by fqdn, type, and + # value? + expected_host = record.fqdn[:-1] + expected_type = record._type + + # TODO: cache here or in Ns1Client + for monitor in self._monitors.values(): + data = self._parse_notes(monitor['notes']) + if expected_host == data['host'] or \ + expected_type == data['type']: + # This monitor does not belong to this record + config = monitor['config'] + value = config['host'] + monitors[value] = monitor return monitors - def _sync_monitor(self, record, value, existing): + @property + def _datasource_id(self): + if self.__datasource_id is None: + name = 'octoDNS NS1 Data Source' + source = None + for candidate in self._client.datasource_list(): + if candidate['name'] == name: + # Found it + source = candidate + break + + if source is None: + # We need to create it + source = self._client \ + .datasource_create(name=name, + sourcetype='nsone_monitoring') + + self.__datasource_id = source['id'] + + return self.__datasource_id + + def _feed_for_monitor(self, monitor): + if self.__feeds_for_monitors is None: + self.__feeds_for_monitors = { + f['config']['jobid']: f['id'] + for f in self._client.datafeed_list(self._datasource_id) + } + + return self.__feeds_for_monitors.get(monitor['id']) + + def _create_monitor(self, monitor): + # TODO: looks like length limit is 64 char + name = '{} - {}'.format(monitor['name'], uuid4().hex[:6]) + + # Create the notify list + notify_list = [{ + 'config': { + 'sourceid': self._datasource_id, + }, + 'type': 'datafeed', + }] + nl = self._client.notifylists_create(name=name, + notify_list=notify_list) + + # Create the monitor + monitor['notify_list'] = nl['id'] + monitor = self._client.monitors_create(**monitor) + + # Create the data feed + config = { + 'jobid': monitor['id'], + } + feed = self._client.datafeed_create(self._datasource_id, name, + config) + + return monitor['id'], feed['id'] + + def _monitor_gen(self, record, value): host = record.fqdn[:-1] _type = record._type request = 'GET {path} HTTP/1.0\\r\\nHost: {host}\\r\\n' \ 'User-agent: NS1\\r\\n\\r\\n'.format(path=record.healthcheck_path, - host=host) + host=record.healthcheck_host) - expected = { + return { 'active': True, 'config': { 'connect_timeout': 2000, @@ -522,177 +658,189 @@ class Ns1Provider(BaseProvider): }], } + def _monitor_is_match(self, expected, have): + # Make sure what we have matches what's in expected exactly. Anything + # else in have will be ignored. + for k, v in expected.items(): + if have.get(k, '--missing--') != v: + return False + + return True + + def _monitor_sync(self, record, value, existing): + expected = self._monitor_gen(record, value) + if existing: monitor_id = existing['id'] - # See if the monitor needs updating - for k, v in expected.items(): - if existing.get(k, '--missing--') != v: - self._client.monitors_update(monitor_id, **expected) - break + + if not self._monitor_is_match(expected, existing): + # Update the monitor to match expected, everything else will be + # left alone and assumed correct + self._client.monitors_update(monitor_id, **expected) + + try: + feed_id = self._feed_for_monitor(existing) + except KeyError: + raise Ns1Exception('Failed to find the feed for {} ({})' + .format(existing['name'], existing['id'])) else: - return self._client.monitors_create(**expected)['id'] + # We don't have an existing monitor create it (and related bits) + monitor_id, feed_id = self._create_monitor(expected) - # TODO: this needs to return the feed - return None + return monitor_id, feed_id - def _params_for_A(self, record): - params = {'ttl': record.ttl} + def _gc_monitors(self, record, active_monitor_ids=None): - if hasattr(record, 'dynamic') and record.dynamic: - pools = record.dynamic.pools + if active_monitor_ids is None: + active_monitor_ids = set() - # Convert rules to regions - regions = {} - for i, rule in enumerate(record.dynamic.rules): - pool_name = rule.data['pool'] + for monitor in self._monitors_for(record).values(): + monitor_id = monitor['id'] + if monitor_id in active_monitor_ids: + continue - notes = { - 'rule-order': i, - } + feed_id = self._feed_for_monitor(monitor) + if feed_id: + self._client.datafeed_delete(self._datasource_id, feed_id) - fallback = pools[pool_name].data.get('fallback', None) - if fallback: - notes['fallback'] = fallback - - country = set() - georegion = set() - us_state = set() - - for geo in rule.data.get('geos', []): - n = len(geo) - if n == 8: - # US state - us_state.add(geo[-2:]) - elif n == 5: - # Country - country.add(geo[-2:]) - else: - # Continent - georegion.update(self._CONTINENT_TO_REGIONS[geo]) - - meta = { - 'note': self._encode_notes(notes), - } - if georegion: - meta['georegion'] = sorted(georegion) - if country: - meta['country'] = sorted(country) - if us_state: - meta['us_state'] = sorted(us_state) - - regions[pool_name] = { - 'meta': meta, - } + self._client.monitors_delete(monitor_id) - existing_monitors = self._monitors_for(record) - - # Build a list of primary values for each pool, including their - # monitor - pool_answers = defaultdict(list) - for pool_name, pool in sorted(pools.items()): - for value in pool.data['values']: - weight = value['weight'] - value = value['value'] - existing = existing_monitors.get(value) - monitor_id = self._sync_monitor(record, value, existing) - pool_answers[pool_name].append({ - 'answer': [value], - 'weight': weight, - 'monitor_id': monitor_id, - }) - - default_answers = [{ - 'answer': [v], - 'weight': 1, - } for v in record.values] - - # Build our list of answers - answers = [] - for pool_name in sorted(pools.keys()): - priority = 1 - - # Dynamic/health checked - current_pool_name = pool_name - while current_pool_name: - pool = pools[current_pool_name] - for answer in pool_answers[current_pool_name]: - answer = { - 'answer': answer['answer'], - 'meta': { - 'priority': priority, - 'note': self._encode_notes({ - 'from': current_pool_name, - }), - 'weight': answer['weight'], - }, - 'up': True, # TODO: this should be a monitor/feed - 'region': pool_name, # the one we're answering - } - answers.append(answer) + notify_list_id = monitor['notify_list'] + self._client.notifylists_delete(notify_list_id) + + def _params_for_dynamic_A(self, record): + pools = record.dynamic.pools + + # Convert rules to regions + regions = {} + for i, rule in enumerate(record.dynamic.rules): + pool_name = rule.data['pool'] + + notes = { + 'rule-order': i, + } - current_pool_name = pool.data.get('fallback', None) - priority += 1 + fallback = pools[pool_name].data.get('fallback', None) + if fallback: + notes['fallback'] = fallback + + country = set() + georegion = set() + us_state = set() + + for geo in rule.data.get('geos', []): + n = len(geo) + if n == 8: + # US state + us_state.add(geo[-2:]) + elif n == 5: + # Country + country.add(geo[-2:]) + else: + # Continent + georegion.update(self._CONTINENT_TO_REGIONS[geo]) + + meta = { + 'note': self._encode_notes(notes), + } + if georegion: + meta['georegion'] = sorted(georegion) + if country: + meta['country'] = sorted(country) + if us_state: + meta['us_state'] = sorted(us_state) + + regions[pool_name] = { + 'meta': meta, + } - # Static/default - for answer in default_answers: + existing_monitors = self._monitors_for(record) + active_monitors = set() + + # Build a list of primary values for each pool, including their + # feed_id (monitor) + pool_answers = defaultdict(list) + for pool_name, pool in sorted(pools.items()): + for value in pool.data['values']: + weight = value['weight'] + value = value['value'] + existing = existing_monitors.get(value) + monitor_id, feed_id = self._monitor_sync(record, value, + existing) + active_monitors.add(monitor_id) + pool_answers[pool_name].append({ + 'answer': [value], + 'weight': weight, + 'feed_id': feed_id, + }) + + default_answers = [{ + 'answer': [v], + 'weight': 1, + } for v in record.values] + + # Build our list of answers + answers = [] + for pool_name in sorted(pools.keys()): + priority = 1 + + # Dynamic/health checked + current_pool_name = pool_name + while current_pool_name: + pool = pools[current_pool_name] + for answer in pool_answers[current_pool_name]: answer = { 'answer': answer['answer'], 'meta': { 'priority': priority, 'note': self._encode_notes({ - 'from': '--default--', + 'from': current_pool_name, }), - 'up': True, - 'weight': 1, + 'up': { + 'feed': answer['feed_id'], + }, + 'weight': answer['weight'], }, 'region': pool_name, # the one we're answering } answers.append(answer) - params.update({ - 'answers': answers, - 'filters': self._DYNAMIC_FILTERS, - 'regions': regions, - }) + current_pool_name = pool.data.get('fallback', None) + priority += 1 + + # Static/default + for answer in default_answers: + answer = { + 'answer': answer['answer'], + 'meta': { + 'priority': priority, + 'note': self._encode_notes({ + 'from': '--default--', + }), + 'up': True, + 'weight': 1, + }, + 'region': pool_name, # the one we're answering + } + answers.append(answer) - return params + return { + 'answers': answers, + 'filters': self._DYNAMIC_FILTERS, + 'regions': regions, + 'ttl': record.ttl, + }, active_monitors + def _params_for_A(self, record): + if getattr(record, 'dynamic', False): + return self._params_for_dynamic_A(record) elif hasattr(record, 'geo'): - # purposefully set non-geo answers to have an empty meta, - # so that we know we did this on purpose if/when troubleshooting - params['answers'] = [{"answer": [x], "meta": {}} - for x in record.values] - has_country = False - for iso_region, target in record.geo.items(): - key = 'iso_region_code' - value = iso_region - if not has_country and \ - len(value.split('-')) > 1: # pragma: nocover - has_country = True - for answer in target.values: - params['answers'].append( - { - 'answer': [answer], - 'meta': {key: [value]}, - }, - ) - params['filters'] = [] - if has_country: - params['filters'].append( - {"filter": "shuffle", "config": {}} - ) - params['filters'].append( - {"filter": "geotarget_country", "config": {}} - ) - params['filters'].append( - {"filter": "select_first_n", - "config": {"N": 1}} - ) - else: - params['answers'] = record.values + return self._params_for_geo_A(record) - self.log.debug("params for A: %s", params) - return params + return { + 'answers': record.values, + 'ttl': record.ttl, + }, None _params_for_AAAA = _params_for_A _params_for_NS = _params_for_A @@ -702,49 +850,81 @@ class Ns1Provider(BaseProvider): # escaped in values so we have to strip them here and add # them when going the other way values = [v.replace('\\;', ';') for v in record.values] - return {'answers': values, 'ttl': record.ttl} + return {'answers': values, 'ttl': record.ttl}, None _params_for_TXT = _params_for_SPF def _params_for_CAA(self, record): values = [(v.flags, v.tag, v.value) for v in record.values] - return {'answers': values, 'ttl': record.ttl} + return {'answers': values, 'ttl': record.ttl}, None + # TODO: dynamic CNAME support def _params_for_CNAME(self, record): - return {'answers': [record.value], 'ttl': record.ttl} + return {'answers': [record.value], 'ttl': record.ttl}, None _params_for_ALIAS = _params_for_CNAME _params_for_PTR = _params_for_CNAME def _params_for_MX(self, record): values = [(v.preference, v.exchange) for v in record.values] - return {'answers': values, 'ttl': record.ttl} + return {'answers': values, 'ttl': record.ttl}, None def _params_for_NAPTR(self, record): values = [(v.order, v.preference, v.flags, v.service, v.regexp, v.replacement) for v in record.values] - return {'answers': values, 'ttl': record.ttl} + return {'answers': values, 'ttl': record.ttl}, None def _params_for_SRV(self, record): values = [(v.priority, v.weight, v.port, v.target) for v in record.values] - return {'answers': values, 'ttl': record.ttl} + return {'answers': values, 'ttl': record.ttl}, None + + def _extra_changes(self, desired, changes, **kwargs): + self.log.debug('_extra_changes: desired=%s', desired.name) + + changed = set([c.record for c in changes]) + + extra = [] + for record in desired.records: + if record in changed or not getattr(record, 'dynamic', False): + # Already changed, or no dynamic , no need to check it + continue + + for have in self._monitors_for(record).values(): + value = have['config']['host'] + expected = self._monitor_gen(record, value) + if not expected: + self.log.info('_extra_changes: monitor missing for %s', + expected['name']) + extra.append(Update(record, record)) + break + if not self._monitor_is_match(expected, have): + self.log.info('_extra_changes: monitor mis-match for %s', + expected['name']) + extra.append(Update(record, record)) + break + + return extra def _apply_Create(self, ns1_zone, change): new = change.new zone = new.zone.name[:-1] domain = new.fqdn[:-1] _type = new._type - params = getattr(self, '_params_for_{}'.format(_type))(new) + params, active_monitor_ids = \ + getattr(self, '_params_for_{}'.format(_type))(new) self._client.records_create(zone, domain, _type, **params) + self._gc_monitors(new, active_monitor_ids) def _apply_Update(self, ns1_zone, change): new = change.new zone = new.zone.name[:-1] domain = new.fqdn[:-1] _type = new._type - params = getattr(self, '_params_for_{}'.format(_type))(new) + params, active_monitor_ids = \ + getattr(self, '_params_for_{}'.format(_type))(new) self._client.records_update(zone, domain, _type, **params) + self._gc_monitors(new, active_monitor_ids) def _apply_Delete(self, ns1_zone, change): existing = change.existing @@ -752,6 +932,7 @@ class Ns1Provider(BaseProvider): domain = existing.fqdn[:-1] _type = existing._type self._client.records_delete(zone, domain, _type) + self._gc_monitors(existing) def _apply(self, plan): desired = plan.desired diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index fedcc2e..539fcfb 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -474,16 +474,16 @@ class TestNs1Provider(TestCase): 'type': 'SPF', 'value': 'foo\\; bar baz\\; blip' }) - self.assertEquals(['foo; bar baz; blip'], - provider._params_for_SPF(record)['answers']) + params, _ = provider._params_for_SPF(record) + self.assertEquals(['foo; bar baz; blip'], params['answers']) record = Record.new(zone, 'txt', { 'ttl': 35, 'type': 'TXT', 'value': 'foo\\; bar baz\\; blip' }) - self.assertEquals(['foo; bar baz; blip'], - provider._params_for_TXT(record)['answers']) + params, _ = provider._params_for_SPF(record) + self.assertEquals(['foo; bar baz; blip'], params['answers']) def test_data_for_CNAME(self): provider = Ns1Provider('test', 'api-key') From c119f2e802a73e19c1918f1b37d2f1e44e5cac13 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Thu, 12 Dec 2019 14:03:09 -0800 Subject: [PATCH 090/155] Move ns1 caching to client where it's much safer/consistent --- octodns/provider/ns1.py | 182 +++++++++++++++++++++------------------- 1 file changed, 97 insertions(+), 85 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index e5cb1ed..321b439 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -38,26 +38,58 @@ class Ns1Client(object): self._datasource = client.datasource() self._datafeed = client.datafeed() - def _try(self, method, *args, **kwargs): - tries = self.retry_count - while True: # We'll raise to break after our tries expire - try: - return method(*args, **kwargs) - except RateLimitException as e: - if tries <= 1: - raise - period = float(e.period) - self.log.warn('rate limit encountered, pausing ' - 'for %ds and trying again, %d remaining', - period, tries) - sleep(period) - tries -= 1 + self._datasource_id = None + self._feeds_for_monitors = None + self._monitors_cache = None + + @property + def datasource_id(self): + if self._datasource_id is None: + name = 'octoDNS NS1 Data Source' + source = None + for candidate in self.datasource_list(): + if candidate['name'] == name: + # Found it + source = candidate + break + + if source is None: + # We need to create it + source = self.datasource_create(name=name, + sourcetype='nsone_monitoring') + + self._datasource_id = source['id'] + + return self._datasource_id + + @property + def feeds_for_monitors(self): + if self._feeds_for_monitors is None: + self._feeds_for_monitors = { + f['config']['jobid']: f['id'] + for f in self.datafeed_list(self.datasource_id) + } + + return self._feeds_for_monitors + + @property + def monitors(self): + if self._monitors_cache is None: + self._monitors_cache = \ + {m['id']: m for m in self.monitors_list()} + return self._monitors_cache def datafeed_create(self, sourceid, name, config): - return self._try(self._datafeed.create, sourceid, name, config) + ret = self._try(self._datafeed.create, sourceid, name, config) + self.feeds_for_monitors[config['jobid']] = ret['id'] + return ret def datafeed_delete(self, sourceid, feedid): - return self._try(self._datafeed.delete, sourceid, feedid) + ret = self._try(self._datafeed.delete, sourceid, feedid) + self._feeds_for_monitors = { + k: v for k, v in self._feeds_for_monitors.items() if v != feedid + } + return ret def datafeed_list(self, sourceid): return self._try(self._datafeed.list, sourceid) @@ -70,10 +102,14 @@ class Ns1Client(object): def monitors_create(self, **params): body = {} - return self._try(self._monitors.create, body, **params) + ret = self._try(self._monitors.create, body, **params) + self.monitors[ret['id']] = ret + return ret def monitors_delete(self, jobid): - return self._try(self._monitors.delete, jobid) + ret = self._try(self._monitors.delete, jobid) + self.monitors.pop(jobid) + return ret def monitors_list(self): return self._try(self._monitors.list) @@ -109,6 +145,21 @@ class Ns1Client(object): def zones_retrieve(self, name): return self._try(self._zones.retrieve, name) + def _try(self, method, *args, **kwargs): + tries = self.retry_count + while True: # We'll raise to break after our tries expire + try: + return method(*args, **kwargs) + except RateLimitException as e: + if tries <= 1: + raise + period = float(e.period) + self.log.warn('rate limit encountered, pausing ' + 'for %ds and trying again, %d remaining', + period, tries) + sleep(period) + tries -= 1 + class Ns1Provider(BaseProvider): ''' @@ -173,9 +224,6 @@ class Ns1Provider(BaseProvider): super(Ns1Provider, self).__init__(id, *args, **kwargs) self._client = Ns1Client(api_key, retry_count) - self.__monitors = None - self.__datasource_id = None - self.__feeds_for_monitors = None def _encode_notes(self, data): return ' '.join(['{}:{}'.format(k, v) @@ -535,25 +583,14 @@ class Ns1Provider(BaseProvider): return params, None - @property - def _monitors(self): - # TODO: cache in sync, here and for others - if self.__monitors is None: - self.__monitors = {m['id']: m - for m in self._client.monitors_list()} - return self.__monitors - def _monitors_for(self, record): monitors = {} if getattr(record, 'dynamic', False): - # TODO: should this just be a global cache by fqdn, type, and - # value? expected_host = record.fqdn[:-1] expected_type = record._type - # TODO: cache here or in Ns1Client - for monitor in self._monitors.values(): + for monitor in self._client.monitors.values(): data = self._parse_notes(monitor['notes']) if expected_host == data['host'] or \ expected_type == data['type']: @@ -564,62 +601,35 @@ class Ns1Provider(BaseProvider): return monitors - @property - def _datasource_id(self): - if self.__datasource_id is None: - name = 'octoDNS NS1 Data Source' - source = None - for candidate in self._client.datasource_list(): - if candidate['name'] == name: - # Found it - source = candidate - break - - if source is None: - # We need to create it - source = self._client \ - .datasource_create(name=name, - sourcetype='nsone_monitoring') - - self.__datasource_id = source['id'] - - return self.__datasource_id + def _create_feed(self, monitor): + # TODO: looks like length limit is 64 char + name = '{} - {}'.format(monitor['name'], uuid4().hex[:6]) - def _feed_for_monitor(self, monitor): - if self.__feeds_for_monitors is None: - self.__feeds_for_monitors = { - f['config']['jobid']: f['id'] - for f in self._client.datafeed_list(self._datasource_id) - } + # Create the data feed + config = { + 'jobid': monitor['id'], + } + feed = self._client.datafeed_create(self._client.datasource_id, name, + config) - return self.__feeds_for_monitors.get(monitor['id']) + return feed['id'] def _create_monitor(self, monitor): - # TODO: looks like length limit is 64 char - name = '{} - {}'.format(monitor['name'], uuid4().hex[:6]) - # Create the notify list notify_list = [{ 'config': { - 'sourceid': self._datasource_id, + 'sourceid': self._client.datasource_id, }, 'type': 'datafeed', }] - nl = self._client.notifylists_create(name=name, + nl = self._client.notifylists_create(name=monitor['name'], notify_list=notify_list) # Create the monitor monitor['notify_list'] = nl['id'] monitor = self._client.monitors_create(**monitor) - # Create the data feed - config = { - 'jobid': monitor['id'], - } - feed = self._client.datafeed_create(self._datasource_id, name, - config) - - return monitor['id'], feed['id'] + return monitor['id'], self._create_feed(monitor) def _monitor_gen(self, record, value): host = record.fqdn[:-1] @@ -678,11 +688,11 @@ class Ns1Provider(BaseProvider): # left alone and assumed correct self._client.monitors_update(monitor_id, **expected) - try: - feed_id = self._feed_for_monitor(existing) - except KeyError: - raise Ns1Exception('Failed to find the feed for {} ({})' - .format(existing['name'], existing['id'])) + feed_id = self._client.feeds_for_monitors.get(monitor_id) + if feed_id is None: + self.log.warn('_monitor_sync: %s (%s) missing feed, creating', + existing['name'], monitor_id) + feed_id = self._create_feed(existing) else: # We don't have an existing monitor create it (and related bits) monitor_id, feed_id = self._create_monitor(expected) @@ -699,9 +709,10 @@ class Ns1Provider(BaseProvider): if monitor_id in active_monitor_ids: continue - feed_id = self._feed_for_monitor(monitor) + feed_id = self._client.feeds_for_monitors.get(monitor_id) if feed_id: - self._client.datafeed_delete(self._datasource_id, feed_id) + self._client.datafeed_delete(self._client.datasource_id, + feed_id) self._client.monitors_delete(monitor_id) @@ -893,16 +904,17 @@ class Ns1Provider(BaseProvider): for have in self._monitors_for(record).values(): value = have['config']['host'] expected = self._monitor_gen(record, value) - if not expected: - self.log.info('_extra_changes: monitor missing for %s', - expected['name']) - extra.append(Update(record, record)) - break + # TODO: find values which have missing monitors if not self._monitor_is_match(expected, have): self.log.info('_extra_changes: monitor mis-match for %s', expected['name']) extra.append(Update(record, record)) break + if not have.get('notify_list'): + self.log.info('_extra_changes: broken monitor no notify ' + 'list %s (%s)', have['name'], have['id']) + extra.append(Update(record, record)) + break return extra From 674c29fb8b905b2754b8e3f93e811168a33f56a7 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Thu, 12 Dec 2019 14:17:42 -0800 Subject: [PATCH 091/155] Debug logging --- octodns/provider/ns1.py | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 321b439..5deb2c9 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -28,6 +28,7 @@ class Ns1Client(object): log = getLogger('NS1Client') def __init__(self, api_key, retry_count=4): + self.log.debug('__init__: retry_count=%d', retry_count) self.retry_count = retry_count client = NS1(apiKey=api_key) @@ -54,9 +55,11 @@ class Ns1Client(object): break if source is None: + self.log.info('datasource_id: creating datasource %s', name) # We need to create it source = self.datasource_create(name=name, sourcetype='nsone_monitoring') + self.log.info('datasource_id: id=%s', source['id']) self._datasource_id = source['id'] @@ -65,6 +68,7 @@ class Ns1Client(object): @property def feeds_for_monitors(self): if self._feeds_for_monitors is None: + self.log.debug('feeds_for_monitors: fetching & building') self._feeds_for_monitors = { f['config']['jobid']: f['id'] for f in self.datafeed_list(self.datasource_id) @@ -75,6 +79,7 @@ class Ns1Client(object): @property def monitors(self): if self._monitors_cache is None: + self.log.debug('monitors: fetching & building') self._monitors_cache = \ {m['id']: m for m in self.monitors_list()} return self._monitors_cache @@ -602,19 +607,24 @@ class Ns1Provider(BaseProvider): return monitors def _create_feed(self, monitor): + monitor_id = monitor['id'] + self.log.debug('_create_feed: monitor=%s', monitor_id) # TODO: looks like length limit is 64 char name = '{} - {}'.format(monitor['name'], uuid4().hex[:6]) # Create the data feed config = { - 'jobid': monitor['id'], + 'jobid': monitor_id, } feed = self._client.datafeed_create(self._client.datasource_id, name, config) + feed_id = feed['id'] + self.log.debug('_create_feed: feed=%s', feed_id) - return feed['id'] + return feed_id def _create_monitor(self, monitor): + self.log.debug('_create_monitor: monitor="%s"', monitor['name']) # Create the notify list notify_list = [{ 'config': { @@ -624,12 +634,16 @@ class Ns1Provider(BaseProvider): }] nl = self._client.notifylists_create(name=monitor['name'], notify_list=notify_list) + nl_id = nl['id'] + self.log.debug('_create_monitor: notify_list=%s', nl_id) # Create the monitor - monitor['notify_list'] = nl['id'] + monitor['notify_list'] = nl_id monitor = self._client.monitors_create(**monitor) + monitor_id = monitor['id'] + self.log.debug('_create_monitor: monitor=%s', monitor_id) - return monitor['id'], self._create_feed(monitor) + return monitor_id, self._create_feed(monitor) def _monitor_gen(self, record, value): host = record.fqdn[:-1] @@ -678,12 +692,16 @@ class Ns1Provider(BaseProvider): return True def _monitor_sync(self, record, value, existing): + self.log.debug('_monitor_sync: record=%s, value=%s', record.fqdn, + value) expected = self._monitor_gen(record, value) if existing: + self.log.debug('_monitor_sync: existing=%s', existing['id']) monitor_id = existing['id'] if not self._monitor_is_match(expected, existing): + self.log.debug('_monitor_sync: existing needs update') # Update the monitor to match expected, everything else will be # left alone and assumed correct self._client.monitors_update(monitor_id, **expected) @@ -694,12 +712,15 @@ class Ns1Provider(BaseProvider): existing['name'], monitor_id) feed_id = self._create_feed(existing) else: + self.log.debug('_monitor_sync: needs create') # We don't have an existing monitor create it (and related bits) monitor_id, feed_id = self._create_monitor(expected) return monitor_id, feed_id def _gc_monitors(self, record, active_monitor_ids=None): + self.log.debug('_gc_monitors: record=%s, active_monitor_ids=%s', + record.fqdn, active_monitor_ids) if active_monitor_ids is None: active_monitor_ids = set() @@ -709,6 +730,8 @@ class Ns1Provider(BaseProvider): if monitor_id in active_monitor_ids: continue + self.log.debug('_gc_monitors: deleting %s', monitor_id) + feed_id = self._client.feeds_for_monitors.get(monitor_id) if feed_id: self._client.datafeed_delete(self._client.datasource_id, From 6c7abe1fd643a3444d2a261dd27391654b9350fe Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Thu, 12 Dec 2019 14:19:16 -0800 Subject: [PATCH 092/155] Ns1 still SUPPORTS_GEO --- octodns/provider/ns1.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 5deb2c9..f18646c 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -174,7 +174,7 @@ class Ns1Provider(BaseProvider): class: octodns.provider.ns1.Ns1Provider api_key: env/NS1_API_KEY ''' - SUPPORTS_GEO = False + SUPPORTS_GEO = True SUPPORTS_DYNAMIC = True SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CAA', 'CNAME', 'MX', 'NAPTR', 'NS', 'PTR', 'SPF', 'SRV', 'TXT')) From d7053a2e92012b7e5bee75952c116144c70bbff3 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 13 Dec 2019 11:58:18 -0800 Subject: [PATCH 093/155] Ns1Client tests for caching and minor logic --- octodns/provider/ns1.py | 11 +- tests/test_octodns_provider_ns1.py | 186 +++++++++++++++++++++++++++++ 2 files changed, 194 insertions(+), 3 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index f18646c..a3bd647 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -121,7 +121,9 @@ class Ns1Client(object): def monitors_update(self, job_id, **params): body = {} - return self._try(self._monitors.update, job_id, body, **params) + ret = self._try(self._monitors.update, job_id, body, **params) + self.monitors[ret['id']] = ret + return ret def notifylists_delete(self, nlid): return self._try(self._notifylists.delete, nlid) @@ -238,8 +240,11 @@ class Ns1Provider(BaseProvider): data = {} if note: for piece in note.split(' '): - k, v = piece.split(':', 1) - data[k] = v + try: + k, v = piece.split(':', 1) + data[k] = v + except ValueError: + pass return data def _data_for_geo_A(self, _type, record): diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index 539fcfb..4849684 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -517,6 +517,24 @@ class TestNs1Provider(TestCase): provider._data_for_CNAME(b_record['type'], b_record)) +class TestNs1ProviderDynamic(TestCase): + + def test_notes(self): + provider = Ns1Provider('test', 'api-key') + + self.assertEquals({}, provider._parse_notes(None)) + self.assertEquals({}, provider._parse_notes('')) + self.assertEquals({}, provider._parse_notes('blah-blah-blah')) + + # Round tripping + data = { + 'key': 'value', + 'priority': '1', + } + notes = provider._encode_notes(data) + self.assertEquals(data, provider._parse_notes(notes)) + + class TestNs1Client(TestCase): @patch('ns1.rest.zones.Zones.retrieve') @@ -558,3 +576,171 @@ class TestNs1Client(TestCase): with self.assertRaises(RateLimitException) as ctx: client.zones_retrieve('unit.tests') self.assertEquals('last', text_type(ctx.exception)) + + @patch('ns1.rest.data.Source.list') + @patch('ns1.rest.data.Source.create') + def test_datasource_id(self, datasource_create_mock, datasource_list_mock): + client = Ns1Client('dummy-key') + + # First invocation with an empty list create + datasource_list_mock.reset_mock() + datasource_create_mock.reset_mock() + datasource_list_mock.side_effect = [[]] + datasource_create_mock.side_effect = [{ + 'id': 'foo', + }] + self.assertEquals('foo', client.datasource_id) + name = 'octoDNS NS1 Data Source' + source_type = 'nsone_monitoring' + datasource_create_mock.assert_has_calls([call(name=name, + sourcetype=source_type)]) + datasource_list_mock.assert_called_once() + + # 2nd invocation is cached + datasource_list_mock.reset_mock() + datasource_create_mock.reset_mock() + self.assertEquals('foo', client.datasource_id) + datasource_create_mock.assert_not_called() + datasource_list_mock.assert_not_called() + + # Reset the client's cache + client._datasource_id = None + + # First invocation with a match in the list finds it and doesn't call + # create + datasource_list_mock.reset_mock() + datasource_create_mock.reset_mock() + datasource_list_mock.side_effect = [[{ + 'id': 'other', + 'name': 'not a match', + }, { + 'id': 'bar', + 'name': name, + }]] + self.assertEquals('bar', client.datasource_id) + datasource_create_mock.assert_not_called() + datasource_list_mock.assert_called_once() + + @patch('ns1.rest.data.Feed.delete') + @patch('ns1.rest.data.Feed.create') + @patch('ns1.rest.data.Feed.list') + def test_feeds_for_monitors(self, datafeed_list_mock, + datafeed_create_mock, + datafeed_delete_mock): + client = Ns1Client('dummy-key') + + # pre-cache datasource_id + client._datasource_id = 'foo' + + # Populate the cache and check the results + datafeed_list_mock.reset_mock() + datafeed_list_mock.side_effect = [[{ + 'config': { + 'jobid': 'the-job', + }, + 'id': 'the-feed', + }, { + 'config': { + 'jobid': 'the-other-job', + }, + 'id': 'the-other-feed', + }]] + expected = { + 'the-job': 'the-feed', + 'the-other-job': 'the-other-feed', + } + self.assertEquals(expected, client.feeds_for_monitors) + datafeed_list_mock.assert_called_once() + + # 2nd call uses cache + datafeed_list_mock.reset_mock() + self.assertEquals(expected, client.feeds_for_monitors) + datafeed_list_mock.assert_not_called() + + # create a feed and make sure it's in the cache/map + datafeed_create_mock.reset_mock() + datafeed_create_mock.side_effect = [{ + 'id': 'new-feed', + }] + client.datafeed_create(client.datasource_id, 'new-name', { + 'jobid': 'new-job', + }) + datafeed_create_mock.assert_has_calls([call('foo', 'new-name', { + 'jobid': 'new-job', + })]) + new_expected = expected.copy() + new_expected['new-job'] = 'new-feed' + self.assertEquals(new_expected, client.feeds_for_monitors) + datafeed_create_mock.assert_called_once() + + # Delete a feed and make sure it's out of the cache/map + datafeed_delete_mock.reset_mock() + client.datafeed_delete(client.datasource_id, 'new-feed') + self.assertEquals(expected, client.feeds_for_monitors) + datafeed_delete_mock.assert_called_once() + + @patch('ns1.rest.monitoring.Monitors.delete') + @patch('ns1.rest.monitoring.Monitors.update') + @patch('ns1.rest.monitoring.Monitors.create') + @patch('ns1.rest.monitoring.Monitors.list') + def test_monitors(self, monitors_list_mock, monitors_create_mock, + monitors_update_mock, monitors_delete_mock): + client = Ns1Client('dummy-key') + + one = { + 'id': 'one', + 'key': 'value', + } + two = { + 'id': 'two', + 'key': 'other-value', + } + + # Populate the cache and check the results + monitors_list_mock.reset_mock() + monitors_list_mock.side_effect = [[one, two]] + expected = { + 'one': one, + 'two': two, + } + self.assertEquals(expected, client.monitors) + monitors_list_mock.assert_called_once() + + # 2nd round pulls it from cache + monitors_list_mock.reset_mock() + self.assertEquals(expected, client.monitors) + monitors_list_mock.assert_not_called() + + # Create a monitor, make sure it's in the list + monitors_create_mock.reset_mock() + monitor = { + 'id': 'new-id', + 'key': 'new-value', + } + monitors_create_mock.side_effect = [monitor] + self.assertEquals(monitor, client.monitors_create(param='eter')) + monitors_create_mock.assert_has_calls([call({}, param='eter')]) + new_expected = expected.copy() + new_expected['new-id'] = monitor + self.assertEquals(new_expected, client.monitors) + + # Update a monitor, make sure it's updated in the cache + monitors_update_mock.reset_mock() + monitor = { + 'id': 'new-id', + 'key': 'changed-value', + } + monitors_update_mock.side_effect = [monitor] + self.assertEquals(monitor, client.monitors_update('new-id', + key='changed-value')) + monitors_update_mock \ + .assert_has_calls([call('new-id', {}, key='changed-value')]) + new_expected['new-id'] = monitor + self.assertEquals(new_expected, client.monitors) + + # Delete a monitor, make sure it's out of the list + monitors_delete_mock.reset_mock() + monitors_delete_mock.side_effect = ['deleted'] + self.assertEquals('deleted', client.monitors_delete('new-id')) + monitors_delete_mock.assert_has_calls([call('new-id')]) + self.assertEquals(expected, client.monitors) From 8ec84f49bb79d6b55a65ca50aec64f5a7b4bac11 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 13 Dec 2019 12:39:14 -0800 Subject: [PATCH 094/155] More ns1 code coverage, bug fix for monitor matching --- octodns/provider/ns1.py | 7 +- tests/test_octodns_provider_ns1.py | 157 +++++++++++++++++++++++++++++ 2 files changed, 162 insertions(+), 2 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index a3bd647..8f13ee5 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -602,7 +602,7 @@ class Ns1Provider(BaseProvider): for monitor in self._client.monitors.values(): data = self._parse_notes(monitor['notes']) - if expected_host == data['host'] or \ + if expected_host == data['host'] and \ expected_type == data['type']: # This monitor does not belong to this record config = monitor['config'] @@ -611,11 +611,14 @@ class Ns1Provider(BaseProvider): return monitors + def _uuid(self): + return uuid4().hex + def _create_feed(self, monitor): monitor_id = monitor['id'] self.log.debug('_create_feed: monitor=%s', monitor_id) # TODO: looks like length limit is 64 char - name = '{} - {}'.format(monitor['name'], uuid4().hex[:6]) + name = '{} - {}'.format(monitor['name'], self._uuid()[:6]) # Create the data feed config = { diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index 4849684..ea7dcf2 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -518,6 +518,36 @@ class TestNs1Provider(TestCase): class TestNs1ProviderDynamic(TestCase): + zone = Zone('unit.tests.', []) + + record = Record.new(zone, '', { + 'dynamic': { + 'pools': { + 'iad': { + 'values': [{ + 'value': '1.2.3.4', + }, { + 'value': '2.3.4.5', + }], + }, + }, + 'rules': [{ + 'pool': 'iad', + }], + }, + 'octodns': { + 'healthcheck': { + 'host': 'send.me', + 'path': '/_ping', + 'port': 80, + 'protocol': 'HTTP', + } + }, + 'ttl': 32, + 'type': 'A', + 'value': '1.2.3.4', + 'meta': {}, + }) def test_notes(self): provider = Ns1Provider('test', 'api-key') @@ -534,6 +564,133 @@ class TestNs1ProviderDynamic(TestCase): notes = provider._encode_notes(data) self.assertEquals(data, provider._parse_notes(notes)) + def test_monitors_for(self): + provider = Ns1Provider('test', 'api-key') + + # pre-populate the client's monitors cache + monitor_one = { + 'config': { + 'host': '1.2.3.4', + }, + 'notes': 'host:unit.tests type:A', + } + monitor_four = { + 'config': { + 'host': '2.3.4.5', + }, + 'notes': 'host:unit.tests type:A', + } + provider._client._monitors_cache = { + 'one': monitor_one, + 'two': { + 'config': { + 'host': '8.8.8.8', + }, + 'notes': 'host:unit.tests type:AAAA', + }, + 'three': { + 'config': { + 'host': '9.9.9.9', + }, + 'notes': 'host:other.unit.tests type:A', + }, + 'four': monitor_four, + } + + # Would match, but won't get there b/c it's not dynamic + record = Record.new(self.zone, '', { + 'ttl': 32, + 'type': 'A', + 'value': '1.2.3.4', + 'meta': {}, + }) + self.assertEquals({}, provider._monitors_for(record)) + + # Will match some records + self.assertEquals({ + '1.2.3.4': monitor_one, + '2.3.4.5': monitor_four, + }, provider._monitors_for(self.record)) + + def test_uuid(self): + # Just a smoke test/for coverage + provider = Ns1Provider('test', 'api-key') + self.assertTrue(provider._uuid()) + + @patch('octodns.provider.ns1.Ns1Provider._uuid') + @patch('ns1.rest.data.Feed.create') + def test_create_feed(self, datafeed_create_mock, uuid_mock): + provider = Ns1Provider('test', 'api-key') + + # pre-fill caches to avoid extranious calls (things we're testing + # elsewhere) + provider._client._datasource_id = 'foo' + provider._client._feeds_for_monitors = {} + + uuid_mock.reset_mock() + datafeed_create_mock.reset_mock() + uuid_mock.side_effect = ['xxxxxxxxxxxxxx'] + feed = { + 'id': 'feed', + } + datafeed_create_mock.side_effect = [feed] + monitor = { + 'id': 'one', + 'name': 'one name', + 'config': { + 'host': '1.2.3.4', + }, + 'notes': 'host:unit.tests type:A', + } + self.assertEquals('feed', provider._create_feed(monitor)) + datafeed_create_mock.assert_has_calls([call('foo', 'one name - xxxxxx', + {'jobid': 'one'})]) + + @patch('octodns.provider.ns1.Ns1Provider._create_feed') + @patch('octodns.provider.ns1.Ns1Client.monitors_create') + @patch('octodns.provider.ns1.Ns1Client.notifylists_create') + def test_create_monitor(self, notifylists_create_mock, + monitors_create_mock, create_feed_mock): + provider = Ns1Provider('test', 'api-key') + + # pre-fill caches to avoid extranious calls (things we're testing + # elsewhere) + provider._client._datasource_id = 'foo' + provider._client._feeds_for_monitors = {} + + notifylists_create_mock.reset_mock() + monitors_create_mock.reset_mock() + create_feed_mock.reset_mock() + notifylists_create_mock.side_effect = [{ + 'id': 'nl-id', + }] + monitors_create_mock.side_effect = [{ + 'id': 'mon-id', + }] + create_feed_mock.side_effect = ['feed-id'] + monitor = { + 'name': 'test monitor', + } + monitor_id, feed_id = provider._create_monitor(monitor) + self.assertEquals('mon-id', monitor_id) + self.assertEquals('feed-id', feed_id) + monitors_create_mock.assert_has_calls([call(name='test monitor', + notify_list='nl-id')]) + + def test_monitor_gen(self): + provider = Ns1Provider('test', 'api-key') + + value = '3.4.5.6' + monitor = provider._monitor_gen(self.record, value) + self.assertEquals(value, monitor['config']['host']) + self.assertTrue('\\nHost: send.me\\r' in monitor['config']['send']) + self.assertFalse(monitor['config']['ssl']) + self.assertEquals('host:unit.tests type:A', monitor['notes']) + + self.record._octodns['healthcheck']['protocol'] = 'HTTPS' + monitor = provider._monitor_gen(self.record, value) + self.assertTrue(monitor['config']['ssl']) + class TestNs1Client(TestCase): From 4022155b72f4f3526145e2c297fe00e2338ec3ff Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 13 Dec 2019 13:07:32 -0800 Subject: [PATCH 095/155] Method naming consistency, test coverage for feeds and monitors --- octodns/provider/ns1.py | 20 ++-- tests/test_octodns_provider_ns1.py | 154 +++++++++++++++++++++++++++-- 2 files changed, 156 insertions(+), 18 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 8f13ee5..5cb7c2e 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -614,9 +614,9 @@ class Ns1Provider(BaseProvider): def _uuid(self): return uuid4().hex - def _create_feed(self, monitor): + def _feed_create(self, monitor): monitor_id = monitor['id'] - self.log.debug('_create_feed: monitor=%s', monitor_id) + self.log.debug('_feed_create: monitor=%s', monitor_id) # TODO: looks like length limit is 64 char name = '{} - {}'.format(monitor['name'], self._uuid()[:6]) @@ -627,12 +627,12 @@ class Ns1Provider(BaseProvider): feed = self._client.datafeed_create(self._client.datasource_id, name, config) feed_id = feed['id'] - self.log.debug('_create_feed: feed=%s', feed_id) + self.log.debug('_feed_create: feed=%s', feed_id) return feed_id - def _create_monitor(self, monitor): - self.log.debug('_create_monitor: monitor="%s"', monitor['name']) + def _monitor_create(self, monitor): + self.log.debug('_monitor_create: monitor="%s"', monitor['name']) # Create the notify list notify_list = [{ 'config': { @@ -643,15 +643,15 @@ class Ns1Provider(BaseProvider): nl = self._client.notifylists_create(name=monitor['name'], notify_list=notify_list) nl_id = nl['id'] - self.log.debug('_create_monitor: notify_list=%s', nl_id) + self.log.debug('_monitor_create: notify_list=%s', nl_id) # Create the monitor monitor['notify_list'] = nl_id monitor = self._client.monitors_create(**monitor) monitor_id = monitor['id'] - self.log.debug('_create_monitor: monitor=%s', monitor_id) + self.log.debug('_monitor_create: monitor=%s', monitor_id) - return monitor_id, self._create_feed(monitor) + return monitor_id, self._feed_create(monitor) def _monitor_gen(self, record, value): host = record.fqdn[:-1] @@ -718,11 +718,11 @@ class Ns1Provider(BaseProvider): if feed_id is None: self.log.warn('_monitor_sync: %s (%s) missing feed, creating', existing['name'], monitor_id) - feed_id = self._create_feed(existing) + feed_id = self._feed_create(existing) else: self.log.debug('_monitor_sync: needs create') # We don't have an existing monitor create it (and related bits) - monitor_id, feed_id = self._create_monitor(expected) + monitor_id, feed_id = self._monitor_create(expected) return monitor_id, feed_id diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index ea7dcf2..994b84a 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -619,7 +619,7 @@ class TestNs1ProviderDynamic(TestCase): @patch('octodns.provider.ns1.Ns1Provider._uuid') @patch('ns1.rest.data.Feed.create') - def test_create_feed(self, datafeed_create_mock, uuid_mock): + def test_feed_create(self, datafeed_create_mock, uuid_mock): provider = Ns1Provider('test', 'api-key') # pre-fill caches to avoid extranious calls (things we're testing @@ -642,15 +642,15 @@ class TestNs1ProviderDynamic(TestCase): }, 'notes': 'host:unit.tests type:A', } - self.assertEquals('feed', provider._create_feed(monitor)) + self.assertEquals('feed', provider._feed_create(monitor)) datafeed_create_mock.assert_has_calls([call('foo', 'one name - xxxxxx', {'jobid': 'one'})]) - @patch('octodns.provider.ns1.Ns1Provider._create_feed') + @patch('octodns.provider.ns1.Ns1Provider._feed_create') @patch('octodns.provider.ns1.Ns1Client.monitors_create') @patch('octodns.provider.ns1.Ns1Client.notifylists_create') - def test_create_monitor(self, notifylists_create_mock, - monitors_create_mock, create_feed_mock): + def test_monitor_create(self, notifylists_create_mock, + monitors_create_mock, feed_create_mock): provider = Ns1Provider('test', 'api-key') # pre-fill caches to avoid extranious calls (things we're testing @@ -660,18 +660,18 @@ class TestNs1ProviderDynamic(TestCase): notifylists_create_mock.reset_mock() monitors_create_mock.reset_mock() - create_feed_mock.reset_mock() + feed_create_mock.reset_mock() notifylists_create_mock.side_effect = [{ 'id': 'nl-id', }] monitors_create_mock.side_effect = [{ 'id': 'mon-id', }] - create_feed_mock.side_effect = ['feed-id'] + feed_create_mock.side_effect = ['feed-id'] monitor = { 'name': 'test monitor', } - monitor_id, feed_id = provider._create_monitor(monitor) + monitor_id, feed_id = provider._monitor_create(monitor) self.assertEquals('mon-id', monitor_id) self.assertEquals('feed-id', feed_id) monitors_create_mock.assert_has_calls([call(name='test monitor', @@ -691,6 +691,144 @@ class TestNs1ProviderDynamic(TestCase): monitor = provider._monitor_gen(self.record, value) self.assertTrue(monitor['config']['ssl']) + def test_monitor_is_match(self): + provider = Ns1Provider('test', 'api-key') + + # Empty matches empty + self.assertTrue(provider._monitor_is_match({}, {})) + + # Anything matches empty + self.assertTrue(provider._monitor_is_match({}, { + 'anything': 'goes' + })) + + # Missing doesn't match + self.assertFalse(provider._monitor_is_match({ + 'exepct': 'this', + }, { + 'anything': 'goes' + })) + + # Identical matches + self.assertTrue(provider._monitor_is_match({ + 'exepct': 'this', + }, { + 'exepct': 'this', + })) + + # Different values don't match + self.assertFalse(provider._monitor_is_match({ + 'exepct': 'this', + }, { + 'exepct': 'that', + })) + + # Different sub-values don't match + self.assertFalse(provider._monitor_is_match({ + 'exepct': { + 'this': 'to-be', + }, + }, { + 'exepct': { + 'this': 'something-else', + }, + })) + + @patch('octodns.provider.ns1.Ns1Provider._feed_create') + @patch('octodns.provider.ns1.Ns1Client.monitors_update') + @patch('octodns.provider.ns1.Ns1Provider._monitor_create') + @patch('octodns.provider.ns1.Ns1Provider._monitor_gen') + def test_monitor_sync(self, monitor_gen_mock, monitor_create_mock, + monitors_update_mock, feed_create_mock): + provider = Ns1Provider('test', 'api-key') + + # pre-fill caches to avoid extranious calls (things we're testing + # elsewhere) + provider._client._datasource_id = 'foo' + provider._client._feeds_for_monitors = { + 'mon-id': 'feed-id', + } + + # No existing monitor + monitor_gen_mock.reset_mock() + monitor_create_mock.reset_mock() + monitors_update_mock.reset_mock() + feed_create_mock.reset_mock() + monitor_gen_mock.side_effect = [{'key': 'value'}] + monitor_create_mock.side_effect = [('mon-id', 'feed-id')] + value = '1.2.3.4' + monitor_id, feed_id = provider._monitor_sync(self.record, value, None) + self.assertEquals('mon-id', monitor_id) + self.assertEquals('feed-id', feed_id) + monitor_gen_mock.assert_has_calls([call(self.record, value)]) + monitor_create_mock.assert_has_calls([call({'key': 'value'})]) + monitors_update_mock.assert_not_called() + feed_create_mock.assert_not_called() + + # Existing monitor that doesn't need updates + monitor_gen_mock.reset_mock() + monitor_create_mock.reset_mock() + monitors_update_mock.reset_mock() + feed_create_mock.reset_mock() + monitor = { + 'id': 'mon-id', + 'key': 'value', + 'name': 'monitor name', + } + monitor_gen_mock.side_effect = [monitor] + monitor_id, feed_id = provider._monitor_sync(self.record, value, + monitor) + self.assertEquals('mon-id', monitor_id) + self.assertEquals('feed-id', feed_id) + monitor_gen_mock.assert_called_once() + monitor_create_mock.assert_not_called() + monitors_update_mock.assert_not_called() + feed_create_mock.assert_not_called() + + # Existing monitor that doesn't need updates, but is missing its feed + monitor_gen_mock.reset_mock() + monitor_create_mock.reset_mock() + monitors_update_mock.reset_mock() + feed_create_mock.reset_mock() + monitor = { + 'id': 'mon-id2', + 'key': 'value', + 'name': 'monitor name', + } + monitor_gen_mock.side_effect = [monitor] + feed_create_mock.side_effect = ['feed-id2'] + monitor_id, feed_id = provider._monitor_sync(self.record, value, + monitor) + self.assertEquals('mon-id2', monitor_id) + self.assertEquals('feed-id2', feed_id) + monitor_gen_mock.assert_called_once() + monitor_create_mock.assert_not_called() + monitors_update_mock.assert_not_called() + feed_create_mock.assert_has_calls([call(monitor)]) + + # Existing monitor that needs updates + monitor_gen_mock.reset_mock() + monitor_create_mock.reset_mock() + monitors_update_mock.reset_mock() + feed_create_mock.reset_mock() + monitor = { + 'id': 'mon-id', + 'key': 'value', + 'name': 'monitor name', + } + gened = { + 'other': 'thing', + } + monitor_gen_mock.side_effect = [gened] + monitor_id, feed_id = provider._monitor_sync(self.record, value, + monitor) + self.assertEquals('mon-id', monitor_id) + self.assertEquals('feed-id', feed_id) + monitor_gen_mock.assert_called_once() + monitor_create_mock.assert_not_called() + monitors_update_mock.assert_has_calls([call('mon-id', other='thing')]) + feed_create_mock.assert_not_called() + class TestNs1Client(TestCase): From 0f298e51bef5b93f0722d335f8f9515964eb51db Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 13 Dec 2019 13:22:54 -0800 Subject: [PATCH 096/155] Tests for ns1 _monitors_gc --- octodns/provider/ns1.py | 12 ++--- tests/test_octodns_provider_ns1.py | 83 ++++++++++++++++++++++++++++++ 2 files changed, 89 insertions(+), 6 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 5cb7c2e..e8b8d17 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -726,8 +726,8 @@ class Ns1Provider(BaseProvider): return monitor_id, feed_id - def _gc_monitors(self, record, active_monitor_ids=None): - self.log.debug('_gc_monitors: record=%s, active_monitor_ids=%s', + def _monitors_gc(self, record, active_monitor_ids=None): + self.log.debug('_monitors_gc: record=%s, active_monitor_ids=%s', record.fqdn, active_monitor_ids) if active_monitor_ids is None: @@ -738,7 +738,7 @@ class Ns1Provider(BaseProvider): if monitor_id in active_monitor_ids: continue - self.log.debug('_gc_monitors: deleting %s', monitor_id) + self.log.debug('_monitors_gc: deleting %s', monitor_id) feed_id = self._client.feeds_for_monitors.get(monitor_id) if feed_id: @@ -957,7 +957,7 @@ class Ns1Provider(BaseProvider): params, active_monitor_ids = \ getattr(self, '_params_for_{}'.format(_type))(new) self._client.records_create(zone, domain, _type, **params) - self._gc_monitors(new, active_monitor_ids) + self._monitors_gc(new, active_monitor_ids) def _apply_Update(self, ns1_zone, change): new = change.new @@ -967,7 +967,7 @@ class Ns1Provider(BaseProvider): params, active_monitor_ids = \ getattr(self, '_params_for_{}'.format(_type))(new) self._client.records_update(zone, domain, _type, **params) - self._gc_monitors(new, active_monitor_ids) + self._monitors_gc(new, active_monitor_ids) def _apply_Delete(self, ns1_zone, change): existing = change.existing @@ -975,7 +975,7 @@ class Ns1Provider(BaseProvider): domain = existing.fqdn[:-1] _type = existing._type self._client.records_delete(zone, domain, _type) - self._gc_monitors(existing) + self._monitors_gc(existing) def _apply(self, plan): desired = plan.desired diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index 994b84a..68e87c9 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -829,6 +829,89 @@ class TestNs1ProviderDynamic(TestCase): monitors_update_mock.assert_has_calls([call('mon-id', other='thing')]) feed_create_mock.assert_not_called() + @patch('octodns.provider.ns1.Ns1Client.notifylists_delete') + @patch('octodns.provider.ns1.Ns1Client.monitors_delete') + @patch('octodns.provider.ns1.Ns1Client.datafeed_delete') + @patch('octodns.provider.ns1.Ns1Provider._monitors_for') + def test_monitors_gc(self, monitors_for_mock, datafeed_delete_mock, + monitors_delete_mock, notifylists_delete_mock): + provider = Ns1Provider('test', 'api-key') + + # pre-fill caches to avoid extranious calls (things we're testing + # elsewhere) + provider._client._datasource_id = 'foo' + provider._client._feeds_for_monitors = { + 'mon-id': 'feed-id', + } + + # No active monitors and no existing, nothing will happen + monitors_for_mock.reset_mock() + datafeed_delete_mock.reset_mock() + monitors_delete_mock.reset_mock() + notifylists_delete_mock.reset_mock() + monitors_for_mock.side_effect = [{}] + provider._monitors_gc(self.record) + monitors_for_mock.assert_has_calls([call(self.record)]) + datafeed_delete_mock.assert_not_called() + monitors_delete_mock.assert_not_called() + notifylists_delete_mock.assert_not_called() + + # No active monitors and one existing, delete all the things + monitors_for_mock.reset_mock() + datafeed_delete_mock.reset_mock() + monitors_delete_mock.reset_mock() + notifylists_delete_mock.reset_mock() + monitors_for_mock.side_effect = [{ + 'x': { + 'id': 'mon-id', + 'notify_list': 'nl-id', + } + }] + provider._monitors_gc(self.record) + monitors_for_mock.assert_has_calls([call(self.record)]) + datafeed_delete_mock.assert_has_calls([call('foo', 'feed-id')]) + monitors_delete_mock.assert_has_calls([call('mon-id')]) + notifylists_delete_mock.assert_has_calls([call('nl-id')]) + + # Same existing, this time in active list, should be noop + monitors_for_mock.reset_mock() + datafeed_delete_mock.reset_mock() + monitors_delete_mock.reset_mock() + notifylists_delete_mock.reset_mock() + monitors_for_mock.side_effect = [{ + 'x': { + 'id': 'mon-id', + 'notify_list': 'nl-id', + } + }] + provider._monitors_gc(self.record, {'mon-id'}) + monitors_for_mock.assert_has_calls([call(self.record)]) + datafeed_delete_mock.assert_not_called() + monitors_delete_mock.assert_not_called() + notifylists_delete_mock.assert_not_called() + + # Non-active monitor w/o a feed, and another monitor that's left alone + # b/c it's active + monitors_for_mock.reset_mock() + datafeed_delete_mock.reset_mock() + monitors_delete_mock.reset_mock() + notifylists_delete_mock.reset_mock() + monitors_for_mock.side_effect = [{ + 'x': { + 'id': 'mon-id', + 'notify_list': 'nl-id', + }, + 'y': { + 'id': 'mon-id2', + 'notify_list': 'nl-id2', + }, + }] + provider._monitors_gc(self.record, {'mon-id'}) + monitors_for_mock.assert_has_calls([call(self.record)]) + datafeed_delete_mock.assert_not_called() + monitors_delete_mock.assert_has_calls([call('mon-id2')]) + notifylists_delete_mock.assert_has_calls([call('nl-id2')]) + class TestNs1Client(TestCase): From ed482c60ca8c1f1c3cb4131da6e1a4cd64651c65 Mon Sep 17 00:00:00 2001 From: Dan Hanks Date: Thu, 19 Dec 2019 10:33:22 -0500 Subject: [PATCH 097/155] Document max_workers flag --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index c3ddb0b..83e28ca 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,9 @@ We start by creating a config file to tell OctoDNS about our providers and the z ```yaml --- +manager: + max_workers: 2 + providers: config: class: octodns.provider.yaml.YamlProvider @@ -80,6 +83,8 @@ zones: Further information can be found in the `docstring` of each source and provider class. +The `max_workers` key in the `manager` section of the config enables threading to parallelize the planning portion of the sync. + Now that we have something to tell OctoDNS about our providers & zones we need to tell it about or records. We'll keep it simple for now and just create a single `A` record at the top-level of the domain. `config/example.com.yaml` From b57f2a64ad4089313073d68f1863b9a8bcf0bb55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20H=C3=BCgli?= Date: Fri, 20 Dec 2019 17:19:40 +0100 Subject: [PATCH 098/155] create/copy test with delegation set support --- tests/test_octodns_provider_route53.py | 96 ++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/tests/test_octodns_provider_route53.py b/tests/test_octodns_provider_route53.py index 7691804..b7e075d 100644 --- a/tests/test_octodns_provider_route53.py +++ b/tests/test_octodns_provider_route53.py @@ -370,6 +370,16 @@ class TestRoute53Provider(TestCase): return (provider, stubber) + def _get_stubbed_delegation_set_provider(self): + provider = Route53Provider('test', 'abc', '123', + delegation_set_id="ABCDEFG123456") + + # Use the stubber + stubber = Stubber(provider._conn) + stubber.activate() + + return (provider, stubber) + def _get_stubbed_fallback_auth_provider(self): provider = Route53Provider('test') @@ -913,6 +923,92 @@ class TestRoute53Provider(TestCase): self.assertEquals(9, provider.apply(plan)) stubber.assert_no_pending_responses() + def test_sync_create_with_delegation_set(self): + provider, stubber = self._get_stubbed_delegation_set_provider() + + got = Zone('unit.tests.', []) + + list_hosted_zones_resp = { + 'HostedZones': [], + 'Marker': 'm', + 'IsTruncated': False, + 'MaxItems': '100', + } + stubber.add_response('list_hosted_zones', list_hosted_zones_resp, + {}) + + plan = provider.plan(self.expected) + self.assertEquals(9, len(plan.changes)) + self.assertFalse(plan.exists) + for change in plan.changes: + self.assertIsInstance(change, Create) + stubber.assert_no_pending_responses() + + create_hosted_zone_resp = { + 'HostedZone': { + 'Name': 'unit.tests.', + 'Id': 'z42', + 'CallerReference': 'abc', + }, + 'ChangeInfo': { + 'Id': 'a12', + 'Status': 'PENDING', + 'SubmittedAt': '2017-01-29T01:02:03Z', + 'Comment': 'hrm', + }, + 'DelegationSet': { + 'Id': 'b23', + 'CallerReference': 'blip', + 'NameServers': [ + 'n12.unit.tests.', + ], + }, + 'Location': 'us-east-1', + } + stubber.add_response('create_hosted_zone', + create_hosted_zone_resp, { + 'Name': got.name, + 'CallerReference': ANY, + 'DelegationSetId': 'ABCDEFG123456' + }) + + list_resource_record_sets_resp = { + 'ResourceRecordSets': [{ + 'Name': 'a.unit.tests.', + 'Type': 'A', + 'GeoLocation': { + 'ContinentCode': 'NA', + }, + 'ResourceRecords': [{ + 'Value': '2.2.3.4', + }], + 'TTL': 61, + }], + 'IsTruncated': False, + 'MaxItems': '100', + } + stubber.add_response('list_resource_record_sets', + list_resource_record_sets_resp, + {'HostedZoneId': 'z42'}) + + stubber.add_response('list_health_checks', + { + 'HealthChecks': self.health_checks, + 'IsTruncated': False, + 'MaxItems': '100', + 'Marker': '', + }) + + stubber.add_response('change_resource_record_sets', + {'ChangeInfo': { + 'Id': 'id', + 'Status': 'PENDING', + 'SubmittedAt': '2017-01-29T01:02:03Z', + }}, {'HostedZoneId': 'z42', 'ChangeBatch': ANY}) + + self.assertEquals(9, provider.apply(plan)) + stubber.assert_no_pending_responses() + def test_health_checks_pagination(self): provider, stubber = self._get_stubbed_provider() From 561a6ca2d98633023a131e491f3b24212ce447fb Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 6 Jan 2020 08:31:35 -0800 Subject: [PATCH 099/155] Test coverage for Ns1Provider _params_for_dynamic_A --- tests/test_octodns_provider_ns1.py | 45 ++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index 68e87c9..bc40f8a 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -523,6 +523,12 @@ class TestNs1ProviderDynamic(TestCase): record = Record.new(zone, '', { 'dynamic': { 'pools': { + 'lhr': { + 'fallback': 'iad', + 'values': [{ + 'value': '3.4.5.6', + }], + }, 'iad': { 'values': [{ 'value': '1.2.3.4', @@ -532,6 +538,13 @@ class TestNs1ProviderDynamic(TestCase): }, }, 'rules': [{ + 'geos': [ + 'AF', + 'EU-GB', + 'NA-US-FL' + ], + 'pool': 'lhr', + }, { 'pool': 'iad', }], }, @@ -912,6 +925,38 @@ class TestNs1ProviderDynamic(TestCase): monitors_delete_mock.assert_has_calls([call('mon-id2')]) notifylists_delete_mock.assert_has_calls([call('nl-id2')]) + @patch('octodns.provider.ns1.Ns1Provider._monitor_sync') + @patch('octodns.provider.ns1.Ns1Provider._monitors_for') + def test_params_for_dynamic(self, monitors_for_mock, monitors_sync_mock): + provider = Ns1Provider('test', 'api-key') + + # pre-fill caches to avoid extranious calls (things we're testing + # elsewhere) + provider._client._datasource_id = 'foo' + provider._client._feeds_for_monitors = { + 'mon-id': 'feed-id', + } + + monitors_for_mock.reset_mock() + monitors_sync_mock.reset_mock() + monitors_for_mock.side_effect = [{ + '3.4.5.6': 'mid-3', + }] + monitors_sync_mock.side_effect = [ + ('mid-1', 'fid-1'), + ('mid-2', 'fid-2'), + ('mid-3', 'fid-3'), + ] + # This indirectly calls into _params_for_dynamic_A and tests the + # handling to get there + provider._params_for_A(self.record) + monitors_for_mock.assert_has_calls([call(self.record)]) + monitors_sync_mock.assert_has_calls([ + call(self.record, '1.2.3.4', None), + call(self.record, '2.3.4.5', None), + call(self.record, '3.4.5.6', 'mid-3'), + ]) + class TestNs1Client(TestCase): From 69cd30a1832c30a998528c4c6549f460ad1ee128 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 6 Jan 2020 09:18:10 -0800 Subject: [PATCH 100/155] Coverage for Ns1Provider _data_for_dynamic_A --- tests/test_octodns_provider_ns1.py | 136 ++++++++++++++++++++++++++++- 1 file changed, 135 insertions(+), 1 deletion(-) diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index bc40f8a..a985c13 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -13,7 +13,7 @@ from six import text_type from unittest import TestCase from octodns.record import Delete, Record, Update -from octodns.provider.ns1 import Ns1Client, Ns1Provider +from octodns.provider.ns1 import Ns1Client, Ns1Exception, Ns1Provider from octodns.zone import Zone @@ -957,6 +957,140 @@ class TestNs1ProviderDynamic(TestCase): call(self.record, '3.4.5.6', 'mid-3'), ]) + def test_data_for_dynamic_A(self): + provider = Ns1Provider('test', 'api-key') + + # Unexpected filters throws an error + ns1_record = { + 'domain': 'unit.tests', + 'filters': [], + } + with self.assertRaises(Ns1Exception) as ctx: + provider._data_for_dynamic_A('A', ns1_record) + self.assertEquals('Unrecognized advanced record', + text_type(ctx.exception)) + + # empty record turns into empty data + ns1_record = { + 'answers': [], + 'domain': 'unit.tests', + 'filters': Ns1Provider._DYNAMIC_FILTERS, + 'regions': {}, + 'ttl': 42, + } + data = provider._data_for_dynamic_A('A', ns1_record) + self.assertEquals({ + 'dynamic': { + 'pools': {}, + 'rules': [], + }, + 'ttl': 42, + 'type': 'A', + 'values': [], + }, data) + + # Test out a small, but realistic setup that covers all the options + ns1_record = { + 'answers': [{ + 'answer': ['3.4.5.6'], + 'meta': { + 'priority': 1, + 'note': 'from:lhr', + }, + 'region': 'lhr', + }, { + 'answer': ['2.3.4.5'], + 'meta': { + 'priority': 2, + 'weight': 12, + 'note': 'from:iad', + }, + 'region': 'lhr', + }, { + 'answer': ['1.2.3.4'], + 'meta': { + 'priority': 3, + 'note': 'from:--default--', + }, + 'region': 'lhr', + }, { + 'answer': ['2.3.4.5'], + 'meta': { + 'priority': 1, + 'weight': 12, + 'note': 'from:iad', + }, + 'region': 'iad', + }, { + 'answer': ['1.2.3.4'], + 'meta': { + 'priority': 2, + 'note': 'from:--default--', + }, + 'region': 'iad', + }], + 'domain': 'unit.tests', + 'filters': Ns1Provider._DYNAMIC_FILTERS, + 'regions': { + 'lhr': { + 'meta': { + 'note': 'rule-order:1 fallback:iad', + 'country': ['CA'], + 'georegion': ['AFRICA'], + 'us_state': ['OR'], + }, + }, + 'iad': { + 'meta': { + 'note': 'rule-order:2', + }, + } + }, + 'tier': 3, + 'ttl': 42, + } + data = provider._data_for_dynamic_A('A', ns1_record) + self.assertEquals({ + 'dynamic': { + 'pools': { + 'iad': { + 'fallback': None, + 'values': [{ + 'value': '2.3.4.5', + 'weight': 12, + }], + }, + 'lhr': { + 'fallback': 'iad', + 'values': [{ + 'weight': 1, + 'value': '3.4.5.6', + }], + }, + }, + 'rules': [{ + '_order': '1', + 'geos': [ + 'AF', + 'NA-CA', + 'NA-US-OR', + ], + 'pool': 'lhr', + }, { + '_order': '2', + 'pool': 'iad', + }], + }, + 'ttl': 42, + 'type': 'A', + 'values': ['1.2.3.4'], + }, data) + + # Same answer if we go through _data_for_A which out sources the job to + # _data_for_dynamic_A + data2 = provider._data_for_A('A', ns1_record) + self.assertEquals(data, data2) + class TestNs1Client(TestCase): From eefd83de80de20ed4937f892f7da235c63d1a0e2 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 6 Jan 2020 10:04:07 -0800 Subject: [PATCH 101/155] Coverage for Ns1Provider _extra_changes --- tests/test_octodns_provider_ns1.py | 107 +++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index a985c13..4c505e4 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -1091,6 +1091,113 @@ class TestNs1ProviderDynamic(TestCase): data2 = provider._data_for_A('A', ns1_record) self.assertEquals(data, data2) + @patch('octodns.provider.ns1.Ns1Provider._monitors_for') + def test_extra_changes(self, monitors_for_mock): + provider = Ns1Provider('test', 'api-key') + + desired = Zone('unit.tests.', []) + + # Empty zone and no changes + monitors_for_mock.reset_mock() + extra = provider._extra_changes(desired, []) + self.assertFalse(extra) + monitors_for_mock.assert_not_called() + + # Simple record, ignored + monitors_for_mock.reset_mock() + simple = Record.new(desired, '', { + 'ttl': 32, + 'type': 'A', + 'value': '1.2.3.4', + 'meta': {}, + }) + desired.add_record(simple) + extra = provider._extra_changes(desired, []) + self.assertFalse(extra) + monitors_for_mock.assert_not_called() + + # Dynamic record, inspectable + dynamic = Record.new(desired, 'dyn', { + 'dynamic': { + 'pools': { + 'iad': { + 'values': [{ + 'value': '1.2.3.4', + }], + }, + }, + 'rules': [{ + 'pool': 'iad', + }], + }, + 'octodns': { + 'healthcheck': { + 'host': 'send.me', + 'path': '/_ping', + 'port': 80, + 'protocol': 'HTTP', + } + }, + 'ttl': 32, + 'type': 'A', + 'value': '1.2.3.4', + 'meta': {}, + }) + desired.add_record(dynamic) + + # untouched, but everything in sync so no change needed + monitors_for_mock.reset_mock() + # Generate what we expect to have + gend = provider._monitor_gen(dynamic, '1.2.3.4') + gend.update({ + 'id': 'mid', # need to add an id + 'notify_list': 'xyz', # need to add a notify list (for now) + }) + monitors_for_mock.side_effect = [{ + '1.2.3.4': gend, + }] + extra = provider._extra_changes(desired, []) + self.assertFalse(extra) + monitors_for_mock.assert_has_calls([call(dynamic)]) + + update = Update(dynamic, dynamic) + + # If we don't have a notify list we're broken and we'll expect to see + # an Update + monitors_for_mock.reset_mock() + del gend['notify_list'] + monitors_for_mock.side_effect = [{ + '1.2.3.4': gend, + }] + extra = provider._extra_changes(desired, []) + self.assertEquals(1, len(extra)) + extra = list(extra)[0] + self.assertIsInstance(extra, Update) + self.assertEquals(dynamic, extra.new) + monitors_for_mock.assert_has_calls([call(dynamic)]) + + # Add notify_list back and change the healthcheck protocol, we'll still + # expect to see an update + monitors_for_mock.reset_mock() + gend['notify_list'] = 'xyz' + dynamic._octodns['healthcheck']['protocol'] = 'HTTPS' + del gend['notify_list'] + monitors_for_mock.side_effect = [{ + '1.2.3.4': gend, + }] + extra = provider._extra_changes(desired, []) + self.assertEquals(1, len(extra)) + extra = list(extra)[0] + self.assertIsInstance(extra, Update) + self.assertEquals(dynamic, extra.new) + monitors_for_mock.assert_has_calls([call(dynamic)]) + + # If it's in the changed list, it'll be ignored + monitors_for_mock.reset_mock() + extra = provider._extra_changes(desired, [update]) + self.assertFalse(extra) + monitors_for_mock.assert_not_called() + class TestNs1Client(TestCase): From f91cac3ef47595f40cb53a96e88c27a59bdf27c4 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 6 Jan 2020 10:13:58 -0800 Subject: [PATCH 102/155] coverage for Ns1Client notifylist methods --- tests/test_octodns_provider_ns1.py | 45 ++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index 4c505e4..0d456fe 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -1408,3 +1408,48 @@ class TestNs1Client(TestCase): self.assertEquals('deleted', client.monitors_delete('new-id')) monitors_delete_mock.assert_has_calls([call('new-id')]) self.assertEquals(expected, client.monitors) + + @patch('ns1.rest.monitoring.NotifyLists.delete') + @patch('ns1.rest.monitoring.NotifyLists.create') + @patch('ns1.rest.monitoring.NotifyLists.list') + def test_notifylists(self, notifylists_list_mock, notifylists_create_mock, + notifylists_delete_mock): + client = Ns1Client('dummy-key') + + notifylists_list_mock.reset_mock() + notifylists_create_mock.reset_mock() + notifylists_delete_mock.reset_mock() + notifylists_create_mock.side_effect = ['bar'] + notify_list = [{ + 'config': { + 'sourceid': 'foo', + }, + 'type': 'datafeed', + }] + nl = client.notifylists_create(name='some name', + notify_list=notify_list) + self.assertEquals('bar', nl) + notifylists_list_mock.assert_not_called() + notifylists_create_mock.assert_has_calls([ + call({'name': 'some name', 'notify_list': notify_list}) + ]) + notifylists_delete_mock.assert_not_called() + + notifylists_list_mock.reset_mock() + notifylists_create_mock.reset_mock() + notifylists_delete_mock.reset_mock() + client.notifylists_delete('nlid') + notifylists_list_mock.assert_not_called() + notifylists_create_mock.assert_not_called() + notifylists_delete_mock.assert_has_calls([call('nlid')]) + + notifylists_list_mock.reset_mock() + notifylists_create_mock.reset_mock() + notifylists_delete_mock.reset_mock() + expected = ['one', 'two', 'three'] + notifylists_list_mock.side_effect = [expected] + nls = client.notifylists_list() + self.assertEquals(expected, nls) + notifylists_list_mock.assert_has_calls([call()]) + notifylists_create_mock.assert_not_called() + notifylists_delete_mock.assert_not_called() From 95f51114871ba552d3e42d408237a8424df30820 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 6 Jan 2020 10:18:56 -0800 Subject: [PATCH 103/155] NS1 geo records will always use 'answers' --- octodns/provider/ns1.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index e8b8d17..e9efd3e 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -256,8 +256,6 @@ class Ns1Provider(BaseProvider): 'type': _type, } values, codes = [], [] - if 'answers' not in record: - values = record['short_answers'] for answer in record.get('answers', []): meta = answer.get('meta', {}) if meta: From e0debc963e2b120d599cff5537a39db053ca8f92 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 6 Jan 2020 10:22:18 -0800 Subject: [PATCH 104/155] Update Dynamic support to include NS1, remove Geo mentions --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c3ddb0b..f884557 100644 --- a/README.md +++ b/README.md @@ -169,7 +169,7 @@ The above command pulled the existing data out of Route53 and placed the results ## Supported providers -| Provider | Requirements | Record Support | Dynamic/Geo Support | Notes | +| Provider | Requirements | Record Support | Dynamic | Notes | |--|--|--|--|--| | [AzureProvider](/octodns/provider/azuredns.py) | azure-mgmt-dns | A, AAAA, CAA, CNAME, MX, NS, PTR, SRV, TXT | No | | | [Akamai](/octodns/provider/fastdns.py) | edgegrid-python | A, AAAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, SSHFP, TXT | No | | @@ -182,7 +182,7 @@ The above command pulled the existing data out of Route53 and placed the results | [EtcHostsProvider](/octodns/provider/etc_hosts.py) | | A, AAAA, ALIAS, CNAME | No | | | [GoogleCloudProvider](/octodns/provider/googlecloud.py) | google-cloud-dns | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | No | | | [MythicBeastsProvider](/octodns/provider/mythicbeasts.py) | Mythic Beasts | A, AAAA, ALIAS, CNAME, MX, NS, SRV, SSHFP, CAA, TXT | No | | -| [Ns1Provider](/octodns/provider/ns1.py) | ns1-python | All | Partial Geo | No health checking for GeoDNS | +| [Ns1Provider](/octodns/provider/ns1.py) | ns1-python | All | Yes | No CNAME support, missing `NA` geo target | | [OVH](/octodns/provider/ovh.py) | ovh | A, AAAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, SSHFP, TXT, DKIM | No | | | [PowerDnsProvider](/octodns/provider/powerdns.py) | | All | No | | | [Rackspace](/octodns/provider/rackspace.py) | | A, AAAA, ALIAS, CNAME, MX, NS, PTR, SPF, TXT | No | | From 391ef583ae3f3a63ddda82f84e00490a7fc66061 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 6 Jan 2020 10:22:41 -0800 Subject: [PATCH 105/155] Ns1 should use geofence_regional to avoid nearest matching --- octodns/provider/ns1.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index e9efd3e..b442271 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -188,7 +188,7 @@ class Ns1Provider(BaseProvider): 'filter': 'up' }, { 'config': {}, - 'filter': u'geotarget_regional' + 'filter': u'geofence_regional' }, { 'config': {}, 'filter': u'select_first_region' From a078ec9d3134535c56d437edd2098d5f996385b3 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 6 Jan 2020 14:16:47 -0800 Subject: [PATCH 106/155] Move to populate_should_replace rather then OverridingYamlProvider --- octodns/provider/yaml.py | 136 ++++++++++++++++------------ tests/test_octodns_provider_yaml.py | 42 ++++----- 2 files changed, 99 insertions(+), 79 deletions(-) diff --git a/octodns/provider/yaml.py b/octodns/provider/yaml.py index aa04528..a010084 100644 --- a/octodns/provider/yaml.py +++ b/octodns/provider/yaml.py @@ -28,7 +28,78 @@ class YamlProvider(BaseProvider): default_ttl: 3600 # Whether or not to enforce sorting order on the yaml config # (optional, default True) - enforce_order: True + enforce_order: true + # Whether duplicate records should replace rather than error + # (optiona, default False) + populate_should_replace: false + + Overriding values can be accomplished using multiple yaml providers in the + `sources` list where subsequent providers have `populate_should_replace` + set to `true`. An example use of this would be a zone that you want to push + to external DNS providers and internally, but you want to modify some of + the records in the internal version. + + config/octodns.com.yaml + --- + other: + type: A + values: + - 192.30.252.115 + - 192.30.252.116 + www: + type: A + values: + - 192.30.252.113 + - 192.30.252.114 + + + internal/octodns.com.yaml + --- + 'www': + type: A + values: + - 10.0.0.12 + - 10.0.0.13 + + external.yaml + --- + providers: + config: + class: octodns.provider.yaml.YamlProvider + directory: ./config + + zones: + + octodns.com.: + sources: + - config + targets: + - route53 + + internal.yaml + --- + providers: + config: + class: octodns.provider.yaml.YamlProvider + directory: ./config + + internal: + class: octodns.provider.yaml.YamlProvider + directory: ./internal + + zones: + + octodns.com.: + sources: + - config + - internal + targets: + - pdns + + You can then sync our records eternally with `--config-file=external.yaml` + and internally (with the custom overrides) with + `--config-file=internal.yaml` + ''' SUPPORTS_GEO = True SUPPORTS_DYNAMIC = True @@ -36,18 +107,20 @@ class YamlProvider(BaseProvider): 'PTR', 'SSHFP', 'SPF', 'SRV', 'TXT')) def __init__(self, id, directory, default_ttl=3600, enforce_order=True, - *args, **kwargs): + populate_should_replace=False, *args, **kwargs): self.log = logging.getLogger('{}[{}]'.format( self.__class__.__name__, id)) self.log.debug('__init__: id=%s, directory=%s, default_ttl=%d, ' - 'enforce_order=%d', id, directory, default_ttl, - enforce_order) + 'enforce_order=%d, populate_should_replace=%d', + id, directory, default_ttl, enforce_order, + populate_should_replace) super(YamlProvider, self).__init__(id, *args, **kwargs) self.directory = directory self.default_ttl = default_ttl self.enforce_order = enforce_order + self.populate_should_replace = populate_should_replace - def _populate_from_file(self, filename, zone, lenient, replace=False): + def _populate_from_file(self, filename, zone, lenient): with open(filename, 'r') as fh: yaml_data = safe_load(fh, enforce_order=self.enforce_order) if yaml_data: @@ -60,7 +133,7 @@ class YamlProvider(BaseProvider): record = Record.new(zone, name, d, source=self, lenient=lenient) zone.add_record(record, lenient=lenient, - replace=replace) + replace=self.populate_should_replace) self.log.debug('_populate_from_file: successfully loaded "%s"', filename) @@ -212,54 +285,3 @@ class SplitYamlProvider(YamlProvider): self.log.debug('_apply: writing catchall filename=%s', filename) with open(filename, 'w') as fh: safe_dump(catchall, fh) - - -class OverridingYamlProvider(YamlProvider): - ''' - Provider that builds on YamlProvider to allow overriding specific records. - - Works identically to YamlProvider with the additional behavior of loading - data from a second zonefile in override_directory if it exists. Records in - this second file will override (replace) those previously seen in the - primary. Records that do not exist in the primary will just be added. There - is currently no mechinism to remove records from the primary zone. - - config: - class: octodns.provider.yaml.OverridingYamlProvider - # The location of yaml config files (required) - directory: ./config - # The location of overriding yaml config files (required) - override_directory: ./config - # The ttl to use for records when not specified in the data - # (optional, default 3600) - default_ttl: 3600 - # Whether or not to enforce sorting order on the yaml config - # (optional, default True) - enforce_order: True - ''' - - def __init__(self, id, directory, override_directory, *args, **kwargs): - super(OverridingYamlProvider, self).__init__(id, directory, *args, - **kwargs) - self.override_directory = override_directory - - def populate(self, zone, target=False, lenient=False): - self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, - target, lenient) - - if target: - # When acting as a target we ignore any existing records so that we - # create a completely new copy - return False - - before = len(zone.records) - filename = join(self.directory, '{}yaml'.format(zone.name)) - self._populate_from_file(filename, zone, lenient) - - filename = join(self.override_directory, '{}yaml'.format(zone.name)) - if isfile(filename): - self._populate_from_file(filename, zone, lenient, replace=True) - - self.log.info('populate: found %s records, exists=False', - len(zone.records) - before) - return False diff --git a/tests/test_octodns_provider_yaml.py b/tests/test_octodns_provider_yaml.py index 0efcee9..f858c05 100644 --- a/tests/test_octodns_provider_yaml.py +++ b/tests/test_octodns_provider_yaml.py @@ -15,7 +15,7 @@ from yaml.constructor import ConstructorError from octodns.record import Create from octodns.provider.base import Plan from octodns.provider.yaml import _list_all_yaml_files, \ - OverridingYamlProvider, SplitYamlProvider, YamlProvider + SplitYamlProvider, YamlProvider from octodns.zone import SubzoneRecordException, Zone from helpers import TemporaryDirectory @@ -377,31 +377,29 @@ class TestOverridingYamlProvider(TestCase): def test_provider(self): config = join(dirname(__file__), 'config') override_config = join(dirname(__file__), 'config', 'override') - source = OverridingYamlProvider('test', config, override_config) - - zone = Zone('unit.tests.', []) - dynamic_zone = Zone('dynamic.tests.', []) - - # With target we don't add anything (same as base) - source.populate(zone, target=source) - self.assertEquals(0, len(zone.records)) - - # without it we see everything - source.populate(zone) - self.assertEquals(18, len(zone.records)) - - # Load the dynamic records - source.populate(dynamic_zone) - - got = {r.name: r for r in dynamic_zone.records} - # We see both the base and override files, 1 extra record + base = YamlProvider('base', config, populate_should_replace=False) + override = YamlProvider('test', override_config, + populate_should_replace=True) + + zone = Zone('dynamic.tests.', []) + + # Load the base, should see the 5 records + base.populate(zone) + got = {r.name: r for r in zone.records} + self.assertEquals(5, len(got)) + # We get the "dynamic" A from the bae config + self.assertTrue('dynamic' in got['a'].data) + # No added + self.assertFalse('added' in got) + + # Load the overrides, should replace one and add 1 + override.populate(zone) + got = {r.name: r for r in zone.records} self.assertEquals(6, len(got)) - # 'a' was replaced with a generic record self.assertEquals({ 'ttl': 3600, 'values': ['4.4.4.4', '5.5.5.5'] }, got['a'].data) - - # And we have a new override + # And we have the new one self.assertTrue('added' in got) From e22a7d2738789f4d30934f9b388c2e60fec25bf6 Mon Sep 17 00:00:00 2001 From: Charles Durieux Date: Wed, 8 Jan 2020 17:45:02 +0100 Subject: [PATCH 107/155] Fix trailing semicolon in dkim for ovh provider Pulling dns records from ovh to a yaml file puts a semicolon at the end. Pushing from yaml to ovh will fail the "dkim-compliant" verification if there is an empty field (and there is one in case of a trailing semicolon). With the current logic, pulling dkim record created with ovh then pushing it back will NOT work. This small patch ignores all empty fields in a dkim records during dkim validation. --- octodns/provider/ovh.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/ovh.py b/octodns/provider/ovh.py index 17aff8d..6bed788 100644 --- a/octodns/provider/ovh.py +++ b/octodns/provider/ovh.py @@ -323,7 +323,7 @@ class OvhProvider(BaseProvider): 'n': lambda _: True, 'g': lambda _: True} - splitted = value.split('\\;') + splitted = list(filter(None, value.split('\\;'))) found_key = False for splitted_value in splitted: sub_split = [x.strip() for x in splitted_value.split("=", 1)] From 4b625eba64f89c37b5ef5866de4d714f8134318b Mon Sep 17 00:00:00 2001 From: Kaari Date: Wed, 8 Jan 2020 19:52:13 +0100 Subject: [PATCH 108/155] Use comprehension for clarity and best practice Co-Authored-By: Ross McFarland --- octodns/provider/ovh.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/ovh.py b/octodns/provider/ovh.py index 6bed788..8a3d492 100644 --- a/octodns/provider/ovh.py +++ b/octodns/provider/ovh.py @@ -323,7 +323,7 @@ class OvhProvider(BaseProvider): 'n': lambda _: True, 'g': lambda _: True} - splitted = list(filter(None, value.split('\\;'))) + splitted = [v for v in value.split('\\;') if v] found_key = False for splitted_value in splitted: sub_split = [x.strip() for x in splitted_value.split("=", 1)] From f1cc392bc41bddd24f480a1178ccd50ab15019b5 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Thu, 9 Jan 2020 07:41:30 -0800 Subject: [PATCH 109/155] Include populate_should_replace in yaml example. --- octodns/provider/yaml.py | 1 + 1 file changed, 1 insertion(+) diff --git a/octodns/provider/yaml.py b/octodns/provider/yaml.py index a010084..10add5a 100644 --- a/octodns/provider/yaml.py +++ b/octodns/provider/yaml.py @@ -86,6 +86,7 @@ class YamlProvider(BaseProvider): internal: class: octodns.provider.yaml.YamlProvider directory: ./internal + populate_should_replace: true zones: From ae9e465d8ddf009762cc1ff1d413fae3816aa991 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Sat, 11 Jan 2020 04:52:11 +0100 Subject: [PATCH 110/155] Add dependabot to periodically refresh dependancies As recommended by @ross at https://github.com/github/octodns/pull/441#discussion_r363515321 --- .dependabot/config.yml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .dependabot/config.yml diff --git a/.dependabot/config.yml b/.dependabot/config.yml new file mode 100644 index 0000000..165af5d --- /dev/null +++ b/.dependabot/config.yml @@ -0,0 +1,6 @@ +version: 1 + +update_configs: + - package_manager: "python" + directory: "/" + update_schedule: "weekly" From fe58c67133d24bb728fc41d46079ffa17434e3d8 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Sat, 11 Jan 2020 16:28:23 +0000 Subject: [PATCH 111/155] Bump python-dateutil from 2.6.1 to 2.8.1 Bumps [python-dateutil](https://github.com/dateutil/dateutil) from 2.6.1 to 2.8.1. - [Release notes](https://github.com/dateutil/dateutil/releases) - [Changelog](https://github.com/dateutil/dateutil/blob/master/NEWS) - [Commits](https://github.com/dateutil/dateutil/compare/2.6.1...2.8.1) Signed-off-by: dependabot-preview[bot] --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 6a26ad3..e227fa4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,7 +18,7 @@ ns1-python==0.13.0 ovh==0.4.8 pycountry-convert==0.7.2 pycountry==19.8.18 -python-dateutil==2.6.1 +python-dateutil==2.8.1 requests==2.22.0 s3transfer==0.1.13 setuptools==40.3.0 diff --git a/setup.py b/setup.py index 4f28232..4858e2b 100644 --- a/setup.py +++ b/setup.py @@ -74,7 +74,7 @@ setup( 'pycountry>=19.8.18', 'pycountry-convert>=0.7.2', # botocore doesn't like >=2.7.0 for some reason - 'python-dateutil>=2.6.0,<2.7.0', + 'python-dateutil>=2.6.0,<2.9.0', 'requests>=2.20.0' ], license='MIT', From c98ba64e8e4037101a6d5c5e5e75aee32f88d6a6 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Sat, 11 Jan 2020 16:29:48 +0000 Subject: [PATCH 112/155] Bump six from 1.12.0 to 1.13.0 Bumps [six](https://github.com/benjaminp/six) from 1.12.0 to 1.13.0. - [Release notes](https://github.com/benjaminp/six/releases) - [Changelog](https://github.com/benjaminp/six/blob/master/CHANGES) - [Commits](https://github.com/benjaminp/six/compare/1.12.0...1.13.0) Signed-off-by: dependabot-preview[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6a26ad3..88ba2b9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,5 +22,5 @@ python-dateutil==2.6.1 requests==2.22.0 s3transfer==0.1.13 setuptools==40.3.0 -six==1.12.0 +six==1.13.0 transip==2.0.0 From 826df247b525137c85ddc8238eaff6ba28909094 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 11 Jan 2020 08:57:05 -0800 Subject: [PATCH 113/155] python-dateutil>=2.8.1 --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 4858e2b..c56aa82 100644 --- a/setup.py +++ b/setup.py @@ -73,8 +73,7 @@ setup( 'natsort>=5.5.0', 'pycountry>=19.8.18', 'pycountry-convert>=0.7.2', - # botocore doesn't like >=2.7.0 for some reason - 'python-dateutil>=2.6.0,<2.9.0', + 'python-dateutil>=2.8.1', 'requests>=2.20.0' ], license='MIT', From 5316dddbc6b63e70b8c31bbdfda53a13cbf50e78 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Sat, 11 Jan 2020 17:05:32 +0000 Subject: [PATCH 114/155] Bump google-cloud-core from 0.28.1 to 1.1.0 Bumps [google-cloud-core](https://github.com/GoogleCloudPlatform/google-cloud-python) from 0.28.1 to 1.1.0. - [Release notes](https://github.com/GoogleCloudPlatform/google-cloud-python/releases) - [Changelog](https://github.com/googleapis/google-cloud-python/blob/master/CHANGELOG.md) - [Commits](https://github.com/GoogleCloudPlatform/google-cloud-python/compare/core-0.28.1...kms-1.1.0) Signed-off-by: dependabot-preview[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 985b044..d06cb3f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,7 @@ docutils==0.14 dyn==1.8.1 edgegrid-python==1.1.1 futures==3.2.0; python_version < '3.0' -google-cloud-core==0.28.1 +google-cloud-core==1.1.0 google-cloud-dns==0.29.0 ipaddress==1.0.22 jmespath==0.9.3 From d4be1036a4f3ffb2e19bcc8e64b9dd036ce7c1ff Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Sat, 11 Jan 2020 17:13:57 +0000 Subject: [PATCH 115/155] Bump s3transfer from 0.1.13 to 0.3.0 Bumps [s3transfer](https://github.com/boto/s3transfer) from 0.1.13 to 0.3.0. - [Release notes](https://github.com/boto/s3transfer/releases) - [Changelog](https://github.com/boto/s3transfer/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/s3transfer/compare/0.1.13...0.3.0) Signed-off-by: dependabot-preview[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d06cb3f..8769223 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,7 +20,7 @@ pycountry-convert==0.7.2 pycountry==19.8.18 python-dateutil==2.8.1 requests==2.22.0 -s3transfer==0.1.13 +s3transfer==0.3.0 setuptools==40.3.0 six==1.13.0 transip==2.0.0 From c28b1a7d7389ff4363229c22d922d212c3881a21 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Sat, 11 Jan 2020 17:17:37 +0000 Subject: [PATCH 116/155] Bump jmespath from 0.9.3 to 0.9.4 Bumps [jmespath](https://github.com/jmespath/jmespath.py) from 0.9.3 to 0.9.4. - [Release notes](https://github.com/jmespath/jmespath.py/releases) - [Changelog](https://github.com/jmespath/jmespath.py/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/jmespath/jmespath.py/compare/0.9.3...0.9.4) Signed-off-by: dependabot-preview[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8769223..5ba5ac2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,7 @@ futures==3.2.0; python_version < '3.0' google-cloud-core==1.1.0 google-cloud-dns==0.29.0 ipaddress==1.0.22 -jmespath==0.9.3 +jmespath==0.9.4 msrestazure==0.6.2 natsort==5.5.0 ns1-python==0.13.0 From 04759a51a02de6ce59829c732b73d1e7055cb490 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Sat, 11 Jan 2020 17:21:33 +0000 Subject: [PATCH 117/155] Bump ovh from 0.4.8 to 0.5.0 Bumps [ovh](https://github.com/ovh/python-ovh) from 0.4.8 to 0.5.0. - [Release notes](https://github.com/ovh/python-ovh/releases) - [Changelog](https://github.com/ovh/python-ovh/blob/master/CHANGELOG.md) - [Commits](https://github.com/ovh/python-ovh/compare/v0.4.8...v0.5.0) Signed-off-by: dependabot-preview[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 5ba5ac2..14a52f0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,7 +15,7 @@ jmespath==0.9.4 msrestazure==0.6.2 natsort==5.5.0 ns1-python==0.13.0 -ovh==0.4.8 +ovh==0.5.0 pycountry-convert==0.7.2 pycountry==19.8.18 python-dateutil==2.8.1 From 09000540a012dc19bc499234dbd17d8054d00b9d Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Sat, 11 Jan 2020 17:26:30 +0000 Subject: [PATCH 118/155] Bump google-cloud-dns from 0.29.0 to 0.31.0 Bumps [google-cloud-dns](https://github.com/GoogleCloudPlatform/google-cloud-python) from 0.29.0 to 0.31.0. - [Release notes](https://github.com/GoogleCloudPlatform/google-cloud-python/releases) - [Changelog](https://github.com/googleapis/google-cloud-python/blob/master/CHANGELOG.md) - [Commits](https://github.com/GoogleCloudPlatform/google-cloud-python/compare/0.29.0...0.31.0) Signed-off-by: dependabot-preview[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 14a52f0..f063d26 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,7 @@ dyn==1.8.1 edgegrid-python==1.1.1 futures==3.2.0; python_version < '3.0' google-cloud-core==1.1.0 -google-cloud-dns==0.29.0 +google-cloud-dns==0.31.0 ipaddress==1.0.22 jmespath==0.9.4 msrestazure==0.6.2 From a33b75911d6a24952cafe586061f16805941771c Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Sat, 11 Jan 2020 17:30:12 +0000 Subject: [PATCH 119/155] Bump pycodestyle from 2.4.0 to 2.5.0 Bumps [pycodestyle](https://github.com/PyCQA/pycodestyle) from 2.4.0 to 2.5.0. - [Release notes](https://github.com/PyCQA/pycodestyle/releases) - [Changelog](https://github.com/PyCQA/pycodestyle/blob/master/CHANGES.txt) - [Commits](https://github.com/PyCQA/pycodestyle/compare/2.4.0...2.5.0) Signed-off-by: dependabot-preview[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index d9888b8..5fb2233 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,7 +1,7 @@ coverage mock nose -pycodestyle==2.4.0 +pycodestyle==2.5.0 pyflakes==1.6.0 readme_renderer[md]==24.0 requests_mock From 79cb88ef2566fc7863d3f71ad078bea081c1c04e Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 11 Jan 2020 09:37:00 -0800 Subject: [PATCH 120/155] Fix pycodestyle overindent warnings w/2.5.0 --- octodns/provider/constellix.py | 4 ++-- octodns/provider/dnsmadeeasy.py | 2 +- tests/test_octodns_provider_azuredns.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/octodns/provider/constellix.py b/octodns/provider/constellix.py index 2ca49e3..0600f80 100644 --- a/octodns/provider/constellix.py +++ b/octodns/provider/constellix.py @@ -429,8 +429,8 @@ class ConstellixProvider(BaseProvider): for record in self.zone_records(zone): if existing.name == record['name'] and \ existing._type == record['type']: - self._client.record_delete(zone.name, record['type'], - record['id']) + self._client.record_delete(zone.name, record['type'], + record['id']) def _apply(self, plan): desired = plan.desired diff --git a/octodns/provider/dnsmadeeasy.py b/octodns/provider/dnsmadeeasy.py index cc10c9a..0bf05a0 100644 --- a/octodns/provider/dnsmadeeasy.py +++ b/octodns/provider/dnsmadeeasy.py @@ -374,7 +374,7 @@ class DnsMadeEasyProvider(BaseProvider): for record in self.zone_records(zone): if existing.name == record['name'] and \ existing._type == record['type']: - self._client.record_delete(zone.name, record['id']) + self._client.record_delete(zone.name, record['id']) def _apply(self, plan): desired = plan.desired diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index 920c502..1769cef 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -321,7 +321,7 @@ class Test_ParseAzureType(TestCase): ['AAAA', 'Microsoft.Network/dnszones/AAAA'], ['NS', 'Microsoft.Network/dnszones/NS'], ['MX', 'Microsoft.Network/dnszones/MX']]: - self.assertEquals(expected, _parse_azure_type(test)) + self.assertEquals(expected, _parse_azure_type(test)) class Test_CheckEndswithDot(TestCase): From d56bf28d8d00f56a9aaa4521e4e63b7a24b7071c Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Sat, 11 Jan 2020 17:42:18 +0000 Subject: [PATCH 121/155] Bump botocore from 1.10.5 to 1.14.0 Bumps [botocore](https://github.com/boto/botocore) from 1.10.5 to 1.14.0. - [Release notes](https://github.com/boto/botocore/releases) - [Changelog](https://github.com/boto/botocore/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/botocore/compare/1.10.5...1.14.0) Signed-off-by: dependabot-preview[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f063d26..e627537 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ PyYaml==4.2b1 azure-common==1.1.23 azure-mgmt-dns==3.0.0 boto3==1.7.5 -botocore==1.10.5 +botocore==1.14.0 dnspython==1.15.0 docutils==0.14 dyn==1.8.1 From eb890f02148aba0593bb5b7e8d57be6f616cc3b9 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 11 Jan 2020 10:16:22 -0800 Subject: [PATCH 122/155] Bump boto3 to 1.11.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e627537..20db7f8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ PyYaml==4.2b1 azure-common==1.1.23 azure-mgmt-dns==3.0.0 -boto3==1.7.5 +boto3==1.11.0 botocore==1.14.0 dnspython==1.15.0 docutils==0.14 From 60ec6e9a288921c8140c3c06d7706e33b322eb73 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 11 Jan 2020 10:16:31 -0800 Subject: [PATCH 123/155] Update Route53 test that pokes at boto internals --- tests/test_octodns_provider_route53.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/test_octodns_provider_route53.py b/tests/test_octodns_provider_route53.py index 7691804..60da3b8 100644 --- a/tests/test_octodns_provider_route53.py +++ b/tests/test_octodns_provider_route53.py @@ -1930,9 +1930,8 @@ class TestRoute53Provider(TestCase): provider = Route53Provider('test', 'abc', '123', client_max_attempts=42) # NOTE: this will break if boto ever changes the impl details... - self.assertEquals(43, provider._conn.meta.events - ._unique_id_handlers['retry-config-route53'] - ['handler']._checker.__dict__['_max_attempts']) + self.assertEquals(42, provider._conn._client_config + .retries['max_attempts']) def test_data_for_dynamic(self): provider = Route53Provider('test', 'abc', '123') From 1573172c36f8316a85fe0322d40370af69d944b8 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Sat, 11 Jan 2020 18:20:20 +0000 Subject: [PATCH 124/155] Bump azure-common from 1.1.23 to 1.1.24 Bumps [azure-common](https://github.com/Azure/azure-sdk-for-python) from 1.1.23 to 1.1.24. - [Release notes](https://github.com/Azure/azure-sdk-for-python/releases) - [Commits](https://github.com/Azure/azure-sdk-for-python/compare/azure-common_1.1.23...azure-common_1.1.24) Signed-off-by: dependabot-preview[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 20db7f8..1dcec72 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ PyYaml==4.2b1 -azure-common==1.1.23 +azure-common==1.1.24 azure-mgmt-dns==3.0.0 boto3==1.11.0 botocore==1.14.0 From e19cc270607c45a54b1fef2d448fa881bd2c0224 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Sat, 11 Jan 2020 18:28:37 +0000 Subject: [PATCH 125/155] Bump ipaddress from 1.0.22 to 1.0.23 Bumps [ipaddress](https://github.com/phihag/ipaddress) from 1.0.22 to 1.0.23. - [Release notes](https://github.com/phihag/ipaddress/releases) - [Commits](https://github.com/phihag/ipaddress/compare/v1.0.22...v1.0.23) Signed-off-by: dependabot-preview[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 1dcec72..95577ce 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ edgegrid-python==1.1.1 futures==3.2.0; python_version < '3.0' google-cloud-core==1.1.0 google-cloud-dns==0.31.0 -ipaddress==1.0.22 +ipaddress==1.0.23 jmespath==0.9.4 msrestazure==0.6.2 natsort==5.5.0 From 8598214edb2db0f1f93bc3445db652c550fd224b Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Sat, 11 Jan 2020 20:31:59 +0000 Subject: [PATCH 126/155] Bump docutils from 0.14 to 0.15.2 Bumps [docutils](http://docutils.sourceforge.net/) from 0.14 to 0.15.2. Signed-off-by: dependabot-preview[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 95577ce..12e413d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ azure-mgmt-dns==3.0.0 boto3==1.11.0 botocore==1.14.0 dnspython==1.15.0 -docutils==0.14 +docutils==0.15.2 dyn==1.8.1 edgegrid-python==1.1.1 futures==3.2.0; python_version < '3.0' From 96dad58a48f2a600a1a5e6fa4e78ef5cd75bacc5 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Sat, 11 Jan 2020 20:35:09 +0000 Subject: [PATCH 127/155] Bump pyyaml from 4.2b1 to 5.3 Bumps [pyyaml](https://github.com/yaml/pyyaml) from 4.2b1 to 5.3. - [Release notes](https://github.com/yaml/pyyaml/releases) - [Changelog](https://github.com/yaml/pyyaml/blob/master/CHANGES) - [Commits](https://github.com/yaml/pyyaml/commits/5.3) Signed-off-by: dependabot-preview[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 12e413d..281a06e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -PyYaml==4.2b1 +PyYaml==5.3 azure-common==1.1.24 azure-mgmt-dns==3.0.0 boto3==1.11.0 From f15358535028a315c89427e0a5a11519f5a12636 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Sat, 11 Jan 2020 20:35:09 +0000 Subject: [PATCH 128/155] Bump dnspython from 1.15.0 to 1.16.0 Bumps [dnspython](https://github.com/rthalley/dnspython) from 1.15.0 to 1.16.0. - [Release notes](https://github.com/rthalley/dnspython/releases) - [Changelog](https://github.com/rthalley/dnspython/blob/master/doc/whatsnew.rst) - [Commits](https://github.com/rthalley/dnspython/compare/v1.15.0...v1.16.0) Signed-off-by: dependabot-preview[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 12e413d..d926f6a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ azure-common==1.1.24 azure-mgmt-dns==3.0.0 boto3==1.11.0 botocore==1.14.0 -dnspython==1.15.0 +dnspython==1.16.0 docutils==0.15.2 dyn==1.8.1 edgegrid-python==1.1.1 From 0e7b986c157339d91a4621af5266c48cce983689 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Sat, 11 Jan 2020 20:38:21 +0000 Subject: [PATCH 129/155] Bump twine from 1.13.0 to 1.15.0 Bumps [twine](https://github.com/pypa/twine) from 1.13.0 to 1.15.0. - [Release notes](https://github.com/pypa/twine/releases) - [Changelog](https://github.com/pypa/twine/blob/master/docs/changelog.rst) - [Commits](https://github.com/pypa/twine/compare/1.13.0...1.15.0) Signed-off-by: dependabot-preview[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 5fb2233..0de9ceb 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,4 +5,4 @@ pycodestyle==2.5.0 pyflakes==1.6.0 readme_renderer[md]==24.0 requests_mock -twine==1.13.0 +twine==1.15.0 From 743516575fd3c9162dac4ad782605abc8a114acb Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Sat, 11 Jan 2020 20:43:20 +0000 Subject: [PATCH 130/155] Bump dnspython from 1.15.0 to 1.16.0 Bumps [dnspython](https://github.com/rthalley/dnspython) from 1.15.0 to 1.16.0. - [Release notes](https://github.com/rthalley/dnspython/releases) - [Changelog](https://github.com/rthalley/dnspython/blob/master/doc/whatsnew.rst) - [Commits](https://github.com/rthalley/dnspython/compare/v1.15.0...v1.16.0) Signed-off-by: dependabot-preview[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 281a06e..fb46bed 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ azure-common==1.1.24 azure-mgmt-dns==3.0.0 boto3==1.11.0 botocore==1.14.0 -dnspython==1.15.0 +dnspython==1.16.0 docutils==0.15.2 dyn==1.8.1 edgegrid-python==1.1.1 From f18c702c59dcf1bd925790e9cd00722b9b6eef64 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Sat, 11 Jan 2020 20:43:23 +0000 Subject: [PATCH 131/155] Bump natsort from 5.5.0 to 6.2.0 Bumps [natsort](https://github.com/SethMMorton/natsort) from 5.5.0 to 6.2.0. - [Release notes](https://github.com/SethMMorton/natsort/releases) - [Changelog](https://github.com/SethMMorton/natsort/blob/master/CHANGELOG.md) - [Commits](https://github.com/SethMMorton/natsort/compare/5.5.0...6.2.0) Signed-off-by: dependabot-preview[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 281a06e..d44fef8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,7 +13,7 @@ google-cloud-dns==0.31.0 ipaddress==1.0.23 jmespath==0.9.4 msrestazure==0.6.2 -natsort==5.5.0 +natsort==6.2.0 ns1-python==0.13.0 ovh==0.5.0 pycountry-convert==0.7.2 From e56051ba55026ba1e372af1970fce1e6c2c3c785 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 11 Jan 2020 13:45:48 -0800 Subject: [PATCH 132/155] Add TTL's to the axfr test files --- tests/zones/invalid.zone. | 2 +- tests/zones/unit.tests. | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/zones/invalid.zone. b/tests/zones/invalid.zone. index c814af6..04748a1 100644 --- a/tests/zones/invalid.zone. +++ b/tests/zones/invalid.zone. @@ -1,5 +1,5 @@ $ORIGIN invalid.zone. -@ IN SOA ns1.invalid.zone. root.invalid.zone. ( +@ 3600 IN SOA ns1.invalid.zone. root.invalid.zone. ( 2018071501 ; Serial 3600 ; Refresh (1 hour) 600 ; Retry (10 minutes) diff --git a/tests/zones/unit.tests. b/tests/zones/unit.tests. index 95828ad..0305e05 100644 --- a/tests/zones/unit.tests. +++ b/tests/zones/unit.tests. @@ -1,5 +1,5 @@ $ORIGIN unit.tests. -@ IN SOA ns1.unit.tests. root.unit.tests. ( +@ 3600 IN SOA ns1.unit.tests. root.unit.tests. ( 2018071501 ; Serial 3600 ; Refresh (1 hour) 600 ; Retry (10 minutes) From a564da68bb04b50730ce87bebbfd2b15990bc484 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Sat, 11 Jan 2020 21:51:52 +0000 Subject: [PATCH 133/155] Bump pyflakes from 1.6.0 to 2.1.1 Bumps [pyflakes](https://github.com/PyCQA/pyflakes) from 1.6.0 to 2.1.1. - [Release notes](https://github.com/PyCQA/pyflakes/releases) - [Changelog](https://github.com/PyCQA/pyflakes/blob/master/NEWS.rst) - [Commits](https://github.com/PyCQA/pyflakes/compare/1.6.0...2.1.1) Signed-off-by: dependabot-preview[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 0de9ceb..3ad1b04 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,7 +2,7 @@ coverage mock nose pycodestyle==2.5.0 -pyflakes==1.6.0 +pyflakes==2.1.1 readme_renderer[md]==24.0 requests_mock twine==1.15.0 From 2ea63959da5ef7f8b6cad0aa5d14de1a11b2943e Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Sat, 11 Jan 2020 21:55:24 +0000 Subject: [PATCH 134/155] Bump setuptools from 40.3.0 to 44.0.0 Bumps [setuptools](https://github.com/pypa/setuptools) from 40.3.0 to 44.0.0. - [Release notes](https://github.com/pypa/setuptools/releases) - [Changelog](https://github.com/pypa/setuptools/blob/master/CHANGES.rst) - [Commits](https://github.com/pypa/setuptools/compare/v40.3.0...v44.0.0) Signed-off-by: dependabot-preview[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d44fef8..9794364 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,6 +21,6 @@ pycountry==19.8.18 python-dateutil==2.8.1 requests==2.22.0 s3transfer==0.3.0 -setuptools==40.3.0 +setuptools==44.0.0 six==1.13.0 transip==2.0.0 From 2d093c479692c8eeca7c425e76ad785385a77ef9 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Sun, 12 Jan 2020 19:50:47 +0000 Subject: [PATCH 135/155] Bump docutils from 0.15.2 to 0.16 Bumps [docutils](http://docutils.sourceforge.net/) from 0.15.2 to 0.16. Signed-off-by: dependabot-preview[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 59b8864..93d8567 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ azure-mgmt-dns==3.0.0 boto3==1.11.0 botocore==1.14.0 dnspython==1.16.0 -docutils==0.15.2 +docutils==0.16 dyn==1.8.1 edgegrid-python==1.1.1 futures==3.2.0; python_version < '3.0' From 01a9fa87b190128c40f6ff6a96a6eb015be4119d Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 13 Jan 2020 07:29:38 -0800 Subject: [PATCH 136/155] Address Ns1Provider review feedback --- octodns/provider/ns1.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index b442271..0694627 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -655,9 +655,9 @@ class Ns1Provider(BaseProvider): host = record.fqdn[:-1] _type = record._type - request = 'GET {path} HTTP/1.0\\r\\nHost: {host}\\r\\n' \ - 'User-agent: NS1\\r\\n\\r\\n'.format(path=record.healthcheck_path, - host=record.healthcheck_host) + request = r'GET {path} HTTP/1.0\r\nHost: {host}\r\n' \ + r'User-agent: NS1\r\n\r\n'.format(path=record.healthcheck_path, + host=record.healthcheck_host) return { 'active': True, @@ -771,13 +771,13 @@ class Ns1Provider(BaseProvider): for geo in rule.data.get('geos', []): n = len(geo) if n == 8: - # US state + # US state, e.g. NA-US-KY us_state.add(geo[-2:]) elif n == 5: - # Country + # Country, e.g. EU-FR country.add(geo[-2:]) else: - # Continent + # Continent, e.g. AS georegion.update(self._CONTINENT_TO_REGIONS[geo]) meta = { @@ -826,7 +826,9 @@ class Ns1Provider(BaseProvider): # Dynamic/health checked current_pool_name = pool_name - while current_pool_name: + seen = set() + while current_pool_name and current_pool_name not in seen: + seen.add(current_pool_name) pool = pools[current_pool_name] for answer in pool_answers[current_pool_name]: answer = { From c950503868ded420ee3d2a2e5df5d2e1d64fba8b Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 13 Jan 2020 07:29:56 -0800 Subject: [PATCH 137/155] Pass through the CHANGELOG --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1cb7b1b..491370f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +## v0.9.10 - ????-??-?? - ??? + +* Added support for dynamic records to Ns1Provider, updated client and rate + limiting implementation +* Moved CI to use GitHub Actions +* Set up dependabot to automatically PR requirements updates +* Pass at bumping all of the requirements + ## v0.9.9 - 2019-11-04 - Python 3.7 Support * Extensive pass through the whole codebase to support Python 3 From c7be8fada2104633b25f8d73319f58728e022a8b Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Thu, 16 Jan 2020 14:37:25 -0800 Subject: [PATCH 138/155] Add a way to configure Ns1Provider monitoring regions for records --- octodns/provider/ns1.py | 27 ++++-- tests/test_octodns_provider_ns1.py | 133 +++++++++++++++++++++++++++++ 2 files changed, 155 insertions(+), 5 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 0694627..a25911d 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -175,6 +175,9 @@ class Ns1Provider(BaseProvider): ns1: class: octodns.provider.ns1.Ns1Provider api_key: env/NS1_API_KEY + # Only required if using dynamic records + monitor_regions: + - lga ''' SUPPORTS_GEO = True SUPPORTS_DYNAMIC = True @@ -224,11 +227,13 @@ class Ns1Provider(BaseProvider): 'NA': ('US-CENTRAL', 'US-EAST', 'US-WEST'), } - def __init__(self, id, api_key, retry_count=4, *args, **kwargs): + def __init__(self, id, api_key, retry_count=4, monitor_regions=None, *args, + **kwargs): self.log = getLogger('Ns1Provider[{}]'.format(id)) - self.log.debug('__init__: id=%s, api_key=***, retry_count=%d', id, - retry_count) + self.log.debug('__init__: id=%s, api_key=***, retry_count=%d, ' + 'monitor_regions=%s', id, retry_count, monitor_regions) super(Ns1Provider, self).__init__(id, *args, **kwargs) + self.monitor_regions = monitor_regions self._client = Ns1Client(api_key, retry_count) @@ -679,8 +684,7 @@ class Ns1Provider(BaseProvider): 'policy': 'quorum', 'rapid_recheck': False, 'region_scope': 'fixed', - # TODO: what should we do here dal, sjc, lga, sin, ams - 'regions': ['lga'], + 'regions': self.monitor_regions, 'rules': [{ 'comparison': 'contains', 'key': 'output', @@ -977,12 +981,25 @@ class Ns1Provider(BaseProvider): self._client.records_delete(zone, domain, _type) self._monitors_gc(existing) + def _has_dynamic(self, changes): + for change in changes: + if getattr(change.record, 'dynamic', False): + return True + + return False + def _apply(self, plan): desired = plan.desired changes = plan.changes self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name, len(changes)) + # Make sure that if we're going to make any dynamic changes that we + # have monitor_regions configured before touching anything so we can + # abort early and not half-apply + if self._has_dynamic(changes) and self.monitor_regions is None: + raise Ns1Exception('Monitored record, but monitor_regions not set') + domain_name = desired.name[:-1] try: ns1_zone = self._client.zones_retrieve(domain_name) diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index 0d456fe..bd4696b 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -14,6 +14,7 @@ from unittest import TestCase from octodns.record import Delete, Record, Update from octodns.provider.ns1 import Ns1Client, Ns1Exception, Ns1Provider +from octodns.provider.plan import Plan from octodns.zone import Zone @@ -1198,6 +1199,138 @@ class TestNs1ProviderDynamic(TestCase): self.assertFalse(extra) monitors_for_mock.assert_not_called() + def test_has_dynamic(self): + provider = Ns1Provider('test', 'api-key') + + desired = Zone('unit.tests.', []) + + simple = Record.new(desired, 'sim', { + 'ttl': 33, + 'type': 'A', + 'value': '1.2.3.4', + }) + + # Dynamic record, inspectable + dynamic = Record.new(desired, 'dyn', { + 'dynamic': { + 'pools': { + 'iad': { + 'values': [{ + 'value': '1.2.3.4', + }], + }, + }, + 'rules': [{ + 'pool': 'iad', + }], + }, + 'octodns': { + 'healthcheck': { + 'host': 'send.me', + 'path': '/_ping', + 'port': 80, + 'protocol': 'HTTP', + } + }, + 'ttl': 32, + 'type': 'A', + 'value': '1.2.3.4', + 'meta': {}, + }) + + simple_update = Update(simple, simple) + dynamic_update = Update(dynamic, dynamic) + + self.assertFalse(provider._has_dynamic([simple_update])) + self.assertTrue(provider._has_dynamic([dynamic_update])) + self.assertTrue(provider._has_dynamic([simple_update, dynamic_update])) + + @patch('octodns.provider.ns1.Ns1Client.zones_retrieve') + @patch('octodns.provider.ns1.Ns1Provider._apply_Update') + def test_apply_monitor_regions(self, apply_update_mock, + zones_retrieve_mock): + provider = Ns1Provider('test', 'api-key') + + desired = Zone('unit.tests.', []) + + simple = Record.new(desired, 'sim', { + 'ttl': 33, + 'type': 'A', + 'value': '1.2.3.4', + }) + + # Dynamic record, inspectable + dynamic = Record.new(desired, 'dyn', { + 'dynamic': { + 'pools': { + 'iad': { + 'values': [{ + 'value': '1.2.3.4', + }], + }, + }, + 'rules': [{ + 'pool': 'iad', + }], + }, + 'octodns': { + 'healthcheck': { + 'host': 'send.me', + 'path': '/_ping', + 'port': 80, + 'protocol': 'HTTP', + } + }, + 'ttl': 32, + 'type': 'A', + 'value': '1.2.3.4', + 'meta': {}, + }) + + simple_update = Update(simple, simple) + simple_plan = Plan(desired, desired, [simple_update], True) + dynamic_update = Update(dynamic, dynamic) + dynamic_plan = Plan(desired, desired, [dynamic_update], True) + both_plan = Plan(desired, desired, [simple_update, dynamic_update], + True) + + # always return foo, we aren't testing this part here + zones_retrieve_mock.side_effect = [ + 'foo', + 'foo', + 'foo', + 'foo', + ] + + # Doesn't blow up, and calls apply once + apply_update_mock.reset_mock() + provider._apply(simple_plan) + apply_update_mock.assert_has_calls([call('foo', simple_update)]) + + # Blows up and apply not called + apply_update_mock.reset_mock() + with self.assertRaises(Ns1Exception) as ctx: + provider._apply(dynamic_plan) + self.assertTrue('monitor_regions not set' in text_type(ctx.exception)) + apply_update_mock.assert_not_called() + + # Blows up and apply not called even though there's a simple + apply_update_mock.reset_mock() + with self.assertRaises(Ns1Exception) as ctx: + provider._apply(both_plan) + self.assertTrue('monitor_regions not set' in text_type(ctx.exception)) + apply_update_mock.assert_not_called() + + # with monitor_regions set + provider.monitor_regions = ['lga'] + + apply_update_mock.reset_mock() + provider._apply(both_plan) + apply_update_mock.assert_has_calls([ + call('foo', dynamic_update), + call('foo', simple_update), + ]) + class TestNs1Client(TestCase): From df6ad6b1f06af7ba907f0a3c3751e8339d8df973 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 20 Jan 2020 08:33:27 +0000 Subject: [PATCH 139/155] Bump s3transfer from 0.3.0 to 0.3.1 Bumps [s3transfer](https://github.com/boto/s3transfer) from 0.3.0 to 0.3.1. - [Release notes](https://github.com/boto/s3transfer/releases) - [Changelog](https://github.com/boto/s3transfer/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/s3transfer/compare/0.3.0...0.3.1) Signed-off-by: dependabot-preview[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 93d8567..3a2aadf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,7 +20,7 @@ pycountry-convert==0.7.2 pycountry==19.8.18 python-dateutil==2.8.1 requests==2.22.0 -s3transfer==0.3.0 +s3transfer==0.3.1 setuptools==44.0.0 six==1.13.0 transip==2.0.0 From e3123be32e88f7aef640c1dab8a1b38da0c835b6 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 21 Jan 2020 14:55:53 +0000 Subject: [PATCH 140/155] Bump boto3 from 1.11.0 to 1.11.6 Bumps [boto3](https://github.com/boto/boto3) from 1.11.0 to 1.11.6. - [Release notes](https://github.com/boto/boto3/releases) - [Changelog](https://github.com/boto/boto3/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/boto3/compare/1.11.0...1.11.6) Signed-off-by: dependabot-preview[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3a2aadf..bc75da0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ PyYaml==5.3 azure-common==1.1.24 azure-mgmt-dns==3.0.0 -boto3==1.11.0 +boto3==1.11.6 botocore==1.14.0 dnspython==1.16.0 docutils==0.16 From ba57724e1b51e5aee0467c8bd1c0fc395d2948d3 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 21 Jan 2020 14:59:48 +0000 Subject: [PATCH 141/155] Bump google-cloud-core from 1.1.0 to 1.2.0 Bumps [google-cloud-core](https://github.com/GoogleCloudPlatform/google-cloud-python) from 1.1.0 to 1.2.0. - [Release notes](https://github.com/GoogleCloudPlatform/google-cloud-python/releases) - [Changelog](https://github.com/googleapis/google-cloud-python/blob/master/CHANGELOG.md) - [Commits](https://github.com/GoogleCloudPlatform/google-cloud-python/compare/kms-1.1.0...kms-1.2.0) Signed-off-by: dependabot-preview[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index bc75da0..2f35502 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,7 @@ docutils==0.16 dyn==1.8.1 edgegrid-python==1.1.1 futures==3.2.0; python_version < '3.0' -google-cloud-core==1.1.0 +google-cloud-core==1.2.0 google-cloud-dns==0.31.0 ipaddress==1.0.23 jmespath==0.9.4 From 2da3c7d8014609235fc69c04267061ac652da5b6 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 21 Jan 2020 15:27:46 +0000 Subject: [PATCH 142/155] Bump six from 1.13.0 to 1.14.0 Bumps [six](https://github.com/benjaminp/six) from 1.13.0 to 1.14.0. - [Release notes](https://github.com/benjaminp/six/releases) - [Changelog](https://github.com/benjaminp/six/blob/master/CHANGES) - [Commits](https://github.com/benjaminp/six/compare/1.13.0...1.14.0) Signed-off-by: dependabot-preview[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2f35502..6f5ab04 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,5 +22,5 @@ python-dateutil==2.8.1 requests==2.22.0 s3transfer==0.3.1 setuptools==44.0.0 -six==1.13.0 +six==1.14.0 transip==2.0.0 From 3a8d8e4627fcd4e8d00ca500287b0b1a6d7cbe24 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 21 Jan 2020 15:32:15 +0000 Subject: [PATCH 143/155] Bump botocore from 1.14.0 to 1.14.6 Bumps [botocore](https://github.com/boto/botocore) from 1.14.0 to 1.14.6. - [Release notes](https://github.com/boto/botocore/releases) - [Changelog](https://github.com/boto/botocore/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/botocore/compare/1.14.0...1.14.6) Signed-off-by: dependabot-preview[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6f5ab04..4f7f349 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ PyYaml==5.3 azure-common==1.1.24 azure-mgmt-dns==3.0.0 boto3==1.11.6 -botocore==1.14.0 +botocore==1.14.6 dnspython==1.16.0 docutils==0.16 dyn==1.8.1 From 01ec880f830935ab5f31903d8ad250351bb1e4b8 Mon Sep 17 00:00:00 2001 From: Charles Date: Tue, 21 Jan 2020 16:44:20 +0100 Subject: [PATCH 144/155] Add CAA record type for ovh provider --- octodns/provider/ovh.py | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/octodns/provider/ovh.py b/octodns/provider/ovh.py index 8a3d492..182639c 100644 --- a/octodns/provider/ovh.py +++ b/octodns/provider/ovh.py @@ -40,8 +40,8 @@ class OvhProvider(BaseProvider): # This variable is also used in populate method to filter which OVH record # types are supported by octodns - SUPPORTS = set(('A', 'AAAA', 'CNAME', 'DKIM', 'MX', 'NAPTR', 'NS', 'PTR', - 'SPF', 'SRV', 'SSHFP', 'TXT')) + SUPPORTS = set(('A', 'AAAA', 'CAA', 'CNAME', 'DKIM', 'MX', 'NAPTR', 'NS', + 'PTR', 'SPF', 'SRV', 'SSHFP', 'TXT')) def __init__(self, id, endpoint, application_key, application_secret, consumer_key, *args, **kwargs): @@ -139,6 +139,22 @@ class OvhProvider(BaseProvider): 'value': record['target'] } + @staticmethod + def _data_for_CAA(_type, records): + values = [] + for record in records: + flags, tag, value = record['target'].split(' ', 2) + values.append({ + 'flags': flags, + 'tag': tag, + 'value': value[1:-1] + }) + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': values + } + @staticmethod def _data_for_MX(_type, records): values = [] @@ -244,6 +260,16 @@ class OvhProvider(BaseProvider): 'fieldType': record._type } + @staticmethod + def _params_for_CAA(record): + for value in record.values: + yield { + 'target': '%d %s "%s"' % (value.flags, value.tag, value.value), + 'subDomain': record.name, + 'ttl': record.ttl, + 'fieldType': record._type + } + @staticmethod def _params_for_MX(record): for value in record.values: From 65840cfbed35b4bc2feedc974810dc2e3c9551f7 Mon Sep 17 00:00:00 2001 From: Charles Date: Tue, 21 Jan 2020 16:45:13 +0100 Subject: [PATCH 145/155] Add test for ovh caa record coverage --- tests/test_octodns_provider_ovh.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/test_octodns_provider_ovh.py b/tests/test_octodns_provider_ovh.py index 924591f..3da4276 100644 --- a/tests/test_octodns_provider_ovh.py +++ b/tests/test_octodns_provider_ovh.py @@ -279,6 +279,24 @@ class TestOvhProvider(TestCase): 'id': 18 }) + # CAA + api_record.append({ + 'fieldType': 'CAA', + 'ttl': 1600, + 'target': '0 issue "ca.unit.tests"', + 'subDomain': 'caa', + 'id': 19 + }) + expected.add(Record.new(zone, 'caa', { + 'ttl': 1600, + 'type': 'CAA', + 'values': [{ + 'flags': 0, + 'tag': 'issue', + 'value': 'ca.unit.tests' + }] + })) + valid_dkim = [valid_dkim_key, 'v=DKIM1 \\; %s' % valid_dkim_key, 'h=sha256 \\; %s' % valid_dkim_key, @@ -404,6 +422,9 @@ class TestOvhProvider(TestCase): call('/domain/zone/unit.tests/record', fieldType='SRV', subDomain='_srv._tcp', target='40 50 60 foo-2.unit.tests.', ttl=800), + call('/domain/zone/unit.tests/record', fieldType='CAA', + subDomain='caa', target='0 issue "ca.unit.tests"', + ttl=1600), call('/domain/zone/unit.tests/record', fieldType='DKIM', subDomain='dkim', target='p=MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCxLaG' From e13d23dc80ff091f3adbc995d94f631dec64d751 Mon Sep 17 00:00:00 2001 From: Charles Date: Tue, 21 Jan 2020 17:09:13 +0100 Subject: [PATCH 146/155] Use python3-friendly syntax --- octodns/provider/ovh.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/octodns/provider/ovh.py b/octodns/provider/ovh.py index 182639c..54f62ac 100644 --- a/octodns/provider/ovh.py +++ b/octodns/provider/ovh.py @@ -264,7 +264,8 @@ class OvhProvider(BaseProvider): def _params_for_CAA(record): for value in record.values: yield { - 'target': '%d %s "%s"' % (value.flags, value.tag, value.value), + 'target': '{} {} "{}"'.format(value.flags, value.tag, + value.value), 'subDomain': record.name, 'ttl': record.ttl, 'fieldType': record._type From fe490636e594a682de8af530bb3098b0bf40136d Mon Sep 17 00:00:00 2001 From: Charles Date: Tue, 21 Jan 2020 17:48:13 +0100 Subject: [PATCH 147/155] Modify Readme: add CAA for ovh provider --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 36dc5e2..6127234 100644 --- a/README.md +++ b/README.md @@ -188,7 +188,7 @@ The above command pulled the existing data out of Route53 and placed the results | [GoogleCloudProvider](/octodns/provider/googlecloud.py) | google-cloud-dns | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | No | | | [MythicBeastsProvider](/octodns/provider/mythicbeasts.py) | Mythic Beasts | A, AAAA, ALIAS, CNAME, MX, NS, SRV, SSHFP, CAA, TXT | No | | | [Ns1Provider](/octodns/provider/ns1.py) | ns1-python | All | Yes | No CNAME support, missing `NA` geo target | -| [OVH](/octodns/provider/ovh.py) | ovh | A, AAAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, SSHFP, TXT, DKIM | No | | +| [OVH](/octodns/provider/ovh.py) | ovh | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, SSHFP, TXT, DKIM | No | | | [PowerDnsProvider](/octodns/provider/powerdns.py) | | All | No | | | [Rackspace](/octodns/provider/rackspace.py) | | A, AAAA, ALIAS, CNAME, MX, NS, PTR, SPF, TXT | No | | | [Route53](/octodns/provider/route53.py) | boto3 | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | Both | CNAME health checks don't support a Host header | From a319453e7e9dcce3874adf5637a1d1d55522d8d9 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 27 Jan 2020 08:28:30 +0000 Subject: [PATCH 148/155] Bump s3transfer from 0.3.1 to 0.3.2 Bumps [s3transfer](https://github.com/boto/s3transfer) from 0.3.1 to 0.3.2. - [Release notes](https://github.com/boto/s3transfer/releases) - [Changelog](https://github.com/boto/s3transfer/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/s3transfer/compare/0.3.1...0.3.2) Signed-off-by: dependabot-preview[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 4f7f349..167307a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,7 +20,7 @@ pycountry-convert==0.7.2 pycountry==19.8.18 python-dateutil==2.8.1 requests==2.22.0 -s3transfer==0.3.1 +s3transfer==0.3.2 setuptools==44.0.0 six==1.14.0 transip==2.0.0 From e12ac1930dac4e9b23ad31f82eb5abef610a93eb Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 27 Jan 2020 13:51:45 +0000 Subject: [PATCH 149/155] Bump botocore from 1.14.6 to 1.14.9 Bumps [botocore](https://github.com/boto/botocore) from 1.14.6 to 1.14.9. - [Release notes](https://github.com/boto/botocore/releases) - [Changelog](https://github.com/boto/botocore/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/botocore/compare/1.14.6...1.14.9) Signed-off-by: dependabot-preview[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 167307a..7f1cd83 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ PyYaml==5.3 azure-common==1.1.24 azure-mgmt-dns==3.0.0 boto3==1.11.6 -botocore==1.14.6 +botocore==1.14.9 dnspython==1.16.0 docutils==0.16 dyn==1.8.1 From 160578fcf1caf7f150c685c899654dace4caa1f6 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 27 Jan 2020 14:17:08 +0000 Subject: [PATCH 150/155] Bump boto3 from 1.11.6 to 1.11.9 Bumps [boto3](https://github.com/boto/boto3) from 1.11.6 to 1.11.9. - [Release notes](https://github.com/boto/boto3/releases) - [Changelog](https://github.com/boto/boto3/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/boto3/compare/1.11.6...1.11.9) Signed-off-by: dependabot-preview[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 7f1cd83..42e3ca1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ PyYaml==5.3 azure-common==1.1.24 azure-mgmt-dns==3.0.0 -boto3==1.11.6 +boto3==1.11.9 botocore==1.14.9 dnspython==1.16.0 docutils==0.16 From ee73cacb5e6f017d7254d6ba38520b0c79d26340 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 27 Jan 2020 08:35:12 -0800 Subject: [PATCH 151/155] DRY up new NS1 dynamic tests --- tests/test_octodns_provider_ns1.py | 122 +++++++++++------------------ 1 file changed, 44 insertions(+), 78 deletions(-) diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index bd4696b..1d4e8f9 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -1199,47 +1199,47 @@ class TestNs1ProviderDynamic(TestCase): self.assertFalse(extra) monitors_for_mock.assert_not_called() - def test_has_dynamic(self): - provider = Ns1Provider('test', 'api-key') - - desired = Zone('unit.tests.', []) + DESIRED = Zone('unit.tests.', []) - simple = Record.new(desired, 'sim', { - 'ttl': 33, - 'type': 'A', - 'value': '1.2.3.4', - }) + SIMPLE = Record.new(DESIRED, 'sim', { + 'ttl': 33, + 'type': 'A', + 'value': '1.2.3.4', + }) - # Dynamic record, inspectable - dynamic = Record.new(desired, 'dyn', { - 'dynamic': { - 'pools': { - 'iad': { - 'values': [{ - 'value': '1.2.3.4', - }], - }, + # Dynamic record, inspectable + DYNAMIC = Record.new(DESIRED, 'dyn', { + 'dynamic': { + 'pools': { + 'iad': { + 'values': [{ + 'value': '1.2.3.4', + }], }, - 'rules': [{ - 'pool': 'iad', - }], - }, - 'octodns': { - 'healthcheck': { - 'host': 'send.me', - 'path': '/_ping', - 'port': 80, - 'protocol': 'HTTP', - } }, - 'ttl': 32, - 'type': 'A', - 'value': '1.2.3.4', - 'meta': {}, - }) + 'rules': [{ + 'pool': 'iad', + }], + }, + 'octodns': { + 'healthcheck': { + 'host': 'send.me', + 'path': '/_ping', + 'port': 80, + 'protocol': 'HTTP', + } + }, + 'ttl': 32, + 'type': 'A', + 'value': '1.2.3.4', + 'meta': {}, + }) + + def test_has_dynamic(self): + provider = Ns1Provider('test', 'api-key') - simple_update = Update(simple, simple) - dynamic_update = Update(dynamic, dynamic) + simple_update = Update(self.SIMPLE, self.SIMPLE) + dynamic_update = Update(self.DYNAMIC, self.DYNAMIC) self.assertFalse(provider._has_dynamic([simple_update])) self.assertTrue(provider._has_dynamic([dynamic_update])) @@ -1251,48 +1251,14 @@ class TestNs1ProviderDynamic(TestCase): zones_retrieve_mock): provider = Ns1Provider('test', 'api-key') - desired = Zone('unit.tests.', []) - - simple = Record.new(desired, 'sim', { - 'ttl': 33, - 'type': 'A', - 'value': '1.2.3.4', - }) - - # Dynamic record, inspectable - dynamic = Record.new(desired, 'dyn', { - 'dynamic': { - 'pools': { - 'iad': { - 'values': [{ - 'value': '1.2.3.4', - }], - }, - }, - 'rules': [{ - 'pool': 'iad', - }], - }, - 'octodns': { - 'healthcheck': { - 'host': 'send.me', - 'path': '/_ping', - 'port': 80, - 'protocol': 'HTTP', - } - }, - 'ttl': 32, - 'type': 'A', - 'value': '1.2.3.4', - 'meta': {}, - }) - - simple_update = Update(simple, simple) - simple_plan = Plan(desired, desired, [simple_update], True) - dynamic_update = Update(dynamic, dynamic) - dynamic_plan = Plan(desired, desired, [dynamic_update], True) - both_plan = Plan(desired, desired, [simple_update, dynamic_update], - True) + simple_update = Update(self.SIMPLE, self.SIMPLE) + simple_plan = Plan(self.DESIRED, self.DESIRED, [simple_update], True) + dynamic_update = Update(self.DYNAMIC, self.DYNAMIC) + dynamic_update = Update(self.DYNAMIC, self.DYNAMIC) + dynamic_plan = Plan(self.DESIRED, self.DESIRED, [dynamic_update], + True) + both_plan = Plan(self.DESIRED, self.DESIRED, [simple_update, + dynamic_update], True) # always return foo, we aren't testing this part here zones_retrieve_mock.side_effect = [ From 6498a1e094a32c910c0312008bc09092393528ab Mon Sep 17 00:00:00 2001 From: Mikalai Radchuk <509198+m1kola@users.noreply.github.com> Date: Wed, 29 Jan 2020 20:53:16 +0000 Subject: [PATCH 152/155] Fixes a typo in log --- octodns/manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/manager.py b/octodns/manager.py index 3c91aa9..f19885f 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -99,7 +99,7 @@ class Manager(object): self.include_meta = include_meta or manager_config.get('include_meta', False) - self.log.info('__init__: max_workers=%s', self.include_meta) + self.log.info('__init__: include_meta=%s', self.include_meta) self.log.debug('__init__: configuring providers') self.providers = {} From 08af9aaab35b351d20cde17cc059cc8761242e7c Mon Sep 17 00:00:00 2001 From: Adam Smith Date: Sat, 18 Jan 2020 20:24:15 -0800 Subject: [PATCH 153/155] ContellixProvider: zone creation and records in one run --- octodns/provider/constellix.py | 13 +++++++++--- tests/test_octodns_provider_constellix.py | 26 +++++++++++------------ 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/octodns/provider/constellix.py b/octodns/provider/constellix.py index 0600f80..5ca89e1 100644 --- a/octodns/provider/constellix.py +++ b/octodns/provider/constellix.py @@ -88,7 +88,7 @@ class ConstellixClient(object): if self._domains is None: zones = [] - resp = self._request('GET', '/').json() + resp = self._request('GET', '').json() zones += resp self._domains = {'{}.'.format(z['name']): z['id'] for z in zones} @@ -96,11 +96,16 @@ class ConstellixClient(object): return self._domains def domain(self, name): - path = '/{}'.format(self.domains.get(name)) + zone_id = self.domains.get(name, False) + if not zone_id: + raise ConstellixClientNotFound() + path = '/{}'.format(zone_id) return self._request('GET', path).json() def domain_create(self, name): - self._request('POST', '/', data={'names': [name]}) + resp = self._request('POST', '/', data={'names': [name]}) + # Add newly created zone to domain cache + self._domains['{}.'.format(name)] = resp.json()[0]['id'] def _absolutize_value(self, value, zone_name): if value == '': @@ -112,6 +117,8 @@ class ConstellixClient(object): def records(self, zone_name): zone_id = self.domains.get(zone_name, False) + if not zone_id: + raise ConstellixClientNotFound() path = '/{}/records'.format(zone_id) resp = self._request('GET', path).json() diff --git a/tests/test_octodns_provider_constellix.py b/tests/test_octodns_provider_constellix.py index 2c5cf26..151d0d4 100644 --- a/tests/test_octodns_provider_constellix.py +++ b/tests/test_octodns_provider_constellix.py @@ -14,13 +14,11 @@ from six import text_type from unittest import TestCase from octodns.record import Record -from octodns.provider.constellix import ConstellixClientNotFound, \ +from octodns.provider.constellix import \ ConstellixProvider from octodns.provider.yaml import YamlProvider from octodns.zone import Zone -import json - class TestConstellixProvider(TestCase): expected = Zone('unit.tests.', []) @@ -102,7 +100,7 @@ class TestConstellixProvider(TestCase): with requests_mock() as mock: base = 'https://api.dns.constellix.com/v1/domains' with open('tests/fixtures/constellix-domains.json') as fh: - mock.get('{}{}'.format(base, '/'), text=fh.read()) + mock.get('{}{}'.format(base, ''), text=fh.read()) with open('tests/fixtures/constellix-records.json') as fh: mock.get('{}{}'.format(base, '/123123/records'), text=fh.read()) @@ -128,15 +126,15 @@ class TestConstellixProvider(TestCase): resp.json = Mock() provider._client._request = Mock(return_value=resp) - with open('tests/fixtures/constellix-domains.json') as fh: - domains = json.load(fh) - # non-existent domain, create everything resp.json.side_effect = [ - ConstellixClientNotFound, # no zone in populate - ConstellixClientNotFound, # no domain during apply - domains + [], # no domains returned during populate + [{ + 'id': 123123, + 'name': 'unit.tests' + }], # domain created in apply ] + plan = provider.plan(self.expected) # No root NS, no ignored, no excluded, no unsupported @@ -145,10 +143,10 @@ class TestConstellixProvider(TestCase): self.assertEquals(n, provider.apply(plan)) provider._client._request.assert_has_calls([ - # created the domain - call('POST', '/', data={'names': ['unit.tests']}), # get all domains to build the cache - call('GET', '/'), + call('GET', ''), + # created the domain + call('POST', '/', data={'names': ['unit.tests']}) ]) # These two checks are broken up so that ordering doesn't break things. # Python3 doesn't make the calls in a consistent order so different @@ -171,7 +169,7 @@ class TestConstellixProvider(TestCase): }), ]) - self.assertEquals(20, provider._client._request.call_count) + self.assertEquals(18, provider._client._request.call_count) provider._client._request.reset_mock() From 8021ce631f5e88eb9ef78d47c495177e31f43cd0 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 3 Feb 2020 08:57:30 +0000 Subject: [PATCH 154/155] Bump google-cloud-core from 1.2.0 to 1.3.0 Bumps [google-cloud-core](https://github.com/GoogleCloudPlatform/google-cloud-python) from 1.2.0 to 1.3.0. - [Release notes](https://github.com/GoogleCloudPlatform/google-cloud-python/releases) - [Changelog](https://github.com/googleapis/google-cloud-python/blob/master/CHANGELOG.md) - [Commits](https://github.com/GoogleCloudPlatform/google-cloud-python/compare/kms-1.2.0...core-1.3.0) Signed-off-by: dependabot-preview[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 42e3ca1..7107b8c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,7 @@ docutils==0.16 dyn==1.8.1 edgegrid-python==1.1.1 futures==3.2.0; python_version < '3.0' -google-cloud-core==1.2.0 +google-cloud-core==1.3.0 google-cloud-dns==0.31.0 ipaddress==1.0.23 jmespath==0.9.4 From e94ade76844b26e4d714b27abdd7b077ac0bfbdc Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 3 Feb 2020 08:58:07 +0000 Subject: [PATCH 155/155] Bump natsort from 6.2.0 to 6.2.1 Bumps [natsort](https://github.com/SethMMorton/natsort) from 6.2.0 to 6.2.1. - [Release notes](https://github.com/SethMMorton/natsort/releases) - [Changelog](https://github.com/SethMMorton/natsort/blob/6.2.1/CHANGELOG.md) - [Commits](https://github.com/SethMMorton/natsort/compare/6.2.0...6.2.1) Signed-off-by: dependabot-preview[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 42e3ca1..21b3a04 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,7 +13,7 @@ google-cloud-dns==0.31.0 ipaddress==1.0.23 jmespath==0.9.4 msrestazure==0.6.2 -natsort==6.2.0 +natsort==6.2.1 ns1-python==0.13.0 ovh==0.5.0 pycountry-convert==0.7.2