From 736d588e86942a215a01607ce3983c3ea1165395 Mon Sep 17 00:00:00 2001 From: Maikel Poot Date: Fri, 20 Sep 2019 10:18:51 +0200 Subject: [PATCH 01/10] Changed requirements to version 2.22.0 Fixes: ERROR: requests 2.20.0 has requirement urllib3<1.25,>=1.21.1, but you'll have urllib3 1.25.5 which is incompatible. --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 75dc1df..f67f2ee 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,7 +18,7 @@ natsort==5.5.0 nsone==0.9.100 ovh==0.4.8 python-dateutil==2.6.1 -requests==2.20.0 +requests==2.22.0 s3transfer==0.1.13 six==1.11.0 setuptools==38.5.2 From bb3f0c0b4a9c0705aad39cfd5aac3cbca8bfab47 Mon Sep 17 00:00:00 2001 From: Maikel Poot Date: Wed, 25 Sep 2019 07:01:06 +0200 Subject: [PATCH 02/10] Added TransIP provider and tests --- octodns/provider/transip.py | 328 +++++++++++++++++++++++++ tests/test_octodns_provider_transip.py | 234 ++++++++++++++++++ 2 files changed, 562 insertions(+) create mode 100644 octodns/provider/transip.py create mode 100644 tests/test_octodns_provider_transip.py diff --git a/octodns/provider/transip.py b/octodns/provider/transip.py new file mode 100644 index 0000000..2ce9180 --- /dev/null +++ b/octodns/provider/transip.py @@ -0,0 +1,328 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from suds import WebFault + +from collections import defaultdict +from .base import BaseProvider +from logging import getLogger +from ..record import Record +from transip.service.domain import DomainService +from transip.service.objects import DnsEntry + + +class TransipProvider(BaseProvider): + ''' + Transip DNS provider + + transip: + class: octodns.provider.transip.TransipProvider + # Your Transip account name (required) + account: yourname + # The api key (required) + key: | + \''' + -----BEGIN PRIVATE KEY----- + ... + -----END PRIVATE KEY----- + \''' + + ''' + SUPPORTS_GEO = False + SUPPORTS_DYNAMIC = False + SUPPORTS = set( + ('A', 'AAAA', 'CNAME', 'MX', 'SRV', 'SPF', 'TXT', 'SSHFP', 'CAA')) + # unsupported by OctoDNS: 'TLSA', 'CAA' + MIN_TTL = 120 + TIMEOUT = 15 + ROOT_RECORD = '@' + + def __init__(self, id, account, key, *args, **kwargs): + self.log = getLogger('TransipProvider[{}]'.format(id)) + self.log.debug('__init__: id=%s, account=%s, token=***', id, + account) + super(TransipProvider, self).__init__(id, *args, **kwargs) + + self._client = DomainService(account, key) + + self.account = account + self.key = key + + self._zones = None + self._zone_records = {} + + self._currentZone = {} + + def populate(self, zone, target=False, lenient=False): + + exists = False + self._currentZone = zone + self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, + target, lenient) + + before = len(zone.records) + try: + zoneInfo = self._client.get_info(zone.name[:-1]) + except WebFault as e: + if e.fault.faultcode == '102' and target == False: + self.log.warning( + 'populate: (%s) Zone %s not found in account ', + e.fault.faultcode, zone.name) + exists = False + return exists + elif e.fault.faultcode == '102' and target == True: + self.log.warning('populate: Transip can\'t create new zones') + raise Exception( + ('populate: ({}) Transip used ' + + 'as target for non-existing zone: {}').format( + e.fault.faultcode, zone.name)) + else: + self.log.error('populate: (%s) %s ', e.fault.faultcode, + e.fault.faultstring) + raise e + + self.log.debug('populate: found %s records for zone %s', + len(zoneInfo.dnsEntries), zone.name) + exists = True + if zoneInfo.dnsEntries: + values = defaultdict(lambda: defaultdict(list)) + for record in zoneInfo.dnsEntries: + name = zone.hostname_from_fqdn(record['name']) + if name == self.ROOT_RECORD: + name = '' + + if record['type'] in self.SUPPORTS: + values[name][record['type']].append(record) + + for name, types in values.items(): + for _type, records in types.items(): + data_for = getattr(self, '_data_for_{}'.format(_type)) + record = Record.new(zone, name, data_for(_type, records), + source=self, lenient=lenient) + zone.add_record(record, lenient=lenient) + self.log.info('populate: found %s records, exists = %s', + len(zone.records) - before, exists) + + self._currentZone = {} + return exists + + def _apply(self, plan): + desired = plan.desired + changes = plan.changes + self.log.debug('apply: zone=%s, changes=%d', desired.name, + len(changes)) + # for change in changes: + # class_name = change.__class__.__name__ + # getattr(self, '_apply_{}'.format(class_name))(change) + + self._currentZone = plan.desired + try: + self._client.get_info(plan.desired.name[:-1]) + except WebFault as e: + self.log.warning('_apply: %s ', e.message) + raise e + + _dns_entries = [] + for record in plan.desired.records: + if record._type in self.SUPPORTS: + entries_for = getattr(self, '_entries_for_{}'.format(record._type)) + + # Root records have '@' as name + name = record.name + if name == '': + name = self.ROOT_RECORD + + _dns_entries.extend(entries_for(name, record)) + + try: + self._client.set_dns_entries(plan.desired.name[:-1], _dns_entries) + except WebFault as e: + self.log.warning(('_apply: Set DNS returned ' + + 'one or more errors: {}').format( + e.fault.faultstring)) + raise Exception(200, e.fault.faultstring) + + self._currentZone = {} + + def _entries_for_multiple(self, name, record): + _entries = [] + + for value in record.values: + _entries.append(DnsEntry(name, record.ttl, record._type, value)) + + return _entries + + def _entries_for_single(self, name, record): + + return [DnsEntry(name, record.ttl, record._type, record.value)] + + _entries_for_A = _entries_for_multiple + _entries_for_AAAA = _entries_for_multiple + _entries_for_NS = _entries_for_multiple + _entries_for_SPF = _entries_for_multiple + _entries_for_CNAME = _entries_for_single + + def _entries_for_MX(self, name, record): + _entries = [] + + for value in record.values: + content = "{} {}".format(value.preference, value.exchange) + _entries.append(DnsEntry(name, record.ttl, record._type, content)) + + return _entries + + def _entries_for_SRV(self, name, record): + _entries = [] + + for value in record.values: + content = "{} {} {} {}".format(value.priority, value.weight, + value.port, value.target) + _entries.append(DnsEntry(name, record.ttl, record._type, content)) + + return _entries + + def _entries_for_SSHFP(self, name, record): + _entries = [] + + for value in record.values: + content = "{} {} {}".format(value.algorithm, value.fingerprint_type, + value.fingerprint) + _entries.append(DnsEntry(name, record.ttl, record._type, content)) + + return _entries + + def _entries_for_CAA(self, name, record): + _entries = [] + + for value in record.values: + content = "{} {} {}".format(value.flags, value.tag, + value.value) + _entries.append(DnsEntry(name, record.ttl, record._type, content)) + + return _entries + + def _entries_for_TXT(self, name, record): + _entries = [] + + for value in record.values: + value = value.replace('\\;', ';') + _entries.append(DnsEntry(name, record.ttl, record._type, value)) + + return _entries + + def _parse_to_fqdn(self, value): + + if (value[-1] != '.'): + self.log.debug('parseToFQDN: changed %s to %s', value, + '{}.{}'.format(value, self._currentZone.name)) + value = '{}.{}'.format(value, self._currentZone.name) + + return value + + def _get_lowest_ttl(self, records): + _ttl = 100000 + for record in records: + _ttl = min(_ttl, record['expire']) + return _ttl + + def _data_for_multiple(self, _type, records): + + _values = [] + for record in records: + _values.append(record['content']) + + return { + 'ttl': self._get_lowest_ttl(records), + 'type': _type, + 'values': _values + } + + _data_for_A = _data_for_multiple + _data_for_AAAA = _data_for_multiple + _data_for_NS = _data_for_multiple + _data_for_SPF = _data_for_multiple + + def _data_for_CNAME(self, _type, records): + return { + 'ttl': records[0]['expire'], + 'type': _type, + 'value': self._parse_to_fqdn(records[0]['content']) + } + + def _data_for_MX(self, _type, records): + values = [] + for record in records: + preference, exchange = record['content'].split(" ", 1) + values.append({ + 'preference': preference, + 'exchange': self._parse_to_fqdn(exchange) + }) + return { + 'ttl': self._get_lowest_ttl(records), + 'type': _type, + 'values': values + } + + def _data_for_SRV(self, _type, records): + values = [] + for record in records: + priority, weight, port, target = record['content'].split(' ', 3) + values.append({ + 'port': port, + 'priority': priority, + 'target': self._parse_to_fqdn(target), + 'weight': weight + }) + + return { + 'type': _type, + 'ttl': self._get_lowest_ttl(records), + 'values': values + } + + def _data_for_SSHFP(self, _type, records): + values = [] + for record in records: + algorithm, fp_type, fingerprint = record['content'].split(' ', 2) + values.append({ + 'algorithm': algorithm, + 'fingerprint': fingerprint.lower(), + 'fingerprint_type': fp_type + }) + + return { + 'type': _type, + 'ttl': self._get_lowest_ttl(records), + 'values': values + } + + def _data_for_CAA(self, _type, records): + values = [] + for record in records: + flags, tag, value = record['content'].split(' ', 2) + values.append({ + 'flags': flags, + 'tag': tag, + 'value': value + }) + + return { + 'type': _type, + 'ttl': self._get_lowest_ttl(records), + 'values': values + } + + def _data_for_TXT(self, _type, records): + values = [] + for record in records: + values.append(record['content'].replace(';', '\\;')) + + return { + 'type': _type, + 'ttl': self._get_lowest_ttl(records), + 'values': values + } diff --git a/tests/test_octodns_provider_transip.py b/tests/test_octodns_provider_transip.py new file mode 100644 index 0000000..dbf7eab --- /dev/null +++ b/tests/test_octodns_provider_transip.py @@ -0,0 +1,234 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +# from mock import Mock, call +from os.path import dirname, join + +from suds import WebFault + +from requests_mock import ANY, mock as requests_mock +from unittest import TestCase + +from octodns.record import Record +from octodns.provider.transip import TransipProvider +from octodns.provider.yaml import YamlProvider +from octodns.zone import Zone +from transip.service.domain import DomainService +from transip.service.objects import DnsEntry + + + + +class MockDomainService(DomainService): + + def __init__(self, *args, **kwargs): + super(MockDomainService, self).__init__('MockDomainService', *args, **kwargs) + self.mockupEntries = [] + + def mockup(self, records): + + provider = TransipProvider('', '', ''); + + _dns_entries = [] + for record in records: + if record._type in provider.SUPPORTS: + entries_for = getattr(provider, '_entries_for_{}'.format(record._type)) + + # Root records have '@' as name + name = record.name + if name == '': + name = provider.ROOT_RECORD + + _dns_entries.extend(entries_for(name, record)) + + _dns_entries.append(DnsEntry('@', '3600', 'NS', 'ns01.transip.nl.')) + + + self.mockupEntries = _dns_entries + + # Skips authentication layer and returns the entries loaded by "Mockup" + def get_info(self, domain_name): + + if str(domain_name) == str('notfound.unit.tests'): + self.raiseZoneNotFound() + + result = lambda: None + setattr(result, "dnsEntries", self.mockupEntries) + return result + + def set_dns_entries(self, domain_name, dns_entries): + if str(domain_name) == str('failsetdns.unit.tests'): + self.raiseSaveError() + + return True + + def raiseZoneNotFound(self): + fault = lambda: None + setattr(fault, "faultstring", '102 is zone not found') + setattr(fault, "faultcode", str('102')) + document = {} + raise WebFault(fault, document) + + def raiseInvalidAuth(self): + fault = lambda: None + setattr(fault, "faultstring", '200 is invalid auth') + setattr(fault, "faultcode", str('200')) + document = {} + raise WebFault(fault, document) + + def raiseSaveError(self): + fault = lambda: None + setattr(fault, "faultstring", '202 error while saving') + setattr(fault, "faultcode", str('202')) + document = {} + raise WebFault(fault, document) + + + +class TestTransipProvider(TestCase): + bogus_key = str("""-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEA0U5HGCkLrz423IyUf3u4cKN2WrNz1x5KNr6PvH2M/zxas+zB +elbxkdT3AQ+wmfcIvOuTmFRTHv35q2um1aBrPxVw+2s+lWo28VwIRttwIB1vIeWu +lSBnkEZQRLyPI2tH0i5QoMX4CVPf9rvij3Uslimi84jdzDfPFIh6jZ6C8nLipOTG +0IMhge1ofVfB0oSy5H+7PYS2858QLAf5ruYbzbAxZRivS402wGmQ0d0Lc1KxraAj +kiMM5yj/CkH/Vm2w9I6+tLFeASE4ub5HCP5G/ig4dbYtqZMQMpqyAbGxd5SOVtyn +UHagAJUxf8DT3I8PyjEHjxdOPUsxNyRtepO/7QIDAQABAoIBAQC7fiZ7gxE/ezjD +2n6PsHFpHVTBLS2gzzZl0dCKZeFvJk6ODJDImaeuHhrh7X8ifMNsEI9XjnojMhl8 +MGPzy88mZHugDNK0H8B19x5G8v1/Fz7dG5WHas660/HFkS+b59cfdXOugYiOOn9O +08HBBpLZNRUOmVUuQfQTjapSwGLG8PocgpyRD4zx0LnldnJcqYCxwCdev+AAsPnq +ibNtOd/MYD37w9MEGcaxLE8wGgkv8yd97aTjkgE+tp4zsM4QE4Rag133tsLLNznT +4Qr/of15M3NW/DXq/fgctyRcJjZpU66eCXLCz2iRTnLyyxxDC2nwlxKbubV+lcS0 +S4hbfd/BAoGBAO8jXxEaiybR0aIhhSR5esEc3ymo8R8vBN3ZMJ+vr5jEPXr/ZuFj +/R4cZ2XV3VoQJG0pvIOYVPZ5DpJM7W+zSXtJ/7bLXy4Bnmh/rc+YYgC+AXQoLSil +iD2OuB2xAzRAK71DVSO0kv8gEEXCersPT2i6+vC2GIlJvLcYbOdRKWGxAoGBAOAQ +aJbRLtKujH+kMdoMI7tRlL8XwI+SZf0FcieEu//nFyerTePUhVgEtcE+7eQ7hyhG +fIXUFx/wALySoqFzdJDLc8U8pTLhbUaoLOTjkwnCTKQVprhnISqQqqh/0U5u47IE +RWzWKN6OHb0CezNTq80Dr6HoxmPCnJHBHn5LinT9AoGAQSpvZpbIIqz8pmTiBl2A +QQ2gFpcuFeRXPClKYcmbXVLkuhbNL1BzEniFCLAt4LQTaRf9ghLJ3FyCxwVlkpHV +zV4N6/8hkcTpKOraL38D/dXJSaEFJVVuee/hZl3tVJjEEpA9rDwx7ooLRSdJEJ6M +ciq55UyKBSdt4KssSiDI2RECgYBL3mJ7xuLy5bWfNsrGiVvD/rC+L928/5ZXIXPw +26oI0Yfun7ulDH4GOroMcDF/GYT/Zzac3h7iapLlR0WYI47xxGI0A//wBZLJ3QIu +krxkDo2C9e3Y/NqnHgsbOQR3aWbiDT4wxydZjIeXS3LKA2fl6Hyc90PN3cTEOb8I +hq2gRQKBgEt0SxhhtyB93SjgTzmUZZ7PiEf0YJatfM6cevmjWHexrZH+x31PB72s +fH2BQyTKKzoCLB1k/6HRaMnZdrWyWSZ7JKz3AHJ8+58d0Hr8LTrzDM1L6BbjeDct +N4OiVz1I3rbZGYa396lpxO6ku8yCglisL1yrSP6DdEUp66ntpKVd +-----END RSA PRIVATE KEY-----""") + + + def make_expected(self): + expected = Zone('unit.tests.', []) + source = YamlProvider('test', join(dirname(__file__), 'config')) + source.populate(expected) + return expected + + def test_populate(self): + + _expected = self.make_expected() + + with self.assertRaises(WebFault) as ctx: + provider = TransipProvider('test', 'unittest', self.bogus_key) + zone = Zone('unit.tests.', []) + provider.populate(zone, True) + + self.assertEquals(str('WebFault'), + str(ctx.exception.__class__.__name__)) + + self.assertEquals(str('200'), ctx.exception.fault.faultcode) + + with self.assertRaises(Exception) as ctx: + provider = TransipProvider('test', 'unittest', self.bogus_key) + provider._client = MockDomainService('unittest', self.bogus_key) + zone = Zone('notfound.unit.tests.', []) + provider.populate(zone, True) + + self.assertEquals(str('Exception'), + str(ctx.exception.__class__.__name__)) + + self.assertEquals('populate: (102) Transip used as target for non-existing zone: notfound.unit.tests.', ctx.exception.message) + + provider = TransipProvider('test', 'unittest', self.bogus_key) + provider._client = MockDomainService('unittest', self.bogus_key) + zone = Zone('notfound.unit.tests.', []) + provider.populate(zone, False) + + provider = TransipProvider('test', 'unittest', self.bogus_key) + provider._client = MockDomainService('unittest', self.bogus_key) + provider._client.mockup(_expected.records) + zone = Zone('unit.tests.', []) + provider.populate(zone, False) + + provider._currentZone = zone + self.assertEquals("www.unit.tests.", provider._parse_to_fqdn("www")) + + + provider = TransipProvider('test', 'unittest', self.bogus_key) + provider._client = MockDomainService('unittest', self.bogus_key) + zone = Zone('unit.tests.', []) + exists = provider.populate(zone, True) + self.assertTrue(exists, 'populate should return true') + + + return + + def test_plan(self): + + _expected = self.make_expected() + + print(_expected.name) + + provider = TransipProvider('test', 'unittest', self.bogus_key) + provider._client = MockDomainService('unittest', self.bogus_key) + plan = provider.plan(_expected) + + self.assertEqual(12, plan.change_counts['Create']) + self.assertEqual(0, plan.change_counts['Update']) + self.assertEqual(0, plan.change_counts['Delete']) + + return + + def test_apply(self): + + _expected = self.make_expected() + + provider = TransipProvider('test', 'unittest', self.bogus_key) + provider._client = MockDomainService('unittest', self.bogus_key) + plan = provider.plan(_expected) + #self.assertEqual(11, plan.changes) + changes = provider.apply(plan) + + + + with self.assertRaises(Exception) as ctx: + provider = TransipProvider('test', 'unittest', self.bogus_key) + provider._client = MockDomainService('unittest', self.bogus_key) + plan = provider.plan(_expected) + plan.desired.name = 'notfound.unit.tests.' + changes = provider.apply(plan) + # self.assertEqual(11, changes) + + self.assertEquals(str('WebFault'), + str(ctx.exception.__class__.__name__)) + + _expected = self.make_expected() + + with self.assertRaises(Exception) as ctx: + provider = TransipProvider('test', 'unittest', self.bogus_key) + provider._client = MockDomainService('unittest', self.bogus_key) + plan = provider.plan(_expected) + plan.desired.name = 'failsetdns.unit.tests.' + changes = provider.apply(plan) + # self.assertEqual(11, changes) + + + #provider = TransipProvider('test', 'unittest', self.bogus_key) + + #plan = provider.plan(_expected) + +# changes = provider.apply(plan) +# self.assertEquals(29, changes) + + From 30c8c4d313d150c2c81324f0b0a7de06f634bdc0 Mon Sep 17 00:00:00 2001 From: Maikel Poot Date: Wed, 25 Sep 2019 08:18:17 +0200 Subject: [PATCH 03/10] Add transip requirement and add provider to readme --- README.md | 1 + requirements.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/README.md b/README.md index be0a4ab..7124b20 100644 --- a/README.md +++ b/README.md @@ -188,6 +188,7 @@ The above command pulled the existing data out of Route53 and placed the results | [Rackspace](/octodns/provider/rackspace.py) | | A, AAAA, ALIAS, CNAME, MX, NS, PTR, SPF, TXT | No | | | [Route53](/octodns/provider/route53.py) | boto3 | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | Both | CNAME health checks don't support a Host header | | [Selectel](/octodns/provider/selectel.py) | | A, AAAA, CNAME, MX, NS, SPF, SRV, TXT | No | | +| [Transip](/octodns/provider/transip.py) | transip | A, AAAA, CNAME, MX, SRV, SPF, TXT, SSHFP, CAA | No | | | [AxfrSource](/octodns/source/axfr.py) | | A, AAAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | read-only | | [ZoneFileSource](/octodns/source/axfr.py) | | A, AAAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | read-only | | [TinyDnsFileSource](/octodns/source/tinydns.py) | | A, CNAME, MX, NS, PTR | No | read-only | diff --git a/requirements.txt b/requirements.txt index f67f2ee..bb373d0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,3 +22,4 @@ requests==2.22.0 s3transfer==0.1.13 six==1.11.0 setuptools==38.5.2 +transip==2.0.0 From 7056d299072f3c03c446ef8b5435f4339b1bb4b4 Mon Sep 17 00:00:00 2001 From: Maikel Poot Date: Wed, 25 Sep 2019 08:18:59 +0200 Subject: [PATCH 04/10] fixes lint warning. --- octodns/provider/transip.py | 20 +++-- tests/test_octodns_provider_transip.py | 110 +++++++++++++++---------- 2 files changed, 79 insertions(+), 51 deletions(-) diff --git a/octodns/provider/transip.py b/octodns/provider/transip.py index 2ce9180..8014310 100644 --- a/octodns/provider/transip.py +++ b/octodns/provider/transip.py @@ -36,7 +36,7 @@ class TransipProvider(BaseProvider): SUPPORTS_DYNAMIC = False SUPPORTS = set( ('A', 'AAAA', 'CNAME', 'MX', 'SRV', 'SPF', 'TXT', 'SSHFP', 'CAA')) - # unsupported by OctoDNS: 'TLSA', 'CAA' + # unsupported by OctoDNS: 'TLSA' MIN_TTL = 120 TIMEOUT = 15 ROOT_RECORD = '@' @@ -68,17 +68,17 @@ class TransipProvider(BaseProvider): try: zoneInfo = self._client.get_info(zone.name[:-1]) except WebFault as e: - if e.fault.faultcode == '102' and target == False: + if e.fault.faultcode == '102' and target is False: self.log.warning( 'populate: (%s) Zone %s not found in account ', e.fault.faultcode, zone.name) exists = False return exists - elif e.fault.faultcode == '102' and target == True: + elif e.fault.faultcode == '102' and target is True: self.log.warning('populate: Transip can\'t create new zones') raise Exception( ('populate: ({}) Transip used ' + - 'as target for non-existing zone: {}').format( + 'as target for non-existing zone: {}').format( e.fault.faultcode, zone.name)) else: self.log.error('populate: (%s) %s ', e.fault.faultcode, @@ -129,7 +129,8 @@ class TransipProvider(BaseProvider): _dns_entries = [] for record in plan.desired.records: if record._type in self.SUPPORTS: - entries_for = getattr(self, '_entries_for_{}'.format(record._type)) + entries_for = getattr(self, + '_entries_for_{}'.format(record._type)) # Root records have '@' as name name = record.name @@ -143,7 +144,7 @@ class TransipProvider(BaseProvider): except WebFault as e: self.log.warning(('_apply: Set DNS returned ' + 'one or more errors: {}').format( - e.fault.faultstring)) + e.fault.faultstring)) raise Exception(200, e.fault.faultstring) self._currentZone = {} @@ -189,8 +190,9 @@ class TransipProvider(BaseProvider): _entries = [] for value in record.values: - content = "{} {} {}".format(value.algorithm, value.fingerprint_type, - value.fingerprint) + content = "{} {} {}".format(value.algorithm, + value.fingerprint_type, + value.fingerprint) _entries.append(DnsEntry(name, record.ttl, record._type, content)) return _entries @@ -200,7 +202,7 @@ class TransipProvider(BaseProvider): for value in record.values: content = "{} {} {}".format(value.flags, value.tag, - value.value) + value.value) _entries.append(DnsEntry(name, record.ttl, record._type, content)) return _entries diff --git a/tests/test_octodns_provider_transip.py b/tests/test_octodns_provider_transip.py index dbf7eab..811c1e2 100644 --- a/tests/test_octodns_provider_transip.py +++ b/tests/test_octodns_provider_transip.py @@ -10,10 +10,8 @@ from os.path import dirname, join from suds import WebFault -from requests_mock import ANY, mock as requests_mock from unittest import TestCase -from octodns.record import Record from octodns.provider.transip import TransipProvider from octodns.provider.yaml import YamlProvider from octodns.zone import Zone @@ -21,22 +19,35 @@ from transip.service.domain import DomainService from transip.service.objects import DnsEntry +class MockFault(object): + faultstring = "" + faultcode = "" + + def __init__(self, code, string, *args, **kwargs): + self.faultstring = string + self.faultcode = code + + +class MockResponse(object): + dnsEntries = [] class MockDomainService(DomainService): def __init__(self, *args, **kwargs): - super(MockDomainService, self).__init__('MockDomainService', *args, **kwargs) + super(MockDomainService, self).__init__('MockDomainService', *args, + **kwargs) self.mockupEntries = [] def mockup(self, records): - provider = TransipProvider('', '', ''); + provider = TransipProvider('', '', '') _dns_entries = [] for record in records: if record._type in provider.SUPPORTS: - entries_for = getattr(provider, '_entries_for_{}'.format(record._type)) + entries_for = getattr(provider, + '_entries_for_{}'.format(record._type)) # Root records have '@' as name name = record.name @@ -45,50 +56,48 @@ class MockDomainService(DomainService): _dns_entries.extend(entries_for(name, record)) - _dns_entries.append(DnsEntry('@', '3600', 'NS', 'ns01.transip.nl.')) - + # NS is not supported as a DNS Entry, + # so it should cover the if statement + _dns_entries.append( + DnsEntry('@', '3600', 'NS', 'ns01.transip.nl.')) self.mockupEntries = _dns_entries # Skips authentication layer and returns the entries loaded by "Mockup" def get_info(self, domain_name): + # Special 'domain' to trigger error if str(domain_name) == str('notfound.unit.tests'): self.raiseZoneNotFound() - result = lambda: None - setattr(result, "dnsEntries", self.mockupEntries) + result = MockResponse() + result.dnsEntries = self.mockupEntries return result def set_dns_entries(self, domain_name, dns_entries): + + # Special 'domain' to trigger error if str(domain_name) == str('failsetdns.unit.tests'): self.raiseSaveError() return True def raiseZoneNotFound(self): - fault = lambda: None - setattr(fault, "faultstring", '102 is zone not found') - setattr(fault, "faultcode", str('102')) + fault = MockFault(str('102'), '102 is zone not found') document = {} raise WebFault(fault, document) def raiseInvalidAuth(self): - fault = lambda: None - setattr(fault, "faultstring", '200 is invalid auth') - setattr(fault, "faultcode", str('200')) + fault = MockFault(str('200'), '200 is invalid auth') document = {} raise WebFault(fault, document) def raiseSaveError(self): - fault = lambda: None - setattr(fault, "faultstring", '202 error while saving') - setattr(fault, "faultcode", str('202')) + fault = MockFault(str('200'), '202 random error') document = {} raise WebFault(fault, document) - class TestTransipProvider(TestCase): bogus_key = str("""-----BEGIN RSA PRIVATE KEY----- MIIEowIBAAKCAQEA0U5HGCkLrz423IyUf3u4cKN2WrNz1x5KNr6PvH2M/zxas+zB @@ -118,7 +127,6 @@ fH2BQyTKKzoCLB1k/6HRaMnZdrWyWSZ7JKz3AHJ8+58d0Hr8LTrzDM1L6BbjeDct N4OiVz1I3rbZGYa396lpxO6ku8yCglisL1yrSP6DdEUp66ntpKVd -----END RSA PRIVATE KEY-----""") - def make_expected(self): expected = Zone('unit.tests.', []) source = YamlProvider('test', join(dirname(__file__), 'config')) @@ -126,9 +134,10 @@ N4OiVz1I3rbZGYa396lpxO6ku8yCglisL1yrSP6DdEUp66ntpKVd return expected def test_populate(self): - _expected = self.make_expected() + # Unhappy Plan - Not authenticated + # Live test against API, will fail in an unauthorized error with self.assertRaises(WebFault) as ctx: provider = TransipProvider('test', 'unittest', self.bogus_key) zone = Zone('unit.tests.', []) @@ -139,6 +148,9 @@ N4OiVz1I3rbZGYa396lpxO6ku8yCglisL1yrSP6DdEUp66ntpKVd self.assertEquals(str('200'), ctx.exception.fault.faultcode) + # Unhappy Plan - Zone does not exists + # Will trigger an exception if provider is used as a target for a + # non-existing zone with self.assertRaises(Exception) as ctx: provider = TransipProvider('test', 'unittest', self.bogus_key) provider._client = MockDomainService('unittest', self.bogus_key) @@ -148,38 +160,48 @@ N4OiVz1I3rbZGYa396lpxO6ku8yCglisL1yrSP6DdEUp66ntpKVd self.assertEquals(str('Exception'), str(ctx.exception.__class__.__name__)) - self.assertEquals('populate: (102) Transip used as target for non-existing zone: notfound.unit.tests.', ctx.exception.message) + self.assertEquals( + 'populate: (102) Transip used as target' + + ' for non-existing zone: notfound.unit.tests.', + ctx.exception.message) + # Happy Plan - Zone does not exists + # Won't trigger an exception if provider is NOT used as a target for a + # non-existing zone. provider = TransipProvider('test', 'unittest', self.bogus_key) provider._client = MockDomainService('unittest', self.bogus_key) zone = Zone('notfound.unit.tests.', []) provider.populate(zone, False) + # Happy Plan - Populate with mockup records provider = TransipProvider('test', 'unittest', self.bogus_key) provider._client = MockDomainService('unittest', self.bogus_key) provider._client.mockup(_expected.records) zone = Zone('unit.tests.', []) provider.populate(zone, False) + # Transip allows relative values for types like cname, mx. + # Test is these are correctly appended with the domain provider._currentZone = zone self.assertEquals("www.unit.tests.", provider._parse_to_fqdn("www")) + self.assertEquals("www.unit.tests.", + provider._parse_to_fqdn("www.unit.tests.")) + self.assertEquals("www.sub.sub.sub.unit.tests.", + provider._parse_to_fqdn("www.sub.sub.sub")) - + # Happy Plan - Even if the zone has no records the zone should exist provider = TransipProvider('test', 'unittest', self.bogus_key) provider._client = MockDomainService('unittest', self.bogus_key) zone = Zone('unit.tests.', []) exists = provider.populate(zone, True) self.assertTrue(exists, 'populate should return true') - return def test_plan(self): - _expected = self.make_expected() - print(_expected.name) - + # Test Happy plan, only create provider = TransipProvider('test', 'unittest', self.bogus_key) provider._client = MockDomainService('unittest', self.bogus_key) plan = provider.plan(_expected) @@ -191,29 +213,38 @@ N4OiVz1I3rbZGYa396lpxO6ku8yCglisL1yrSP6DdEUp66ntpKVd return def test_apply(self): - _expected = self.make_expected() + # Test happy flow. Create all supoorted records provider = TransipProvider('test', 'unittest', self.bogus_key) provider._client = MockDomainService('unittest', self.bogus_key) plan = provider.plan(_expected) - #self.assertEqual(11, plan.changes) + self.assertEqual(12, len(plan.changes)) changes = provider.apply(plan) + self.assertEqual(changes, len(plan.changes)) - - + # Test unhappy flow. Trigger 'not found error' in apply stage + # This should normally not happen as populate will capture it first + # but just in case. + changes = [] # reset changes with self.assertRaises(Exception) as ctx: provider = TransipProvider('test', 'unittest', self.bogus_key) provider._client = MockDomainService('unittest', self.bogus_key) plan = provider.plan(_expected) plan.desired.name = 'notfound.unit.tests.' changes = provider.apply(plan) - # self.assertEqual(11, changes) + + # Changes should not be set due to an Exception + self.assertEqual([], changes) self.assertEquals(str('WebFault'), str(ctx.exception.__class__.__name__)) - _expected = self.make_expected() + self.assertEquals(str('102'), ctx.exception.fault.faultcode) + + # Test unhappy flow. Trigger a unrecoverable error while saving + _expected = self.make_expected() # reset expected + changes = [] # reset changes with self.assertRaises(Exception) as ctx: provider = TransipProvider('test', 'unittest', self.bogus_key) @@ -221,14 +252,9 @@ N4OiVz1I3rbZGYa396lpxO6ku8yCglisL1yrSP6DdEUp66ntpKVd plan = provider.plan(_expected) plan.desired.name = 'failsetdns.unit.tests.' changes = provider.apply(plan) - # self.assertEqual(11, changes) - - - #provider = TransipProvider('test', 'unittest', self.bogus_key) - - #plan = provider.plan(_expected) - -# changes = provider.apply(plan) -# self.assertEquals(29, changes) + # Changes should not be set due to an Exception + self.assertEqual([], changes) + self.assertEquals(str('Exception'), + str(ctx.exception.__class__.__name__)) From 59e44b865cfa5e7d3a1f40d34f41d5b2dd285aac Mon Sep 17 00:00:00 2001 From: Maikel Poot Date: Wed, 25 Sep 2019 11:24:13 +0200 Subject: [PATCH 05/10] Added detection for edge case that could happen with existing records where the value is '@' TransIP allows '@' as value to alias the root record. '@' was on populate appended with the zone, which trigger an unneeded update. '@' => '@.example.com.' -> 'example.com' This fix will stop the unneeded update --- octodns/provider/transip.py | 7 ++++++- tests/test_octodns_provider_transip.py | 2 ++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/octodns/provider/transip.py b/octodns/provider/transip.py index 8014310..adde617 100644 --- a/octodns/provider/transip.py +++ b/octodns/provider/transip.py @@ -218,7 +218,12 @@ class TransipProvider(BaseProvider): def _parse_to_fqdn(self, value): - if (value[-1] != '.'): + # TransIP allows '@' as value to alias the root record. + # this provider won't set an '@' value, but can be an existing record + if value == self.ROOT_RECORD: + value = self._currentZone.name + + if value[-1] != '.': self.log.debug('parseToFQDN: changed %s to %s', value, '{}.{}'.format(value, self._currentZone.name)) value = '{}.{}'.format(value, self._currentZone.name) diff --git a/tests/test_octodns_provider_transip.py b/tests/test_octodns_provider_transip.py index 811c1e2..d6bcaa7 100644 --- a/tests/test_octodns_provider_transip.py +++ b/tests/test_octodns_provider_transip.py @@ -188,6 +188,8 @@ N4OiVz1I3rbZGYa396lpxO6ku8yCglisL1yrSP6DdEUp66ntpKVd provider._parse_to_fqdn("www.unit.tests.")) self.assertEquals("www.sub.sub.sub.unit.tests.", provider._parse_to_fqdn("www.sub.sub.sub")) + self.assertEquals("unit.tests.", + provider._parse_to_fqdn("@")) # Happy Plan - Even if the zone has no records the zone should exist provider = TransipProvider('test', 'unittest', self.bogus_key) From cebc629a06295a9cae3c5821b448e11737eddbbe Mon Sep 17 00:00:00 2001 From: Maikel Poot Date: Wed, 25 Sep 2019 14:33:49 +0200 Subject: [PATCH 06/10] Enforce values as basic string to fix yaml export error Fixes an exception in combination with the yamlProvider as a target The unmodified value object isn't represented as string while building the yaml output The Exception: yaml.representer.RepresenterError: ('cannot represent an object', 1.1.1.1) yaml/representer.py@249, represent_undefined() --- octodns/provider/transip.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/octodns/provider/transip.py b/octodns/provider/transip.py index adde617..050398a 100644 --- a/octodns/provider/transip.py +++ b/octodns/provider/transip.py @@ -218,6 +218,9 @@ class TransipProvider(BaseProvider): def _parse_to_fqdn(self, value): + # Enforce switch from suds.sax.text.Text to string + value = ''+value + # TransIP allows '@' as value to alias the root record. # this provider won't set an '@' value, but can be an existing record if value == self.ROOT_RECORD: @@ -240,7 +243,8 @@ class TransipProvider(BaseProvider): _values = [] for record in records: - _values.append(record['content']) + # Enforce switch from suds.sax.text.Text to string + _values.append(''+record['content']) return { 'ttl': self._get_lowest_ttl(records), From 9cab94a83a4621334880059b8007fe7bdd2bd8ba Mon Sep 17 00:00:00 2001 From: Maikel Poot Date: Wed, 25 Sep 2019 14:45:41 +0200 Subject: [PATCH 07/10] Some codestyle review changes. --- octodns/provider/transip.py | 37 +++++++++++++++---------------------- 1 file changed, 15 insertions(+), 22 deletions(-) diff --git a/octodns/provider/transip.py b/octodns/provider/transip.py index 050398a..aa5b5b3 100644 --- a/octodns/provider/transip.py +++ b/octodns/provider/transip.py @@ -52,9 +52,6 @@ class TransipProvider(BaseProvider): self.account = account self.key = key - self._zones = None - self._zone_records = {} - self._currentZone = {} def populate(self, zone, target=False, lenient=False): @@ -72,7 +69,6 @@ class TransipProvider(BaseProvider): self.log.warning( 'populate: (%s) Zone %s not found in account ', e.fault.faultcode, zone.name) - exists = False return exists elif e.fault.faultcode == '102' and target is True: self.log.warning('populate: Transip can\'t create new zones') @@ -115,9 +111,6 @@ class TransipProvider(BaseProvider): changes = plan.changes self.log.debug('apply: zone=%s, changes=%d', desired.name, len(changes)) - # for change in changes: - # class_name = change.__class__.__name__ - # getattr(self, '_apply_{}'.format(class_name))(change) self._currentZone = plan.desired try: @@ -265,24 +258,24 @@ class TransipProvider(BaseProvider): } def _data_for_MX(self, _type, records): - values = [] + _values = [] for record in records: preference, exchange = record['content'].split(" ", 1) - values.append({ + _values.append({ 'preference': preference, 'exchange': self._parse_to_fqdn(exchange) }) return { 'ttl': self._get_lowest_ttl(records), 'type': _type, - 'values': values + 'values': _values } def _data_for_SRV(self, _type, records): - values = [] + _values = [] for record in records: priority, weight, port, target = record['content'].split(' ', 3) - values.append({ + _values.append({ 'port': port, 'priority': priority, 'target': self._parse_to_fqdn(target), @@ -292,14 +285,14 @@ class TransipProvider(BaseProvider): return { 'type': _type, 'ttl': self._get_lowest_ttl(records), - 'values': values + 'values': _values } def _data_for_SSHFP(self, _type, records): - values = [] + _values = [] for record in records: algorithm, fp_type, fingerprint = record['content'].split(' ', 2) - values.append({ + _values.append({ 'algorithm': algorithm, 'fingerprint': fingerprint.lower(), 'fingerprint_type': fp_type @@ -308,14 +301,14 @@ class TransipProvider(BaseProvider): return { 'type': _type, 'ttl': self._get_lowest_ttl(records), - 'values': values + 'values': _values } def _data_for_CAA(self, _type, records): - values = [] + _values = [] for record in records: flags, tag, value = record['content'].split(' ', 2) - values.append({ + _values.append({ 'flags': flags, 'tag': tag, 'value': value @@ -324,16 +317,16 @@ class TransipProvider(BaseProvider): return { 'type': _type, 'ttl': self._get_lowest_ttl(records), - 'values': values + 'values': _values } def _data_for_TXT(self, _type, records): - values = [] + _values = [] for record in records: - values.append(record['content'].replace(';', '\\;')) + _values.append(record['content'].replace(';', '\\;')) return { 'type': _type, 'ttl': self._get_lowest_ttl(records), - 'values': values + 'values': _values } From 71f215932d94b323cddba6ed6e0b8f35ba3b9a4d Mon Sep 17 00:00:00 2001 From: Maikel Poot Date: Wed, 25 Sep 2019 14:51:53 +0200 Subject: [PATCH 08/10] whitespaces around operators to make /script/lint happy again --- octodns/provider/transip.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/octodns/provider/transip.py b/octodns/provider/transip.py index aa5b5b3..64692ee 100644 --- a/octodns/provider/transip.py +++ b/octodns/provider/transip.py @@ -212,7 +212,7 @@ class TransipProvider(BaseProvider): def _parse_to_fqdn(self, value): # Enforce switch from suds.sax.text.Text to string - value = ''+value + value = '' + value # TransIP allows '@' as value to alias the root record. # this provider won't set an '@' value, but can be an existing record @@ -237,7 +237,7 @@ class TransipProvider(BaseProvider): _values = [] for record in records: # Enforce switch from suds.sax.text.Text to string - _values.append(''+record['content']) + _values.append('' + record['content']) return { 'ttl': self._get_lowest_ttl(records), From a035ee8c84955d2dcdf40c0ea15f1901224ae000 Mon Sep 17 00:00:00 2001 From: Maikel Poot Date: Thu, 26 Sep 2019 14:49:14 +0200 Subject: [PATCH 09/10] Give the option to use a private_key_file. Transip sdk also supports a private_key_file, so forwarding that option to the provider. Could be handy in combination with k8s secrets. --- octodns/provider/transip.py | 14 +++++++++++--- tests/test_octodns_provider_transip.py | 13 +++++++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/octodns/provider/transip.py b/octodns/provider/transip.py index 64692ee..92d607d 100644 --- a/octodns/provider/transip.py +++ b/octodns/provider/transip.py @@ -23,13 +23,16 @@ class TransipProvider(BaseProvider): class: octodns.provider.transip.TransipProvider # Your Transip account name (required) account: yourname - # The api key (required) + # Path to a private key file (required if key is not used) + key_file: /path/to/file + # The api key as string (required if key_file is not used) key: | \''' -----BEGIN PRIVATE KEY----- ... -----END PRIVATE KEY----- \''' + # if both `key_file` and `key` are presented `key_file` is used ''' SUPPORTS_GEO = False @@ -41,13 +44,18 @@ class TransipProvider(BaseProvider): TIMEOUT = 15 ROOT_RECORD = '@' - def __init__(self, id, account, key, *args, **kwargs): + def __init__(self, id, account, key=None, key_file=None, *args, **kwargs): self.log = getLogger('TransipProvider[{}]'.format(id)) self.log.debug('__init__: id=%s, account=%s, token=***', id, account) super(TransipProvider, self).__init__(id, *args, **kwargs) - self._client = DomainService(account, key) + if key_file is not None: + self._client = DomainService(account, private_key_file=key_file) + elif key is not None: + self._client = DomainService(account, private_key=key) + else: + raise Exception('Missing `key` of `key_file` parameter in config') self.account = account self.key = key diff --git a/tests/test_octodns_provider_transip.py b/tests/test_octodns_provider_transip.py index d6bcaa7..8d85e1f 100644 --- a/tests/test_octodns_provider_transip.py +++ b/tests/test_octodns_provider_transip.py @@ -133,6 +133,19 @@ N4OiVz1I3rbZGYa396lpxO6ku8yCglisL1yrSP6DdEUp66ntpKVd source.populate(expected) return expected + def test_init(self): + with self.assertRaises(Exception) as ctx: + TransipProvider('test', 'unittest') + + self.assertEquals( + str('Missing `key` of `key_file` parameter in config'), + str(ctx.exception)) + + TransipProvider('test', 'unittest', key=self.bogus_key) + + # Existence and content of the key is tested in the SDK on client call + TransipProvider('test', 'unittest', key_file='/fake/path') + def test_populate(self): _expected = self.make_expected() From 637c2547782b15e21473f51960eb1ab4b3a94c1d Mon Sep 17 00:00:00 2001 From: Maikel Poot Date: Mon, 30 Sep 2019 13:18:57 +0200 Subject: [PATCH 10/10] Handling PR Review comments. - Added Specific exceptions - str() instead of concatenation - removed zone not found warning --- octodns/provider/transip.py | 29 +++++++++++++++++++------- tests/test_octodns_provider_transip.py | 4 ++-- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/octodns/provider/transip.py b/octodns/provider/transip.py index 92d607d..09920a9 100644 --- a/octodns/provider/transip.py +++ b/octodns/provider/transip.py @@ -15,6 +15,18 @@ from transip.service.domain import DomainService from transip.service.objects import DnsEntry +class TransipException(Exception): + pass + + +class TransipConfigException(TransipException): + pass + + +class TransipNewZoneException(TransipException): + pass + + class TransipProvider(BaseProvider): ''' Transip DNS provider @@ -55,7 +67,9 @@ class TransipProvider(BaseProvider): elif key is not None: self._client = DomainService(account, private_key=key) else: - raise Exception('Missing `key` of `key_file` parameter in config') + raise TransipConfigException( + 'Missing `key` of `key_file` parameter in config' + ) self.account = account self.key = key @@ -74,13 +88,12 @@ class TransipProvider(BaseProvider): zoneInfo = self._client.get_info(zone.name[:-1]) except WebFault as e: if e.fault.faultcode == '102' and target is False: - self.log.warning( - 'populate: (%s) Zone %s not found in account ', - e.fault.faultcode, zone.name) + # Zone not found in account, and not a target so just + # leave an empty zone. return exists elif e.fault.faultcode == '102' and target is True: self.log.warning('populate: Transip can\'t create new zones') - raise Exception( + raise TransipNewZoneException( ('populate: ({}) Transip used ' + 'as target for non-existing zone: {}').format( e.fault.faultcode, zone.name)) @@ -146,7 +159,7 @@ class TransipProvider(BaseProvider): self.log.warning(('_apply: Set DNS returned ' + 'one or more errors: {}').format( e.fault.faultstring)) - raise Exception(200, e.fault.faultstring) + raise TransipException(200, e.fault.faultstring) self._currentZone = {} @@ -220,7 +233,7 @@ class TransipProvider(BaseProvider): def _parse_to_fqdn(self, value): # Enforce switch from suds.sax.text.Text to string - value = '' + value + value = str(value) # TransIP allows '@' as value to alias the root record. # this provider won't set an '@' value, but can be an existing record @@ -245,7 +258,7 @@ class TransipProvider(BaseProvider): _values = [] for record in records: # Enforce switch from suds.sax.text.Text to string - _values.append('' + record['content']) + _values.append(str(record['content'])) return { 'ttl': self._get_lowest_ttl(records), diff --git a/tests/test_octodns_provider_transip.py b/tests/test_octodns_provider_transip.py index 8d85e1f..c56509a 100644 --- a/tests/test_octodns_provider_transip.py +++ b/tests/test_octodns_provider_transip.py @@ -170,7 +170,7 @@ N4OiVz1I3rbZGYa396lpxO6ku8yCglisL1yrSP6DdEUp66ntpKVd zone = Zone('notfound.unit.tests.', []) provider.populate(zone, True) - self.assertEquals(str('Exception'), + self.assertEquals(str('TransipNewZoneException'), str(ctx.exception.__class__.__name__)) self.assertEquals( @@ -271,5 +271,5 @@ N4OiVz1I3rbZGYa396lpxO6ku8yCglisL1yrSP6DdEUp66ntpKVd # Changes should not be set due to an Exception self.assertEqual([], changes) - self.assertEquals(str('Exception'), + self.assertEquals(str('TransipException'), str(ctx.exception.__class__.__name__))