From cfe33e543c0caf6c746f215f8d3f5c526505d017 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 14 Jan 2022 14:17:49 -0800 Subject: [PATCH] Extract TransipProvider from octoDNS core --- CHANGELOG.md | 1 + README.md | 3 +- octodns/provider/transip.py | 273 ++-------------- requirements.txt | 1 - tests/test_octodns_provider_transip.py | 425 +------------------------ 5 files changed, 31 insertions(+), 672 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cf12578..480d2c4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,7 @@ * [Route53Provider](https://github.com/octodns/octodns-route53/) also AwsAcmMangingProcessor * [SelectelProvider](https://github.com/octodns/octodns-selectel/) + * [TransipProvider](https://github.com/octodns/octodns-transip/) * NS1 provider has received improvements to the dynamic record implementation. As a result, if octoDNS is downgraded from this version, any dynamic records created or updated using this version will show an update. diff --git a/README.md b/README.md index a8e8437..5147aa2 100644 --- a/README.md +++ b/README.md @@ -214,8 +214,7 @@ The table below lists the providers octoDNS supports. We're currently in the pro | [RackspaceProvider](https://github.com/octodns/octodns-rackspace/) | [octodns_rackspace](https://github.com/octodns/octodns-rackspace/) | | | | | | [Route53Provider](https://github.com/octodns/octodns-route53) | [octodns_route53](https://github.com/octodns/octodns-route53) | | | | | | [SelectelProvider](https://github.com/octodns/octodns-selectel/) | [octodns_selectel](https://github.com/octodns/octodns-selectel/) | | | | | -| [Selectel](/octodns/provider/selectel.py) | | | A, AAAA, CNAME, MX, NS, SPF, SRV, TXT | No | | -| [Transip](/octodns/provider/transip.py) | | transip | A, AAAA, CNAME, MX, NS, SRV, SPF, TXT, SSHFP, CAA | No | | +| [TransipProvider](https://github.com/octodns/octodns-transip/) | [octodns_transip](https://github.com/octodns/octodns-transip/) | | | | | | [UltraDns](/octodns/provider/ultra.py) | | | A, AAAA, CAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | | | [AxfrSource](/octodns/source/axfr.py) | | | A, AAAA, CAA, CNAME, LOC, MX, NS, PTR, SPF, SRV, TXT | No | read-only | | [ZoneFileSource](/octodns/source/axfr.py) | | | A, AAAA, CAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | read-only | diff --git a/octodns/provider/transip.py b/octodns/provider/transip.py index f4c0fe2..2e17c24 100644 --- a/octodns/provider/transip.py +++ b/octodns/provider/transip.py @@ -1,258 +1,21 @@ -from __future__ import (absolute_import, division, print_function, - unicode_literals) +# +# +# -from collections import defaultdict, namedtuple -from logging import getLogger - -from transip import TransIP -from transip.exceptions import TransIPHTTPError -from transip.v6.objects import DnsEntry - -from . import ProviderException -from ..record import Record -from .base import BaseProvider - -DNSEntry = namedtuple('DNSEntry', ('name', 'expire', 'type', 'content')) - - -class TransipException(ProviderException): - pass - - -class TransipConfigException(TransipException): - pass - - -class TransipNewZoneException(TransipException): - pass - - -class TransipProvider(BaseProvider): - ''' - Transip DNS provider - - transip: - class: octodns.provider.transip.TransipProvider - # Your Transip account name (required) - account: yourname - # Path to a private key file (required if key is not used) - key_file: /path/to/file - # The api key as string (required if key_file is not used) - key: | - \''' - -----BEGIN PRIVATE KEY----- - ... - -----END PRIVATE KEY----- - \''' - # if both `key_file` and `key` are presented `key_file` is used - - ''' - - SUPPORTS_GEO = False - SUPPORTS_DYNAMIC = False - SUPPORTS = set(('A', 'AAAA', 'CNAME', 'MX', 'NS', 'SRV', 'SPF', 'TXT', - 'SSHFP', 'CAA')) - # unsupported by OctoDNS: 'TLSA' - MIN_TTL = 120 - TIMEOUT = 15 - ROOT_RECORD = '@' - - def __init__(self, id, account, key=None, key_file=None, *args, **kwargs): - self.log = getLogger('TransipProvider[{}]'.format(id)) - self.log.debug('__init__: id=%s, account=%s, token=***', id, - account) - super(TransipProvider, self).__init__(id, *args, **kwargs) - - if key_file is not None: - self._client = TransIP(login=account, private_key_file=key_file) - elif key is not None: - self._client = TransIP(login=account, private_key=key) - else: - raise TransipConfigException( - 'Missing `key` or `key_file` parameter in config' - ) - - def populate(self, zone, target=False, lenient=False): - ''' - Populate the zone with records in-place. - ''' - self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, - target, lenient) - - before = len(zone.records) - - try: - domain = self._client.domains.get(zone.name.strip('.')) - records = domain.dns.list() - except TransIPHTTPError as e: - if e.response_code == 404 and target is False: - # Zone not found in account, and not a target so just - # leave an empty zone. - return False - elif e.response_code == 404 and target is True: - self.log.warning('populate: Transip can\'t create new zones') - raise TransipNewZoneException( - ('populate: ({}) Transip used ' + - 'as target for non-existing zone: {}').format( - e.response_code, zone.name)) - else: - self.log.error( - 'populate: (%s) %s ', e.response_code, e.message - ) - raise TransipException( - 'Unhandled error: ({}) {}'.format( - e.response_code, e.message - ) - ) - - self.log.debug( - 'populate: found %s records for zone %s', len(records), zone.name - ) - if records: - values = defaultdict(lambda: defaultdict(list)) - for record in records: - name = zone.hostname_from_fqdn(record.name) - if name == self.ROOT_RECORD: - name = '' - - if record.type in self.SUPPORTS: - values[name][record.type].append(record) - - for name, types in values.items(): - for _type, records in types.items(): - record = Record.new( - zone, - name, - _data_for(_type, records, zone), - source=self, - lenient=lenient, - ) - zone.add_record(record, lenient=lenient) - self.log.info('populate: found %s records', - len(zone.records) - before) - - return True - - def _apply(self, plan): - desired = plan.desired - changes = plan.changes - self.log.debug('apply: zone=%s, changes=%d', desired.name, - len(changes)) - - try: - domain = self._client.domains.get(plan.desired.name[:-1]) - except TransIPHTTPError as e: - self.log.exception('_apply: getting the domain failed') - raise TransipException( - 'Unhandled error: ({}) {}'.format(e.response_code, e.message) - ) - - records = [] - for record in plan.desired.records: - if record._type in self.SUPPORTS: - # Root records have '@' as name - name = record.name - if name == '': - name = self.ROOT_RECORD - - records.extend(_entries_for(name, record)) - - # Transform DNSEntry namedtuples into transip.v6.objects.DnsEntry - # objects, which is a bit ugly because it's quite a magical object. - api_records = [DnsEntry(domain.dns, r._asdict()) for r in records] - try: - domain.dns.replace(api_records) - except TransIPHTTPError as e: - self.log.warning( - '_apply: Set DNS returned one or more errors: {}'.format(e) - ) - raise TransipException( - 'Unhandled error: ({}) {}'.format(e.response_code, e.message) - ) - - -def _data_for(type_, records, current_zone): - if type_ == 'CNAME': - return { - 'type': type_, - 'ttl': records[0].expire, - 'value': _parse_to_fqdn(records[0].content, current_zone), - } - - def format_mx(record): - preference, exchange = record.content.split(' ', 1) - return { - 'preference': preference, - 'exchange': _parse_to_fqdn(exchange, current_zone), - } - - def format_srv(record): - priority, weight, port, target = record.content.split(' ', 3) - return { - 'port': port, - 'priority': priority, - 'target': _parse_to_fqdn(target, current_zone), - 'weight': weight, - } - - def format_sshfp(record): - algorithm, fp_type, fingerprint = record.content.split(' ', 2) - return { - 'algorithm': algorithm, - 'fingerprint': fingerprint.lower(), - 'fingerprint_type': fp_type, - } - - def format_caa(record): - flags, tag, value = record.content.split(' ', 2) - return {'flags': flags, 'tag': tag, 'value': value} - - def format_txt(record): - return record.content.replace(';', '\\;') - - value_formatter = { - 'MX': format_mx, - 'SRV': format_srv, - 'SSHFP': format_sshfp, - 'CAA': format_caa, - 'TXT': format_txt, - }.get(type_, lambda r: r.content) - - return { - 'type': type_, - 'ttl': _get_lowest_ttl(records), - 'values': [value_formatter(r) for r in records], - } - - -def _parse_to_fqdn(value, current_zone): - # TransIP allows '@' as value to alias the root record. - # this provider won't set an '@' value, but can be an existing record - if value == TransipProvider.ROOT_RECORD: - value = current_zone.name - - if value[-1] != '.': - value = '{}.{}'.format(value, current_zone.name) - - return value - - -def _get_lowest_ttl(records): - return min([r.expire for r in records] + [100000]) +from __future__ import absolute_import, division, print_function, \ + unicode_literals +from logging import getLogger -def _entries_for(name, record): - values = record.values if hasattr(record, 'values') else [record.value] - formatter = { - 'MX': lambda v: f'{v.preference} {v.exchange}', - 'SRV': lambda v: f'{v.priority} {v.weight} {v.port} {v.target}', - 'SSHFP': lambda v: ( - f'{v.algorithm} {v.fingerprint_type} {v.fingerprint}' - ), - 'CAA': lambda v: f'{v.flags} {v.tag} {v.value}', - 'TXT': lambda v: v.replace('\\;', ';'), - }.get(record._type, lambda r: r) - return [ - DNSEntry(name, record.ttl, record._type, formatter(value)) - for value in values - ] +logger = getLogger('Transip') +try: + logger.warning('octodns_transip shimmed. Update your provider class to ' + 'octodns_transip.TransipProvider. ' + 'Shim will be removed in 1.0') + from octodns_transip import TransipProvider + TransipProvider # pragma: no cover +except ModuleNotFoundError: + logger.exception('TransipProvider has been moved into a seperate module, ' + 'octodns_transip is now required. Provider class should ' + 'be updated to octodns_transip.TransipProvider') + raise diff --git a/requirements.txt b/requirements.txt index 1ba51d9..653572a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,4 +9,3 @@ pycountry==22.1.10 python-dateutil==2.8.1 requests==2.25.1 setuptools==60.5.0 -python-transip==0.5.0 diff --git a/tests/test_octodns_provider_transip.py b/tests/test_octodns_provider_transip.py index d1bd29f..faa90b3 100644 --- a/tests/test_octodns_provider_transip.py +++ b/tests/test_octodns_provider_transip.py @@ -1,419 +1,16 @@ -from __future__ import (absolute_import, division, print_function, - unicode_literals) +# +# +# -from operator import itemgetter -from os.path import dirname, join -from unittest import TestCase -from unittest.mock import Mock, patch - -from octodns.provider.transip import (DNSEntry, TransipConfigException, - TransipException, - TransipNewZoneException, TransipProvider, - _entries_for, _parse_to_fqdn) -from octodns.provider.yaml import YamlProvider -from octodns.zone import Zone -from transip.exceptions import TransIPHTTPError - - -def make_expected(): - expected = Zone("unit.tests.", []) - source = YamlProvider("test", join(dirname(__file__), "config")) - source.populate(expected) - return expected - - -def make_mock(): - zone = make_expected() - - # Turn Zone.records into TransIP DNSEntries - api_entries = [] - for record in zone.records: - if record._type in TransipProvider.SUPPORTS: - # Root records have '@' as name - name = record.name - if name == "": - name = TransipProvider.ROOT_RECORD - - api_entries.extend(_entries_for(name, record)) - - # Append bogus entry so test for record type not being in SUPPORTS is - # executed. For 100% test coverage. - api_entries.append(DNSEntry("@", "3600", "BOGUS", "ns.transip.nl")) - - return zone, api_entries - - -def make_mock_empty(): - mock = Mock() - mock.return_value.domains.get.return_value.dns.list.return_value = [] - return mock - - -def make_failing_mock(response_code): - mock = Mock() - mock.return_value.domains.get.side_effect = [ - TransIPHTTPError(str(response_code), response_code) - ] - return mock - - -class TestTransipProvider(TestCase): - - bogus_key = "-----BEGIN RSA PRIVATE KEY-----Z-----END RSA PRIVATE KEY-----" - - @patch("octodns.provider.transip.TransIP", make_mock_empty()) - def test_init(self): - with self.assertRaises(TransipConfigException) as ctx: - TransipProvider("test", "unittest") - - self.assertEqual( - "Missing `key` or `key_file` parameter in config", - str(ctx.exception), - ) - - # Those should work - TransipProvider("test", "unittest", key=self.bogus_key) - TransipProvider("test", "unittest", key_file="/fake/path") - - @patch("octodns.provider.transip.TransIP", make_failing_mock(401)) - def test_populate_unauthenticated(self): - # Unhappy Plan - Not authenticated - provider = TransipProvider("test", "unittest", self.bogus_key) - zone = Zone("unit.tests.", []) - with self.assertRaises(TransipException): - provider.populate(zone, True) - - @patch("octodns.provider.transip.TransIP", make_failing_mock(404)) - def test_populate_new_zone_as_target(self): - # Unhappy Plan - Zone does not exists - # Will trigger an exception if provider is used as a target for a - # non-existing zone - provider = TransipProvider("test", "unittest", self.bogus_key) - zone = Zone("notfound.unit.tests.", []) - with self.assertRaises(TransipNewZoneException): - provider.populate(zone, True) - - @patch("octodns.provider.transip.TransIP", make_mock_empty()) - def test_populate_new_zone_not_target(self): - # Happy Plan - Zone does not exists - # Won't trigger an exception if provider is NOT used as a target for a - # non-existing zone. - provider = TransipProvider("test", "unittest", self.bogus_key) - zone = Zone("notfound.unit.tests.", []) - provider.populate(zone, False) +from __future__ import absolute_import, division, print_function, \ + unicode_literals - @patch("octodns.provider.transip.TransIP", make_failing_mock(404)) - def test_populate_zone_does_not_exist(self): - # Happy Plan - Zone does not exists - # Won't trigger an exception if provider is NOT used as a target for a - # non-existing zone. - provider = TransipProvider("test", "unittest", self.bogus_key) - zone = Zone("notfound.unit.tests.", []) - provider.populate(zone, False) - - @patch("octodns.provider.transip.TransIP") - def test_populate_zone_exists_not_target(self, mock_client): - # Happy Plan - Populate - source_zone, api_records = make_mock() - mock_client.return_value.domains.get.return_value.dns.list. \ - return_value = api_records - provider = TransipProvider("test", "unittest", self.bogus_key) - zone = Zone("unit.tests.", []) - - exists = provider.populate(zone, False) - - self.assertTrue(exists, "populate should return True") - - # Due to the implementation of Record._equality_tuple() we can't do a - # normal compare, as that ingores ttl's for example. We therefor use - # the __repr__ to compare. We do need to filter out `.geo` attributes - # that Transip doesn't support. - expected = set() - for r in source_zone.records: - if r._type in TransipProvider.SUPPORTS: - if hasattr(r, "geo"): - r.geo = None - expected.add(r.__repr__()) - self.assertEqual({r.__repr__() for r in zone.records}, expected) - - @patch("octodns.provider.transip.TransIP", make_mock_empty()) - def test_populate_zone_exists_as_target(self): - # Happy Plan - Even if the zone has no records the zone should exist - provider = TransipProvider("test", "unittest", self.bogus_key) - zone = Zone("unit.tests.", []) - exists = provider.populate(zone, True) - self.assertTrue(exists, "populate should return True") - - @patch("octodns.provider.transip.TransIP", make_mock_empty()) - def test_plan(self): - # Test happy plan, only create - provider = TransipProvider("test", "unittest", self.bogus_key) - - plan = provider.plan(make_expected()) - - self.assertIsNotNone(plan) - self.assertEqual(15, plan.change_counts["Create"]) - self.assertEqual(0, plan.change_counts["Update"]) - self.assertEqual(0, plan.change_counts["Delete"]) - - @patch("octodns.provider.transip.TransIP") - def test_apply(self, client_mock): - # Test happy flow. Create all supported records - domain_mock = Mock() - client_mock.return_value.domains.get.return_value = domain_mock - domain_mock.dns.list.return_value = [] - provider = TransipProvider("test", "unittest", self.bogus_key) - - plan = provider.plan(make_expected()) - self.assertIsNotNone(plan) - provider.apply(plan) - - domain_mock.dns.replace.assert_called_once() - - # These are the supported ones from tests/config/unit.test.yaml - expected_entries = [ - { - "name": "ignored", - "expire": 3600, - "type": "A", - "content": "9.9.9.9", - }, - { - "name": "@", - "expire": 3600, - "type": "CAA", - "content": "0 issue ca.unit.tests", - }, - { - "name": "sub", - "expire": 3600, - "type": "NS", - "content": "6.2.3.4.", - }, - { - "name": "sub", - "expire": 3600, - "type": "NS", - "content": "7.2.3.4.", - }, - { - "name": "spf", - "expire": 600, - "type": "SPF", - "content": "v=spf1 ip4:192.168.0.1/16-all", - }, - { - "name": "_srv._tcp", - "expire": 600, - "type": "SRV", - "content": "10 20 30 foo-1.unit.tests.", - }, - { - "name": "_srv._tcp", - "expire": 600, - "type": "SRV", - "content": "12 20 30 foo-2.unit.tests.", - }, - { - "name": "_pop3._tcp", - "expire": 600, - "type": "SRV", - "content": "0 0 0 .", - }, - { - "name": "_imap._tcp", - "expire": 600, - "type": "SRV", - "content": "0 0 0 .", - }, - { - "name": "txt", - "expire": 600, - "type": "TXT", - "content": "Bah bah black sheep", - }, - { - "name": "txt", - "expire": 600, - "type": "TXT", - "content": "have you any wool.", - }, - { - "name": "txt", - "expire": 600, - "type": "TXT", - "content": ( - "v=DKIM1;k=rsa;s=email;h=sha256;" - "p=A/kinda+of/long/string+with+numb3rs" - ), - }, - {"name": "@", "expire": 3600, "type": "NS", "content": "6.2.3.4."}, - {"name": "@", "expire": 3600, "type": "NS", "content": "7.2.3.4."}, - { - "name": "cname", - "expire": 300, - "type": "CNAME", - "content": "unit.tests.", - }, - { - "name": "excluded", - "expire": 3600, - "type": "CNAME", - "content": "unit.tests.", - }, - { - "name": "www.sub", - "expire": 300, - "type": "A", - "content": "2.2.3.6", - }, - { - "name": "included", - "expire": 3600, - "type": "CNAME", - "content": "unit.tests.", - }, - { - "name": "mx", - "expire": 300, - "type": "MX", - "content": "10 smtp-4.unit.tests.", - }, - { - "name": "mx", - "expire": 300, - "type": "MX", - "content": "20 smtp-2.unit.tests.", - }, - { - "name": "mx", - "expire": 300, - "type": "MX", - "content": "30 smtp-3.unit.tests.", - }, - { - "name": "mx", - "expire": 300, - "type": "MX", - "content": "40 smtp-1.unit.tests.", - }, - { - "name": "aaaa", - "expire": 600, - "type": "AAAA", - "content": "2601:644:500:e210:62f8:1dff:feb8:947a", - }, - {"name": "@", "expire": 300, "type": "A", "content": "1.2.3.4"}, - {"name": "@", "expire": 300, "type": "A", "content": "1.2.3.5"}, - {"name": "www", "expire": 300, "type": "A", "content": "2.2.3.6"}, - { - "name": "@", - "expire": 3600, - "type": "SSHFP", - "content": "1 1 7491973e5f8b39d5327cd4e08bc81b05f7710b49", - }, - { - "name": "@", - "expire": 3600, - "type": "SSHFP", - "content": "1 1 bf6b6825d2977c511a475bbefb88aad54a92ac73", - }, - ] - # Unpack from the transip library magic structure... - seen_entries = [ - e.__dict__["_attrs"] - for e in domain_mock.dns.replace.mock_calls[0][1][0] - ] - self.assertEqual( - sorted(seen_entries, key=itemgetter("name", "type", "expire")), - sorted(expected_entries, key=itemgetter("name", "type", "expire")), - ) - - @patch("octodns.provider.transip.TransIP") - def test_apply_unsupported(self, client_mock): - # This triggers the if supported statement to give 100% code coverage - domain_mock = Mock() - client_mock.return_value.domains.get.return_value = domain_mock - domain_mock.dns.list.return_value = [] - provider = TransipProvider("test", "unittest", self.bogus_key) - - plan = provider.plan(make_expected()) - self.assertIsNotNone(plan) - - # Test apply with only support for A records - provider.SUPPORTS = set(("A")) - - provider.apply(plan) - seen_entries = [ - e.__dict__["_attrs"] - for e in domain_mock.dns.replace.mock_calls[0][1][0] - ] - expected_entries = [ - { - "name": "ignored", - "expire": 3600, - "type": "A", - "content": "9.9.9.9", - }, - { - "name": "www.sub", - "expire": 300, - "type": "A", - "content": "2.2.3.6", - }, - {"name": "@", "expire": 300, "type": "A", "content": "1.2.3.4"}, - {"name": "@", "expire": 300, "type": "A", "content": "1.2.3.5"}, - {"name": "www", "expire": 300, "type": "A", "content": "2.2.3.6"}, - ] - self.assertEqual( - sorted(seen_entries, key=itemgetter("name", "type", "expire")), - sorted(expected_entries, key=itemgetter("name", "type", "expire")), - ) - - @patch("octodns.provider.transip.TransIP") - def test_apply_failure_on_not_found(self, client_mock): - # Test unhappy flow. Trigger 'not found error' in apply stage - # This should normally not happen as populate will capture it first - # but just in case. - domain_mock = Mock() - domain_mock.dns.list.return_value = [] - client_mock.return_value.domains.get.side_effect = [ - domain_mock, - TransIPHTTPError("Not Found", 404), - ] - provider = TransipProvider("test", "unittest", self.bogus_key) - - plan = provider.plan(make_expected()) - - with self.assertRaises(TransipException): - provider.apply(plan) - - @patch("octodns.provider.transip.TransIP") - def test_apply_failure_on_error(self, client_mock): - # Test unhappy flow. Trigger a unrecoverable error while saving - domain_mock = Mock() - domain_mock.dns.list.return_value = [] - domain_mock.dns.replace.side_effect = [ - TransIPHTTPError("Not Found", 500) - ] - client_mock.return_value.domains.get.return_value = domain_mock - provider = TransipProvider("test", "unittest", self.bogus_key) - - plan = provider.plan(make_expected()) +from unittest import TestCase - with self.assertRaises(TransipException): - provider.apply(plan) +class TestTransipShim(TestCase): -class TestParseFQDN(TestCase): - def test_parse_fqdn(self): - zone = Zone("unit.tests.", []) - self.assertEqual("www.unit.tests.", _parse_to_fqdn("www", zone)) - self.assertEqual( - "www.unit.tests.", _parse_to_fqdn("www.unit.tests.", zone) - ) - self.assertEqual( - "www.sub.sub.sub.unit.tests.", - _parse_to_fqdn("www.sub.sub.sub", zone), - ) - self.assertEqual("unit.tests.", _parse_to_fqdn("@", zone)) + def test_missing(self): + with self.assertRaises(ModuleNotFoundError): + from octodns.provider.transip import TransipProvider + TransipProvider