Merge pull request #858 from octodns/extract-transip

Extract TransipProvider from octoDNS core
4 years ago · e60390207a
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -28,6 +28,7 @@
   * [Route53Provider](https://github.com/octodns/octodns-route53/) also
     AwsAcmMangingProcessor
   * [SelectelProvider](https://github.com/octodns/octodns-selectel/)
   * [TransipProvider](https://github.com/octodns/octodns-transip/)
 * NS1 provider has received improvements to the dynamic record implementation.
  As a result, if octoDNS is downgraded from this version, any dynamic records
  created or updated using this version will show an update.
--- a/README.md
+++ b/README.md
@ -212,9 +212,9 @@ The table below lists the providers octoDNS supports. We're currently in the pro
 | [NS1](https://ns1.com/products/managed-dns) | [octodns_ns1](https://github.com/octodns/octodns-ns1/) | | | | |
 | [OVHcloud DNS](https://www.ovhcloud.com/en/domains/dns-subdomain/) | [octodns_ovh](https://github.com/octodns/octodns-ovh/) | | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, SSHFP, TXT, DKIM | No | |
 | [PowerDNS](https://www.powerdns.com/) | [octodns_powerdns](https://github.com/octodns/octodns-powerdns/) | | | | |
 | [Rackspace](/octodns/provider/rackspace.py) | [octodns_rackspace](https://github.com/octodns/octodns-rackspace/) | | | |  |
 | [Selectel](/octodns/provider/selectel.py) | [octodns_selectel](https://github.com/octodns/octodns-selectel/) | | | | |
 | [Transip](/octodns/provider/transip.py) | | transip | A, AAAA, CNAME, MX, NS, SRV, SPF, TXT, SSHFP, CAA | No | |
 | [Rackspace](https://www.rackspace.com/library/what-is-dns) | [octodns_rackspace](https://github.com/octodns/octodns-rackspace/) | | | |  |
 | [Selectel](https://selectel.ru/en/services/additional/dns/) | [octodns_selectel](https://github.com/octodns/octodns-selectel/) | | | | |
 | [Transip](https://www.transip.eu/knowledgebase/entry/155-dns-and-nameservers/) | [octodns_transip](https://github.com/octodns/octodns-transip/) | | | | |
 | [UltraDns](/octodns/provider/ultra.py) | | | A, AAAA, CAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | |
 | [AxfrSource](/octodns/source/axfr.py) | | | A, AAAA, CAA, CNAME, LOC, MX, NS, PTR, SPF, SRV, TXT | No | read-only |
 | [ZoneFileSource](/octodns/source/axfr.py) | | | A, AAAA, CAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | read-only |
--- a/octodns/provider/transip.py
+++ b/octodns/provider/transip.py
@ -1,258 +1,21 @@
 from __future__ import (absolute_import, division, print_function,
                        unicode_literals)
 #
 #
 #

 from collections import defaultdict, namedtuple
 from logging import getLogger

 from transip import TransIP
 from transip.exceptions import TransIPHTTPError
 from transip.v6.objects import DnsEntry

 from . import ProviderException
 from ..record import Record
 from .base import BaseProvider

 DNSEntry = namedtuple('DNSEntry', ('name', 'expire', 'type', 'content'))


 class TransipException(ProviderException):
    pass


 class TransipConfigException(TransipException):
    pass


 class TransipNewZoneException(TransipException):
    pass


 class TransipProvider(BaseProvider):
    '''
    Transip DNS provider

    transip:
        class: octodns.provider.transip.TransipProvider
        # Your Transip account name (required)
        account: yourname
        # Path to a private key file (required if key is not used)
        key_file: /path/to/file
        # The api key as string (required if key_file is not used)
        key: |
            \'''
            -----BEGIN PRIVATE KEY-----
            ...
            -----END PRIVATE KEY-----
            \'''
        # if both `key_file` and `key` are presented `key_file` is used

    '''

    SUPPORTS_GEO = False
    SUPPORTS_DYNAMIC = False
    SUPPORTS = set(('A', 'AAAA', 'CNAME', 'MX', 'NS', 'SRV', 'SPF', 'TXT',
                    'SSHFP', 'CAA'))
    # unsupported by OctoDNS: 'TLSA'
    MIN_TTL = 120
    TIMEOUT = 15
    ROOT_RECORD = '@'

    def __init__(self, id, account, key=None, key_file=None, *args, **kwargs):
        self.log = getLogger('TransipProvider[{}]'.format(id))
        self.log.debug('__init__: id=%s, account=%s, token=***', id,
                       account)
        super(TransipProvider, self).__init__(id, *args, **kwargs)

        if key_file is not None:
            self._client = TransIP(login=account, private_key_file=key_file)
        elif key is not None:
            self._client = TransIP(login=account, private_key=key)
        else:
            raise TransipConfigException(
                'Missing `key` or `key_file` parameter in config'
            )

    def populate(self, zone, target=False, lenient=False):
        '''
        Populate the zone with records in-place.
        '''
        self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name,
                       target, lenient)

        before = len(zone.records)

        try:
            domain = self._client.domains.get(zone.name.strip('.'))
            records = domain.dns.list()
        except TransIPHTTPError as e:
            if e.response_code == 404 and target is False:
                # Zone not found in account, and not a target so just
                # leave an empty zone.
                return False
            elif e.response_code == 404 and target is True:
                self.log.warning('populate: Transip can\'t create new zones')
                raise TransipNewZoneException(
                    ('populate: ({}) Transip used ' +
                     'as target for non-existing zone: {}').format(
                        e.response_code, zone.name))
            else:
                self.log.error(
                    'populate: (%s) %s ', e.response_code, e.message
                )
                raise TransipException(
                    'Unhandled error: ({}) {}'.format(
                        e.response_code, e.message
                    )
                )

        self.log.debug(
            'populate: found %s records for zone %s', len(records), zone.name
        )
        if records:
            values = defaultdict(lambda: defaultdict(list))
            for record in records:
                name = zone.hostname_from_fqdn(record.name)
                if name == self.ROOT_RECORD:
                    name = ''

                if record.type in self.SUPPORTS:
                    values[name][record.type].append(record)

            for name, types in values.items():
                for _type, records in types.items():
                    record = Record.new(
                        zone,
                        name,
                        _data_for(_type, records, zone),
                        source=self,
                        lenient=lenient,
                    )
                    zone.add_record(record, lenient=lenient)
        self.log.info('populate:   found %s records',
                      len(zone.records) - before)

        return True

    def _apply(self, plan):
        desired = plan.desired
        changes = plan.changes
        self.log.debug('apply: zone=%s, changes=%d', desired.name,
                       len(changes))

        try:
            domain = self._client.domains.get(plan.desired.name[:-1])
        except TransIPHTTPError as e:
            self.log.exception('_apply: getting the domain failed')
            raise TransipException(
                'Unhandled error: ({}) {}'.format(e.response_code, e.message)
            )

        records = []
        for record in plan.desired.records:
            if record._type in self.SUPPORTS:
                # Root records have '@' as name
                name = record.name
                if name == '':
                    name = self.ROOT_RECORD

                records.extend(_entries_for(name, record))

        # Transform DNSEntry namedtuples into transip.v6.objects.DnsEntry
        # objects, which is a bit ugly because it's quite a magical object.
        api_records = [DnsEntry(domain.dns, r._asdict()) for r in records]
        try:
            domain.dns.replace(api_records)
        except TransIPHTTPError as e:
            self.log.warning(
                '_apply: Set DNS returned one or more errors: {}'.format(e)
            )
            raise TransipException(
                'Unhandled error: ({}) {}'.format(e.response_code, e.message)
            )


 def _data_for(type_, records, current_zone):
    if type_ == 'CNAME':
        return {
            'type': type_,
            'ttl': records[0].expire,
            'value': _parse_to_fqdn(records[0].content, current_zone),
        }

    def format_mx(record):
        preference, exchange = record.content.split(' ', 1)
        return {
            'preference': preference,
            'exchange': _parse_to_fqdn(exchange, current_zone),
        }

    def format_srv(record):
        priority, weight, port, target = record.content.split(' ', 3)
        return {
            'port': port,
            'priority': priority,
            'target': _parse_to_fqdn(target, current_zone),
            'weight': weight,
        }

    def format_sshfp(record):
        algorithm, fp_type, fingerprint = record.content.split(' ', 2)
        return {
            'algorithm': algorithm,
            'fingerprint': fingerprint.lower(),
            'fingerprint_type': fp_type,
        }

    def format_caa(record):
        flags, tag, value = record.content.split(' ', 2)
        return {'flags': flags, 'tag': tag, 'value': value}

    def format_txt(record):
        return record.content.replace(';', '\\;')

    value_formatter = {
        'MX': format_mx,
        'SRV': format_srv,
        'SSHFP': format_sshfp,
        'CAA': format_caa,
        'TXT': format_txt,
    }.get(type_, lambda r: r.content)

    return {
        'type': type_,
        'ttl': _get_lowest_ttl(records),
        'values': [value_formatter(r) for r in records],
    }


 def _parse_to_fqdn(value, current_zone):
    # TransIP allows '@' as value to alias the root record.
    # this provider won't set an '@' value, but can be an existing record
    if value == TransipProvider.ROOT_RECORD:
        value = current_zone.name

    if value[-1] != '.':
        value = '{}.{}'.format(value, current_zone.name)

    return value


 def _get_lowest_ttl(records):
    return min([r.expire for r in records] + [100000])
 from __future__ import absolute_import, division, print_function, \
    unicode_literals

 from logging import getLogger

 def _entries_for(name, record):
    values = record.values if hasattr(record, 'values') else [record.value]
    formatter = {
        'MX': lambda v: f'{v.preference} {v.exchange}',
        'SRV': lambda v: f'{v.priority} {v.weight} {v.port} {v.target}',
        'SSHFP': lambda v: (
            f'{v.algorithm} {v.fingerprint_type} {v.fingerprint}'
        ),
        'CAA': lambda v: f'{v.flags} {v.tag} {v.value}',
        'TXT': lambda v: v.replace('\\;', ';'),
    }.get(record._type, lambda r: r)
    return [
        DNSEntry(name, record.ttl, record._type, formatter(value))
        for value in values
    ]
 logger = getLogger('Transip')
 try:
    logger.warning('octodns_transip shimmed. Update your provider class to '
                   'octodns_transip.TransipProvider. '
                   'Shim will be removed in 1.0')
    from octodns_transip import TransipProvider
    TransipProvider  # pragma: no cover
 except ModuleNotFoundError:
    logger.exception('TransipProvider has been moved into a seperate module, '
                     'octodns_transip is now required. Provider class should '
                     'be updated to octodns_transip.TransipProvider')
    raise
--- a/requirements.txt
+++ b/requirements.txt
@ -9,4 +9,3 @@ pycountry==22.1.10
 python-dateutil==2.8.1
 requests==2.25.1
 setuptools==60.5.0
 python-transip==0.5.0
--- a/tests/test_octodns_provider_transip.py
+++ b/tests/test_octodns_provider_transip.py
@ -1,419 +1,16 @@
 from __future__ import (absolute_import, division, print_function,
                        unicode_literals)
 #
 #
 #

 from operator import itemgetter
 from os.path import dirname, join
 from unittest import TestCase
 from unittest.mock import Mock, patch

 from octodns.provider.transip import (DNSEntry, TransipConfigException,
                                      TransipException,
                                      TransipNewZoneException, TransipProvider,
                                      _entries_for, _parse_to_fqdn)
 from octodns.provider.yaml import YamlProvider
 from octodns.zone import Zone
 from transip.exceptions import TransIPHTTPError


 def make_expected():
    expected = Zone("unit.tests.", [])
    source = YamlProvider("test", join(dirname(__file__), "config"))
    source.populate(expected)
    return expected


 def make_mock():
    zone = make_expected()

    # Turn Zone.records into TransIP DNSEntries
    api_entries = []
    for record in zone.records:
        if record._type in TransipProvider.SUPPORTS:
            # Root records have '@' as name
            name = record.name
            if name == "":
                name = TransipProvider.ROOT_RECORD

            api_entries.extend(_entries_for(name, record))

    # Append bogus entry so test for record type not being in SUPPORTS is
    # executed. For 100% test coverage.
    api_entries.append(DNSEntry("@", "3600", "BOGUS", "ns.transip.nl"))

    return zone, api_entries


 def make_mock_empty():
    mock = Mock()
    mock.return_value.domains.get.return_value.dns.list.return_value = []
    return mock


 def make_failing_mock(response_code):
    mock = Mock()
    mock.return_value.domains.get.side_effect = [
        TransIPHTTPError(str(response_code), response_code)
    ]
    return mock


 class TestTransipProvider(TestCase):

    bogus_key = "-----BEGIN RSA PRIVATE KEY-----Z-----END RSA PRIVATE KEY-----"

    @patch("octodns.provider.transip.TransIP", make_mock_empty())
    def test_init(self):
        with self.assertRaises(TransipConfigException) as ctx:
            TransipProvider("test", "unittest")

        self.assertEqual(
            "Missing `key` or `key_file` parameter in config",
            str(ctx.exception),
        )

        # Those should work
        TransipProvider("test", "unittest", key=self.bogus_key)
        TransipProvider("test", "unittest", key_file="/fake/path")

    @patch("octodns.provider.transip.TransIP", make_failing_mock(401))
    def test_populate_unauthenticated(self):
        # Unhappy Plan - Not authenticated
        provider = TransipProvider("test", "unittest", self.bogus_key)
        zone = Zone("unit.tests.", [])
        with self.assertRaises(TransipException):
            provider.populate(zone, True)

    @patch("octodns.provider.transip.TransIP", make_failing_mock(404))
    def test_populate_new_zone_as_target(self):
        # Unhappy Plan - Zone does not exists
        # Will trigger an exception if provider is used as a target for a
        # non-existing zone
        provider = TransipProvider("test", "unittest", self.bogus_key)
        zone = Zone("notfound.unit.tests.", [])
        with self.assertRaises(TransipNewZoneException):
            provider.populate(zone, True)

    @patch("octodns.provider.transip.TransIP", make_mock_empty())
    def test_populate_new_zone_not_target(self):
        # Happy Plan - Zone does not exists
        # Won't trigger an exception if provider is NOT used as a target for a
        # non-existing zone.
        provider = TransipProvider("test", "unittest", self.bogus_key)
        zone = Zone("notfound.unit.tests.", [])
        provider.populate(zone, False)
 from __future__ import absolute_import, division, print_function, \
    unicode_literals

    @patch("octodns.provider.transip.TransIP", make_failing_mock(404))
    def test_populate_zone_does_not_exist(self):
        # Happy Plan - Zone does not exists
        # Won't trigger an exception if provider is NOT used as a target for a
        # non-existing zone.
        provider = TransipProvider("test", "unittest", self.bogus_key)
        zone = Zone("notfound.unit.tests.", [])
        provider.populate(zone, False)

    @patch("octodns.provider.transip.TransIP")
    def test_populate_zone_exists_not_target(self, mock_client):
        # Happy Plan - Populate
        source_zone, api_records = make_mock()
        mock_client.return_value.domains.get.return_value.dns.list. \
            return_value = api_records
        provider = TransipProvider("test", "unittest", self.bogus_key)
        zone = Zone("unit.tests.", [])

        exists = provider.populate(zone, False)

        self.assertTrue(exists, "populate should return True")

        # Due to the implementation of Record._equality_tuple() we can't do a
        # normal compare, as that ingores ttl's for example. We therefor use
        # the __repr__ to compare. We do need to filter out `.geo` attributes
        # that Transip doesn't support.
        expected = set()
        for r in source_zone.records:
            if r._type in TransipProvider.SUPPORTS:
                if hasattr(r, "geo"):
                    r.geo = None
                expected.add(r.__repr__())
        self.assertEqual({r.__repr__() for r in zone.records}, expected)

    @patch("octodns.provider.transip.TransIP", make_mock_empty())
    def test_populate_zone_exists_as_target(self):
        # Happy Plan - Even if the zone has no records the zone should exist
        provider = TransipProvider("test", "unittest", self.bogus_key)
        zone = Zone("unit.tests.", [])
        exists = provider.populate(zone, True)
        self.assertTrue(exists, "populate should return True")

    @patch("octodns.provider.transip.TransIP", make_mock_empty())
    def test_plan(self):
        # Test happy plan, only create
        provider = TransipProvider("test", "unittest", self.bogus_key)

        plan = provider.plan(make_expected())

        self.assertIsNotNone(plan)
        self.assertEqual(15, plan.change_counts["Create"])
        self.assertEqual(0, plan.change_counts["Update"])
        self.assertEqual(0, plan.change_counts["Delete"])

    @patch("octodns.provider.transip.TransIP")
    def test_apply(self, client_mock):
        # Test happy flow. Create all supported records
        domain_mock = Mock()
        client_mock.return_value.domains.get.return_value = domain_mock
        domain_mock.dns.list.return_value = []
        provider = TransipProvider("test", "unittest", self.bogus_key)

        plan = provider.plan(make_expected())
        self.assertIsNotNone(plan)
        provider.apply(plan)

        domain_mock.dns.replace.assert_called_once()

        # These are the supported ones from tests/config/unit.test.yaml
        expected_entries = [
            {
                "name": "ignored",
                "expire": 3600,
                "type": "A",
                "content": "9.9.9.9",
            },
            {
                "name": "@",
                "expire": 3600,
                "type": "CAA",
                "content": "0 issue ca.unit.tests",
            },
            {
                "name": "sub",
                "expire": 3600,
                "type": "NS",
                "content": "6.2.3.4.",
            },
            {
                "name": "sub",
                "expire": 3600,
                "type": "NS",
                "content": "7.2.3.4.",
            },
            {
                "name": "spf",
                "expire": 600,
                "type": "SPF",
                "content": "v=spf1 ip4:192.168.0.1/16-all",
            },
            {
                "name": "_srv._tcp",
                "expire": 600,
                "type": "SRV",
                "content": "10 20 30 foo-1.unit.tests.",
            },
            {
                "name": "_srv._tcp",
                "expire": 600,
                "type": "SRV",
                "content": "12 20 30 foo-2.unit.tests.",
            },
            {
                "name": "_pop3._tcp",
                "expire": 600,
                "type": "SRV",
                "content": "0 0 0 .",
            },
            {
                "name": "_imap._tcp",
                "expire": 600,
                "type": "SRV",
                "content": "0 0 0 .",
            },
            {
                "name": "txt",
                "expire": 600,
                "type": "TXT",
                "content": "Bah bah black sheep",
            },
            {
                "name": "txt",
                "expire": 600,
                "type": "TXT",
                "content": "have you any wool.",
            },
            {
                "name": "txt",
                "expire": 600,
                "type": "TXT",
                "content": (
                    "v=DKIM1;k=rsa;s=email;h=sha256;"
                    "p=A/kinda+of/long/string+with+numb3rs"
                ),
            },
            {"name": "@", "expire": 3600, "type": "NS", "content": "6.2.3.4."},
            {"name": "@", "expire": 3600, "type": "NS", "content": "7.2.3.4."},
            {
                "name": "cname",
                "expire": 300,
                "type": "CNAME",
                "content": "unit.tests.",
            },
            {
                "name": "excluded",
                "expire": 3600,
                "type": "CNAME",
                "content": "unit.tests.",
            },
            {
                "name": "www.sub",
                "expire": 300,
                "type": "A",
                "content": "2.2.3.6",
            },
            {
                "name": "included",
                "expire": 3600,
                "type": "CNAME",
                "content": "unit.tests.",
            },
            {
                "name": "mx",
                "expire": 300,
                "type": "MX",
                "content": "10 smtp-4.unit.tests.",
            },
            {
                "name": "mx",
                "expire": 300,
                "type": "MX",
                "content": "20 smtp-2.unit.tests.",
            },
            {
                "name": "mx",
                "expire": 300,
                "type": "MX",
                "content": "30 smtp-3.unit.tests.",
            },
            {
                "name": "mx",
                "expire": 300,
                "type": "MX",
                "content": "40 smtp-1.unit.tests.",
            },
            {
                "name": "aaaa",
                "expire": 600,
                "type": "AAAA",
                "content": "2601:644:500:e210:62f8:1dff:feb8:947a",
            },
            {"name": "@", "expire": 300, "type": "A", "content": "1.2.3.4"},
            {"name": "@", "expire": 300, "type": "A", "content": "1.2.3.5"},
            {"name": "www", "expire": 300, "type": "A", "content": "2.2.3.6"},
            {
                "name": "@",
                "expire": 3600,
                "type": "SSHFP",
                "content": "1 1 7491973e5f8b39d5327cd4e08bc81b05f7710b49",
            },
            {
                "name": "@",
                "expire": 3600,
                "type": "SSHFP",
                "content": "1 1 bf6b6825d2977c511a475bbefb88aad54a92ac73",
            },
        ]
        # Unpack from the transip library magic structure...
        seen_entries = [
            e.__dict__["_attrs"]
            for e in domain_mock.dns.replace.mock_calls[0][1][0]
        ]
        self.assertEqual(
            sorted(seen_entries, key=itemgetter("name", "type", "expire")),
            sorted(expected_entries, key=itemgetter("name", "type", "expire")),
        )

    @patch("octodns.provider.transip.TransIP")
    def test_apply_unsupported(self, client_mock):
        # This triggers the if supported statement to give 100% code coverage
        domain_mock = Mock()
        client_mock.return_value.domains.get.return_value = domain_mock
        domain_mock.dns.list.return_value = []
        provider = TransipProvider("test", "unittest", self.bogus_key)

        plan = provider.plan(make_expected())
        self.assertIsNotNone(plan)

        # Test apply with only support for A records
        provider.SUPPORTS = set(("A"))

        provider.apply(plan)
        seen_entries = [
            e.__dict__["_attrs"]
            for e in domain_mock.dns.replace.mock_calls[0][1][0]
        ]
        expected_entries = [
            {
                "name": "ignored",
                "expire": 3600,
                "type": "A",
                "content": "9.9.9.9",
            },
            {
                "name": "www.sub",
                "expire": 300,
                "type": "A",
                "content": "2.2.3.6",
            },
            {"name": "@", "expire": 300, "type": "A", "content": "1.2.3.4"},
            {"name": "@", "expire": 300, "type": "A", "content": "1.2.3.5"},
            {"name": "www", "expire": 300, "type": "A", "content": "2.2.3.6"},
        ]
        self.assertEqual(
            sorted(seen_entries, key=itemgetter("name", "type", "expire")),
            sorted(expected_entries, key=itemgetter("name", "type", "expire")),
        )

    @patch("octodns.provider.transip.TransIP")
    def test_apply_failure_on_not_found(self, client_mock):
        # Test unhappy flow. Trigger 'not found error' in apply stage
        # This should normally not happen as populate will capture it first
        # but just in case.
        domain_mock = Mock()
        domain_mock.dns.list.return_value = []
        client_mock.return_value.domains.get.side_effect = [
            domain_mock,
            TransIPHTTPError("Not Found", 404),
        ]
        provider = TransipProvider("test", "unittest", self.bogus_key)

        plan = provider.plan(make_expected())

        with self.assertRaises(TransipException):
            provider.apply(plan)

    @patch("octodns.provider.transip.TransIP")
    def test_apply_failure_on_error(self, client_mock):
        # Test unhappy flow. Trigger a unrecoverable error while saving
        domain_mock = Mock()
        domain_mock.dns.list.return_value = []
        domain_mock.dns.replace.side_effect = [
            TransIPHTTPError("Not Found", 500)
        ]
        client_mock.return_value.domains.get.return_value = domain_mock
        provider = TransipProvider("test", "unittest", self.bogus_key)

        plan = provider.plan(make_expected())
 from unittest import TestCase

        with self.assertRaises(TransipException):
            provider.apply(plan)

 class TestTransipShim(TestCase):

 class TestParseFQDN(TestCase):
    def test_parse_fqdn(self):
        zone = Zone("unit.tests.", [])
        self.assertEqual("www.unit.tests.", _parse_to_fqdn("www", zone))
        self.assertEqual(
            "www.unit.tests.", _parse_to_fqdn("www.unit.tests.", zone)
        )
        self.assertEqual(
            "www.sub.sub.sub.unit.tests.",
            _parse_to_fqdn("www.sub.sub.sub", zone),
        )
        self.assertEqual("unit.tests.", _parse_to_fqdn("@", zone))
    def test_missing(self):
        with self.assertRaises(ModuleNotFoundError):
            from octodns.provider.transip import TransipProvider
            TransipProvider