diff --git a/CHANGELOG.md b/CHANGELOG.md index adb1f8c..6e2b243 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,12 @@ +## v0.9.6 - 2019-07-16 - The little one that fixes stuff from the big one + +* Reduced dynamic record value weight range to 0-15 so that Dyn and Route53 + match up behaviors. Dyn is limited to 0-15 and scaling that up would lose + resolution that couldn't be recovered during populate. +* Addressed issues with Route53 change set ordering for dynamic records +* Ignore unsupported record types in DigitalOceanProvider +* Fix bugs in Route53 extra changes handling and health check managagement + ## v0.9.5 - 2019-05-06 - The big one, with all the dynamic stuff * dynamic record support, essentially a v2 version of geo records with a lot diff --git a/README.md b/README.md index a3f3eae..aa9950e 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ - + ## DNS as code - Tools for managing DNS across multiple providers @@ -179,6 +179,7 @@ The above command pulled the existing data out of Route53 and placed the results | [DynProvider](/octodns/provider/dyn.py) | dyn | All | Both | | | [EtcHostsProvider](/octodns/provider/etc_hosts.py) | | A, AAAA, ALIAS, CNAME | No | | | [GoogleCloudProvider](/octodns/provider/googlecloud.py) | google-cloud-dns | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | No | | +| [MythicBeastsProvider](/octodns/provider/mythicbeasts.py) | Mythic Beasts | A, AAAA, ALIAS, CNAME, MX, NS, SRV, SSHFP, CAA, TXT | No | | | [Ns1Provider](/octodns/provider/ns1.py) | nsone | All | Partial Geo | No health checking for GeoDNS | | [OVH](/octodns/provider/ovh.py) | ovh | A, AAAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, SSHFP, TXT, DKIM | No | | | [PowerDnsProvider](/octodns/provider/powerdns.py) | | All | No | | @@ -275,4 +276,4 @@ GitHub® and its stylized versions and the Invertocat mark are GitHub's Trademar ## Authors -OctoDNS was designed and authored by [Ross McFarland](https://github.com/ross) and [Joe Williams](https://github.com/joewilliams). It is now maintained, reviewed, and tested by Ross, Joe, and the rest of the Site Reliability Engineering team at GitHub. +OctoDNS was designed and authored by [Ross McFarland](https://github.com/ross) and [Joe Williams](https://github.com/joewilliams). It is now maintained, reviewed, and tested by Traffic Engineering team at GitHub. diff --git a/octodns/__init__.py b/octodns/__init__.py index 939c293..6422577 100644 --- a/octodns/__init__.py +++ b/octodns/__init__.py @@ -3,4 +3,4 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -__VERSION__ = '0.9.5' +__VERSION__ = '0.9.6' diff --git a/octodns/provider/mythicbeasts.py b/octodns/provider/mythicbeasts.py new file mode 100644 index 0000000..17029db --- /dev/null +++ b/octodns/provider/mythicbeasts.py @@ -0,0 +1,474 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +import re + +from requests import Session +from logging import getLogger + +from ..record import Record +from .base import BaseProvider + +from collections import defaultdict + + +def add_trailing_dot(value): + ''' + Add trailing dots to values + ''' + assert value, 'Missing value' + assert value[-1] != '.', 'Value already has trailing dot' + return value + '.' + + +def remove_trailing_dot(value): + ''' + Remove trailing dots from values + ''' + assert value, 'Missing value' + assert value[-1] == '.', 'Value already missing trailing dot' + return value[:-1] + + +class MythicBeastsUnauthorizedException(Exception): + def __init__(self, zone, *args): + self.zone = zone + self.message = 'Mythic Beasts unauthorized for zone: {}'.format( + self.zone + ) + + super(MythicBeastsUnauthorizedException, self).__init__( + self.message, self.zone, *args) + + +class MythicBeastsRecordException(Exception): + def __init__(self, zone, command, *args): + self.zone = zone + self.command = command + self.message = 'Mythic Beasts could not action command: {} {}'.format( + self.zone, + self.command, + ) + + super(MythicBeastsRecordException, self).__init__( + self.message, self.zone, self.command, *args) + + +class MythicBeastsProvider(BaseProvider): + ''' + Mythic Beasts DNS API Provider + + Config settings: + + --- + providers: + config: + ... + mythicbeasts: + class: octodns.provider.mythicbeasts.MythicBeastsProvider + passwords: + my.domain.: 'password' + + zones: + my.domain.: + targets: + - mythic + ''' + + RE_MX = re.compile(r'^(?P[0-9]+)\s+(?P\S+)$', + re.IGNORECASE) + + RE_SRV = re.compile(r'^(?P[0-9]+)\s+(?P[0-9]+)\s+' + r'(?P[0-9]+)\s+(?P\S+)$', + re.IGNORECASE) + + RE_SSHFP = re.compile(r'^(?P[0-9]+)\s+' + r'(?P[0-9]+)\s+' + r'(?P\S+)$', + re.IGNORECASE) + + RE_CAA = re.compile(r'^(?P[0-9]+)\s+' + r'(?Pissue|issuewild|iodef)\s+' + r'(?P\S+)$', + re.IGNORECASE) + + RE_POPLINE = re.compile(r'^(?P\S+)\s+(?P\d+)\s+' + r'(?P\S+)\s+(?P.*)$', + re.IGNORECASE) + + SUPPORTS_GEO = False + SUPPORTS_DYNAMIC = False + SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CNAME', 'MX', 'NS', + 'SRV', 'SSHFP', 'CAA', 'TXT')) + BASE = 'https://dnsapi.mythic-beasts.com/' + + def __init__(self, identifier, passwords, *args, **kwargs): + self.log = getLogger('MythicBeastsProvider[{}]'.format(identifier)) + + assert isinstance(passwords, dict), 'Passwords must be a dictionary' + + self.log.debug( + '__init__: id=%s, registered zones; %s', + identifier, + passwords.keys()) + super(MythicBeastsProvider, self).__init__(identifier, *args, **kwargs) + + self._passwords = passwords + sess = Session() + self._sess = sess + + def _request(self, method, path, data=None): + self.log.debug('_request: method=%s, path=%s data=%s', + method, path, data) + + resp = self._sess.request(method, path, data=data) + self.log.debug( + '_request: status=%d data=%s', + resp.status_code, + resp.text[:20]) + + if resp.status_code == 401: + raise MythicBeastsUnauthorizedException(data['domain']) + + if resp.status_code == 400: + raise MythicBeastsRecordException( + data['domain'], + data['command'] + ) + resp.raise_for_status() + return resp + + def _post(self, data=None): + return self._request('POST', self.BASE, data=data) + + def records(self, zone): + assert zone in self._passwords, 'Missing password for domain: {}' \ + .format(remove_trailing_dot(zone)) + + return self._post({ + 'domain': remove_trailing_dot(zone), + 'password': self._passwords[zone], + 'showall': 0, + 'command': 'LIST', + }) + + @staticmethod + def _data_for_single(_type, data): + return { + 'type': _type, + 'value': data['raw_values'][0]['value'], + 'ttl': data['raw_values'][0]['ttl'] + } + + @staticmethod + def _data_for_multiple(_type, data): + return { + 'type': _type, + 'values': + [raw_values['value'] for raw_values in data['raw_values']], + 'ttl': + max([raw_values['ttl'] for raw_values in data['raw_values']]), + } + + @staticmethod + def _data_for_TXT(_type, data): + return { + 'type': _type, + 'values': + [ + str(raw_values['value']).replace(';', '\\;') + for raw_values in data['raw_values'] + ], + 'ttl': + max([raw_values['ttl'] for raw_values in data['raw_values']]), + } + + @staticmethod + def _data_for_MX(_type, data): + ttl = max([raw_values['ttl'] for raw_values in data['raw_values']]) + values = [] + + for raw_value in \ + [raw_values['value'] for raw_values in data['raw_values']]: + match = MythicBeastsProvider.RE_MX.match(raw_value) + + assert match is not None, 'Unable to parse MX data' + + exchange = match.group('exchange') + + if not exchange.endswith('.'): + exchange = '{}.{}'.format(exchange, data['zone']) + + values.append({ + 'preference': match.group('preference'), + 'exchange': exchange, + }) + + return { + 'type': _type, + 'values': values, + 'ttl': ttl, + } + + @staticmethod + def _data_for_CNAME(_type, data): + ttl = data['raw_values'][0]['ttl'] + value = data['raw_values'][0]['value'] + if not value.endswith('.'): + value = '{}.{}'.format(value, data['zone']) + + return MythicBeastsProvider._data_for_single( + _type, + {'raw_values': [ + {'value': value, 'ttl': ttl} + ]}) + + @staticmethod + def _data_for_ANAME(_type, data): + ttl = data['raw_values'][0]['ttl'] + value = data['raw_values'][0]['value'] + return MythicBeastsProvider._data_for_single( + 'ALIAS', + {'raw_values': [ + {'value': value, 'ttl': ttl} + ]}) + + @staticmethod + def _data_for_SRV(_type, data): + ttl = max([raw_values['ttl'] for raw_values in data['raw_values']]) + values = [] + + for raw_value in \ + [raw_values['value'] for raw_values in data['raw_values']]: + + match = MythicBeastsProvider.RE_SRV.match(raw_value) + + assert match is not None, 'Unable to parse SRV data' + + target = match.group('target') + if not target.endswith('.'): + target = '{}.{}'.format(target, data['zone']) + + values.append({ + 'priority': match.group('priority'), + 'weight': match.group('weight'), + 'port': match.group('port'), + 'target': target, + }) + + return { + 'type': _type, + 'values': values, + 'ttl': ttl, + } + + @staticmethod + def _data_for_SSHFP(_type, data): + ttl = max([raw_values['ttl'] for raw_values in data['raw_values']]) + values = [] + + for raw_value in \ + [raw_values['value'] for raw_values in data['raw_values']]: + match = MythicBeastsProvider.RE_SSHFP.match(raw_value) + + assert match is not None, 'Unable to parse SSHFP data' + + values.append({ + 'algorithm': match.group('algorithm'), + 'fingerprint_type': match.group('fingerprint_type'), + 'fingerprint': match.group('fingerprint'), + }) + + return { + 'type': _type, + 'values': values, + 'ttl': ttl, + } + + @staticmethod + def _data_for_CAA(_type, data): + ttl = data['raw_values'][0]['ttl'] + raw_value = data['raw_values'][0]['value'] + + match = MythicBeastsProvider.RE_CAA.match(raw_value) + + assert match is not None, 'Unable to parse CAA data' + + value = { + 'flags': match.group('flags'), + 'tag': match.group('tag'), + 'value': match.group('value'), + } + + return MythicBeastsProvider._data_for_single( + 'CAA', + {'raw_values': [{'value': value, 'ttl': ttl}]}) + + _data_for_NS = _data_for_multiple + _data_for_A = _data_for_multiple + _data_for_AAAA = _data_for_multiple + + def populate(self, zone, target=False, lenient=False): + self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, + target, lenient) + + resp = self.records(zone.name) + + before = len(zone.records) + exists = False + data = defaultdict(lambda: defaultdict(lambda: { + 'raw_values': [], + 'name': None, + 'zone': None, + })) + + exists = True + for line in resp.content.splitlines(): + match = MythicBeastsProvider.RE_POPLINE.match(line) + + if match is None: + self.log.debug('failed to match line: %s', line) + continue + + if match.group(1) == '@': + _name = '' + else: + _name = match.group('name') + + _type = match.group('type') + _ttl = int(match.group('ttl')) + _value = match.group('value').strip() + + if hasattr(self, '_data_for_{}'.format(_type)): + if _name not in data[_type]: + data[_type][_name] = { + 'raw_values': [{'value': _value, 'ttl': _ttl}], + 'name': _name, + 'zone': zone.name, + } + + else: + data[_type][_name].get('raw_values').append( + {'value': _value, 'ttl': _ttl} + ) + else: + self.log.debug('skipping %s as not supported', _type) + + for _type in data: + for _name in data[_type]: + data_for = getattr(self, '_data_for_{}'.format(_type)) + + record = Record.new( + zone, + _name, + data_for(_type, data[_type][_name]), + source=self + ) + zone.add_record(record, lenient=lenient) + + self.log.debug('populate: found %s records, exists=%s', + len(zone.records) - before, exists) + + return exists + + def _compile_commands(self, action, record): + commands = [] + + hostname = remove_trailing_dot(record.fqdn) + ttl = record.ttl + _type = record._type + + if _type == 'ALIAS': + _type = 'ANAME' + + if hasattr(record, 'values'): + values = record.values + else: + values = [record.value] + + base = '{} {} {} {}'.format(action, hostname, ttl, _type) + + # Unescape TXT records + if _type == 'TXT': + values = [value.replace('\\;', ';') for value in values] + + # Handle specific types or default + if _type == 'SSHFP': + data = values[0].data + commands.append('{} {} {} {}'.format( + base, + data['algorithm'], + data['fingerprint_type'], + data['fingerprint'] + )) + + elif _type == 'SRV': + for value in values: + data = value.data + commands.append('{} {} {} {} {}'.format( + base, + data['priority'], + data['weight'], + data['port'], + data['target'])) + + elif _type == 'MX': + for value in values: + data = value.data + commands.append('{} {} {}'.format( + base, + data['preference'], + data['exchange'])) + + else: + if hasattr(self, '_data_for_{}'.format(_type)): + for value in values: + commands.append('{} {}'.format(base, value)) + else: + self.log.debug('skipping %s as not supported', _type) + + return commands + + def _apply_Create(self, change): + zone = change.new.zone + commands = self._compile_commands('ADD', change.new) + + for command in commands: + self._post({ + 'domain': remove_trailing_dot(zone.name), + 'origin': '.', + 'password': self._passwords[zone.name], + 'command': command, + }) + return True + + def _apply_Update(self, change): + self._apply_Delete(change) + self._apply_Create(change) + + def _apply_Delete(self, change): + zone = change.existing.zone + commands = self._compile_commands('DELETE', change.existing) + + for command in commands: + self._post({ + 'domain': remove_trailing_dot(zone.name), + 'origin': '.', + 'password': self._passwords[zone.name], + 'command': command, + }) + return True + + def _apply(self, plan): + desired = plan.desired + changes = plan.changes + self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name, + len(changes)) + + for change in changes: + class_name = change.__class__.__name__ + getattr(self, '_apply_{}'.format(class_name))(change) diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index 1516f43..4cf7c99 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -519,37 +519,71 @@ class _Route53GeoRecord(_Route53Record): self.values) -_mod_keyer_action_order = { - 'DELETE': 0, # Delete things first - 'CREATE': 1, # Then Create things - 'UPSERT': 2, # Upsert things last -} - - def _mod_keyer(mod): rrset = mod['ResourceRecordSet'] - action_order = _mod_keyer_action_order[mod['Action']] - # We're sorting by 3 "columns", the action, the rrset type, and finally the - # name/id of the rrset. This ensures that Route53 won't see a RRSet that - # targets another that hasn't been seen yet. I.e. targets must come before - # things that target them. We sort on types of things rather than - # explicitly looking for targeting relationships since that's sufficent and - # easier to grok/do. + # Route53 requires that changes are ordered such that a target of an + # AliasTarget is created or upserted prior to the record that targets it. + # This is complicated by "UPSERT" appearing to be implemented as "DELETE" + # before all changes, followed by a "CREATE", internally in the AWS API. + # Because of this, we order changes as follows: + # - Delete any records that we wish to delete that are GEOS + # (because they are never targetted by anything) + # - Delete any records that we wish to delete that are SECONDARY + # (because they are no longer targetted by GEOS) + # - Delete any records that we wish to delete that are PRIMARY + # (because they are no longer targetted by SECONDARY) + # - Delete any records that we wish to delete that are VALUES + # (because they are no longer targetted by PRIMARY) + # - CREATE/UPSERT any records that are VALUES + # (because they don't depend on other records) + # - CREATE/UPSERT any records that are PRIMARY + # (because they always point to VALUES which now exist) + # - CREATE/UPSERT any records that are SECONDARY + # (because they now have PRIMARY records to target) + # - CREATE/UPSERT any records that are GEOS + # (because they now have all their PRIMARY pools to target) + # - :tada: + # + # In theory we could also do this based on actual target reference + # checking, but that's more complex. Since our rules have a known + # dependency order, we just rely on that. + + # Get the unique ID from the name/id to get a consistent ordering. + if rrset.get('GeoLocation', False): + unique_id = rrset['SetIdentifier'] + else: + unique_id = rrset['Name'] + # Prioritise within the action_priority, ensuring targets come first. if rrset.get('GeoLocation', False): - return (action_order, 3, rrset['SetIdentifier']) + # Geos reference pools, so they come last. + record_priority = 3 elif rrset.get('AliasTarget', False): # We use an alias if rrset.get('Failover', False) == 'SECONDARY': - # We're a secondary we'll ref primaries - return (action_order, 2, rrset['Name']) + # We're a secondary, which reference the primary (failover, P1). + record_priority = 2 else: - # We're a primary we'll ref values - return (action_order, 1, rrset['Name']) - - # We're just a plain value, these come first - return (action_order, 0, rrset['Name']) + # We're a primary, we reference values (P0). + record_priority = 1 + else: + # We're just a plain value, has no dependencies so first. + record_priority = 0 + + if mod['Action'] == 'DELETE': + # Delete things first, so we can never trounce our own additions + action_priority = 0 + # Delete in the reverse order of priority, e.g. start with the deepest + # reference and work back to the values, rather than starting at the + # values (still ref'd). + record_priority = -record_priority + else: + # For CREATE and UPSERT, Route53 seems to treat them the same, so + # interleave these, keeping the reference order described above. + action_priority = 1 + + return (action_priority, record_priority, unique_id) def _parse_pool_name(n): diff --git a/octodns/record/__init__.py b/octodns/record/__init__.py index dca6100..83632bc 100644 --- a/octodns/record/__init__.py +++ b/octodns/record/__init__.py @@ -514,7 +514,7 @@ class _DynamicMixin(object): try: weight = value['weight'] weight = int(weight) - if weight < 1 or weight > 255: + if weight < 1 or weight > 15: reasons.append('invalid weight "{}" in pool "{}" ' 'value {}'.format(weight, _id, value_num)) diff --git a/requirements-dev.txt b/requirements-dev.txt index 1afee06..77dd50c 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,4 +6,4 @@ pycountry>=18.12.8 pycountry_convert>=0.7.2 pyflakes==1.6.0 requests_mock -twine==1.11.0 +twine==1.13.0 diff --git a/setup.py b/setup.py index 7a9348e..75a39d7 100644 --- a/setup.py +++ b/setup.py @@ -41,6 +41,7 @@ setup( ], license='MIT', long_description=open('README.md').read(), + long_description_content_type='text/markdown', name='octodns', packages=find_packages(), url='https://github.com/github/octodns', diff --git a/tests/config/dynamic.tests.yaml b/tests/config/dynamic.tests.yaml index 3d806f9..f826880 100644 --- a/tests/config/dynamic.tests.yaml +++ b/tests/config/dynamic.tests.yaml @@ -19,7 +19,7 @@ a: - value: 6.6.6.6 weight: 10 - value: 5.5.5.5 - weight: 25 + weight: 15 rules: - geos: - EU-GB @@ -90,9 +90,9 @@ cname: sea: values: - value: target-sea-1.unit.tests. - weight: 100 + weight: 10 - value: target-sea-2.unit.tests. - weight: 175 + weight: 14 rules: - geos: - EU-GB diff --git a/tests/config/split/dynamic.tests./a.yaml b/tests/config/split/dynamic.tests./a.yaml index fd748b4..f182df6 100644 --- a/tests/config/split/dynamic.tests./a.yaml +++ b/tests/config/split/dynamic.tests./a.yaml @@ -23,7 +23,7 @@ a: fallback: null values: - value: 5.5.5.5 - weight: 25 + weight: 15 - value: 6.6.6.6 weight: 10 rules: diff --git a/tests/config/split/dynamic.tests./cname.yaml b/tests/config/split/dynamic.tests./cname.yaml index a84c202..ff85955 100644 --- a/tests/config/split/dynamic.tests./cname.yaml +++ b/tests/config/split/dynamic.tests./cname.yaml @@ -21,9 +21,9 @@ cname: fallback: null values: - value: target-sea-1.unit.tests. - weight: 100 + weight: 10 - value: target-sea-2.unit.tests. - weight: 175 + weight: 14 rules: - geos: - EU-GB diff --git a/tests/fixtures/mythicbeasts-list.txt b/tests/fixtures/mythicbeasts-list.txt new file mode 100644 index 0000000..ed4ea4c --- /dev/null +++ b/tests/fixtures/mythicbeasts-list.txt @@ -0,0 +1,25 @@ +@ 3600 NS 6.2.3.4. +@ 3600 NS 7.2.3.4. +@ 300 A 1.2.3.4 +@ 300 A 1.2.3.5 +@ 3600 SSHFP 1 1 bf6b6825d2977c511a475bbefb88aad54a92ac73 +@ 3600 SSHFP 1 1 7491973e5f8b39d5327cd4e08bc81b05f7710b49 +@ 3600 CAA 0 issue ca.unit.tests +_srv._tcp 600 SRV 10 20 30 foo-1.unit.tests. +_srv._tcp 600 SRV 12 20 30 foo-2.unit.tests. +aaaa 600 AAAA 2601:644:500:e210:62f8:1dff:feb8:947a +cname 300 CNAME unit.tests. +excluded 300 CNAME unit.tests. +ignored 300 A 9.9.9.9 +included 3600 CNAME unit.tests. +mx 300 MX 10 smtp-4.unit.tests. +mx 300 MX 20 smtp-2.unit.tests. +mx 300 MX 30 smtp-3.unit.tests. +mx 300 MX 40 smtp-1.unit.tests. +sub 3600 NS 6.2.3.4. +sub 3600 NS 7.2.3.4. +txt 600 TXT "Bah bah black sheep" +txt 600 TXT "have you any wool." +txt 600 TXT "v=DKIM1;k=rsa;s=email;h=sha256;p=A/kinda+of/long/string+with+numb3rs" +www 300 A 2.2.3.6 +www.sub 300 A 2.2.3.6 diff --git a/tests/test_octodns_provider_mythicbeasts.py b/tests/test_octodns_provider_mythicbeasts.py new file mode 100644 index 0000000..5acbc55 --- /dev/null +++ b/tests/test_octodns_provider_mythicbeasts.py @@ -0,0 +1,451 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from os.path import dirname, join + +from requests_mock import ANY, mock as requests_mock +from unittest import TestCase + +from octodns.provider.mythicbeasts import MythicBeastsProvider, \ + add_trailing_dot, remove_trailing_dot +from octodns.provider.yaml import YamlProvider +from octodns.zone import Zone +from octodns.record import Create, Update, Delete, Record + + +class TestMythicBeastsProvider(TestCase): + expected = Zone('unit.tests.', []) + source = YamlProvider('test_expected', join(dirname(__file__), 'config')) + source.populate(expected) + + # Dump anything we don't support from expected + for record in list(expected.records): + if record._type not in MythicBeastsProvider.SUPPORTS: + expected._remove_record(record) + + def test_trailing_dot(self): + with self.assertRaises(AssertionError) as err: + add_trailing_dot('unit.tests.') + self.assertEquals('Value already has trailing dot', + err.exception.message) + + with self.assertRaises(AssertionError) as err: + remove_trailing_dot('unit.tests') + self.assertEquals('Value already missing trailing dot', + err.exception.message) + + self.assertEquals(add_trailing_dot('unit.tests'), 'unit.tests.') + self.assertEquals(remove_trailing_dot('unit.tests.'), 'unit.tests') + + def test_data_for_single(self): + test_data = { + 'raw_values': [{'value': 'a:a::c', 'ttl': 0}], + 'zone': 'unit.tests.', + } + test_single = MythicBeastsProvider._data_for_single('', test_data) + self.assertTrue(isinstance(test_single, dict)) + self.assertEquals('a:a::c', test_single['value']) + + def test_data_for_multiple(self): + test_data = { + 'raw_values': [ + {'value': 'b:b::d', 'ttl': 60}, + {'value': 'a:a::c', 'ttl': 60}], + 'zone': 'unit.tests.', + } + test_multiple = MythicBeastsProvider._data_for_multiple('', test_data) + self.assertTrue(isinstance(test_multiple, dict)) + self.assertEquals(2, len(test_multiple['values'])) + + def test_data_for_txt(self): + test_data = { + 'raw_values': [ + {'value': 'v=DKIM1; k=rsa; p=prawf', 'ttl': 60}, + {'value': 'prawf prawf dyma prawf', 'ttl': 300}], + 'zone': 'unit.tests.', + } + test_txt = MythicBeastsProvider._data_for_TXT('', test_data) + self.assertTrue(isinstance(test_txt, dict)) + self.assertEquals(2, len(test_txt['values'])) + self.assertEquals('v=DKIM1\\; k=rsa\\; p=prawf', test_txt['values'][0]) + + def test_data_for_MX(self): + test_data = { + 'raw_values': [ + {'value': '10 un.unit', 'ttl': 60}, + {'value': '20 dau.unit', 'ttl': 60}, + {'value': '30 tri.unit', 'ttl': 60}], + 'zone': 'unit.tests.', + } + test_MX = MythicBeastsProvider._data_for_MX('', test_data) + self.assertTrue(isinstance(test_MX, dict)) + self.assertEquals(3, len(test_MX['values'])) + + with self.assertRaises(AssertionError) as err: + test_MX = MythicBeastsProvider._data_for_MX( + '', + {'raw_values': [{'value': '', 'ttl': 0}]} + ) + self.assertEquals('Unable to parse MX data', + err.exception.message) + + def test_data_for_CNAME(self): + test_data = { + 'raw_values': [{'value': 'cname', 'ttl': 60}], + 'zone': 'unit.tests.', + } + test_cname = MythicBeastsProvider._data_for_CNAME('', test_data) + self.assertTrue(isinstance(test_cname, dict)) + self.assertEquals('cname.unit.tests.', test_cname['value']) + + def test_data_for_ANAME(self): + test_data = { + 'raw_values': [{'value': 'aname', 'ttl': 60}], + 'zone': 'unit.tests.', + } + test_aname = MythicBeastsProvider._data_for_ANAME('', test_data) + self.assertTrue(isinstance(test_aname, dict)) + self.assertEquals('aname', test_aname['value']) + + def test_data_for_SRV(self): + test_data = { + 'raw_values': [ + {'value': '10 20 30 un.srv.unit', 'ttl': 60}, + {'value': '20 30 40 dau.srv.unit', 'ttl': 60}, + {'value': '30 30 50 tri.srv.unit', 'ttl': 60}], + 'zone': 'unit.tests.', + } + test_SRV = MythicBeastsProvider._data_for_SRV('', test_data) + self.assertTrue(isinstance(test_SRV, dict)) + self.assertEquals(3, len(test_SRV['values'])) + + with self.assertRaises(AssertionError) as err: + test_SRV = MythicBeastsProvider._data_for_SRV( + '', + {'raw_values': [{'value': '', 'ttl': 0}]} + ) + self.assertEquals('Unable to parse SRV data', + err.exception.message) + + def test_data_for_SSHFP(self): + test_data = { + 'raw_values': [ + {'value': '1 1 0123456789abcdef', 'ttl': 60}, + {'value': '1 2 0123456789abcdef', 'ttl': 60}, + {'value': '2 3 0123456789abcdef', 'ttl': 60}], + 'zone': 'unit.tests.', + } + test_SSHFP = MythicBeastsProvider._data_for_SSHFP('', test_data) + self.assertTrue(isinstance(test_SSHFP, dict)) + self.assertEquals(3, len(test_SSHFP['values'])) + + with self.assertRaises(AssertionError) as err: + test_SSHFP = MythicBeastsProvider._data_for_SSHFP( + '', + {'raw_values': [{'value': '', 'ttl': 0}]} + ) + self.assertEquals('Unable to parse SSHFP data', + err.exception.message) + + def test_data_for_CAA(self): + test_data = { + 'raw_values': [{'value': '1 issue letsencrypt.org', 'ttl': 60}], + 'zone': 'unit.tests.', + } + test_CAA = MythicBeastsProvider._data_for_CAA('', test_data) + self.assertTrue(isinstance(test_CAA, dict)) + self.assertEquals(3, len(test_CAA['value'])) + + with self.assertRaises(AssertionError) as err: + test_CAA = MythicBeastsProvider._data_for_CAA( + '', + {'raw_values': [{'value': '', 'ttl': 0}]} + ) + self.assertEquals('Unable to parse CAA data', + err.exception.message) + + def test_command_generation(self): + zone = Zone('unit.tests.', []) + zone.add_record(Record.new(zone, 'prawf-alias', { + 'ttl': 60, + 'type': 'ALIAS', + 'value': 'alias.unit.tests.', + })) + zone.add_record(Record.new(zone, 'prawf-ns', { + 'ttl': 300, + 'type': 'NS', + 'values': [ + 'alias.unit.tests.', + 'alias2.unit.tests.', + ], + })) + zone.add_record(Record.new(zone, 'prawf-a', { + 'ttl': 60, + 'type': 'A', + 'values': [ + '1.2.3.4', + '5.6.7.8', + ], + })) + zone.add_record(Record.new(zone, 'prawf-aaaa', { + 'ttl': 60, + 'type': 'AAAA', + 'values': [ + 'a:a::a', + 'b:b::b', + 'c:c::c:c', + ], + })) + zone.add_record(Record.new(zone, 'prawf-txt', { + 'ttl': 60, + 'type': 'TXT', + 'value': 'prawf prawf dyma prawf', + })) + zone.add_record(Record.new(zone, 'prawf-txt2', { + 'ttl': 60, + 'type': 'TXT', + 'value': 'v=DKIM1\\; k=rsa\\; p=prawf', + })) + with requests_mock() as mock: + mock.post(ANY, status_code=200, text='') + + provider = MythicBeastsProvider('test', { + 'unit.tests.': 'mypassword' + }) + + plan = provider.plan(zone) + changes = plan.changes + generated_commands = [] + + for change in changes: + generated_commands.extend( + provider._compile_commands('ADD', change.new) + ) + + expected_commands = [ + 'ADD prawf-alias.unit.tests 60 ANAME alias.unit.tests.', + 'ADD prawf-ns.unit.tests 300 NS alias.unit.tests.', + 'ADD prawf-ns.unit.tests 300 NS alias2.unit.tests.', + 'ADD prawf-a.unit.tests 60 A 1.2.3.4', + 'ADD prawf-a.unit.tests 60 A 5.6.7.8', + 'ADD prawf-aaaa.unit.tests 60 AAAA a:a::a', + 'ADD prawf-aaaa.unit.tests 60 AAAA b:b::b', + 'ADD prawf-aaaa.unit.tests 60 AAAA c:c::c:c', + 'ADD prawf-txt.unit.tests 60 TXT prawf prawf dyma prawf', + 'ADD prawf-txt2.unit.tests 60 TXT v=DKIM1; k=rsa; p=prawf', + ] + + generated_commands.sort() + expected_commands.sort() + + self.assertEquals( + generated_commands, + expected_commands + ) + + # Now test deletion + existing = 'prawf-txt 300 TXT prawf prawf dyma prawf\n' \ + 'prawf-txt2 300 TXT v=DKIM1; k=rsa; p=prawf\n' \ + 'prawf-a 60 A 1.2.3.4' + + with requests_mock() as mock: + mock.post(ANY, status_code=200, text=existing) + wanted = Zone('unit.tests.', []) + + plan = provider.plan(wanted) + changes = plan.changes + generated_commands = [] + + for change in changes: + generated_commands.extend( + provider._compile_commands('DELETE', change.existing) + ) + + expected_commands = [ + 'DELETE prawf-a.unit.tests 60 A 1.2.3.4', + 'DELETE prawf-txt.unit.tests 300 TXT prawf prawf dyma prawf', + 'DELETE prawf-txt2.unit.tests 300 TXT v=DKIM1; k=rsa; p=prawf', + ] + + generated_commands.sort() + expected_commands.sort() + + self.assertEquals( + generated_commands, + expected_commands + ) + + def test_fake_command_generation(self): + class FakeChangeRecord(object): + def __init__(self): + self.__fqdn = 'prawf.unit.tests.' + self._type = 'NOOP' + self.value = 'prawf' + self.ttl = 60 + + @property + def record(self): + return self + + @property + def fqdn(self): + return self.__fqdn + + with requests_mock() as mock: + mock.post(ANY, status_code=200, text='') + + provider = MythicBeastsProvider('test', { + 'unit.tests.': 'mypassword' + }) + record = FakeChangeRecord() + command = provider._compile_commands('ADD', record) + self.assertEquals([], command) + + def test_populate(self): + provider = None + + # Null passwords dict + with self.assertRaises(AssertionError) as err: + provider = MythicBeastsProvider('test', None) + self.assertEquals('Passwords must be a dictionary', + err.exception.message) + + # Missing password + with requests_mock() as mock: + mock.post(ANY, status_code=401, text='ERR Not authenticated') + + with self.assertRaises(AssertionError) as err: + provider = MythicBeastsProvider('test', dict()) + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals( + 'Missing password for domain: unit.tests', + err.exception.message) + + # Failed authentication + with requests_mock() as mock: + mock.post(ANY, status_code=401, text='ERR Not authenticated') + + with self.assertRaises(Exception) as err: + provider = MythicBeastsProvider('test', { + 'unit.tests.': 'mypassword' + }) + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals( + 'Mythic Beasts unauthorized for zone: unit.tests', + err.exception.message) + + # Check unmatched lines are ignored + test_data = 'This should not match' + with requests_mock() as mock: + mock.post(ANY, status_code=200, text=test_data) + + provider = MythicBeastsProvider('test', { + 'unit.tests.': 'mypassword' + }) + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals(0, len(zone.records)) + + # Check unsupported records are skipped + test_data = '@ 60 NOOP prawf\n@ 60 SPF prawf prawf prawf' + with requests_mock() as mock: + mock.post(ANY, status_code=200, text=test_data) + + provider = MythicBeastsProvider('test', { + 'unit.tests.': 'mypassword' + }) + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals(0, len(zone.records)) + + # Check no changes between what we support and what's parsed + # from the unit.tests. config YAML. Also make sure we see the same + # for both after we've thrown away records we don't support + with requests_mock() as mock: + with open('tests/fixtures/mythicbeasts-list.txt') as file_handle: + mock.post(ANY, status_code=200, text=file_handle.read()) + + provider = MythicBeastsProvider('test', { + 'unit.tests.': 'mypassword' + }) + zone = Zone('unit.tests.', []) + provider.populate(zone) + + self.assertEquals(15, len(zone.records)) + self.assertEquals(15, len(self.expected.records)) + changes = self.expected.changes(zone, provider) + self.assertEquals(0, len(changes)) + + def test_apply(self): + provider = MythicBeastsProvider('test', { + 'unit.tests.': 'mypassword' + }) + zone = Zone('unit.tests.', []) + + # Create blank zone + with requests_mock() as mock: + mock.post(ANY, status_code=200, text='') + provider.populate(zone) + + self.assertEquals(0, len(zone.records)) + + # Record change failed + with requests_mock() as mock: + mock.post(ANY, status_code=200, text='') + provider.populate(zone) + zone.add_record(Record.new(zone, 'prawf', { + 'ttl': 300, + 'type': 'TXT', + 'value': 'prawf', + })) + plan = provider.plan(zone) + + with requests_mock() as mock: + mock.post(ANY, status_code=400, text='NADD 300 TXT prawf') + + with self.assertRaises(Exception) as err: + provider.apply(plan) + self.assertEquals( + 'Mythic Beasts could not action command: unit.tests ' + 'ADD prawf.unit.tests 300 TXT prawf', + err.exception.message) + + # Check deleting and adding/changing test record + existing = 'prawf 300 TXT prawf prawf prawf\ndileu 300 TXT dileu' + + with requests_mock() as mock: + mock.post(ANY, status_code=200, text=existing) + + # Mash up a new zone with records so a plan + # is generated with changes and applied. For some reason + # passing self.expected, or just changing each record's zone + # doesn't work. Nor does this without a single add_record after + wanted = Zone('unit.tests.', []) + for record in list(self.expected.records): + data = {'type': record._type} + data.update(record.data) + wanted.add_record(Record.new(wanted, record.name, data)) + + wanted.add_record(Record.new(wanted, 'prawf', { + 'ttl': 60, + 'type': 'TXT', + 'value': 'prawf yw e', + })) + + plan = provider.plan(wanted) + + # Octo ignores NS records (15-1) + self.assertEquals(1, len(filter(lambda u: isinstance(u, Update), + plan.changes))) + self.assertEquals(1, len(filter(lambda d: isinstance(d, Delete), + plan.changes))) + self.assertEquals(14, len(filter(lambda c: isinstance(c, Create), + plan.changes))) + self.assertEquals(16, provider.apply(plan)) + self.assertTrue(plan.exists) diff --git a/tests/test_octodns_provider_route53.py b/tests/test_octodns_provider_route53.py index 849ea2b..265a0a7 100644 --- a/tests/test_octodns_provider_route53.py +++ b/tests/test_octodns_provider_route53.py @@ -700,18 +700,6 @@ class TestRoute53Provider(TestCase): 'TTL': 61, 'Type': 'A' } - }, { - 'Action': 'CREATE', - 'ResourceRecordSet': { - 'GeoLocation': {'CountryCode': 'US', - 'SubdivisionCode': 'CA'}, - 'HealthCheckId': u'44', - 'Name': 'unit.tests.', - 'ResourceRecords': [{'Value': '7.2.3.4'}], - 'SetIdentifier': 'NA-US-CA', - 'TTL': 61, - 'Type': 'A' - } }, { 'Action': 'UPSERT', 'ResourceRecordSet': { @@ -735,6 +723,18 @@ class TestRoute53Provider(TestCase): 'TTL': 61, 'Type': 'A' } + }, { + 'Action': 'CREATE', + 'ResourceRecordSet': { + 'GeoLocation': {'CountryCode': 'US', + 'SubdivisionCode': 'CA'}, + 'HealthCheckId': u'44', + 'Name': 'unit.tests.', + 'ResourceRecords': [{'Value': '7.2.3.4'}], + 'SetIdentifier': 'NA-US-CA', + 'TTL': 61, + 'Type': 'A' + } }, { 'Action': 'UPSERT', 'ResourceRecordSet': { @@ -2426,7 +2426,7 @@ class TestModKeyer(TestCase): def test_mod_keyer(self): - # First "column" + # First "column" is the action priority for C/R/U # Deletes come first self.assertEquals((0, 0, 'something'), _mod_keyer({ @@ -2444,8 +2444,8 @@ class TestModKeyer(TestCase): } })) - # Then upserts - self.assertEquals((2, 0, 'last'), _mod_keyer({ + # Upserts are the same as creates + self.assertEquals((1, 0, 'last'), _mod_keyer({ 'Action': 'UPSERT', 'ResourceRecordSet': { 'Name': 'last', @@ -2455,7 +2455,7 @@ class TestModKeyer(TestCase): # Second "column" value records tested above # AliasTarget primary second (to value) - self.assertEquals((0, 1, 'thing'), _mod_keyer({ + self.assertEquals((0, -1, 'thing'), _mod_keyer({ 'Action': 'DELETE', 'ResourceRecordSet': { 'AliasTarget': 'some-target', @@ -2464,8 +2464,17 @@ class TestModKeyer(TestCase): } })) + self.assertEquals((1, 1, 'thing'), _mod_keyer({ + 'Action': 'UPSERT', + 'ResourceRecordSet': { + 'AliasTarget': 'some-target', + 'Failover': 'PRIMARY', + 'Name': 'thing', + } + })) + # AliasTarget secondary third - self.assertEquals((0, 2, 'thing'), _mod_keyer({ + self.assertEquals((0, -2, 'thing'), _mod_keyer({ 'Action': 'DELETE', 'ResourceRecordSet': { 'AliasTarget': 'some-target', @@ -2474,8 +2483,17 @@ class TestModKeyer(TestCase): } })) + self.assertEquals((1, 2, 'thing'), _mod_keyer({ + 'Action': 'UPSERT', + 'ResourceRecordSet': { + 'AliasTarget': 'some-target', + 'Failover': 'SECONDARY', + 'Name': 'thing', + } + })) + # GeoLocation fourth - self.assertEquals((0, 3, 'some-id'), _mod_keyer({ + self.assertEquals((0, -3, 'some-id'), _mod_keyer({ 'Action': 'DELETE', 'ResourceRecordSet': { 'GeoLocation': 'some-target', @@ -2483,4 +2501,12 @@ class TestModKeyer(TestCase): } })) + self.assertEquals((1, 3, 'some-id'), _mod_keyer({ + 'Action': 'UPSERT', + 'ResourceRecordSet': { + 'GeoLocation': 'some-target', + 'SetIdentifier': 'some-id', + } + })) + # The third "column" has already been tested above, Name/SetIdentifier diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index 53bc5e7..2b11364 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -2460,7 +2460,7 @@ class TestDynamicRecords(TestCase): 'weight': 1, 'value': '6.6.6.6', }, { - 'weight': 256, + 'weight': 16, 'value': '7.7.7.7', }], }, @@ -2484,7 +2484,7 @@ class TestDynamicRecords(TestCase): } with self.assertRaises(ValidationError) as ctx: Record.new(self.zone, 'bad', a_data) - self.assertEquals(['invalid weight "256" in pool "three" value 2'], + self.assertEquals(['invalid weight "16" in pool "three" value 2'], ctx.exception.reasons) # invalid non-int weight