diff --git a/README.md b/README.md index 28d9e7f..ebd7111 100644 --- a/README.md +++ b/README.md @@ -195,7 +195,7 @@ The above command pulled the existing data out of Route53 and placed the results | [AzureProvider](/octodns/provider/azuredns.py) | azure-identity, azure-mgmt-dns, azure-mgmt-trafficmanager | A, AAAA, CAA, CNAME, MX, NS, PTR, SRV, TXT | Alpha (A, AAAA, CNAME) | | | [Akamai](/octodns/provider/edgedns.py) | edgegrid-python | A, AAAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, SSHFP, TXT | No | | | [CloudflareProvider](/octodns/provider/cloudflare.py) | | A, AAAA, ALIAS, CAA, CNAME, LOC, MX, NS, PTR, SPF, SRV, TXT | No | CAA tags restricted | -| [ConstellixProvider](/octodns/provider/constellix.py) | | A, AAAA, ALIAS (ANAME), CAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | CAA tags restricted | +| [ConstellixProvider](/octodns/provider/constellix.py) | | A, AAAA, ALIAS (ANAME), CAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | Yes | CAA tags restricted | | [DigitalOceanProvider](/octodns/provider/digitalocean.py) | | A, AAAA, CAA, CNAME, MX, NS, TXT, SRV | No | CAA tags restricted | | [DnsMadeEasyProvider](/octodns/provider/dnsmadeeasy.py) | | A, AAAA, ALIAS (ANAME), CAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | CAA tags restricted | | [DnsimpleProvider](/octodns/provider/dnsimple.py) | | All | No | CAA tags restricted | diff --git a/octodns/provider/constellix.py b/octodns/provider/constellix.py index cc65ae1..3f75650 100644 --- a/octodns/provider/constellix.py +++ b/octodns/provider/constellix.py @@ -9,6 +9,7 @@ from collections import defaultdict from requests import Session from base64 import b64encode from six import string_types +from pycountry_convert import country_alpha2_to_continent_code import hashlib import hmac import logging @@ -53,7 +54,8 @@ class ConstellixClient(object): self._sess = Session() self._sess.headers.update({'x-cnsdns-apiKey': self.api_key}) self._domains = None - self._pools = None + self._pools = {'A': None, 'AAAA': None, 'CNAME': None} + self._geofilters = None def _current_time(self): return str(int(time.time() * 1000)) @@ -100,19 +102,29 @@ class ConstellixClient(object): zone_id = self.domains.get(name, False) if not zone_id: raise ConstellixClientNotFound() - path = '/domains/{}'.format(zone_id) + path = f'/domains/{zone_id}' return self._request('GET', path).json() def domain_create(self, name): resp = self._request('POST', '/domains', data={'names': [name]}) # Add newly created zone to domain cache - self._domains['{}.'.format(name)] = resp.json()[0]['id'] + self._domains[f'{name}.'] = resp.json()[0]['id'] + + def domain_enable_geoip(self, domain_name): + domain = self.domain(domain_name) + if domain['hasGeoIP'] is False: + domain_id = self.domains[domain_name] + self._request( + 'PUT', + f'/domains/{domain_id}', + data={'hasGeoIP': True} + ) def _absolutize_value(self, value, zone_name): if value == '': value = zone_name elif not value.endswith('.'): - value = '{}.{}'.format(value, zone_name) + value = f'{value}.{zone_name}' return value @@ -120,7 +132,7 @@ class ConstellixClient(object): zone_id = self.domains.get(zone_name, False) if not zone_id: raise ConstellixClientNotFound() - path = '/domains/{}/records'.format(zone_id) + path = f'/domains/{zone_id}/records' resp = self._request('GET', path).json() for record in resp: @@ -147,7 +159,7 @@ class ConstellixClient(object): record_type = 'ANAME' zone_id = self.domains.get(zone_name, False) - path = '/domains/{}/records/{}'.format(zone_id, record_type) + path = f'/domains/{zone_id}/records/{record_type}' self._request('POST', path, data=params) @@ -157,18 +169,17 @@ class ConstellixClient(object): record_type = 'ANAME' zone_id = self.domains.get(zone_name, False) - path = '/domains/{}/records/{}/{}'.format(zone_id, record_type, - record_id) + path = f'/domains/{zone_id}/records/{record_type}/{record_id}' self._request('DELETE', path) def pools(self, pool_type): - if self._pools is None: - self._pools = {} - path = '/pools/{}'.format(pool_type) + if self._pools[pool_type] is None: + self._pools[pool_type] = {} + path = f'/pools/{pool_type}' response = self._request('GET', path).json() for pool in response: - self._pools[pool['id']] = pool - return self._pools.values() + self._pools[pool_type][pool['id']] = pool + return self._pools[pool_type].values() def pool(self, pool_type, pool_name): pools = self.pools(pool_type) @@ -186,11 +197,11 @@ class ConstellixClient(object): def pool_create(self, data): path = '/pools/{}'.format(data.get('type')) # This returns a list of items, we want the first one - response = self._request('POST', path, data=data).json()[0] + response = self._request('POST', path, data=data).json() - # Invalidate our cache - self._pools = None - return response + # Update our cache + self._pools[data.get('type')][response[0]['id']] = response[0] + return response[0] def pool_update(self, pool_id, data): path = '/pools/{}/{}'.format(data.get('type'), pool_id) @@ -203,6 +214,63 @@ class ConstellixClient(object): raise e return data + def pool_delete(self, pool_type, pool_id): + path = f'/pools/{pool_type}/{pool_id}' + self._request('DELETE', path) + + # Update our cache + if self._pools[pool_type] is not None: + self._pools[pool_type].pop(pool_id, None) + + def geofilters(self): + if self._geofilters is None: + self._geofilters = {} + path = '/geoFilters' + response = self._request('GET', path).json() + for geofilter in response: + self._geofilters[geofilter['id']] = geofilter + return self._geofilters.values() + + def geofilter(self, geofilter_name): + geofilters = self.geofilters() + for geofilter in geofilters: + if geofilter['name'] == geofilter_name: + return geofilter + return None + + def geofilter_by_id(self, geofilter_id): + geofilters = self.geofilters() + for geofilter in geofilters: + if geofilter['id'] == geofilter_id: + return geofilter + + def geofilter_create(self, data): + path = '/geoFilters' + response = self._request('POST', path, data=data).json() + + # Update our cache + self._geofilters[response[0]['id']] = response[0] + return response[0] + + def geofilter_update(self, geofilter_id, data): + path = f'/geoFilters/{geofilter_id}' + try: + self._request('PUT', path, data=data).json() + + except ConstellixClientBadRequest as e: + message = str(e) + if not message or "no changes to save" not in message: + raise e + return data + + def geofilter_delete(self, geofilter_id): + path = f'/geoFilters/{geofilter_id}' + self._request('DELETE', path) + + # Update our cache + if self._geofilters is not None: + self._geofilters.pop(geofilter_id, None) + class ConstellixProvider(BaseProvider): ''' @@ -225,7 +293,7 @@ class ConstellixProvider(BaseProvider): def __init__(self, id, api_key, secret_key, ratelimit_delay=0.0, *args, **kwargs): - self.log = logging.getLogger('ConstellixProvider[{}]'.format(id)) + self.log = logging.getLogger(f'ConstellixProvider[{id}]') self.log.debug('__init__: id=%s, api_key=***, secret_key=***', id) super(ConstellixProvider, self).__init__(id, *args, **kwargs) self._client = ConstellixClient(api_key, secret_key, ratelimit_delay) @@ -234,39 +302,95 @@ class ConstellixProvider(BaseProvider): def _data_for_multiple(self, _type, records): record = records[0] if record['recordOption'] == 'pools': - return self._data_for_pool(_type, record) + return self._data_for_pool(_type, records) return { 'ttl': record['ttl'], 'type': _type, 'values': record['value'] } - def _data_for_pool(self, _type, record): - pool_id = record['pools'][0] - pool = self._client.pool_by_id(_type, pool_id) - pool_name = pool['name'].split(':')[-1] + def _data_for_pool(self, _type, records): + default_values = [] + fallback_pool_name = None pools = {} - values = [] - pools[pool_name] = { - 'values': [] - } - for value in pool['values']: - pools[pool_name]['values'].append({ - 'value': value['value'], - 'weight': value['weight'] - }) - values.append(value['value']) - return { + rules = [] + + for record in records: + # fetch record pool data + pool_id = record['pools'][0] + pool = self._client.pool_by_id(_type, pool_id) + + geofilter_id = 1 + if 'geolocation' in record.keys() \ + and record['geolocation'] is not None: + # fetch record geofilter data + geofilter_id = record['geolocation']['geoipFilter'] + geofilter = self._client.geofilter_by_id(geofilter_id) + + pool_name = pool['name'].split(':')[-1] + + # fetch default values from the World Default pool + if geofilter_id == 1: + fallback_pool_name = pool_name + for value in pool['values']: + default_values.append(value['value']) + + # populate pools + pools[pool_name] = { + 'fallback': None, + 'values': [] + } + for value in pool['values']: + pools[pool_name]['values'].append({ + 'value': value['value'], + 'weight': value['weight'] + }) + + # populate rules + if geofilter_id == 1: + rules.append({'pool': pool_name}) + else: + geos = [] + + if 'geoipContinents' in geofilter.keys(): + for continent_code in geofilter['geoipContinents']: + geos.append(continent_code) + + if 'geoipCountries' in geofilter.keys(): + for country_code in geofilter['geoipCountries']: + geos.append('{}-{}'.format( + country_alpha2_to_continent_code(country_code), + country_code + )) + + if 'regions' in geofilter.keys(): + for region in geofilter['regions']: + geos.append('{}-{}-{}'.format( + region['continentCode'], + region['countryCode'], + region['regionCode'])) + + rules.append({ + 'pool': pool_name, + 'geos': sorted(geos) + }) + + # set fallback pool + for pool_name in pools: + if pool_name != fallback_pool_name: + pools[pool_name]['fallback'] = fallback_pool_name + + res = { 'ttl': record['ttl'], 'type': _type, 'dynamic': { - 'pools': pools, - 'rules': [{ - 'pool': pool_name - }] + 'pools': dict( + sorted(pools.items(), key=lambda t: t[0])), + 'rules': sorted(rules, key=lambda t: t['pool']) }, - 'value': values + 'values': default_values } + return res _data_for_A = _data_for_multiple _data_for_AAAA = _data_for_multiple @@ -381,7 +505,7 @@ class ConstellixProvider(BaseProvider): before = len(zone.records) for name, types in values.items(): for _type, records in types.items(): - data_for = getattr(self, '_data_for_{}'.format(_type)) + data_for = getattr(self, f'_data_for_{_type}') record = Record.new(zone, name, data_for(_type, records), source=self, lenient=lenient) zone.add_record(record, lenient=lenient) @@ -491,34 +615,70 @@ class ConstellixProvider(BaseProvider): def _handle_pools(self, record): # If we don't have dynamic, then there's no pools if not getattr(record, 'dynamic', False): - return None - - # Get our first entry in the rules that references a pool - rules = list(filter( - lambda rule: 'pool' in rule.data, - record.dynamic.rules - )) - - pool_name = rules[0].data.get('pool') - - pool = record.dynamic.pools.get(pool_name) - values = pool.data.get('values') - - # Make a pool name based on zone, record, type and name - pool_name = '{}:{}:{}:{}'.format( - record.zone.name, - record.name, - record._type, - pool_name - ) - - # OK, pool is valid, let's create it or update it - return self._create_update_pool( - pool_name = pool_name, - pool_type = record._type, - ttl = record.ttl, - values = values - ) + return [] + + res_pools = [] + + for i, rule in enumerate(record.dynamic.rules): + pool_name = rule.data.get('pool') + pool = record.dynamic.pools.get(pool_name) + values = pool.data.get('values') + + # Make a pool name based on zone, record, type and name + generated_pool_name = '{}:{}:{}:{}'.format( + record.zone.name, + record.name, + record._type, + pool_name + ) + + # OK, pool is valid, let's create it or update it + self.log.debug("Creating pool %s", generated_pool_name) + pool_obj = self._create_update_pool( + pool_name = generated_pool_name, + pool_type = record._type, + ttl = record.ttl, + values = values + ) + + # Now will crate GeoFilter for the pool + continents = [] + countries = [] + regions = [] + + for geo in rule.data.get('geos', []): + codes = geo.split('-') + n = len(geo) + if n == 2: + continents.append(geo) + elif n == 5: + countries.append(codes[1]) + else: + regions.append({ + 'continentCode': codes[0], + 'countryCode': codes[1], + 'regionCode': codes[2] + }) + + if len(continents) == 0 and \ + len(countries) == 0 and \ + len(regions) == 0: + pool_obj['geofilter'] = 1 + else: + self.log.debug( + "Creating geofilter %s", + generated_pool_name + ) + geofilter_obj = self._create_update_geofilter( + generated_pool_name, + continents, + countries, + regions + ) + pool_obj['geofilter'] = geofilter_obj['id'] + + res_pools.append(pool_obj) + return res_pools def _create_update_pool(self, pool_name, pool_type, ttl, values): pool = { @@ -538,29 +698,173 @@ class ConstellixProvider(BaseProvider): updated_pool['id'] = pool_id return updated_pool - def _apply_Create(self, change): + def _create_update_geofilter( + self, + geofilter_name, + continents, + countries, + regions): + geofilter = { + 'filterRulesLimit': 100, + 'name': geofilter_name, + 'geoipContinents': continents, + 'geoipCountries': countries, + 'regions': regions + } + if len(regions) == 0: + geofilter.pop('regions', None) + + existing_geofilter = self._client.geofilter(geofilter_name) + if not existing_geofilter: + return self._client.geofilter_create(geofilter) + + geofilter_id = existing_geofilter['id'] + updated_geofilter = self._client.geofilter_update( + geofilter_id, geofilter) + updated_geofilter['id'] = geofilter_id + return updated_geofilter + + def _apply_Create(self, change, domain_name): new = change.new params_for = getattr(self, '_params_for_{}'.format(new._type)) - pool = self._handle_pools(new) + pools = self._handle_pools(new) for params in params_for(new): - if pool: - params['pools'] = [pool['id']] + if len(pools) == 0: + self._client.record_create(new.zone.name, new._type, params) + elif len(pools) == 1: + params['pools'] = [pools[0]['id']] params['recordOption'] = 'pools' - self._client.record_create(new.zone.name, new._type, params) - - def _apply_Update(self, change): - self._apply_Delete(change) - self._apply_Create(change) - - def _apply_Delete(self, change): + params.pop('roundRobin', None) + self.log.debug( + "Creating record %s %s", + new.zone.name, + new._type + ) + self._client.record_create( + new.zone.name, + new._type, + params + ) + else: + # To use GeoIPFilter feature we need to enable it for domain + self.log.debug("Enabling domain %s geo support", domain_name) + self._client.domain_enable_geoip(domain_name) + + # First we need to create World Default (1) Record + for pool in pools: + if pool['geofilter'] != 1: + continue + params['pools'] = [pool['id']] + params['recordOption'] = 'pools' + params['geolocation'] = { + 'geoipUserRegion': [pool['geofilter']] + } + params.pop('roundRobin', None) + self.log.debug( + "Creating record %s %s", + new.zone.name, + new._type) + self._client.record_create( + new.zone.name, + new._type, + params + ) + + # Now we can create the rest of records + for pool in pools: + if pool['geofilter'] == 1: + continue + params['pools'] = [pool['id']] + params['recordOption'] = 'pools' + params['geolocation'] = { + 'geoipUserRegion': [pool['geofilter']] + } + params.pop('roundRobin', None) + self.log.debug( + "Creating record %s %s", + new.zone.name, + new._type) + self._client.record_create( + new.zone.name, + new._type, + params) + + def _apply_Update(self, change, domain_name): + self._apply_Delete(change, domain_name) + self._apply_Create(change, domain_name) + + def _apply_Delete(self, change, domain_name): existing = change.existing zone = existing.zone + + # if it is dynamic pools record, we need to delete World Default last + world_default_record = None + for record in self.zone_records(zone): if existing.name == record['name'] and \ existing._type == record['type']: - self._client.record_delete(zone.name, record['type'], - record['id']) + + # handle dynamic record + if record['recordOption'] == 'pools': + if record['geolocation'] is None: + world_default_record = record + else: + if record['geolocation']['geoipFilter'] == 1: + world_default_record = record + else: + # delete record + self.log.debug( + "Deleting record %s %s", + zone.name, + record['type']) + self._client.record_delete( + zone.name, + record['type'], + record['id']) + # delete geofilter + self.log.debug( + "Deleting geofilter %s", + zone.name) + self._client.geofilter_delete( + record['geolocation']['geoipFilter']) + + # delete pool + self.log.debug( + "Deleting pool %s %s", + zone.name, + record['type']) + self._client.pool_delete( + record['type'], + record['pools'][0]) + + # for all the rest records + else: + self._client.record_delete( + zone.name, record['type'], record['id']) + # delete World Default + if world_default_record: + # delete record + self.log.debug( + "Deleting record %s %s", + zone.name, + world_default_record['type'] + ) + self._client.record_delete( + zone.name, + world_default_record['type'], + world_default_record['id'] + ) + # delete pool + self.log.debug( + "Deleting pool %s %s", + zone.name, + world_default_record['type'] + ) + self._client.pool_delete( + world_default_record['type'], + world_default_record['pools'][0] + ) def _apply(self, plan): desired = plan.desired @@ -576,7 +880,9 @@ class ConstellixProvider(BaseProvider): for change in changes: class_name = change.__class__.__name__ - getattr(self, '_apply_{}'.format(class_name))(change) + getattr(self, f'_apply_{class_name}')( + change, + desired.name) # Clear out the cache if any self._zone_records.pop(desired.name, None) diff --git a/tests/fixtures/constellix-geofilters.json b/tests/fixtures/constellix-geofilters.json new file mode 100644 index 0000000..eef17a3 --- /dev/null +++ b/tests/fixtures/constellix-geofilters.json @@ -0,0 +1,34 @@ +[ + { + "id": 6303, + "name": "some.other", + "filterRulesLimit": 100, + "createdTs": "2021-08-19T14:47:47Z", + "modifiedTs": "2021-08-19T14:47:47Z", + "geoipContinents": ["AS", "OC"], + "geoipCountries": ["ES", "SE", "UA"], + "regions": [ + { + "continentCode": "NA", + "countryCode": "CA", + "regionCode": "NL" + } + ] + }, + { + "id": 5303, + "name": "unit.tests.:www.dynamic:A:one", + "filterRulesLimit": 100, + "createdTs": "2021-08-19T14:47:47Z", + "modifiedTs": "2021-08-19T14:47:47Z", + "geoipContinents": ["AS", "OC"], + "geoipCountries": ["ES", "SE", "UA"], + "regions": [ + { + "continentCode": "NA", + "countryCode": "CA", + "regionCode": "NL" + } + ] + } +] diff --git a/tests/fixtures/constellix-pools.json b/tests/fixtures/constellix-pools.json index 8fbd491..8d90bd4 100644 --- a/tests/fixtures/constellix-pools.json +++ b/tests/fixtures/constellix-pools.json @@ -28,5 +28,35 @@ "weight": 1 } ] + }, + { + "id": 1808522, + "name": "unit.tests.:www.dynamic:A:one", + "type": "A", + "numReturn": 1, + "minAvailableFailover": 1, + "createdTs": "2020-09-12T00:44:35Z", + "modifiedTs": "2020-09-12T00:44:35Z", + "appliedDomains": [ + { + "id": 123123, + "name": "unit.tests", + "recordOption": "pools" + } + ], + "appliedTemplates": null, + "unlinkedDomains": [], + "unlinkedTemplates": null, + "itoEnabled": false, + "values": [ + { + "value": "1.2.3.6", + "weight": 1 + }, + { + "value": "1.2.3.7", + "weight": 1 + } + ] } ] \ No newline at end of file diff --git a/tests/fixtures/constellix-records.json b/tests/fixtures/constellix-records.json index f509fe7..c5cdf8e 100644 --- a/tests/fixtures/constellix-records.json +++ b/tests/fixtures/constellix-records.json @@ -65,10 +65,10 @@ "pools": [], "poolsDetail": [] }, { - "id": 1898527, + "id": 1808527, "type": "SRV", "recordType": "srv", - "name": "_imap._tcp", + "name": "_srv._tcp", "recordOption": "roundRobin", "noAnswer": false, "note": "", @@ -79,24 +79,36 @@ "source": "Domain", "modifiedTs": 1565149714387, "value": [{ - "value": ".", - "priority": 0, - "weight": 0, - "port": 0, + "value": "foo-1.unit.tests.", + "priority": 10, + "weight": 20, + "port": 30, + "disableFlag": false + }, { + "value": "foo-2.unit.tests.", + "priority": 12, + "weight": 20, + "port": 30, "disableFlag": false }], "roundRobin": [{ - "value": ".", - "priority": 0, - "weight": 0, - "port": 0, + "value": "foo-1.unit.tests.", + "priority": 10, + "weight": 20, + "port": 30, + "disableFlag": false + }, { + "value": "foo-2.unit.tests.", + "priority": 12, + "weight": 20, + "port": 30, "disableFlag": false }] }, { - "id": 1898528, + "id": 1808527, "type": "SRV", "recordType": "srv", - "name": "_pop3._tcp", + "name": "_imap._tcp", "recordOption": "roundRobin", "noAnswer": false, "note": "", @@ -124,7 +136,7 @@ "id": 1808527, "type": "SRV", "recordType": "srv", - "name": "_srv._tcp", + "name": "_pop3._tcp", "recordOption": "roundRobin", "noAnswer": false, "note": "", @@ -135,29 +147,17 @@ "source": "Domain", "modifiedTs": 1565149714387, "value": [{ - "value": "foo-1.unit.tests.", - "priority": 10, - "weight": 20, - "port": 30, - "disableFlag": false - }, { - "value": "foo-2.unit.tests.", - "priority": 12, - "weight": 20, - "port": 30, + "value": ".", + "priority": 0, + "weight": 0, + "port": 0, "disableFlag": false }], "roundRobin": [{ - "value": "foo-1.unit.tests.", - "priority": 10, - "weight": 20, - "port": 30, - "disableFlag": false - }, { - "value": "foo-2.unit.tests.", - "priority": 12, - "weight": 20, - "port": 30, + "value": ".", + "priority": 0, + "weight": 0, + "port": 0, "disableFlag": false }] }, { @@ -630,7 +630,9 @@ "modifiedTs": 1565150090588, "value": [], "roundRobin": [], - "geolocation": null, + "geolocation": { + "geoipFilter": 1 + }, "recordFailover": { "disabled": false, "failoverType": 1, @@ -651,4 +653,44 @@ "id": 1808521, "name": "unit.tests.:www.dynamic:A:two" }] +}, +{ + "id": 1808521, + "type": "A", + "recordType": "a", + "name": "www.dynamic", + "recordOption": "pools", + "noAnswer": false, + "note": "", + "ttl": 300, + "gtdRegion": 1, + "parentId": 123123, + "parent": "domain", + "source": "Domain", + "modifiedTs": 1565150090588, + "value": [], + "roundRobin": [], + "geolocation": { + "geoipFilter": 5303 + }, + "recordFailover": { + "disabled": false, + "failoverType": 1, + "failoverTypeStr": "Normal (always lowest level)", + "values": [] + }, + "failover": { + "disabled": false, + "failoverType": 1, + "failoverTypeStr": "Normal (always lowest level)", + "values": [] + }, + "roundRobinFailover": [], + "pools": [ + 1808522 + ], + "poolsDetail": [{ + "id": 1808522, + "name": "unit.tests.:www.dynamic:A:one" + }] }] diff --git a/tests/test_octodns_provider_constellix.py b/tests/test_octodns_provider_constellix.py index d15f611..75aa07f 100644 --- a/tests/test_octodns_provider_constellix.py +++ b/tests/test_octodns_provider_constellix.py @@ -42,22 +42,78 @@ class TestConstellixProvider(TestCase): 'value': 'aname.unit.tests.' })) - expected.add_record(Record.new(expected, 'sub', { + # Add a dynamic record + expected.add_record(Record.new(expected, 'www.dynamic', { + 'ttl': 300, + 'type': 'A', + 'values': [ + '1.2.3.4', + '1.2.3.5' + ], + 'dynamic': { + 'pools': { + 'two': { + 'values': [{ + 'value': '1.2.3.4', + 'weight': 1 + }, { + 'value': '1.2.3.5', + 'weight': 1 + }], + }, + }, + 'rules': [{ + 'pool': 'two', + }], + }, + })) + + for record in list(expected.records): + if record.name == 'sub' and record._type == 'NS': + expected._remove_record(record) + break + + expected_dynamic = Zone('unit.tests.', []) + source = YamlProvider('test', join(dirname(__file__), 'config')) + source.populate(expected_dynamic) + + # Our test suite differs a bit, add our NS and remove the simple one + expected_dynamic.add_record(Record.new(expected_dynamic, 'under', { + 'ttl': 3600, + 'type': 'NS', + 'values': [ + 'ns1.unit.tests.', + 'ns2.unit.tests.', + ] + })) + + # Add some ALIAS records + expected_dynamic.add_record(Record.new(expected_dynamic, '', { 'ttl': 1800, 'type': 'ALIAS', 'value': 'aname.unit.tests.' })) # Add a dynamic record - expected.add_record(Record.new(expected, 'www.dynamic', { + expected_dynamic.add_record(Record.new(expected_dynamic, 'www.dynamic', { 'ttl': 300, 'type': 'A', - 'value': [ + 'values': [ '1.2.3.4', '1.2.3.5' ], 'dynamic': { 'pools': { + 'one': { + 'fallback': 'two', + 'values': [{ + 'value': '1.2.3.6', + 'weight': 1 + }, { + 'value': '1.2.3.7', + 'weight': 1 + }], + }, 'two': { 'values': [{ 'value': '1.2.3.4', @@ -69,14 +125,24 @@ class TestConstellixProvider(TestCase): }, }, 'rules': [{ + 'geos': [ + 'AS', + 'EU-ES', + 'EU-UA', + 'EU-SE', + 'NA-CA-NL', + 'OC' + ], + 'pool': 'one' + }, { 'pool': 'two', }], - }, + } })) - for record in list(expected.records): + for record in list(expected_dynamic.records): if record.name == 'sub' and record._type == 'NS': - expected._remove_record(record) + expected_dynamic._remove_record(record) break def test_populate(self): @@ -126,24 +192,28 @@ class TestConstellixProvider(TestCase): with requests_mock() as mock: base = 'https://api.dns.constellix.com/v1' with open('tests/fixtures/constellix-domains.json') as fh: - mock.get('{}{}'.format(base, '/domains'), text=fh.read()) + mock.get('{}{}'.format(base, '/domains'), + text=fh.read()) with open('tests/fixtures/constellix-records.json') as fh: mock.get('{}{}'.format(base, '/domains/123123/records'), text=fh.read()) with open('tests/fixtures/constellix-pools.json') as fh: mock.get('{}{}'.format(base, '/pools/A'), text=fh.read()) + with open('tests/fixtures/constellix-geofilters.json') as fh: + mock.get('{}{}'.format(base, '/geoFilters'), + text=fh.read()) zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals(16, len(zone.records)) - changes = self.expected.changes(zone, provider) + self.assertEquals(17, len(zone.records)) + changes = self.expected_dynamic.changes(zone, provider) self.assertEquals(0, len(changes)) # 2nd populate makes no network calls/all from cache again = Zone('unit.tests.', []) provider.populate(again) - self.assertEquals(16, len(again.records)) + self.assertEquals(17, len(again.records)) # bust the cache del provider._zone_records[zone.name] @@ -225,7 +295,7 @@ class TestConstellixProvider(TestCase): }), ]) - self.assertEquals(21, provider._client._request.call_count) + self.assertEquals(22, provider._client._request.call_count) provider._client._request.reset_mock() @@ -262,6 +332,7 @@ class TestConstellixProvider(TestCase): "id": 1808520, "type": "A", "name": "www.dynamic", + "geolocation": None, "recordOption": "pools", "ttl": 300, "value": [], @@ -289,8 +360,16 @@ class TestConstellixProvider(TestCase): # Domain exists, we don't care about return resp.json.side_effect = [ - ['{}'], - ['{}'], + [], # no domains returned during populate + [{ + 'id': 123123, + 'name': 'unit.tests' + }], # domain created in apply + [], # No pools returned during populate + [{ + "id": 1808521, + "name": "unit.tests.:www.dynamic:A:one" + }] # pool created in apply ] wanted = Zone('unit.tests.', []) @@ -303,7 +382,7 @@ class TestConstellixProvider(TestCase): wanted.add_record(Record.new(wanted, 'www.dynamic', { 'ttl': 300, 'type': 'A', - 'value': [ + 'values': [ '1.2.3.4' ], 'dynamic': { @@ -340,17 +419,393 @@ class TestConstellixProvider(TestCase): 'numReturn': 1, 'minAvailableFailover': 1, 'ttl': 300, - 'id': 1808521, 'values': [{ "value": "1.2.3.4", "weight": 1 - }] + }], + 'id': 1808521, + 'geofilter': 1 }), call('DELETE', '/domains/123123/records/A/11189897'), call('DELETE', '/domains/123123/records/A/11189898'), call('DELETE', '/domains/123123/records/ANAME/11189899'), ], any_order=True) + def test_apply_dunamic(self): + provider = ConstellixProvider('test', 'api', 'secret') + + resp = Mock() + resp.json = Mock() + provider._client._request = Mock(return_value=resp) + + # non-existent domain, create everything + resp.json.side_effect = [ + [], # no domains returned during populate + [{ + 'id': 123123, + 'name': 'unit.tests' + }], # domain created in apply + [], # No pools returned during populate + [{ + "id": 1808521, + "name": "unit.tests.:www.dynamic:A:one" + }], # pool created in apply + [], # no geofilters returned during populate + [{ + "id": 5303, + "name": "unit.tests.:www.dynamic:A:one", + "filterRulesLimit": 100, + "geoipContinents": ["AS", "OC"], + "geoipCountries": ["ES", "SE", "UA"], + "regions": [ + { + "continentCode": "NA", + "countryCode": "CA", + "regionCode": "NL" + } + ] + }], # geofilters created in applly + [{ + "id": 1808520, + "name": "unit.tests.:www.dynamic:A:two", + }], # pool created in apply + { + 'id': 123123, + 'name': 'unit.tests', + 'hasGeoIP': False + }, # domain listed for enabling geo + [] # enabling geo + ] + + plan = provider.plan(self.expected_dynamic) + + # No root NS, no ignored, no excluded, no unsupported + n = len(self.expected_dynamic.records) - 8 + self.assertEquals(n, len(plan.changes)) + self.assertEquals(n, provider.apply(plan)) + + provider._client._request.assert_has_calls([ + # get all domains to build the cache + call('GET', '/domains'), + # created the domain + call('POST', '/domains', data={'names': ['unit.tests']}) + ]) +# + # Check we tried to get our pool + provider._client._request.assert_has_calls([ + call('GET', '/pools/A'), + call('POST', '/pools/A', data={ + 'name': 'unit.tests.:www.dynamic:A:one', + 'type': 'A', + 'numReturn': 1, + 'minAvailableFailover': 1, + 'ttl': 300, + 'values': [{ + 'value': '1.2.3.6', + 'weight': 1 + }, { + 'value': '1.2.3.7', + 'weight': 1}] + }), + call('GET', '/geoFilters'), + call('POST', '/geoFilters', data={ + 'filterRulesLimit': 100, + 'name': 'unit.tests.:www.dynamic:A:one', + 'geoipContinents': ['AS', 'OC'], + 'geoipCountries': ['ES', 'SE', 'UA'], + 'regions': [{ + 'continentCode': 'NA', + 'countryCode': 'CA', + 'regionCode': 'NL'}] + }), + call('POST', '/pools/A', data={ + 'name': 'unit.tests.:www.dynamic:A:two', + 'type': 'A', + 'numReturn': 1, + 'minAvailableFailover': 1, + 'ttl': 300, + 'values': [{ + 'value': '1.2.3.4', + 'weight': 1 + }, { + 'value': '1.2.3.5', + 'weight': 1}] + }) + ]) + + # These two checks are broken up so that ordering doesn't break things. + # Python3 doesn't make the calls in a consistent order so different + # things follow the GET / on different runs + provider._client._request.assert_has_calls([ + call('POST', '/domains/123123/records/SRV', data={ + 'roundRobin': [{ + 'priority': 10, + 'weight': 20, + 'value': 'foo-1.unit.tests.', + 'port': 30 + }, { + 'priority': 12, + 'weight': 20, + 'value': 'foo-2.unit.tests.', + 'port': 30 + }], + 'name': '_srv._tcp', + 'ttl': 600, + }), + ]) + + self.assertEquals(28, provider._client._request.call_count) + + provider._client._request.reset_mock() + + provider._client.records = Mock(return_value=[ + { + 'id': 11189897, + 'type': 'A', + 'name': 'www', + 'ttl': 300, + 'recordOption': 'roundRobin', + 'value': [ + '1.2.3.4', + '2.2.3.4', + ] + }, { + 'id': 11189898, + 'type': 'A', + 'name': 'ttl', + 'ttl': 600, + 'recordOption': 'roundRobin', + 'value': [ + '3.2.3.4' + ] + }, { + 'id': 11189899, + 'type': 'ALIAS', + 'name': 'alias', + 'ttl': 600, + 'recordOption': 'roundRobin', + 'value': [{ + 'value': 'aname.unit.tests.' + }] + }, { + "id": 1808520, + "type": "A", + "name": "www.dynamic", + "geolocation": { + "geoipFilter": 1 + }, + "recordOption": "pools", + "ttl": 300, + "value": [], + "pools": [ + 1808521 + ] + }, { + "id": 1808521, + "type": "A", + "name": "www.dynamic", + "geolocation": { + "geoipFilter": 5303 + }, + "recordOption": "pools", + "ttl": 300, + "value": [], + "pools": [ + 1808522 + ] + } + ]) + + provider._client.pools = Mock(return_value=[ + { + "id": 1808521, + "name": "unit.tests.:www.dynamic:A:two", + "type": "A", + "values": [ + { + "value": "1.2.3.4", + "weight": 1 + }, + { + "value": "1.2.3.5", + "weight": 1 + } + ] + }, + { + "id": 1808522, + "name": "unit.tests.:www.dynamic:A:one", + "type": "A", + "values": [ + { + "value": "1.2.3.6", + "weight": 1 + }, + { + "value": "1.2.3.7", + "weight": 1 + } + ] + } + ]) + + provider._client.geofilters = Mock(return_value=[ + { + "id": 5303, + "name": "unit.tests.:www.dynamic:A:one", + "filterRulesLimit": 100, + "geoipContinents": ["AS", "OC"], + "geoipCountries": ["ES", "SE", "UA"], + "regions": [ + { + "continentCode": "NA", + "countryCode": "CA", + "regionCode": "NL" + } + ] + } + ]) + + # Domain exists, we don't care about return + resp.json.side_effect = [ + [], + [], + [], + [], + { + 'id': 123123, + 'name': 'unit.tests', + 'hasGeoIP': True + } # domain listed for enabling geo + ] + + wanted = Zone('unit.tests.', []) + wanted.add_record(Record.new(wanted, 'ttl', { + 'ttl': 300, + 'type': 'A', + 'value': '3.2.3.4' + })) + + wanted.add_record(Record.new(wanted, 'www.dynamic', { + 'ttl': 300, + 'type': 'A', + 'values': [ + '1.2.3.4' + ], + 'dynamic': { + 'pools': { + 'one': { + 'fallback': 'two', + 'values': [{ + 'value': '1.2.3.6', + 'weight': 1 + }, { + 'value': '1.2.3.7', + 'weight': 1 + }], + }, + 'two': { + 'values': [{ + 'value': '1.2.3.4', + 'weight': 1 + }], + }, + }, + 'rules': [{ + 'geos': [ + 'AS', + 'EU-ES', + 'EU-UA', + 'EU-SE', + 'NA-CA-NL', + 'OC' + ], + 'pool': 'one' + }, { + 'pool': 'two', + }], + }, + })) + + plan = provider.plan(wanted) + self.assertEquals(4, len(plan.changes)) + self.assertEquals(4, provider.apply(plan)) + + # recreate for update, and deletes for the 2 parts of the other + provider._client._request.assert_has_calls([ + call('POST', '/domains/123123/records/A', data={ + 'roundRobin': [{ + 'value': '3.2.3.4' + }], + 'name': 'ttl', + 'ttl': 300 + }), + + call('DELETE', '/domains/123123/records/A/1808521'), + call('DELETE', '/geoFilters/5303'), + call('DELETE', '/pools/A/1808522'), + call('DELETE', '/domains/123123/records/A/1808520'), + call('DELETE', '/pools/A/1808521'), + call('DELETE', '/domains/123123/records/ANAME/11189899'), + + call('PUT', '/pools/A/1808522', data={ + 'name': 'unit.tests.:www.dynamic:A:one', + 'type': 'A', + 'numReturn': 1, + 'minAvailableFailover': 1, + 'ttl': 300, + 'values': [ + {'value': '1.2.3.6', 'weight': 1}, + {'value': '1.2.3.7', 'weight': 1}], + 'id': 1808522, + 'geofilter': 5303 + }), + + call('PUT', '/geoFilters/5303', data={ + 'filterRulesLimit': 100, + 'name': 'unit.tests.:www.dynamic:A:one', + 'geoipContinents': ['AS', 'OC'], + 'geoipCountries': ['ES', 'SE', 'UA'], + 'regions': [{ + 'continentCode': 'NA', + 'countryCode': 'CA', + 'regionCode': 'NL'}], + 'id': 5303 + }), + + call('PUT', '/pools/A/1808521', data={ + 'name': 'unit.tests.:www.dynamic:A:two', + 'type': 'A', + 'numReturn': 1, + 'minAvailableFailover': 1, + 'ttl': 300, + 'values': [{'value': '1.2.3.4', 'weight': 1}], + 'id': 1808521, + 'geofilter': 1 + }), + + call('GET', '/domains/123123'), + call('POST', '/domains/123123/records/A', data={ + 'name': 'www.dynamic', + 'ttl': 300, + 'pools': [1808522], + 'recordOption': 'pools', + 'geolocation': { + 'geoipUserRegion': [5303] + } + }), + + call('POST', '/domains/123123/records/A', data={ + 'name': 'www.dynamic', + 'ttl': 300, + 'pools': [1808522], + 'recordOption': 'pools', + 'geolocation': { + 'geoipUserRegion': [5303] + } + }) + ], any_order=True) + def test_dynamic_record_failures(self): provider = ConstellixProvider('test', 'api', 'secret') @@ -367,6 +822,7 @@ class TestConstellixProvider(TestCase): "id": 1808520, "type": "A", "name": "www.dynamic", + "geolocation": None, "recordOption": "pools", "ttl": 300, "value": [], @@ -388,6 +844,8 @@ class TestConstellixProvider(TestCase): ] }]) + provider._client.geofilters = Mock(return_value=[]) + wanted = Zone('unit.tests.', []) resp.json.side_effect = [ @@ -397,7 +855,7 @@ class TestConstellixProvider(TestCase): wanted.add_record(Record.new(wanted, 'www.dynamic', { 'ttl': 300, 'type': 'A', - 'value': [ + 'values': [ '1.2.3.4' ], 'dynamic': { @@ -428,44 +886,133 @@ class TestConstellixProvider(TestCase): "id": 1808520, "type": "A", "name": "www.dynamic", + "geolocation": { + "geoipFilter": 1 + }, "recordOption": "pools", "ttl": 300, "value": [], "pools": [ 1808521 ] + }, { + "id": 1808521, + "type": "A", + "name": "www.dynamic", + "geolocation": { + "geoipFilter": 5303 + }, + "recordOption": "pools", + "ttl": 300, + "value": [], + "pools": [ + 1808522 + ] } ]) - provider._client.pools = Mock(return_value=[{ - "id": 1808521, - "name": "unit.tests.:www.dynamic:A:two", - "type": "A", - "values": [ - { - "value": "1.2.3.4", - "weight": 1 - } - ] - }]) + provider._client.pools = Mock(return_value=[ + { + "id": 1808521, + "name": "unit.tests.:www.dynamic:A:two", + "type": "A", + "values": [ + { + "value": "1.2.3.4", + "weight": 1 + }, + { + "value": "1.2.3.5", + "weight": 1 + } + ] + }, + { + "id": 1808522, + "name": "unit.tests.:www.dynamic:A:one", + "type": "A", + "values": [ + { + "value": "1.2.3.6", + "weight": 1 + }, + { + "value": "1.2.3.7", + "weight": 1 + } + ] + } + ]) + + provider._client.geofilters = Mock(return_value=[ + { + "id": 6303, + "name": "some.other", + "filterRulesLimit": 100, + "createdTs": "2021-08-19T14:47:47Z", + "modifiedTs": "2021-08-19T14:47:47Z", + "geoipContinents": ["AS", "OC"], + "geoipCountries": ["ES", "SE", "UA"], + "regions": [ + { + "continentCode": "NA", + "countryCode": "CA", + "regionCode": "NL" + } + ] + }, { + "id": 5303, + "name": "unit.tests.:www.dynamic:A:one", + "filterRulesLimit": 100, + "geoipContinents": ["AS", "OC"], + "geoipCountries": ["ES", "SE", "UA"], + "regions": [ + { + "continentCode": "NA", + "countryCode": "CA", + "regionCode": "NL" + } + ] + } + ]) wanted = Zone('unit.tests.', []) wanted.add_record(Record.new(wanted, 'www.dynamic', { 'ttl': 300, 'type': 'A', - 'value': [ + 'values': [ '1.2.3.4' ], 'dynamic': { 'pools': { + 'one': { + 'fallback': 'two', + 'values': [{ + 'value': '1.2.3.6', + 'weight': 1 + }, { + 'value': '1.2.3.7', + 'weight': 1 + }], + }, 'two': { 'values': [{ - 'value': '1.2.3.5' + 'value': '1.2.3.4', + 'weight': 1 }], }, }, 'rules': [{ + 'geos': [ + 'AS', + 'EU-ES', + 'EU-UA', + 'EU-SE', + 'OC' + ], + 'pool': 'one' + }, { 'pool': 'two', }], }, @@ -473,13 +1020,25 @@ class TestConstellixProvider(TestCase): # Try an error we can handle with requests_mock() as mock: - mock.get(ANY, status_code=200, - text='{}') + mock.get( + "https://api.dns.constellix.com/v1/domains", + status_code=200, + text='[{"id": 1234, "name": "unit.tests", "hasGeoIP": true}]') + mock.get( + "https://api.dns.constellix.com/v1/domains/1234", + status_code=200, + text='{"id": 1234, "name": "unit.tests", "hasGeoIP": true}') mock.delete(ANY, status_code=200, text='{}') mock.put("https://api.dns.constellix.com/v1/pools/A/1808521", status_code=400, text='{"errors": [\"no changes to save\"]}') + mock.put("https://api.dns.constellix.com/v1/pools/A/1808522", + status_code=400, + text='{"errors": [\"no changes to save\"]}') + mock.put("https://api.dns.constellix.com/v1/geoFilters/5303", + status_code=400, + text='{"errors": [\"no changes to save\"]}') mock.post(ANY, status_code=200, text='[{"id": 1234}]') @@ -487,15 +1046,89 @@ class TestConstellixProvider(TestCase): self.assertEquals(1, len(plan.changes)) self.assertEquals(1, provider.apply(plan)) + provider._client.geofilters = Mock(return_value=[ + { + "id": 5303, + "name": "unit.tests.:www.dynamic:A:one", + "filterRulesLimit": 100, + "regions": [ + { + "continentCode": "NA", + "countryCode": "CA", + "regionCode": "NL" + } + ] + } + ]) + + plan = provider.plan(wanted) + self.assertEquals(1, len(plan.changes)) + self.assertEquals(1, provider.apply(plan)) + + provider._client.geofilters = Mock(return_value=[ + { + "id": 5303, + "name": "unit.tests.:www.dynamic:A:one", + "filterRulesLimit": 100, + "geoipContinents": ["AS", "OC"], + } + ]) + + plan = provider.plan(wanted) + self.assertEquals(1, len(plan.changes)) + self.assertEquals(1, provider.apply(plan)) + + # Now what happens if an error happens that we can't handle + # geofilter case + with requests_mock() as mock: + mock.get( + "https://api.dns.constellix.com/v1/domains", + status_code=200, + text='[{"id": 1234, "name": "unit.tests", "hasGeoIP": true}]') + mock.get( + "https://api.dns.constellix.com/v1/domains/1234", + status_code=200, + text='{"id": 1234, "name": "unit.tests", "hasGeoIP": true}') + mock.delete(ANY, status_code=200, + text='{}') + mock.put("https://api.dns.constellix.com/v1/pools/A/1808521", + status_code=400, + text='{"errors": [\"no changes to save\"]}') + mock.put("https://api.dns.constellix.com/v1/pools/A/1808522", + status_code=400, + text='{"errors": [\"no changes to save\"]}') + mock.put("https://api.dns.constellix.com/v1/geoFilters/5303", + status_code=400, + text='{"errors": [\"generic error\"]}') + mock.post(ANY, status_code=200, + text='[{"id": 1234}]') + + plan = provider.plan(wanted) + self.assertEquals(1, len(plan.changes)) + with self.assertRaises(ConstellixClientBadRequest): + provider.apply(plan) + # Now what happens if an error happens that we can't handle with requests_mock() as mock: - mock.get(ANY, status_code=200, - text='{}') + mock.get( + "https://api.dns.constellix.com/v1/domains", + status_code=200, + text='[{"id": 1234, "name": "unit.tests", "hasGeoIP": true}]') + mock.get( + "https://api.dns.constellix.com/v1/domains/1234", + status_code=200, + text='{"id": 1234, "name": "unit.tests", "hasGeoIP": true}') mock.delete(ANY, status_code=200, text='{}') mock.put("https://api.dns.constellix.com/v1/pools/A/1808521", status_code=400, text='{"errors": [\"generic error\"]}') + mock.put("https://api.dns.constellix.com/v1/pools/A/1808522", + status_code=400, + text='{"errors": [\"generic error\"]}') + mock.put("https://api.dns.constellix.com/v1/geoFilters/5303", + status_code=400, + text='{"errors": [\"generic error\"]}') mock.post(ANY, status_code=200, text='[{"id": 1234}]')