|
|
|
@ -9,11 +9,13 @@ from boto3 import client |
|
|
|
from botocore.config import Config |
|
|
|
from collections import defaultdict |
|
|
|
from incf.countryutils.transformations import cca_to_ctca2 |
|
|
|
from ipaddress import AddressValueError, ip_address |
|
|
|
from uuid import uuid4 |
|
|
|
import logging |
|
|
|
import re |
|
|
|
|
|
|
|
from ..record import Record, Update |
|
|
|
from ..record.geo import GeoCodes |
|
|
|
from .base import BaseProvider |
|
|
|
|
|
|
|
|
|
|
|
@ -29,19 +31,105 @@ def _octal_replace(s): |
|
|
|
class _Route53Record(object): |
|
|
|
|
|
|
|
@classmethod |
|
|
|
def new(self, provider, record, creating): |
|
|
|
def _new_dynamic(cls, provider, record, hosted_zone_id, creating): |
|
|
|
# Creates the RRSets that correspond to the given dynamic record |
|
|
|
ret = set() |
|
|
|
if getattr(record, 'geo', False): |
|
|
|
ret.add(_Route53GeoDefault(provider, record, creating)) |
|
|
|
for ident, geo in record.geo.items(): |
|
|
|
ret.add(_Route53GeoRecord(provider, record, ident, geo, |
|
|
|
creating)) |
|
|
|
else: |
|
|
|
ret.add(_Route53Record(provider, record, creating)) |
|
|
|
|
|
|
|
# HostedZoneId wants just the last bit, but the place we're getting |
|
|
|
# this from looks like /hostedzone/Z424CArX3BB224 |
|
|
|
hosted_zone_id = hosted_zone_id.split('/', 2)[-1] |
|
|
|
|
|
|
|
# Create the default pool which comes from the base `values` of the |
|
|
|
# record object. Its only used if all other values fail their |
|
|
|
# healthchecks, which hopefully never happens. |
|
|
|
fqdn = record.fqdn |
|
|
|
ret.add(_Route53Record(provider, record, creating, |
|
|
|
'_octodns-default-pool.{}'.format(fqdn))) |
|
|
|
|
|
|
|
# Pools |
|
|
|
for pool_name, pool in record.dynamic.pools.items(): |
|
|
|
|
|
|
|
# Create the primary, this will be the rrset that geo targeted |
|
|
|
# rrsets will point to when they want to use a pool of values. It's |
|
|
|
# a primary and observes target health so if all the values for |
|
|
|
# this pool go red, we'll use the fallback/SECONDARY just below |
|
|
|
ret.add(_Route53DynamicPool(provider, hosted_zone_id, record, |
|
|
|
pool_name, creating)) |
|
|
|
|
|
|
|
# Create the fallback for this pool |
|
|
|
fallback = pool.data.get('fallback', False) |
|
|
|
if fallback: |
|
|
|
# We have an explicitly configured fallback, another pool to |
|
|
|
# use if all our values go red. This RRSet configures that pool |
|
|
|
# as the next best option |
|
|
|
ret.add(_Route53DynamicPool(provider, hosted_zone_id, record, |
|
|
|
pool_name, creating, |
|
|
|
target_name=fallback)) |
|
|
|
else: |
|
|
|
# We fallback on the default, no explicit fallback so if all of |
|
|
|
# this pool's values go red we'll fallback to the base |
|
|
|
# (non-health-checked) default pool of values |
|
|
|
ret.add(_Route53DynamicPool(provider, hosted_zone_id, record, |
|
|
|
pool_name, creating, |
|
|
|
target_name='default')) |
|
|
|
|
|
|
|
# Create the values for this pool. These are health checked and in |
|
|
|
# general each unique value will have an associated healthcheck. |
|
|
|
# The PRIMARY pool up above will point to these RRSets which will |
|
|
|
# be served out according to their weights |
|
|
|
for i, value in enumerate(pool.data['values']): |
|
|
|
weight = value['weight'] |
|
|
|
value = value['value'] |
|
|
|
ret.add(_Route53DynamicValue(provider, record, pool_name, |
|
|
|
value, weight, i, creating)) |
|
|
|
|
|
|
|
# Rules |
|
|
|
for i, rule in enumerate(record.dynamic.rules): |
|
|
|
pool_name = rule.data['pool'] |
|
|
|
geos = rule.data.get('geos', []) |
|
|
|
if geos: |
|
|
|
for geo in geos: |
|
|
|
# Create a RRSet for each geo in each rule that uses the |
|
|
|
# desired target pool |
|
|
|
ret.add(_Route53DynamicRule(provider, hosted_zone_id, |
|
|
|
record, pool_name, i, |
|
|
|
creating, geo=geo)) |
|
|
|
else: |
|
|
|
# There's no geo's for this rule so it's the catchall that will |
|
|
|
# just point things that don't match any geo rules to the |
|
|
|
# specified pool |
|
|
|
ret.add(_Route53DynamicRule(provider, hosted_zone_id, record, |
|
|
|
pool_name, i, creating)) |
|
|
|
|
|
|
|
return ret |
|
|
|
|
|
|
|
@classmethod |
|
|
|
def _new_geo(cls, provider, record, creating): |
|
|
|
# Creates the RRSets that correspond to the given geo record |
|
|
|
ret = set() |
|
|
|
|
|
|
|
ret.add(_Route53GeoDefault(provider, record, creating)) |
|
|
|
for ident, geo in record.geo.items(): |
|
|
|
ret.add(_Route53GeoRecord(provider, record, ident, geo, |
|
|
|
creating)) |
|
|
|
|
|
|
|
return ret |
|
|
|
|
|
|
|
def __init__(self, provider, record, creating): |
|
|
|
self.fqdn = record.fqdn |
|
|
|
@classmethod |
|
|
|
def new(cls, provider, record, hosted_zone_id, creating): |
|
|
|
# Creates the RRSets that correspond to the given record |
|
|
|
|
|
|
|
if getattr(record, 'dynamic', False): |
|
|
|
ret = cls._new_dynamic(provider, record, hosted_zone_id, creating) |
|
|
|
return ret |
|
|
|
elif getattr(record, 'geo', False): |
|
|
|
return cls._new_geo(provider, record, creating) |
|
|
|
|
|
|
|
# Its a simple record that translates into a single RRSet |
|
|
|
return set((_Route53Record(provider, record, creating),)) |
|
|
|
|
|
|
|
def __init__(self, provider, record, creating, fqdn_override=None): |
|
|
|
self.fqdn = fqdn_override or record.fqdn |
|
|
|
self._type = record._type |
|
|
|
self.ttl = record.ttl |
|
|
|
|
|
|
|
@ -83,6 +171,15 @@ class _Route53Record(object): |
|
|
|
return '_Route53Record<{} {} {} {}>'.format(self.fqdn, self._type, |
|
|
|
self.ttl, self.values) |
|
|
|
|
|
|
|
def _value_convert_value(self, value, record): |
|
|
|
return value |
|
|
|
|
|
|
|
_value_convert_A = _value_convert_value |
|
|
|
_value_convert_AAAA = _value_convert_value |
|
|
|
_value_convert_NS = _value_convert_value |
|
|
|
_value_convert_CNAME = _value_convert_value |
|
|
|
_value_convert_PTR = _value_convert_value |
|
|
|
|
|
|
|
def _values_for_values(self, record): |
|
|
|
return record.values |
|
|
|
|
|
|
|
@ -90,9 +187,11 @@ class _Route53Record(object): |
|
|
|
_values_for_AAAA = _values_for_values |
|
|
|
_values_for_NS = _values_for_values |
|
|
|
|
|
|
|
def _value_convert_CAA(self, value, record): |
|
|
|
return '{} {} "{}"'.format(value.flags, value.tag, value.value) |
|
|
|
|
|
|
|
def _values_for_CAA(self, record): |
|
|
|
return ['{} {} "{}"'.format(v.flags, v.tag, v.value) |
|
|
|
for v in record.values] |
|
|
|
return [self._value_convert_CAA(v, record) for v in record.values] |
|
|
|
|
|
|
|
def _values_for_value(self, record): |
|
|
|
return [record.value] |
|
|
|
@ -100,18 +199,28 @@ class _Route53Record(object): |
|
|
|
_values_for_CNAME = _values_for_value |
|
|
|
_values_for_PTR = _values_for_value |
|
|
|
|
|
|
|
def _value_convert_MX(self, value, record): |
|
|
|
return '{} {}'.format(value.preference, value.exchange) |
|
|
|
|
|
|
|
def _values_for_MX(self, record): |
|
|
|
return ['{} {}'.format(v.preference, v.exchange) |
|
|
|
for v in record.values] |
|
|
|
return [self._value_convert_MX(v, record) for v in record.values] |
|
|
|
|
|
|
|
def _value_convert_NAPTR(self, value, record): |
|
|
|
return '{} {} "{}" "{}" "{}" {}' \ |
|
|
|
.format(value.order, value.preference, |
|
|
|
value.flags if value.flags else '', |
|
|
|
value.service if value.service else '', |
|
|
|
value.regexp if value.regexp else '', |
|
|
|
value.replacement) |
|
|
|
|
|
|
|
def _values_for_NAPTR(self, record): |
|
|
|
return ['{} {} "{}" "{}" "{}" {}' |
|
|
|
.format(v.order, v.preference, |
|
|
|
v.flags if v.flags else '', |
|
|
|
v.service if v.service else '', |
|
|
|
v.regexp if v.regexp else '', |
|
|
|
v.replacement) |
|
|
|
for v in record.values] |
|
|
|
return [self._value_convert_NAPTR(v, record) for v in record.values] |
|
|
|
|
|
|
|
def _value_convert_quoted(self, value, record): |
|
|
|
return record.chunked_value(value) |
|
|
|
|
|
|
|
_value_convert_SPF = _value_convert_quoted |
|
|
|
_value_convert_TXT = _value_convert_quoted |
|
|
|
|
|
|
|
def _values_for_quoted(self, record): |
|
|
|
return record.chunked_values |
|
|
|
@ -119,10 +228,178 @@ class _Route53Record(object): |
|
|
|
_values_for_SPF = _values_for_quoted |
|
|
|
_values_for_TXT = _values_for_quoted |
|
|
|
|
|
|
|
def _value_for_SRV(self, value, record): |
|
|
|
return '{} {} {} {}'.format(value.priority, value.weight, |
|
|
|
value.port, value.target) |
|
|
|
|
|
|
|
def _values_for_SRV(self, record): |
|
|
|
return ['{} {} {} {}'.format(v.priority, v.weight, v.port, |
|
|
|
v.target) |
|
|
|
for v in record.values] |
|
|
|
return [self._value_for_SRV(v, record) for v in record.values] |
|
|
|
|
|
|
|
|
|
|
|
class _Route53DynamicPool(_Route53Record): |
|
|
|
|
|
|
|
def __init__(self, provider, hosted_zone_id, record, pool_name, creating, |
|
|
|
target_name=None): |
|
|
|
fqdn_override = '_octodns-{}-pool.{}'.format(pool_name, record.fqdn) |
|
|
|
super(_Route53DynamicPool, self) \ |
|
|
|
.__init__(provider, record, creating, fqdn_override=fqdn_override) |
|
|
|
|
|
|
|
self.hosted_zone_id = hosted_zone_id |
|
|
|
self.pool_name = pool_name |
|
|
|
|
|
|
|
self.target_name = target_name |
|
|
|
if target_name: |
|
|
|
# We're pointing down the chain |
|
|
|
self.target_dns_name = '_octodns-{}-pool.{}'.format(target_name, |
|
|
|
record.fqdn) |
|
|
|
else: |
|
|
|
# We're a paimary, point at our values |
|
|
|
self.target_dns_name = '_octodns-{}-value.{}'.format(pool_name, |
|
|
|
record.fqdn) |
|
|
|
|
|
|
|
@property |
|
|
|
def mode(self): |
|
|
|
return 'Secondary' if self.target_name else 'Primary' |
|
|
|
|
|
|
|
@property |
|
|
|
def identifer(self): |
|
|
|
if self.target_name: |
|
|
|
return '{}-{}-{}'.format(self.pool_name, self.mode, |
|
|
|
self.target_name) |
|
|
|
return '{}-{}'.format(self.pool_name, self.mode) |
|
|
|
|
|
|
|
def mod(self, action): |
|
|
|
return { |
|
|
|
'Action': action, |
|
|
|
'ResourceRecordSet': { |
|
|
|
'AliasTarget': { |
|
|
|
'DNSName': self.target_dns_name, |
|
|
|
'EvaluateTargetHealth': True, |
|
|
|
'HostedZoneId': self.hosted_zone_id, |
|
|
|
}, |
|
|
|
'Failover': 'SECONDARY' if self.target_name else 'PRIMARY', |
|
|
|
'Name': self.fqdn, |
|
|
|
'SetIdentifier': self.identifer, |
|
|
|
'Type': self._type, |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
def __hash__(self): |
|
|
|
return '{}:{}:{}'.format(self.fqdn, self._type, |
|
|
|
self.identifer).__hash__() |
|
|
|
|
|
|
|
def __repr__(self): |
|
|
|
return '_Route53DynamicPool<{} {} {} {}>' \ |
|
|
|
.format(self.fqdn, self._type, self.mode, self.target_dns_name) |
|
|
|
|
|
|
|
|
|
|
|
class _Route53DynamicRule(_Route53Record): |
|
|
|
|
|
|
|
def __init__(self, provider, hosted_zone_id, record, pool_name, index, |
|
|
|
creating, geo=None): |
|
|
|
super(_Route53DynamicRule, self).__init__(provider, record, creating) |
|
|
|
|
|
|
|
self.hosted_zone_id = hosted_zone_id |
|
|
|
self.geo = geo |
|
|
|
self.pool_name = pool_name |
|
|
|
self.index = index |
|
|
|
|
|
|
|
self.target_dns_name = '_octodns-{}-pool.{}'.format(pool_name, |
|
|
|
record.fqdn) |
|
|
|
|
|
|
|
@property |
|
|
|
def identifer(self): |
|
|
|
return '{}-{}-{}'.format(self.index, self.pool_name, self.geo) |
|
|
|
|
|
|
|
def mod(self, action): |
|
|
|
rrset = { |
|
|
|
'AliasTarget': { |
|
|
|
'DNSName': self.target_dns_name, |
|
|
|
'EvaluateTargetHealth': True, |
|
|
|
'HostedZoneId': self.hosted_zone_id, |
|
|
|
}, |
|
|
|
'GeoLocation': { |
|
|
|
'CountryCode': '*' |
|
|
|
}, |
|
|
|
'Name': self.fqdn, |
|
|
|
'SetIdentifier': self.identifer, |
|
|
|
'Type': self._type, |
|
|
|
} |
|
|
|
|
|
|
|
if self.geo: |
|
|
|
geo = GeoCodes.parse(self.geo) |
|
|
|
|
|
|
|
if geo['province_code']: |
|
|
|
rrset['GeoLocation'] = { |
|
|
|
'CountryCode': geo['country_code'], |
|
|
|
'SubdivisionCode': geo['province_code'], |
|
|
|
} |
|
|
|
elif geo['country_code']: |
|
|
|
rrset['GeoLocation'] = { |
|
|
|
'CountryCode': geo['country_code'] |
|
|
|
} |
|
|
|
else: |
|
|
|
rrset['GeoLocation'] = { |
|
|
|
'ContinentCode': geo['continent_code'], |
|
|
|
} |
|
|
|
|
|
|
|
return { |
|
|
|
'Action': action, |
|
|
|
'ResourceRecordSet': rrset, |
|
|
|
} |
|
|
|
|
|
|
|
def __hash__(self): |
|
|
|
return '{}:{}:{}'.format(self.fqdn, self._type, |
|
|
|
self.identifer).__hash__() |
|
|
|
|
|
|
|
def __repr__(self): |
|
|
|
return '_Route53DynamicRule<{} {} {} {} {}>' \ |
|
|
|
.format(self.fqdn, self._type, self.index, self.geo, |
|
|
|
self.target_dns_name) |
|
|
|
|
|
|
|
|
|
|
|
class _Route53DynamicValue(_Route53Record): |
|
|
|
|
|
|
|
def __init__(self, provider, record, pool_name, value, weight, index, |
|
|
|
creating): |
|
|
|
fqdn_override = '_octodns-{}-value.{}'.format(pool_name, record.fqdn) |
|
|
|
super(_Route53DynamicValue, self).__init__(provider, record, creating, |
|
|
|
fqdn_override=fqdn_override) |
|
|
|
|
|
|
|
self.pool_name = pool_name |
|
|
|
self.index = index |
|
|
|
value_convert = getattr(self, '_value_convert_{}'.format(record._type)) |
|
|
|
self.value = value_convert(value, record) |
|
|
|
self.weight = weight |
|
|
|
|
|
|
|
self.health_check_id = provider.get_health_check_id(record, self.value, |
|
|
|
creating) |
|
|
|
|
|
|
|
@property |
|
|
|
def identifer(self): |
|
|
|
return '{}-{:03d}'.format(self.pool_name, self.index) |
|
|
|
|
|
|
|
def mod(self, action): |
|
|
|
return { |
|
|
|
'Action': action, |
|
|
|
'ResourceRecordSet': { |
|
|
|
'HealthCheckId': self.health_check_id, |
|
|
|
'Name': self.fqdn, |
|
|
|
'ResourceRecords': [{'Value': self.value}], |
|
|
|
'SetIdentifier': self.identifer, |
|
|
|
'TTL': self.ttl, |
|
|
|
'Type': self._type, |
|
|
|
'Weight': self.weight, |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
def __hash__(self): |
|
|
|
return '{}:{}:{}'.format(self.fqdn, self._type, |
|
|
|
self.identifer).__hash__() |
|
|
|
|
|
|
|
def __repr__(self): |
|
|
|
return '_Route53DynamicValue<{} {} {} {}>' \ |
|
|
|
.format(self.fqdn, self._type, self.identifer, self.value) |
|
|
|
|
|
|
|
|
|
|
|
class _Route53GeoDefault(_Route53Record): |
|
|
|
@ -156,8 +433,9 @@ class _Route53GeoRecord(_Route53Record): |
|
|
|
super(_Route53GeoRecord, self).__init__(provider, record, creating) |
|
|
|
self.geo = geo |
|
|
|
|
|
|
|
self.health_check_id = provider.get_health_check_id(record, ident, |
|
|
|
geo, creating) |
|
|
|
value = geo.values[0] |
|
|
|
self.health_check_id = provider.get_health_check_id(record, value, |
|
|
|
creating) |
|
|
|
|
|
|
|
def mod(self, action): |
|
|
|
geo = self.geo |
|
|
|
@ -211,6 +489,44 @@ class _Route53GeoRecord(_Route53Record): |
|
|
|
self.values) |
|
|
|
|
|
|
|
|
|
|
|
_mod_keyer_action_order = { |
|
|
|
'DELETE': 0, # Delete things first |
|
|
|
'CREATE': 1, # Then Create things |
|
|
|
'UPSERT': 2, # Upsert things last |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
def _mod_keyer(mod): |
|
|
|
rrset = mod['ResourceRecordSet'] |
|
|
|
action_order = _mod_keyer_action_order[mod['Action']] |
|
|
|
|
|
|
|
# We're sorting by 3 "columns", the action, the rrset type, and finally the |
|
|
|
# name/id of the rrset. This ensures that Route53 won't see a RRSet that |
|
|
|
# targets another that hasn't been seen yet. I.e. targets must come before |
|
|
|
# things that target them. We sort on types of things rather than |
|
|
|
# explicitly looking for targeting relationships since that's sufficent and |
|
|
|
# easier to grok/do. |
|
|
|
|
|
|
|
if rrset.get('GeoLocation', False): |
|
|
|
return (action_order, 3, rrset['SetIdentifier']) |
|
|
|
elif rrset.get('AliasTarget', False): |
|
|
|
# We use an alias |
|
|
|
if rrset.get('Failover', False) == 'SECONDARY': |
|
|
|
# We're a secondary we'll ref primaries |
|
|
|
return (action_order, 2, rrset['Name']) |
|
|
|
else: |
|
|
|
# We're a primary we'll ref values |
|
|
|
return (action_order, 1, rrset['Name']) |
|
|
|
|
|
|
|
# We're just a plain value, these come first |
|
|
|
return (action_order, 0, rrset['Name']) |
|
|
|
|
|
|
|
|
|
|
|
def _parse_pool_name(n): |
|
|
|
# Parse the pool name out of _octodns-<pool-name>-pool... |
|
|
|
return n.split('.', 1)[0][9:-5] |
|
|
|
|
|
|
|
|
|
|
|
class Route53Provider(BaseProvider): |
|
|
|
''' |
|
|
|
AWS Route53 Provider |
|
|
|
@ -232,8 +548,7 @@ class Route53Provider(BaseProvider): |
|
|
|
In general the account used will need full permissions on Route53. |
|
|
|
''' |
|
|
|
SUPPORTS_GEO = True |
|
|
|
# TODO: dynamic |
|
|
|
SUPPORTS_DYNAMIC = False |
|
|
|
SUPPORTS_DYNAMIC = True |
|
|
|
SUPPORTS = set(('A', 'AAAA', 'CAA', 'CNAME', 'MX', 'NAPTR', 'NS', 'PTR', |
|
|
|
'SPF', 'SRV', 'TXT')) |
|
|
|
|
|
|
|
@ -465,6 +780,79 @@ class Route53Provider(BaseProvider): |
|
|
|
|
|
|
|
return self._r53_rrsets[zone_id] |
|
|
|
|
|
|
|
def _data_for_dynamic(self, name, _type, rrsets): |
|
|
|
# This converts a bunch of RRSets into their corresponding dynamic |
|
|
|
# Record. It's used by populate. |
|
|
|
pools = defaultdict(lambda: {'values': []}) |
|
|
|
# Data to build our rules will be collected here and "converted" into |
|
|
|
# their final form below |
|
|
|
rules = defaultdict(lambda: {'pool': None, 'geos': []}) |
|
|
|
# Base/empty data |
|
|
|
data = { |
|
|
|
'dynamic': { |
|
|
|
'pools': pools, |
|
|
|
'rules': [], |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
# For all the rrsets that comprise this dynamic record |
|
|
|
for rrset in rrsets: |
|
|
|
name = rrset['Name'] |
|
|
|
if '-pool.' in name: |
|
|
|
# This is a pool rrset |
|
|
|
pool_name = _parse_pool_name(name) |
|
|
|
if pool_name == 'default': |
|
|
|
# default becomes the base for the record and its |
|
|
|
# value(s) will fill the non-dynamic values |
|
|
|
data_for = getattr(self, '_data_for_{}'.format(_type)) |
|
|
|
data.update(data_for(rrset)) |
|
|
|
elif rrset['Failover'] == 'SECONDARY': |
|
|
|
# This is a failover record, we'll ignore PRIMARY, but |
|
|
|
# SECONDARY will tell us what the pool's fallback is |
|
|
|
fallback_name = \ |
|
|
|
_parse_pool_name(rrset['AliasTarget']['DNSName']) |
|
|
|
# Don't care about default fallbacks, anything else |
|
|
|
# we'll record |
|
|
|
if fallback_name != 'default': |
|
|
|
pools[pool_name]['fallback'] = fallback_name |
|
|
|
elif 'GeoLocation' in rrset: |
|
|
|
# These are rules |
|
|
|
_id = rrset['SetIdentifier'] |
|
|
|
# We record rule index as the first part of set-id, the 2nd |
|
|
|
# part just ensures uniqueness across geos and is ignored |
|
|
|
i = int(_id.split('-', 1)[0]) |
|
|
|
target_pool = _parse_pool_name(rrset['AliasTarget']['DNSName']) |
|
|
|
# Record the pool |
|
|
|
rules[i]['pool'] = target_pool |
|
|
|
# Record geo if we have one |
|
|
|
geo = self._parse_geo(rrset) |
|
|
|
if geo: |
|
|
|
rules[i]['geos'].append(geo) |
|
|
|
else: |
|
|
|
# These are the pool value(s) |
|
|
|
# Grab the pool name out of the SetIdentifier, format looks |
|
|
|
# like ...-000 where 000 is a zero-padded index for the value |
|
|
|
# it's ignored only used to make sure the value is unique |
|
|
|
pool_name = rrset['SetIdentifier'][:-4] |
|
|
|
value = rrset['ResourceRecords'][0]['Value'] |
|
|
|
pools[pool_name]['values'].append({ |
|
|
|
'value': value, |
|
|
|
'weight': rrset['Weight'], |
|
|
|
}) |
|
|
|
|
|
|
|
# Convert our map of rules into an ordered list now that we have all |
|
|
|
# the data |
|
|
|
for _, rule in sorted(rules.items()): |
|
|
|
r = { |
|
|
|
'pool': rule['pool'], |
|
|
|
} |
|
|
|
geos = sorted(rule['geos']) |
|
|
|
if geos: |
|
|
|
r['geos'] = geos |
|
|
|
data['dynamic']['rules'].append(r) |
|
|
|
|
|
|
|
return data |
|
|
|
|
|
|
|
def populate(self, zone, target=False, lenient=False): |
|
|
|
self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, |
|
|
|
target, lenient) |
|
|
|
@ -476,21 +864,46 @@ class Route53Provider(BaseProvider): |
|
|
|
if zone_id: |
|
|
|
exists = True |
|
|
|
records = defaultdict(lambda: defaultdict(list)) |
|
|
|
dynamic = defaultdict(lambda: defaultdict(list)) |
|
|
|
|
|
|
|
for rrset in self._load_records(zone_id): |
|
|
|
record_name = zone.hostname_from_fqdn(rrset['Name']) |
|
|
|
record_name = _octal_replace(record_name) |
|
|
|
record_type = rrset['Type'] |
|
|
|
if record_type not in self.SUPPORTS: |
|
|
|
# Skip stuff we don't support |
|
|
|
continue |
|
|
|
if record_name.startswith('_octodns-'): |
|
|
|
# Part of a dynamic record |
|
|
|
try: |
|
|
|
record_name = record_name.split('.', 1)[1] |
|
|
|
except IndexError: |
|
|
|
record_name = '' |
|
|
|
dynamic[record_name][record_type].append(rrset) |
|
|
|
continue |
|
|
|
if 'AliasTarget' in rrset: |
|
|
|
# Alias records are Route53 specific and are not |
|
|
|
# portable, so we need to skip them |
|
|
|
self.log.warning("%s is an Alias record. Skipping..." |
|
|
|
% rrset['Name']) |
|
|
|
elif 'AliasTarget' in rrset: |
|
|
|
if rrset['AliasTarget']['DNSName'].startswith('_octodns-'): |
|
|
|
# Part of a dynamic record |
|
|
|
dynamic[record_name][record_type].append(rrset) |
|
|
|
else: |
|
|
|
# Alias records are Route53 specific and are not |
|
|
|
# portable, so we need to skip them |
|
|
|
self.log.warning("%s is an Alias record. Skipping..." |
|
|
|
% rrset['Name']) |
|
|
|
continue |
|
|
|
# A basic record (potentially including geo) |
|
|
|
data = getattr(self, '_data_for_{}'.format(record_type))(rrset) |
|
|
|
records[record_name][record_type].append(data) |
|
|
|
|
|
|
|
# Convert the dynamic rrsets to Records |
|
|
|
for name, types in dynamic.items(): |
|
|
|
for _type, rrsets in types.items(): |
|
|
|
data = self._data_for_dynamic(name, _type, rrsets) |
|
|
|
record = Record.new(zone, name, data, source=self, |
|
|
|
lenient=lenient) |
|
|
|
zone.add_record(record, lenient=lenient) |
|
|
|
|
|
|
|
# Convert the basic (potentially with geo) rrsets to records |
|
|
|
for name, types in records.items(): |
|
|
|
for _type, data in types.items(): |
|
|
|
if len(data) > 1: |
|
|
|
@ -537,6 +950,7 @@ class Route53Provider(BaseProvider): |
|
|
|
# ignore anything else |
|
|
|
continue |
|
|
|
checks[health_check['Id']] = health_check |
|
|
|
|
|
|
|
more = resp['IsTruncated'] |
|
|
|
start['Marker'] = resp.get('NextMarker', None) |
|
|
|
|
|
|
|
@ -551,27 +965,44 @@ class Route53Provider(BaseProvider): |
|
|
|
.get('measure_latency', True) |
|
|
|
|
|
|
|
def _health_check_equivilent(self, host, path, protocol, port, |
|
|
|
measure_latency, health_check, |
|
|
|
first_value=None): |
|
|
|
measure_latency, health_check, value=None): |
|
|
|
config = health_check['HealthCheckConfig'] |
|
|
|
|
|
|
|
# So interestingly Route53 normalizes IPAddress which will cause us to |
|
|
|
# fail to find see things as equivalent. To work around this we'll |
|
|
|
# ip_address's returned object for equivalence |
|
|
|
# E.g 2001:4860:4860::8842 -> 2001:4860:4860:0:0:0:0:8842 |
|
|
|
if value: |
|
|
|
value = ip_address(unicode(value)) |
|
|
|
config_ip_address = ip_address(unicode(config['IPAddress'])) |
|
|
|
else: |
|
|
|
# No value so give this a None to match value's |
|
|
|
config_ip_address = None |
|
|
|
|
|
|
|
return host == config['FullyQualifiedDomainName'] and \ |
|
|
|
path == config['ResourcePath'] and protocol == config['Type'] \ |
|
|
|
and port == config['Port'] and \ |
|
|
|
measure_latency == config['MeasureLatency'] and \ |
|
|
|
(first_value is None or first_value == config['IPAddress']) |
|
|
|
value == config_ip_address |
|
|
|
|
|
|
|
def get_health_check_id(self, record, ident, geo, create): |
|
|
|
def get_health_check_id(self, record, value, create): |
|
|
|
# fqdn & the first value are special, we use them to match up health |
|
|
|
# checks to their records. Route53 health checks check a single ip and |
|
|
|
# we're going to assume that ips are interchangeable to avoid |
|
|
|
# health-checking each one independently |
|
|
|
fqdn = record.fqdn |
|
|
|
first_value = geo.values[0] |
|
|
|
self.log.debug('get_health_check_id: fqdn=%s, type=%s, geo=%s, ' |
|
|
|
'first_value=%s', fqdn, record._type, ident, |
|
|
|
first_value) |
|
|
|
self.log.debug('get_health_check_id: fqdn=%s, type=%s, value=%s', |
|
|
|
fqdn, record._type, value) |
|
|
|
|
|
|
|
try: |
|
|
|
ip_address(unicode(value)) |
|
|
|
# We're working with an IP, host is the Host header |
|
|
|
healthcheck_host = record.healthcheck_host |
|
|
|
except (AddressValueError, ValueError): |
|
|
|
# This isn't an IP, host is the value, value should be None |
|
|
|
healthcheck_host = value |
|
|
|
value = None |
|
|
|
|
|
|
|
healthcheck_host = record.healthcheck_host |
|
|
|
healthcheck_path = record.healthcheck_path |
|
|
|
healthcheck_protocol = record.healthcheck_protocol |
|
|
|
healthcheck_port = record.healthcheck_port |
|
|
|
@ -591,7 +1022,7 @@ class Route53Provider(BaseProvider): |
|
|
|
healthcheck_port, |
|
|
|
healthcheck_latency, |
|
|
|
health_check, |
|
|
|
first_value=first_value): |
|
|
|
value=value): |
|
|
|
# this is the health check we're looking for |
|
|
|
self.log.debug('get_health_check_id: found match id=%s', id) |
|
|
|
return id |
|
|
|
@ -606,28 +1037,44 @@ class Route53Provider(BaseProvider): |
|
|
|
'EnableSNI': healthcheck_protocol == 'HTTPS', |
|
|
|
'FailureThreshold': 6, |
|
|
|
'FullyQualifiedDomainName': healthcheck_host, |
|
|
|
'IPAddress': first_value, |
|
|
|
'MeasureLatency': healthcheck_latency, |
|
|
|
'Port': healthcheck_port, |
|
|
|
'RequestInterval': 10, |
|
|
|
'ResourcePath': healthcheck_path, |
|
|
|
'Type': healthcheck_protocol, |
|
|
|
} |
|
|
|
if value: |
|
|
|
config['IPAddress'] = value |
|
|
|
|
|
|
|
ref = '{}:{}:{}:{}'.format(self.HEALTH_CHECK_VERSION, record._type, |
|
|
|
record.fqdn, uuid4().hex[:12]) |
|
|
|
resp = self._conn.create_health_check(CallerReference=ref, |
|
|
|
HealthCheckConfig=config) |
|
|
|
health_check = resp['HealthCheck'] |
|
|
|
id = health_check['Id'] |
|
|
|
|
|
|
|
# Set a Name for the benefit of the UI |
|
|
|
name = '{}:{} - {}'.format(record.fqdn, record._type, |
|
|
|
value or healthcheck_host) |
|
|
|
self._conn.change_tags_for_resource(ResourceType='healthcheck', |
|
|
|
ResourceId=id, |
|
|
|
AddTags=[{ |
|
|
|
'Key': 'Name', |
|
|
|
'Value': name, |
|
|
|
}]) |
|
|
|
# Manually add it to our cache |
|
|
|
health_check['Tags'] = { |
|
|
|
'Name': name |
|
|
|
} |
|
|
|
|
|
|
|
# store the new health check so that we'll be able to find it in the |
|
|
|
# future |
|
|
|
self._health_checks[id] = health_check |
|
|
|
self.log.info('get_health_check_id: created id=%s, host=%s, ' |
|
|
|
'path=%s, protocol=%s, port=%d, measure_latency=%r, ' |
|
|
|
'first_value=%s', |
|
|
|
id, healthcheck_host, healthcheck_path, |
|
|
|
'value=%s', id, healthcheck_host, healthcheck_path, |
|
|
|
healthcheck_protocol, healthcheck_port, |
|
|
|
healthcheck_latency, first_value) |
|
|
|
healthcheck_latency, value) |
|
|
|
return id |
|
|
|
|
|
|
|
def _gc_health_checks(self, record, new): |
|
|
|
@ -664,25 +1111,26 @@ class Route53Provider(BaseProvider): |
|
|
|
id) |
|
|
|
self._conn.delete_health_check(HealthCheckId=id) |
|
|
|
|
|
|
|
def _gen_records(self, record, creating=False): |
|
|
|
def _gen_records(self, record, zone_id, creating=False): |
|
|
|
''' |
|
|
|
Turns an octodns.Record into one or more `_Route53*`s |
|
|
|
''' |
|
|
|
return _Route53Record.new(self, record, creating) |
|
|
|
return _Route53Record.new(self, record, zone_id, creating) |
|
|
|
|
|
|
|
def _mod_Create(self, change): |
|
|
|
def _mod_Create(self, change, zone_id): |
|
|
|
# New is the stuff that needs to be created |
|
|
|
new_records = self._gen_records(change.new, creating=True) |
|
|
|
new_records = self._gen_records(change.new, zone_id, creating=True) |
|
|
|
# Now is a good time to clear out any unused health checks since we |
|
|
|
# know what we'll be using going forward |
|
|
|
self._gc_health_checks(change.new, new_records) |
|
|
|
return self._gen_mods('CREATE', new_records) |
|
|
|
|
|
|
|
def _mod_Update(self, change): |
|
|
|
def _mod_Update(self, change, zone_id): |
|
|
|
# See comments in _Route53Record for how the set math is made to do our |
|
|
|
# bidding here. |
|
|
|
existing_records = self._gen_records(change.existing, creating=False) |
|
|
|
new_records = self._gen_records(change.new, creating=True) |
|
|
|
existing_records = self._gen_records(change.existing, zone_id, |
|
|
|
creating=False) |
|
|
|
new_records = self._gen_records(change.new, zone_id, creating=True) |
|
|
|
# Now is a good time to clear out any unused health checks since we |
|
|
|
# know what we'll be using going forward |
|
|
|
self._gc_health_checks(change.new, new_records) |
|
|
|
@ -704,14 +1152,91 @@ class Route53Provider(BaseProvider): |
|
|
|
self._gen_mods('CREATE', creates) + \ |
|
|
|
self._gen_mods('UPSERT', upserts) |
|
|
|
|
|
|
|
def _mod_Delete(self, change): |
|
|
|
def _mod_Delete(self, change, zone_id): |
|
|
|
# Existing is the thing that needs to be deleted |
|
|
|
existing_records = self._gen_records(change.existing, creating=False) |
|
|
|
existing_records = self._gen_records(change.existing, zone_id, |
|
|
|
creating=False) |
|
|
|
# Now is a good time to clear out all the health checks since we know |
|
|
|
# we're done with them |
|
|
|
self._gc_health_checks(change.existing, []) |
|
|
|
return self._gen_mods('DELETE', existing_records) |
|
|
|
|
|
|
|
def _extra_changes_update_needed(self, record, rrset): |
|
|
|
healthcheck_host = record.healthcheck_host |
|
|
|
healthcheck_path = record.healthcheck_path |
|
|
|
healthcheck_protocol = record.healthcheck_protocol |
|
|
|
healthcheck_port = record.healthcheck_port |
|
|
|
healthcheck_latency = self._healthcheck_measure_latency(record) |
|
|
|
|
|
|
|
try: |
|
|
|
health_check_id = rrset['HealthCheckId'] |
|
|
|
health_check = self.health_checks[health_check_id] |
|
|
|
caller_ref = health_check['CallerReference'] |
|
|
|
if caller_ref.startswith(self.HEALTH_CHECK_VERSION): |
|
|
|
if self._health_check_equivilent(healthcheck_host, |
|
|
|
healthcheck_path, |
|
|
|
healthcheck_protocol, |
|
|
|
healthcheck_port, |
|
|
|
healthcheck_latency, |
|
|
|
health_check): |
|
|
|
# it has the right health check |
|
|
|
return False |
|
|
|
except (IndexError, KeyError): |
|
|
|
# no health check id or one that isn't the right version |
|
|
|
pass |
|
|
|
|
|
|
|
# no good, doesn't have the right health check, needs an update |
|
|
|
self.log.info('_extra_changes_update_needed: health-check caused ' |
|
|
|
'update of %s:%s', record.fqdn, record._type) |
|
|
|
return True |
|
|
|
|
|
|
|
def _extra_changes_geo_needs_update(self, zone_id, record): |
|
|
|
# OK this is a record we don't have change for that does have geo |
|
|
|
# information. We need to look and see if it needs to be updated b/c of |
|
|
|
# a health check version bump or other mismatch |
|
|
|
self.log.debug('_extra_changes_geo_needs_update: inspecting=%s, %s', |
|
|
|
record.fqdn, record._type) |
|
|
|
|
|
|
|
fqdn = record.fqdn |
|
|
|
|
|
|
|
# loop through all the r53 rrsets |
|
|
|
for rrset in self._load_records(zone_id): |
|
|
|
if fqdn == rrset['Name'] and record._type == rrset['Type'] and \ |
|
|
|
rrset.get('GeoLocation', {}).get('CountryCode', False) != '*' \ |
|
|
|
and self._extra_changes_update_needed(record, rrset): |
|
|
|
# no good, doesn't have the right health check, needs an update |
|
|
|
self.log.info('_extra_changes_geo_needs_update: health-check ' |
|
|
|
'caused update of %s:%s', record.fqdn, |
|
|
|
record._type) |
|
|
|
return True |
|
|
|
|
|
|
|
return False |
|
|
|
|
|
|
|
def _extra_changes_dynamic_needs_update(self, zone_id, record): |
|
|
|
# OK this is a record we don't have change for that does have dynamic |
|
|
|
# information. We need to look and see if it needs to be updated b/c of |
|
|
|
# a health check version bump or other mismatch |
|
|
|
self.log.debug('_extra_changes_dynamic_needs_update: inspecting=%s, ' |
|
|
|
'%s', record.fqdn, record._type) |
|
|
|
|
|
|
|
fqdn = record.fqdn |
|
|
|
|
|
|
|
# loop through all the r53 rrsets |
|
|
|
for rrset in self._load_records(zone_id): |
|
|
|
name = rrset['Name'] |
|
|
|
|
|
|
|
if record._type == rrset['Type'] and name.endswith(fqdn) and \ |
|
|
|
name.startswith('_octodns-') and '-value.' in name and \ |
|
|
|
'-default-' not in name and \ |
|
|
|
self._extra_changes_update_needed(record, rrset): |
|
|
|
# no good, doesn't have the right health check, needs an update |
|
|
|
self.log.info('_extra_changes_dynamic_needs_update: ' |
|
|
|
'health-check caused update of %s:%s', |
|
|
|
record.fqdn, record._type) |
|
|
|
return True |
|
|
|
|
|
|
|
return False |
|
|
|
|
|
|
|
def _extra_changes(self, desired, changes, **kwargs): |
|
|
|
self.log.debug('_extra_changes: desired=%s', desired.name) |
|
|
|
zone_id = self._get_zone_id(desired.name) |
|
|
|
@ -722,61 +1247,20 @@ class Route53Provider(BaseProvider): |
|
|
|
changed = set([c.record for c in changes]) |
|
|
|
# ok, now it's time for the reason we're here, we need to go over all |
|
|
|
# the desired records |
|
|
|
extra = [] |
|
|
|
extras = [] |
|
|
|
for record in desired.records: |
|
|
|
if record in changed: |
|
|
|
# already have a change for it, skipping |
|
|
|
continue |
|
|
|
if not getattr(record, 'geo', False): |
|
|
|
# record doesn't support geo, we don't need to inspect it |
|
|
|
continue |
|
|
|
# OK this is a record we don't have change for that does have geo |
|
|
|
# information. We need to look and see if it needs to be updated |
|
|
|
# b/c of a health check version bump |
|
|
|
self.log.debug('_extra_changes: inspecting=%s, %s', record.fqdn, |
|
|
|
record._type) |
|
|
|
|
|
|
|
healthcheck_host = record.healthcheck_host |
|
|
|
healthcheck_path = record.healthcheck_path |
|
|
|
healthcheck_protocol = record.healthcheck_protocol |
|
|
|
healthcheck_port = record.healthcheck_port |
|
|
|
healthcheck_latency = self._healthcheck_measure_latency(record) |
|
|
|
fqdn = record.fqdn |
|
|
|
|
|
|
|
# loop through all the r53 rrsets |
|
|
|
for rrset in self._load_records(zone_id): |
|
|
|
if fqdn != rrset['Name'] or record._type != rrset['Type']: |
|
|
|
# not a name and type match |
|
|
|
continue |
|
|
|
if rrset.get('GeoLocation', {}) \ |
|
|
|
.get('CountryCode', False) == '*': |
|
|
|
# it's a default record |
|
|
|
continue |
|
|
|
# we expect a healthcheck now |
|
|
|
try: |
|
|
|
health_check_id = rrset['HealthCheckId'] |
|
|
|
health_check = self.health_checks[health_check_id] |
|
|
|
caller_ref = health_check['CallerReference'] |
|
|
|
if caller_ref.startswith(self.HEALTH_CHECK_VERSION): |
|
|
|
if self._health_check_equivilent(healthcheck_host, |
|
|
|
healthcheck_path, |
|
|
|
healthcheck_protocol, |
|
|
|
healthcheck_port, |
|
|
|
healthcheck_latency, |
|
|
|
health_check): |
|
|
|
# it has the right health check |
|
|
|
continue |
|
|
|
except (IndexError, KeyError): |
|
|
|
# no health check id or one that isn't the right version |
|
|
|
pass |
|
|
|
# no good, doesn't have the right health check, needs an update |
|
|
|
self.log.info('_extra_changes: health-check caused ' |
|
|
|
'update of %s:%s', record.fqdn, record._type) |
|
|
|
extra.append(Update(record, record)) |
|
|
|
# We don't need to process this record any longer |
|
|
|
break |
|
|
|
if getattr(record, 'geo', False): |
|
|
|
if self._extra_changes_geo_needs_update(zone_id, record): |
|
|
|
extras.append(Update(record, record)) |
|
|
|
elif getattr(record, 'dynamic', False): |
|
|
|
if self._extra_changes_dynamic_needs_update(zone_id, record): |
|
|
|
extras.append(Update(record, record)) |
|
|
|
|
|
|
|
return extra |
|
|
|
return extras |
|
|
|
|
|
|
|
def _apply(self, plan): |
|
|
|
desired = plan.desired |
|
|
|
@ -788,9 +1272,17 @@ class Route53Provider(BaseProvider): |
|
|
|
batch_rs_count = 0 |
|
|
|
zone_id = self._get_zone_id(desired.name, True) |
|
|
|
for c in changes: |
|
|
|
mods = getattr(self, '_mod_{}'.format(c.__class__.__name__))(c) |
|
|
|
# Generate the mods for this change |
|
|
|
mod_type = getattr(self, '_mod_{}'.format(c.__class__.__name__)) |
|
|
|
mods = mod_type(c, zone_id) |
|
|
|
|
|
|
|
# Order our mods to make sure targets exist before alises point to |
|
|
|
# them and we CRUD in the desired order |
|
|
|
mods.sort(key=_mod_keyer) |
|
|
|
|
|
|
|
mods_rs_count = sum( |
|
|
|
[len(m['ResourceRecordSet']['ResourceRecords']) for m in mods] |
|
|
|
[len(m['ResourceRecordSet'].get('ResourceRecords', '')) |
|
|
|
for m in mods] |
|
|
|
) |
|
|
|
|
|
|
|
if mods_rs_count > self.max_changes: |
|
|
|
|
Ross McFarland
7 years ago
GitHub