Merge branch 'master' into powerdns-4.3.x-support

* master: (21 commits)
  Add Canadian provinces to geo_data.py
  Fix comment < 80 chars
  Add support for geo-targeting of CA provinces - For providers that support such
  Update geo_data to pick up a couple renames
  Ignore E741, flags single-letter var names in comprehensions which I want to allow
  Bump pycodestyle from 2.5.0 to 2.6.0
  Bump boto3 from 1.13.0 to 1.13.19
  Bump botocore from 1.16.0 to 1.16.19
  Bump six from 1.14.0 to 1.15.0
  Bump ns1-python from 0.15.0 to 0.16.0
  Bump setuptools from 44.1.0 to 44.1.1
  Cloudflare: Rename _try to _try_request
  Cloudflare: Add Support for Rate Limit
  Cloudflare: Add Support for PTR Records
  Update NS1 _REGION_FILTER to include remove_no_georegion in config
  Fix code coverage for NS1
  Docs and changelog for TCP health check support
  Fix Dyn python3 error with dict_values that needed a list
  TCP healthcheck support for Route53
  NS1 support for TCP healthchecks
  ...

CHANGELOG.md

@@ -1,3 +1,7 @@
+## v0.9.11 - 2020-??-?? - ???????????????
+
+* Added support for TCP health checking to dynamic records
+
 ## v0.9.10 - 2020-04-20 - Dynamic NS1 and lots of misc
 
 * Added support for dynamic records to Ns1Provider, updated client and rate

README.md

@@ -11,6 +11,8 @@ It is similar to [Netflix/denominator](https://github.com/Netflix/denominator).
 
 ## Table of Contents
 
+- [DNS as code - Tools for managing DNS across multiple providers](#dns-as-code---tools-for-managing-dns-across-multiple-providers)
+- [Table of Contents](#table-of-contents)
 - [Getting started](#getting-started)
   - [Workspace](#workspace)
   - [Config](#config)
@@ -178,7 +180,7 @@ The above command pulled the existing data out of Route53 and placed the results
 |--|--|--|--|--|
 | [AzureProvider](/octodns/provider/azuredns.py) | azure-mgmt-dns | A, AAAA, CAA, CNAME, MX, NS, PTR, SRV, TXT | No | |
 | [Akamai](/octodns/provider/edgedns.py) | edgegrid-python | A, AAAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, SSHFP, TXT | No | |
-| [CloudflareProvider](/octodns/provider/cloudflare.py) | | A, AAAA, ALIAS, CAA, CNAME, MX, NS, SPF, SRV, TXT | No | CAA tags restricted |
+| [CloudflareProvider](/octodns/provider/cloudflare.py) | | A, AAAA, ALIAS, CAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | CAA tags restricted |
 | [ConstellixProvider](/octodns/provider/constellix.py) | | A, AAAA, ALIAS (ANAME), CAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | CAA tags restricted |
 | [DigitalOceanProvider](/octodns/provider/digitalocean.py) | | A, AAAA, CAA, CNAME, MX, NS, TXT, SRV | No | CAA tags restricted |
 | [DnsMadeEasyProvider](/octodns/provider/dnsmadeeasy.py) | | A, AAAA, ALIAS (ANAME), CAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | CAA tags restricted |

docs/dynamic_records.md

@@ -103,7 +103,7 @@ test:
 | host | FQDN for host header and SNI | - |
 | path | path to check | _dns |
 | port | port to check | 443 |
-| protocol | HTTP/HTTPS | HTTPS |
+| protocol | HTTP/HTTPS/TCP | HTTPS |
 
 #### Route53 Healtch Check Options
 

octodns/provider/cloudflare.py

@@ -9,6 +9,7 @@ from collections import defaultdict
 from copy import deepcopy
 from logging import getLogger
 from requests import Session
+from time import sleep
 
 from ..record import Record, Update
 from .base import BaseProvider
@@ -18,7 +19,7 @@ class CloudflareError(Exception):
     def __init__(self, data):
         try:
             message = data['errors'][0]['message']
-        except (IndexError, KeyError):
+        except (IndexError, KeyError, TypeError):
             message = 'Cloudflare error'
         super(CloudflareError, self).__init__(message)
 
@@ -28,6 +29,11 @@ class CloudflareAuthenticationError(CloudflareError):
         CloudflareError.__init__(self, data)
 
 
+class CloudflareRateLimitError(CloudflareError):
+    def __init__(self, data):
+        CloudflareError.__init__(self, data)
+
+
 _PROXIABLE_RECORD_TYPES = {'A', 'AAAA', 'ALIAS', 'CNAME'}
 
 
@@ -47,6 +53,11 @@ class CloudflareProvider(BaseProvider):
         #
         # See: https://support.cloudflare.com/hc/en-us/articles/115000830351
         cdn: false
+        # Optional. Default: 4. Number of times to retry if a 429 response
+        # is received.
+        retry_count: 4
+        # Optional. Default: 300. Number of seconds to wait before retrying.
+        retry_period: 300
 
     Note: The "proxied" flag of "A", "AAAA" and "CNAME" records can be managed
           via the YAML provider like so:
@@ -60,13 +71,14 @@ class CloudflareProvider(BaseProvider):
     '''
     SUPPORTS_GEO = False
     SUPPORTS_DYNAMIC = False
-    SUPPORTS = set(('ALIAS', 'A', 'AAAA', 'CAA', 'CNAME', 'MX', 'NS', 'SRV',
-                    'SPF', 'TXT'))
+    SUPPORTS = set(('ALIAS', 'A', 'AAAA', 'CAA', 'CNAME', 'MX', 'NS', 'PTR',
+                    'SRV', 'SPF', 'TXT'))
 
     MIN_TTL = 120
     TIMEOUT = 15
 
-    def __init__(self, id, email=None, token=None, cdn=False, *args, **kwargs):
+    def __init__(self, id, email=None, token=None, cdn=False, retry_count=4,
+                 retry_period=300, *args, **kwargs):
         self.log = getLogger('CloudflareProvider[{}]'.format(id))
         self.log.debug('__init__: id=%s, email=%s, token=***, cdn=%s', id,
                        email, cdn)
@@ -85,11 +97,27 @@ class CloudflareProvider(BaseProvider):
                 'Authorization': 'Bearer {}'.format(token),
             })
         self.cdn = cdn
+        self.retry_count = retry_count
+        self.retry_period = retry_period
         self._sess = sess
 
         self._zones = None
         self._zone_records = {}
 
+    def _try_request(self, *args, **kwargs):
+        tries = self.retry_count
+        while True:  # We'll raise to break after our tries expire
+            try:
+                return self._request(*args, **kwargs)
+            except CloudflareRateLimitError:
+                if tries <= 1:
+                    raise
+                tries -= 1
+                self.log.warn('rate limit encountered, pausing '
+                              'for %ds and trying again, %d remaining',
+                              self.retry_period, tries)
+                sleep(self.retry_period)
+
     def _request(self, method, path, params=None, data=None):
         self.log.debug('_request: method=%s, path=%s', method, path)
 
@@ -101,6 +129,8 @@ class CloudflareProvider(BaseProvider):
             raise CloudflareError(resp.json())
         if resp.status_code == 403:
             raise CloudflareAuthenticationError(resp.json())
+        if resp.status_code == 429:
+            raise CloudflareRateLimitError(resp.json())
 
         resp.raise_for_status()
         return resp.json()
@@ -111,7 +141,8 @@ class CloudflareProvider(BaseProvider):
             page = 1
             zones = []
             while page:
-                resp = self._request('GET', '/zones', params={'page': page})
+                resp = self._try_request('GET', '/zones',
+                                         params={'page': page})
                 zones += resp['result']
                 info = resp['result_info']
                 if info['count'] > 0 and info['count'] == info['per_page']:
@@ -173,6 +204,7 @@ class CloudflareProvider(BaseProvider):
         }
 
     _data_for_ALIAS = _data_for_CNAME
+    _data_for_PTR = _data_for_CNAME
 
     def _data_for_MX(self, _type, records):
         values = []
@@ -219,7 +251,7 @@ class CloudflareProvider(BaseProvider):
             path = '/zones/{}/dns_records'.format(zone_id)
             page = 1
             while page:
-                resp = self._request('GET', path, params={'page': page})
+                resp = self._try_request('GET', path, params={'page': page})
                 records += resp['result']
                 info = resp['result_info']
                 if info['count'] > 0 and info['count'] == info['per_page']:
@@ -339,6 +371,8 @@ class CloudflareProvider(BaseProvider):
     def _contents_for_CNAME(self, record):
         yield {'content': record.value}
 
+    _contents_for_PTR = _contents_for_CNAME
+
     def _contents_for_MX(self, record):
         for value in record.values:
             yield {
@@ -430,7 +464,7 @@ class CloudflareProvider(BaseProvider):
         zone_id = self.zones[new.zone.name]
         path = '/zones/{}/dns_records'.format(zone_id)
         for content in self._gen_data(new):
-            self._request('POST', path, data=content)
+            self._try_request('POST', path, data=content)
 
     def _apply_Update(self, change):
         zone = change.new.zone
@@ -519,7 +553,7 @@ class CloudflareProvider(BaseProvider):
         path = '/zones/{}/dns_records'.format(zone_id)
         for _, data in sorted(creates.items()):
             self.log.debug('_apply_Update: creating %s', data)
-            self._request('POST', path, data=data)
+            self._try_request('POST', path, data=data)
 
         # Updates
         for _, info in sorted(updates.items()):
@@ -529,7 +563,7 @@ class CloudflareProvider(BaseProvider):
             path = '/zones/{}/dns_records/{}'.format(zone_id, record_id)
             self.log.debug('_apply_Update: updating %s, %s -> %s',
                            record_id, data, old_data)
-            self._request('PUT', path, data=data)
+            self._try_request('PUT', path, data=data)
 
         # Deletes
         for _, info in sorted(deletes.items()):
@@ -538,7 +572,7 @@ class CloudflareProvider(BaseProvider):
             path = '/zones/{}/dns_records/{}'.format(zone_id, record_id)
             self.log.debug('_apply_Update: removing %s, %s', record_id,
                            old_data)
-            self._request('DELETE', path)
+            self._try_request('DELETE', path)
 
     def _apply_Delete(self, change):
         existing = change.existing
@@ -551,7 +585,7 @@ class CloudflareProvider(BaseProvider):
                existing_type == record['type']:
                 path = '/zones/{}/dns_records/{}'.format(record['zone_id'],
                                                          record['id'])
-                self._request('DELETE', path)
+                self._try_request('DELETE', path)
 
     def _apply(self, plan):
         desired = plan.desired
@@ -566,7 +600,7 @@ class CloudflareProvider(BaseProvider):
                 'name': name[:-1],
                 'jump_start': False,
             }
-            resp = self._request('POST', '/zones', data=data)
+            resp = self._try_request('POST', '/zones', data=data)
             zone_id = resp['result']['id']
             self.zones[name] = zone_id
             self._zone_records[name] = {}

octodns/provider/dyn.py

@@ -1141,7 +1141,7 @@ class DynProvider(BaseProvider):
                 pools[rpid] = get_response_pool(rpid, td)
         # now that we have full objects for the complete set of existing pools,
         # a list will be more useful
-        pools = pools.values()
+        pools = list(pools.values())
 
         # Rulesets
 

octodns/provider/ns1.py

@@ -253,7 +253,9 @@ class Ns1Provider(BaseProvider):
 
     def _REGION_FILTER(self, with_disabled):
         return self._update_filter({
-            'config': {},
+            'config': {
+                'remove_no_georegion': True
+            },
             'filter': u'geofence_regional'
         }, with_disabled)
 
@@ -766,8 +768,7 @@ class Ns1Provider(BaseProvider):
         for iso_region, target in record.geo.items():
             key = 'iso_region_code'
             value = iso_region
-            if not has_country and \
-               len(value.split('-')) > 1:  # pragma: nocover
+            if not has_country and len(value.split('-')) > 1:
                 has_country = True
             for answer in target.values:
                 params['answers'].append(
@@ -856,18 +857,13 @@ class Ns1Provider(BaseProvider):
         host = record.fqdn[:-1]
         _type = record._type
 
-        request = r'GET {path} HTTP/1.0\r\nHost: {host}\r\n' \
-            r'User-agent: NS1\r\n\r\n'.format(path=record.healthcheck_path,
-                                              host=record.healthcheck_host)
-
-        return {
+        ret = {
             'active': True,
             'config': {
                 'connect_timeout': 2000,
                 'host': value,
                 'port': record.healthcheck_port,
                 'response_timeout': 10000,
-                'send': request,
                 'ssl': record.healthcheck_protocol == 'HTTPS',
             },
             'frequency': 60,
@@ -881,12 +877,23 @@ class Ns1Provider(BaseProvider):
             'rapid_recheck': False,
             'region_scope': 'fixed',
             'regions': self.monitor_regions,
-            'rules': [{
+        }
+
+        if record.healthcheck_protocol != 'TCP':
+            # IF it's HTTP we need to send the request string
+            path = record.healthcheck_path
+            host = record.healthcheck_host
+            request = r'GET {path} HTTP/1.0\r\nHost: {host}\r\n' \
+                r'User-agent: NS1\r\n\r\n'.format(path=path, host=host)
+            ret['config']['send'] = request
+            # We'll also expect a HTTP response
+            ret['rules'] = [{
                 'comparison': 'contains',
                 'key': 'output',
                 'value': '200 OK',
-            }],
-        }
+            }]
+
+        return ret
 
     def _monitor_is_match(self, expected, have):
         # Make sure what we have matches what's in expected exactly. Anything

octodns/provider/route53.py

@@ -1046,8 +1046,11 @@ class Route53Provider(BaseProvider):
             # No value so give this a None to match value's
             config_ip_address = None
 
-        return host == config['FullyQualifiedDomainName'] and \
-            path == config['ResourcePath'] and protocol == config['Type'] \
+        fully_qualified_domain_name = config.get('FullyQualifiedDomainName',
+                                                 None)
+        resource_path = config.get('ResourcePath', None)
+        return host == fully_qualified_domain_name and \
+            path == resource_path and protocol == config['Type'] \
             and port == config['Port'] and \
             measure_latency == config['MeasureLatency'] and \
             value == config_ip_address
@@ -1103,13 +1106,14 @@ class Route53Provider(BaseProvider):
         config = {
             'EnableSNI': healthcheck_protocol == 'HTTPS',
             'FailureThreshold': 6,
-            'FullyQualifiedDomainName': healthcheck_host,
             'MeasureLatency': healthcheck_latency,
             'Port': healthcheck_port,
             'RequestInterval': 10,
-            'ResourcePath': healthcheck_path,
             'Type': healthcheck_protocol,
         }
+        if healthcheck_protocol != 'TCP':
+            config['FullyQualifiedDomainName'] = healthcheck_host
+            config['ResourcePath'] = healthcheck_path
         if value:
             config['IPAddress'] = value
 

octodns/record/__init__.py

@@ -137,7 +137,7 @@ class Record(EqualityTupleMixin):
             reasons.append('missing ttl')
         try:
             if data['octodns']['healthcheck']['protocol'] \
-               not in ('HTTP', 'HTTPS'):
+               not in ('HTTP', 'HTTPS', 'TCP'):
                 reasons.append('invalid healthcheck protocol')
         except KeyError:
             pass
@@ -181,15 +181,21 @@ class Record(EqualityTupleMixin):
 
     @property
     def healthcheck_host(self):
+        healthcheck = self._octodns.get('healthcheck', {})
+        if healthcheck.get('protocol', None) == 'TCP':
+            return None
         try:
-            return self._octodns['healthcheck']['host']
+            return healthcheck['host']
         except KeyError:
             return self.fqdn[:-1]
 
     @property
     def healthcheck_path(self):
+        healthcheck = self._octodns.get('healthcheck', {})
+        if healthcheck.get('protocol', None) == 'TCP':
+            return None
         try:
-            return self._octodns['healthcheck']['path']
+            return healthcheck['path']
         except KeyError:
             return '/_dns'
 

octodns/record/geo.py

@@ -63,9 +63,15 @@ class GeoCodes(object):
 
     @classmethod
     def province_to_code(cls, province):
-        # We get to cheat on this one since we only support provinces in NA-US
-        if province not in geo_data['NA']['US']['provinces']:
+        # We cheat on this one a little since we only support provinces in
+        # NA-US, NA-CA
+        if (province not in geo_data['NA']['US']['provinces'] and
+                province not in geo_data['NA']['CA']['provinces']):
             cls.log.warn('country_to_code: unrecognized province "%s"',
                          province)
             return
-        return 'NA-US-{}'.format(province)
+        if province in geo_data['NA']['US']['provinces']:
+            country = 'US'
+        if province in geo_data['NA']['CA']['provinces']:
+            country = 'CA'
+        return 'NA-{}-{}'.format(country, province)

octodns/record/geo_data.py

@@ -55,7 +55,7 @@ geo_data = \
             'SO': {'name': 'Somalia'},
             'SS': {'name': 'South Sudan'},
             'ST': {'name': 'Sao Tome and Principe'},
-            'SZ': {'name': 'Swaziland'},
+            'SZ': {'name': 'Eswatini'},
             'TD': {'name': 'Chad'},
             'TG': {'name': 'Togo'},
             'TN': {'name': 'Tunisia'},
@@ -157,7 +157,7 @@ geo_data = \
             'MC': {'name': 'Monaco'},
             'MD': {'name': 'Moldova, Republic of'},
             'ME': {'name': 'Montenegro'},
-            'MK': {'name': 'Macedonia, Republic of'},
+            'MK': {'name': 'North Macedonia'},
             'MT': {'name': 'Malta'},
             'NL': {'name': 'Netherlands'},
             'NO': {'name': 'Norway'},
@@ -183,7 +183,20 @@ geo_data = \
             'BQ': {'name': 'Bonaire, Sint Eustatius and Saba'},
             'BS': {'name': 'Bahamas'},
             'BZ': {'name': 'Belize'},
-            'CA': {'name': 'Canada'},
+            'CA': {'name': 'Canada',
+                   'provinces': {'AB': {'name': 'Alberta'},
+                                 'BC': {'name': 'British Columbia'},
+                                 'MB': {'name': 'Manitoba'},
+                                 'NB': {'name': 'New Brunswick'},
+                                 'NL': {'name': 'Newfoundland and Labrador'},
+                                 'NS': {'name': 'Nova Scotia'},
+                                 'NT': {'name': 'Northwest Territories'},
+                                 'NU': {'name': 'Nunavut'},
+                                 'ON': {'name': 'Ontario'},
+                                 'PE': {'name': 'Prince Edward Island'},
+                                 'QC': {'name': 'Quebec'},
+                                 'SK': {'name': 'Saskatchewan'},
+                                 'YT': {'name': 'Yukon Territory'}}},
             'CR': {'name': 'Costa Rica'},
             'CU': {'name': 'Cuba'},
             'CW': {'name': 'Curaçao'},

requirements-dev.txt

@@ -1,7 +1,7 @@
 coverage
 mock
 nose
-pycodestyle==2.5.0
+pycodestyle==2.6.0
 pyflakes==2.2.0
 readme_renderer[md]==26.0
 requests_mock

requirements.txt

@@ -1,8 +1,8 @@
 PyYaml==5.3.1
 azure-common==1.1.25
 azure-mgmt-dns==3.0.0
-boto3==1.13.0
-botocore==1.16.0
+boto3==1.13.19
+botocore==1.16.19
 dnspython==1.16.0
 docutils==0.16
 dyn==1.8.1
@@ -14,13 +14,13 @@ ipaddress==1.0.23
 jmespath==0.9.5
 msrestazure==0.6.3
 natsort==6.2.1
-ns1-python==0.15.0
+ns1-python==0.16.0
 ovh==0.5.0
 pycountry-convert==0.7.2
 pycountry==19.8.18
 python-dateutil==2.8.1
 requests==2.23.0
 s3transfer==0.3.3
-setuptools==44.1.0
-six==1.14.0
+setuptools==44.1.1
+six==1.15.0
 transip==2.1.2

script/coverage

@@ -26,6 +26,12 @@ export DYN_PASSWORD=
 export DYN_USERNAME=
 export GOOGLE_APPLICATION_CREDENTIALS=
 
+# Don't allow disabling coverage
+grep -r -I --line-number "# pragma: nocover" octodns && {
+  echo "Code coverage should not be disabled"
+  exit 1
+}
+
 coverage run --branch --source=octodns --omit=octodns/cmds/* "$(command -v nosetests)" --with-xunit "$@"
 coverage html
 coverage xml

script/generate-geo-data

@@ -8,8 +8,8 @@ from pycountry_convert import country_alpha2_to_continent_code
 
 subs = defaultdict(dict)
 for subdivision in subdivisions:
-    # Route53 only supports US states, Dyn supports US states and CA provinces, but for now we'll just do US
-    if subdivision.country_code not in ('US'):
+    # Route53 only supports US states, Dyn (and others) support US states and CA provinces
+    if subdivision.country_code not in ('US', 'CA'):
         continue
     subs[subdivision.country_code][subdivision.code[3:]] = {
         'name': subdivision.name

script/lint

@@ -17,5 +17,5 @@ fi
 
 SOURCES="*.py octodns/*.py octodns/*/*.py tests/*.py"
 
-pycodestyle --ignore=E221,E241,E251,E722,W504 $SOURCES
+pycodestyle --ignore=E221,E241,E251,E722,E741,W504 $SOURCES
 pyflakes $SOURCES

tests/fixtures/cloudflare-dns_records-page-1.json

@@ -180,7 +180,7 @@
     "per_page": 10,
     "total_pages": 2,
     "count": 10,
-    "total_count": 19
+    "total_count": 20
   },
   "success": true,
   "errors": [],

tests/fixtures/cloudflare-dns_records-page-2.json

@@ -157,6 +157,23 @@
         "auto_added": false
       }
     },
+    {
+      "id": "fc12ab34cd5611334422ab3322997677",
+      "type": "PTR",
+      "name": "ptr.unit.tests",
+      "content": "foo.bar.com",
+      "proxiable": true,
+      "proxied": false,
+      "ttl": 300,
+      "locked": false,
+      "zone_id": "ff12ab34cd5611334422ab3322997650",
+      "zone_name": "unit.tests",
+      "modified_on": "2017-03-11T18:01:43.940682Z",
+      "created_on": "2017-03-11T18:01:43.940682Z",
+      "meta": {
+        "auto_added": false
+      }
+    },
     {
       "id": "fc12ab34cd5611334422ab3322997656",
       "type": "SRV",
@@ -212,8 +229,8 @@
     "page": 2,
     "per_page": 11,
     "total_pages": 2,
-    "count": 9,
-    "total_count": 21
+    "count": 10,
+    "total_count": 20
   },
   "success": true,
   "errors": [],

tests/test_octodns_provider_cloudflare.py

@@ -14,7 +14,8 @@ from unittest import TestCase
 
 from octodns.record import Record, Update
 from octodns.provider.base import Plan
-from octodns.provider.cloudflare import CloudflareProvider
+from octodns.provider.cloudflare import CloudflareProvider, \
+    CloudflareRateLimitError
 from octodns.provider.yaml import YamlProvider
 from octodns.zone import Zone
 
@@ -52,7 +53,7 @@ class TestCloudflareProvider(TestCase):
     empty = {'result': [], 'result_info': {'count': 0, 'per_page': 0}}
 
     def test_populate(self):
-        provider = CloudflareProvider('test', 'email', 'token')
+        provider = CloudflareProvider('test', 'email', 'token', retry_period=0)
 
         # Bad requests
         with requests_mock() as mock:
@@ -103,6 +104,36 @@ class TestCloudflareProvider(TestCase):
                 provider.populate(zone)
             self.assertEquals(502, ctx.exception.response.status_code)
 
+        # Rate Limit error
+        with requests_mock() as mock:
+            mock.get(ANY, status_code=429,
+                     text='{"success":false,"errors":[{"code":10100,'
+                     '"message":"More than 1200 requests per 300 seconds '
+                     'reached. Please wait and consider throttling your '
+                     'request speed"}],"messages":[],"result":null}')
+
+            with self.assertRaises(Exception) as ctx:
+                zone = Zone('unit.tests.', [])
+                provider.populate(zone)
+
+            self.assertEquals('CloudflareRateLimitError',
+                              type(ctx.exception).__name__)
+            self.assertEquals('More than 1200 requests per 300 seconds '
+                              'reached. Please wait and consider throttling '
+                              'your request speed', text_type(ctx.exception))
+
+        # Rate Limit error, unknown resp
+        with requests_mock() as mock:
+            mock.get(ANY, status_code=429, text='{}')
+
+            with self.assertRaises(Exception) as ctx:
+                zone = Zone('unit.tests.', [])
+                provider.populate(zone)
+
+            self.assertEquals('CloudflareRateLimitError',
+                              type(ctx.exception).__name__)
+            self.assertEquals('Cloudflare error', text_type(ctx.exception))
+
         # Non-existent zone doesn't populate anything
         with requests_mock() as mock:
             mock.get(ANY, status_code=200, json=self.empty)
@@ -149,7 +180,7 @@ class TestCloudflareProvider(TestCase):
 
             zone = Zone('unit.tests.', [])
             provider.populate(zone)
-            self.assertEquals(12, len(zone.records))
+            self.assertEquals(13, len(zone.records))
 
             changes = self.expected.changes(zone, provider)
 
@@ -158,10 +189,10 @@ class TestCloudflareProvider(TestCase):
         # re-populating the same zone/records comes out of cache, no calls
         again = Zone('unit.tests.', [])
         provider.populate(again)
-        self.assertEquals(12, len(again.records))
+        self.assertEquals(13, len(again.records))
 
     def test_apply(self):
-        provider = CloudflareProvider('test', 'email', 'token')
+        provider = CloudflareProvider('test', 'email', 'token', retry_period=0)
 
         provider._request = Mock()
 
@@ -172,12 +203,12 @@ class TestCloudflareProvider(TestCase):
                     'id': 42,
                 }
             },  # zone create
-        ] + [None] * 20  # individual record creates
+        ] + [None] * 22  # individual record creates
 
         # non-existent zone, create everything
         plan = provider.plan(self.expected)
-        self.assertEquals(12, len(plan.changes))
-        self.assertEquals(12, provider.apply(plan))
+        self.assertEquals(13, len(plan.changes))
+        self.assertEquals(13, provider.apply(plan))
         self.assertFalse(plan.exists)
 
         provider._request.assert_has_calls([
@@ -203,7 +234,7 @@ class TestCloudflareProvider(TestCase):
             }),
         ], True)
         # expected number of total calls
-        self.assertEquals(22, provider._request.call_count)
+        self.assertEquals(23, provider._request.call_count)
 
         provider._request.reset_mock()
 
@@ -280,7 +311,11 @@ class TestCloudflareProvider(TestCase):
 
         # we don't care about the POST/create return values
         provider._request.return_value = {}
-        provider._request.side_effect = None
+
+        # Test out the create rate-limit handling, then 9 successes
+        provider._request.side_effect = [
+            CloudflareRateLimitError('{}'),
+        ] + ([None] * 3)
 
         wanted = Zone('unit.tests.', [])
         wanted.add_record(Record.new(wanted, 'nc', {
@@ -316,7 +351,7 @@ class TestCloudflareProvider(TestCase):
         ])
 
     def test_update_add_swap(self):
-        provider = CloudflareProvider('test', 'email', 'token')
+        provider = CloudflareProvider('test', 'email', 'token', retry_period=0)
 
         provider.zone_records = Mock(return_value=[
             {
@@ -357,6 +392,7 @@ class TestCloudflareProvider(TestCase):
 
         provider._request = Mock()
         provider._request.side_effect = [
+            CloudflareRateLimitError('{}'),
             self.empty,  # no zones
             {
                 'result': {
@@ -423,7 +459,7 @@ class TestCloudflareProvider(TestCase):
     def test_update_delete(self):
         # We need another run so that we can delete, we can't both add and
         # delete in one go b/c of swaps
-        provider = CloudflareProvider('test', 'email', 'token')
+        provider = CloudflareProvider('test', 'email', 'token', retry_period=0)
 
         provider.zone_records = Mock(return_value=[
             {
@@ -464,6 +500,7 @@ class TestCloudflareProvider(TestCase):
 
         provider._request = Mock()
         provider._request.side_effect = [
+            CloudflareRateLimitError('{}'),
             self.empty,  # no zones
             {
                 'result': {
@@ -510,6 +547,25 @@ class TestCloudflareProvider(TestCase):
                  'fc12ab34cd5611334422ab3322997653')
         ])
 
+    def test_ptr(self):
+        provider = CloudflareProvider('test', 'email', 'token')
+
+        zone = Zone('unit.tests.', [])
+        # PTR record
+        ptr_record = Record.new(zone, 'ptr', {
+            'ttl': 300,
+            'type': 'PTR',
+            'value': 'foo.bar.com.'
+        })
+
+        ptr_record_contents = provider._gen_data(ptr_record)
+        self.assertEquals({
+            'name': 'ptr.unit.tests',
+            'ttl': 300,
+            'type': 'PTR',
+            'content': 'foo.bar.com.'
+        }, list(ptr_record_contents)[0])
+
     def test_srv(self):
         provider = CloudflareProvider('test', 'email', 'token')
 
@@ -1223,3 +1279,64 @@ class TestCloudflareProvider(TestCase):
         provider = CloudflareProvider('test', token='token 123')
         headers = provider._sess.headers
         self.assertEquals('Bearer token 123', headers['Authorization'])
+
+    def test_retry_behavior(self):
+        provider = CloudflareProvider('test', token='token 123',
+                                      email='email 234', retry_period=0)
+        result = {
+            "success": True,
+            "errors": [],
+            "messages": [],
+            "result": [],
+            "result_info": {
+                "count": 1,
+                "per_page": 50
+            }
+        }
+        zone = Zone('unit.tests.', [])
+        provider._request = Mock()
+
+        # No retry required, just calls and is returned
+        provider._zones = None
+        provider._request.reset_mock()
+        provider._request.side_effect = [result]
+        self.assertEquals([], provider.zone_records(zone))
+        provider._request.assert_has_calls([call('GET', '/zones',
+                                           params={'page': 1})])
+
+        # One retry required
+        provider._zones = None
+        provider._request.reset_mock()
+        provider._request.side_effect = [
+            CloudflareRateLimitError('{}'),
+            result
+        ]
+        self.assertEquals([], provider.zone_records(zone))
+        provider._request.assert_has_calls([call('GET', '/zones',
+                                           params={'page': 1})])
+
+        # Two retries required
+        provider._zones = None
+        provider._request.reset_mock()
+        provider._request.side_effect = [
+            CloudflareRateLimitError('{}'),
+            CloudflareRateLimitError('{}'),
+            result
+        ]
+        self.assertEquals([], provider.zone_records(zone))
+        provider._request.assert_has_calls([call('GET', '/zones',
+                                           params={'page': 1})])
+
+        # # Exhaust our retries
+        provider._zones = None
+        provider._request.reset_mock()
+        provider._request.side_effect = [
+            CloudflareRateLimitError({"errors": [{"message": "first"}]}),
+            CloudflareRateLimitError({"errors": [{"message": "boo"}]}),
+            CloudflareRateLimitError({"errors": [{"message": "boo"}]}),
+            CloudflareRateLimitError({"errors": [{"message": "boo"}]}),
+            CloudflareRateLimitError({"errors": [{"message": "last"}]}),
+        ]
+        with self.assertRaises(CloudflareRateLimitError) as ctx:
+            provider.zone_records(zone)
+            self.assertEquals('last', text_type(ctx.exception))

tests/test_octodns_provider_ns1.py

@@ -717,6 +717,13 @@ class TestNs1ProviderDynamic(TestCase):
         monitor = provider._monitor_gen(self.record, value)
         self.assertTrue(monitor['config']['ssl'])
 
+        self.record._octodns['healthcheck']['protocol'] = 'TCP'
+        monitor = provider._monitor_gen(self.record, value)
+        # No http send done
+        self.assertFalse('send' in monitor['config'])
+        # No http response expected
+        self.assertFalse('rules' in monitor)
+
     def test_monitor_is_match(self):
         provider = Ns1Provider('test', 'api-key')
 
@@ -1059,6 +1066,16 @@ class TestNs1ProviderDynamic(TestCase):
             call(self.record, '3.4.5.6', 'mid-3'),
         ])
 
+        record = Record.new(self.zone, 'geo', {
+            'ttl': 34,
+            'type': 'A',
+            'values': ['101.102.103.104', '101.102.103.105'],
+            'geo': {'EU': ['201.202.203.204']},
+            'meta': {},
+        })
+        params, _ = provider._params_for_geo_A(record)
+        self.assertEquals([], params['filters'])
+
     def test_data_for_dynamic_A(self):
         provider = Ns1Provider('test', 'api-key')
 

tests/test_octodns_provider_route53.py

@@ -1213,6 +1213,35 @@ class TestRoute53Provider(TestCase):
         self.assertEquals('42', id)
         stubber.assert_no_pending_responses()
 
+        # TCP health check
+
+        health_check_config = {
+            'EnableSNI': False,
+            'FailureThreshold': 6,
+            'MeasureLatency': True,
+            'Port': 8080,
+            'RequestInterval': 10,
+            'Type': 'TCP'
+        }
+        stubber.add_response('create_health_check', {
+            'HealthCheck': {
+                'Id': '42',
+                'CallerReference': self.caller_ref,
+                'HealthCheckConfig': health_check_config,
+                'HealthCheckVersion': 1,
+            },
+            'Location': 'http://url',
+        }, {
+            'CallerReference': ANY,
+            'HealthCheckConfig': health_check_config,
+        })
+        stubber.add_response('change_tags_for_resource', {})
+
+        record._octodns['healthcheck']['protocol'] = 'TCP'
+        id = provider.get_health_check_id(record, 'target-1.unit.tests.', True)
+        self.assertEquals('42', id)
+        stubber.assert_no_pending_responses()
+
     def test_health_check_measure_latency(self):
         provider, stubber = self._get_stubbed_provider()
         record_true = Record.new(self.expected, 'a', {

tests/test_octodns_record.py

@@ -886,6 +886,40 @@ class TestRecord(TestCase):
         self.assertEquals('HTTPS', new.healthcheck_protocol)
         self.assertEquals(443, new.healthcheck_port)
 
+    def test_healthcheck_tcp(self):
+        new = Record.new(self.zone, 'a', {
+            'ttl': 44,
+            'type': 'A',
+            'value': '1.2.3.4',
+            'octodns': {
+                'healthcheck': {
+                    'path': '/ignored',
+                    'host': 'completely.ignored',
+                    'protocol': 'TCP',
+                    'port': 8080,
+                }
+            }
+        })
+        self.assertIsNone(new.healthcheck_path)
+        self.assertIsNone(new.healthcheck_host)
+        self.assertEquals('TCP', new.healthcheck_protocol)
+        self.assertEquals(8080, new.healthcheck_port)
+
+        new = Record.new(self.zone, 'a', {
+            'ttl': 44,
+            'type': 'A',
+            'value': '1.2.3.4',
+            'octodns': {
+                'healthcheck': {
+                    'protocol': 'TCP',
+                }
+            }
+        })
+        self.assertIsNone(new.healthcheck_path)
+        self.assertIsNone(new.healthcheck_host)
+        self.assertEquals('TCP', new.healthcheck_protocol)
+        self.assertEquals(443, new.healthcheck_port)
+
     def test_inored(self):
         new = Record.new(self.zone, 'txt', {
             'ttl': 44,

tests/test_octodns_record_geo.py

@@ -77,4 +77,6 @@ class TestRecordGeoCodes(TestCase):
     def test_province_to_code(self):
         self.assertEquals('NA-US-OR', GeoCodes.province_to_code('OR'))
         self.assertEquals('NA-US-KY', GeoCodes.province_to_code('KY'))
+        self.assertEquals('NA-CA-AB', GeoCodes.province_to_code('AB'))
+        self.assertEquals('NA-CA-BC', GeoCodes.province_to_code('BC'))
         self.assertFalse(GeoCodes.province_to_code('XX'))