Merge branch 'master' into dont-escape-semicolon-in-azure

CHANGELOG.md

@@ -1,3 +1,11 @@
+## v0.9.3 - 2018-10-29 - Misc. stuff sort of release
+
+* ZoneFile source added
+* Major rework/improvements to the Cloudflare record update process, fixed bugs
+  and optimized it quite a bit
+* Add ability to manage Cloudflare proxy flag
+* Bump requests version to 2.20.0
+
 ## v0.9.2 - 2018-08-20 - More sources
 
 * EtcHostsProvider implementation to create static/emergency best effort

README.md

@@ -163,6 +163,7 @@ The above command pulled the existing data out of Route53 and placed the results
 | [Rackspace](/octodns/provider/rackspace.py) | | A, AAAA, ALIAS, CNAME, MX, NS, PTR, SPF, TXT | No |  |
 | [Route53](/octodns/provider/route53.py) | boto3 | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | Yes | |
 | [AxfrSource](/octodns/source/axfr.py) | | A, AAAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | read-only |
+| [ZoneFileSource](/octodns/source/axfr.py) | | A, AAAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | read-only |
 | [TinyDnsFileSource](/octodns/source/tinydns.py) | | A, CNAME, MX, NS, PTR | No | read-only |
 | [YamlProvider](/octodns/provider/yaml.py) | | All | Yes | config |
 

octodns/__init__.py

@@ -3,4 +3,4 @@
 from __future__ import absolute_import, division, print_function, \
     unicode_literals
 
-__VERSION__ = '0.9.2'
+__VERSION__ = '0.9.3'

octodns/cmds/args.py

@@ -11,6 +11,8 @@ from logging import DEBUG, INFO, WARN, Formatter, StreamHandler, \
 from logging.handlers import SysLogHandler
 from sys import stderr, stdout
 
+from octodns import __VERSION__
+
 
 class ArgumentParser(_Base):
     '''
@@ -23,6 +25,9 @@ class ArgumentParser(_Base):
         super(ArgumentParser, self).__init__(*args, **kwargs)
 
     def parse_args(self, default_log_level=INFO):
+        version = 'octoDNS {}'.format(__VERSION__)
+        self.add_argument('--version', action='version', version=version,
+                          help='Print octoDNS version and exit')
         self.add_argument('--log-stream-stdout', action='store_true',
                           default=False,
                           help='Log to stdout instead of stderr')

octodns/provider/cloudflare.py

@@ -6,8 +6,8 @@ from __future__ import absolute_import, division, print_function, \
     unicode_literals
 
 from collections import defaultdict
+from copy import deepcopy
 from logging import getLogger
-from json import dumps
 from requests import Session
 
 from ..record import Record, Update
@@ -28,6 +28,9 @@ class CloudflareAuthenticationError(CloudflareError):
         CloudflareError.__init__(self, data)
 
 
+_PROXIABLE_RECORD_TYPES = {'A', 'AAAA', 'ALIAS', 'CNAME'}
+
+
 class CloudflareProvider(BaseProvider):
     '''
     Cloudflare DNS provider
@@ -44,6 +47,16 @@ class CloudflareProvider(BaseProvider):
         #
         # See: https://support.cloudflare.com/hc/en-us/articles/115000830351
         cdn: false
+
+    Note: The "proxied" flag of "A", "AAAA" and "CNAME" records can be managed
+          via the YAML provider like so:
+              name:
+                octodns:
+                  cloudflare:
+                    proxied: true
+                ttl: 120
+                type: A
+                value: 1.2.3.4
     '''
     SUPPORTS_GEO = False
     SUPPORTS = set(('ALIAS', 'A', 'AAAA', 'CAA', 'CNAME', 'MX', 'NS', 'SRV',
@@ -222,7 +235,14 @@ class CloudflareProvider(BaseProvider):
             data_for = getattr(self, '_data_for_{}'.format(_type))
             data = data_for(_type, records)
 
-        return Record.new(zone, name, data, source=self, lenient=lenient)
+        record = Record.new(zone, name, data, source=self, lenient=lenient)
+
+        if _type in _PROXIABLE_RECORD_TYPES:
+            record._octodns['cloudflare'] = {
+                'proxied': records[0].get('proxied', False)
+            }
+
+        return record
 
     def populate(self, zone, target=False, lenient=False):
         self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name,
@@ -261,8 +281,18 @@ class CloudflareProvider(BaseProvider):
 
     def _include_change(self, change):
         if isinstance(change, Update):
-            existing = change.existing.data
             new = change.new.data
+
+            # Cloudflare manages TTL of proxied records, so we should exclude
+            # TTL from the comparison (to prevent false-positives).
+            if self._record_is_proxied(change.existing):
+                existing = deepcopy(change.existing.data)
+                existing.update({
+                    'ttl': new['ttl']
+                })
+            else:
+                existing = change.existing.data
+
             new['ttl'] = max(self.MIN_TTL, new['ttl'])
             if new == existing:
                 return False
@@ -323,7 +353,13 @@ class CloudflareProvider(BaseProvider):
                 }
             }
 
-    def _gen_contents(self, record):
+    def _record_is_proxied(self, record):
+        return (
+            not self.cdn and
+            record._octodns.get('cloudflare', {}).get('proxied', False)
+        )
+
+    def _gen_data(self, record):
         name = record.fqdn[:-1]
         _type = record._type
         ttl = max(self.MIN_TTL, record.ttl)
@@ -339,94 +375,151 @@ class CloudflareProvider(BaseProvider):
                 'type': _type,
                 'ttl': ttl,
             })
+
+            if _type in _PROXIABLE_RECORD_TYPES:
+                content.update({
+                    'proxied': self._record_is_proxied(record)
+                })
+
             yield content
 
+    def _gen_key(self, data):
+        # Note that most CF record data has a `content` field the value of
+        # which is a unique/hashable string for the record's. It includes all
+        # the "value" bits, but not the secondary stuff like TTL's. E.g.  for
+        # an A it'll include the value, for a CAA it'll include the flags, tag,
+        # and value, ... We'll take advantage of this to try and match up old &
+        # new records cleanly. In general when there are multiple records for a
+        # name & type each will have a distinct/consistent `content` that can
+        # serve as a unique identifier.
+        # BUT... there are exceptions. MX, CAA, and SRV don't have a simple
+        # content as things are currently implemented so we need to handle
+        # those explicitly and create unique/hashable strings for them.
+        _type = data['type']
+        if _type == 'MX':
+            return '{priority} {content}'.format(**data)
+        elif _type == 'CAA':
+            data = data['data']
+            return '{flags} {tag} {value}'.format(**data)
+        elif _type == 'SRV':
+            data = data['data']
+            return '{port} {priority} {target} {weight}'.format(**data)
+        return data['content']
+
     def _apply_Create(self, change):
         new = change.new
         zone_id = self.zones[new.zone.name]
         path = '/zones/{}/dns_records'.format(zone_id)
-        for content in self._gen_contents(new):
+        for content in self._gen_data(new):
             self._request('POST', path, data=content)
 
-    def _hash_content(self, content):
-        # Some of the dicts are nested so this seems about as good as any
-        # option we have for consistently hashing them (within a single run)
-        return hash(dumps(content, sort_keys=True))
-
     def _apply_Update(self, change):
-
-        # Ugh, this is pretty complicated and ugly, mainly due to the
-        # sub-optimal API/semantics. Ideally we'd have a batch change API like
-        # Route53's to make this 100% clean and safe without all this PITA, but
-        # we don't so we'll have to work around that and manually do it as
-        # safely as possible. Note this still isn't perfect as we don't/can't
-        # practically take into account things like the different "types" of
-        # CAA records so when we "swap" there may be brief periods where things
-        # are invalid or even worse Cloudflare may update their validations to
-        # prevent dups. I see no clean way around that short of making this
-        # understand 100% of the details of each record type and develop an
-        # individual/specific ordering of changes that prevents it. That'd
-        # probably result in more code than this whole provider currently has
-        # so... :-(
-
-        existing_contents = {
-            self._hash_content(c): c
-            for c in self._gen_contents(change.existing)
-        }
-        new_contents = {
-            self._hash_content(c): c
-            for c in self._gen_contents(change.new)
-        }
-
-        # Find the things we need to add
-        adds = []
-        for k, content in new_contents.items():
-            try:
-                existing_contents.pop(k)
-                self.log.debug('_apply_Update: leaving %s', content)
-            except KeyError:
-                adds.append(content)
-
         zone = change.new.zone
         zone_id = self.zones[zone.name]
-
-        # Find things we need to remove
         hostname = zone.hostname_from_fqdn(change.new.fqdn[:-1])
         _type = change.new._type
-        # OK, work through each record from the zone
+
+        existing = {}
+        # Find all of the existing CF records for this name & type
         for record in self.zone_records(zone):
             name = zone.hostname_from_fqdn(record['name'])
             # Use the _record_for so that we include all of standard
             # conversion logic
             r = self._record_for(zone, name, record['type'], [record], True)
             if hostname == r.name and _type == r._type:
-
                 # Round trip the single value through a record to contents flow
-                # to get a consistent _gen_contents result that matches what
+                # to get a consistent _gen_data result that matches what
                 # went in to new_contents
-                content = self._gen_contents(r).next()
-
-                # If the hash of that dict isn't in new this record isn't
-                # needed
-                if self._hash_content(content) not in new_contents:
-                    rid = record['id']
-                    path = '/zones/{}/dns_records/{}'.format(record['zone_id'],
-                                                             rid)
-                    try:
-                        add_content = adds.pop(0)
-                        self.log.debug('_apply_Update: swapping %s -> %s, %s',
-                                       content, add_content, rid)
-                        self._request('PUT', path, data=add_content)
-                    except IndexError:
-                        self.log.debug('_apply_Update: removing %s, %s',
-                                       content, rid)
-                        self._request('DELETE', path)
-
-        # Any remaining adds just need to be created
+                data = self._gen_data(r).next()
+
+                # Record the record_id and data for this existing record
+                key = self._gen_key(data)
+                existing[key] = {
+                    'record_id': record['id'],
+                    'data': data,
+                }
+
+        # Build up a list of new CF records for this Update
+        new = {
+            self._gen_key(d): d for d in self._gen_data(change.new)
+        }
+
+        # OK we now have a picture of the old & new CF records, our next step
+        # is to figure out which records need to be deleted
+        deletes = {}
+        for key, info in existing.items():
+            if key not in new:
+                deletes[key] = info
+        # Now we need to figure out which records will need to be created
+        creates = {}
+        # And which will be updated
+        updates = {}
+        for key, data in new.items():
+            if key in existing:
+                # To update we need to combine the new data and existing's
+                # record_id. old_data is just for debugging/logging purposes
+                old_info = existing[key]
+                updates[key] = {
+                    'record_id': old_info['record_id'],
+                    'data': data,
+                    'old_data': old_info['data'],
+                }
+            else:
+                creates[key] = data
+
+        # To do this as safely as possible we'll add new things first, update
+        # existing things, and then remove old things. This should (try) and
+        # ensure that we have as many value CF records in their system as
+        # possible at any given time. Ideally we'd have a "batch" API that
+        # would allow create, delete, and upsert style stuff so operations
+        # could be done atomically, but that's not available so we made the
+        # best of it...
+
+        # However, there are record types like CNAME that can only have a
+        # single value. B/c of that our create and then delete approach isn't
+        # actually viable. To address this we'll convert as many creates &
+        # deletes as we can to updates. This will have a minor upside of
+        # resulting in fewer ops and in the case of things like CNAME where
+        # there's a single create and delete result in a single update instead.
+        create_keys = sorted(creates.keys())
+        delete_keys = sorted(deletes.keys())
+        for i in range(0, min(len(create_keys), len(delete_keys))):
+            create_key = create_keys[i]
+            create_data = creates.pop(create_key)
+            delete_info = deletes.pop(delete_keys[i])
+            updates[create_key] = {
+                'record_id': delete_info['record_id'],
+                'data': create_data,
+                'old_data': delete_info['data'],
+            }
+
+        # The sorts ensure a consistent order of operations, they're not
+        # otherwise required, just makes things deterministic
+
+        # Creates
         path = '/zones/{}/dns_records'.format(zone_id)
-        for content in adds:
-            self.log.debug('_apply_Update: adding %s', content)
-            self._request('POST', path, data=content)
+        for _, data in sorted(creates.items()):
+            self.log.debug('_apply_Update: creating %s', data)
+            self._request('POST', path, data=data)
+
+        # Updates
+        for _, info in sorted(updates.items()):
+            record_id = info['record_id']
+            data = info['data']
+            old_data = info['old_data']
+            path = '/zones/{}/dns_records/{}'.format(zone_id, record_id)
+            self.log.debug('_apply_Update: updating %s, %s -> %s',
+                           record_id, data, old_data)
+            self._request('PUT', path, data=data)
+
+        # Deletes
+        for _, info in sorted(deletes.items()):
+            record_id = info['record_id']
+            old_data = info['data']
+            path = '/zones/{}/dns_records/{}'.format(zone_id, record_id)
+            self.log.debug('_apply_Update: removing %s, %s', record_id,
+                           old_data)
+            self._request('DELETE', path)
 
     def _apply_Delete(self, change):
         existing = change.existing
@@ -462,3 +555,23 @@ class CloudflareProvider(BaseProvider):
 
         # clear the cache
         self._zone_records.pop(name, None)
+
+    def _extra_changes(self, existing, desired, changes):
+        extra_changes = []
+
+        existing_records = {r: r for r in existing.records}
+        changed_records = {c.record for c in changes}
+
+        for desired_record in desired.records:
+            if desired_record not in existing.records:  # Will be created
+                continue
+            elif desired_record in changed_records:  # Already being updated
+                continue
+
+            existing_record = existing_records[desired_record]
+
+            if (self._record_is_proxied(existing_record) !=
+                    self._record_is_proxied(desired_record)):
+                extra_changes.append(Update(existing_record, desired_record))
+
+        return extra_changes

octodns/provider/dnsimple.py

@@ -112,7 +112,14 @@ class DnsimpleProvider(BaseProvider):
     _data_for_A = _data_for_multiple
     _data_for_AAAA = _data_for_multiple
     _data_for_SPF = _data_for_multiple
-    _data_for_TXT = _data_for_multiple
+
+    def _data_for_TXT(self, _type, records):
+        return {
+            'ttl': records[0]['ttl'],
+            'type': _type,
+            # escape semicolons
+            'values': [r['content'].replace(';', '\\;') for r in records]
+        }
 
     def _data_for_CAA(self, _type, records):
         values = []
@@ -290,7 +297,16 @@ class DnsimpleProvider(BaseProvider):
     _params_for_AAAA = _params_for_multiple
     _params_for_NS = _params_for_multiple
     _params_for_SPF = _params_for_multiple
-    _params_for_TXT = _params_for_multiple
+
+    def _params_for_TXT(self, record):
+        for value in record.values:
+            yield {
+                # un-escape semicolons
+                'content': value.replace('\\', ''),
+                'name': record.name,
+                'ttl': record.ttl,
+                'type': record._type,
+            }
 
     def _params_for_CAA(self, record):
         for value in record.values:

octodns/provider/dyn.py

@@ -529,6 +529,7 @@ class DynProvider(BaseProvider):
         return [{
             'flags': v.flags,
             'tag': v.tag,
+            'ttl': record.ttl,
             'value': v.value,
         } for v in record.values]
 

octodns/provider/route53.py

@@ -221,9 +221,13 @@ class Route53Provider(BaseProvider):
         access_key_id:
         # The AWS secret access key
         secret_access_key:
+        # The AWS session token (optional)
+        # Only needed if using temporary security credentials
+        session_token:
 
-    Alternatively, you may leave out access_key_id and secret_access_key,
-    this will result in boto3 deciding authentication dynamically.
+    Alternatively, you may leave out access_key_id, secret_access_key
+    and session_token.
+    This will result in boto3 deciding authentication dynamically.
 
     In general the account used will need full permissions on Route53.
     '''
@@ -236,10 +240,14 @@ class Route53Provider(BaseProvider):
     HEALTH_CHECK_VERSION = '0001'
 
     def __init__(self, id, access_key_id=None, secret_access_key=None,
-                 max_changes=1000, client_max_attempts=None, *args, **kwargs):
+                 max_changes=1000, client_max_attempts=None,
+                 session_token=None, *args, **kwargs):
         self.max_changes = max_changes
-        _msg = 'access_key_id={}, secret_access_key=***'.format(access_key_id)
-        if access_key_id is None and secret_access_key is None:
+        _msg = 'access_key_id={}, secret_access_key=***, ' \
+               'session_token=***'.format(access_key_id)
+        use_fallback_auth = access_key_id is None and \
+            secret_access_key is None and session_token is None
+        if use_fallback_auth:
             _msg = 'auth=fallback'
         self.log = logging.getLogger('Route53Provider[{}]'.format(id))
         self.log.debug('__init__: id=%s, %s', id, _msg)
@@ -251,11 +259,12 @@ class Route53Provider(BaseProvider):
                           client_max_attempts)
             config = Config(retries={'max_attempts': client_max_attempts})
 
-        if access_key_id is None and secret_access_key is None:
+        if use_fallback_auth:
             self._conn = client('route53', config=config)
         else:
             self._conn = client('route53', aws_access_key_id=access_key_id,
                                 aws_secret_access_key=secret_access_key,
+                                aws_session_token=session_token,
                                 config=config)
 
         self._r53_zones = None

octodns/provider/yaml.py

@@ -89,6 +89,8 @@ class YamlProvider(BaseProvider):
             if record.ttl == self.default_ttl:
                 # ttl is the default, we don't need to store it
                 del d['ttl']
+            if record._octodns:
+                d['octodns'] = record._octodns
             data[record.name].append(d)
 
         # Flatten single element lists

octodns/source/axfr.py

@@ -13,6 +13,8 @@ import dns.rdatatype
 from dns.exception import DNSException
 
 from collections import defaultdict
+from os import listdir
+from os.path import join
 import logging
 
 from ..record import Record
@@ -160,3 +162,73 @@ class AxfrSource(AxfrBaseSource):
             })
 
         return records
+
+
+class ZoneFileSourceException(Exception):
+    pass
+
+
+class ZoneFileSourceNotFound(ZoneFileSourceException):
+
+    def __init__(self):
+        super(ZoneFileSourceNotFound, self).__init__(
+            'Zone file not found')
+
+
+class ZoneFileSourceLoadFailure(ZoneFileSourceException):
+
+    def __init__(self, error):
+        super(ZoneFileSourceLoadFailure, self).__init__(
+            error.message)
+
+
+class ZoneFileSource(AxfrBaseSource):
+    '''
+    Bind compatible zone file source
+
+    zonefile:
+        class: octodns.source.axfr.ZoneFileSource
+        # The directory holding the zone files
+        # Filenames should match zone name (eg. example.com.)
+        directory: ./zonefiles
+    '''
+    def __init__(self, id, directory):
+        self.log = logging.getLogger('ZoneFileSource[{}]'.format(id))
+        self.log.debug('__init__: id=%s, directory=%s', id, directory)
+        super(ZoneFileSource, self).__init__(id)
+        self.directory = directory
+
+        self._zone_records = {}
+
+    def _load_zone_file(self, zone_name):
+        zonefiles = listdir(self.directory)
+        if zone_name in zonefiles:
+            try:
+                z = dns.zone.from_file(join(self.directory, zone_name),
+                                       zone_name, relativize=False)
+            except DNSException as error:
+                raise ZoneFileSourceLoadFailure(error)
+        else:
+            raise ZoneFileSourceNotFound()
+
+        return z
+
+    def zone_records(self, zone):
+        if zone.name not in self._zone_records:
+            try:
+                z = self._load_zone_file(zone.name)
+                records = []
+                for (name, ttl, rdata) in z.iterate_rdatas():
+                    rdtype = dns.rdatatype.to_text(rdata.rdtype)
+                    records.append({
+                        "name": name.to_text(),
+                        "ttl": ttl,
+                        "type": rdtype,
+                        "value": rdata.to_text()
+                    })
+
+                self._zone_records[zone.name] = records
+            except ZoneFileSourceNotFound:
+                return []
+
+        return self._zone_records[zone.name]

requirements.txt

@@ -13,11 +13,11 @@ incf.countryutils==1.0
 ipaddress==1.0.22
 jmespath==0.9.3
 msrestazure==0.4.27
-natsort==5.2.0
+natsort==5.5.0
 nsone==0.9.100
 ovh==0.4.8
 python-dateutil==2.6.1
-requests==2.18.4
+requests==2.20.0
 s3transfer==0.1.13
 six==1.11.0
 setuptools==38.5.2

setup.py

@@ -34,10 +34,10 @@ setup(
         'futures>=3.2.0',
         'incf.countryutils>=1.0',
         'ipaddress>=1.0.22',
-        'natsort>=5.2.0',
+        'natsort>=5.5.0',
         # botocore doesn't like >=2.7.0 for some reason
         'python-dateutil>=2.6.0,<2.7.0',
-        'requests>=2.18.4'
+        'requests>=2.20.0'
     ],
     license='MIT',
     long_description=open('README.md').read(),

tests/fixtures/dnsimple-page-2.json

@@ -133,7 +133,7 @@
       "zone_id": "unit.tests",
       "parent_id": null,
       "name": "txt",
-      "content": "v=DKIM1\\;k=rsa\\;s=email\\;h=sha256\\;p=A/kinda+of/long/string+with+numb3rs",
+      "content": "v=DKIM1;k=rsa;s=email;h=sha256;p=A/kinda+of/long/string+with+numb3rs",
       "ttl": 600,
       "priority": null,
       "type": "TXT",

tests/test_octodns_provider_cloudflare.py

@@ -18,6 +18,17 @@ from octodns.provider.yaml import YamlProvider
 from octodns.zone import Zone
 
 
+def set_record_proxied_flag(record, proxied):
+    try:
+        record._octodns['cloudflare']['proxied'] = proxied
+    except KeyError:
+        record._octodns['cloudflare'] = {
+            'proxied': proxied
+        }
+
+    return record
+
+
 class TestCloudflareProvider(TestCase):
     expected = Zone('unit.tests.', [])
     source = YamlProvider('test', join(dirname(__file__), 'config'))
@@ -287,14 +298,16 @@ class TestCloudflareProvider(TestCase):
         self.assertEquals(2, len(plan.changes))
         self.assertEquals(2, provider.apply(plan))
         self.assertTrue(plan.exists)
-        # recreate for update, and deletes for the 2 parts of the other
+        # creates a the new value and then deletes all the old
         provider._request.assert_has_calls([
-            call('PUT', '/zones/ff12ab34cd5611334422ab3322997650/dns_records/'
-                 'fc12ab34cd5611334422ab3322997655',
-                 data={'content': '3.2.3.4',
-                       'type': 'A',
-                       'name': 'ttl.unit.tests',
-                       'ttl': 300}),
+            call('PUT', '/zones/42/dns_records/'
+                 'fc12ab34cd5611334422ab3322997655', data={
+                     'content': '3.2.3.4',
+                     'type': 'A',
+                     'name': 'ttl.unit.tests',
+                     'proxied': False,
+                     'ttl': 300
+                 }),
             call('DELETE', '/zones/ff12ab34cd5611334422ab3322997650/'
                  'dns_records/fc12ab34cd5611334422ab3322997653'),
             call('DELETE', '/zones/ff12ab34cd5611334422ab3322997650/'
@@ -351,6 +364,8 @@ class TestCloudflareProvider(TestCase):
             },  # zone create
             None,
             None,
+            None,
+            None,
         ]
 
         # Add something and delete something
@@ -371,17 +386,37 @@ class TestCloudflareProvider(TestCase):
         plan = Plan(zone, zone, [change], True)
         provider._apply(plan)
 
+        # get the list of zones, create a zone, add some records, update
+        # something, and delete something
         provider._request.assert_has_calls([
             call('GET', '/zones', params={'page': 1}),
-            call('POST', '/zones', data={'jump_start': False,
-                                         'name': 'unit.tests'}),
-            call('PUT', '/zones/ff12ab34cd5611334422ab3322997650/dns_records/'
-                 'fc12ab34cd5611334422ab3322997653',
-                 data={'content': '4.4.4.4', 'type': 'A', 'name':
-                       'a.unit.tests', 'ttl': 300}),
-            call('POST', '/zones/42/dns_records',
-                 data={'content': '3.3.3.3', 'type': 'A',
-                       'name': 'a.unit.tests', 'ttl': 300})
+            call('POST', '/zones', data={
+                'jump_start': False,
+                'name': 'unit.tests'
+            }),
+            call('POST', '/zones/42/dns_records', data={
+                'content': '4.4.4.4',
+                'type': 'A',
+                'name': 'a.unit.tests',
+                'proxied': False,
+                'ttl': 300
+            }),
+            call('PUT', '/zones/42/dns_records/'
+                 'fc12ab34cd5611334422ab3322997654', data={
+                     'content': '2.2.2.2',
+                     'type': 'A',
+                     'name': 'a.unit.tests',
+                     'proxied': False,
+                     'ttl': 300
+                 }),
+            call('PUT', '/zones/42/dns_records/'
+                 'fc12ab34cd5611334422ab3322997653', data={
+                     'content': '3.3.3.3',
+                     'type': 'A',
+                     'name': 'a.unit.tests',
+                     'proxied': False,
+                     'ttl': 300
+                 }),
         ])
 
     def test_update_delete(self):
@@ -456,12 +491,22 @@ class TestCloudflareProvider(TestCase):
         plan = Plan(zone, zone, [change], True)
         provider._apply(plan)
 
+        # Get zones, create zone, create a record, delete a record
         provider._request.assert_has_calls([
             call('GET', '/zones', params={'page': 1}),
-            call('POST', '/zones',
-                 data={'jump_start': False, 'name': 'unit.tests'}),
-            call('DELETE', '/zones/ff12ab34cd5611334422ab3322997650/'
-                 'dns_records/fc12ab34cd5611334422ab3322997653')
+            call('POST', '/zones', data={
+                'jump_start': False,
+                'name': 'unit.tests'
+            }),
+            call('PUT', '/zones/42/dns_records/'
+                 'fc12ab34cd5611334422ab3322997654', data={
+                     'content': 'ns2.foo.bar.',
+                     'type': 'NS',
+                     'name': 'unit.tests',
+                     'ttl': 300
+                 }),
+            call('DELETE', '/zones/42/dns_records/'
+                 'fc12ab34cd5611334422ab3322997653')
         ])
 
     def test_alias(self):
@@ -498,14 +543,48 @@ class TestCloudflareProvider(TestCase):
         self.assertEquals('www.unit.tests.', record.value)
 
         # Make sure we transform back to CNAME going the other way
-        contents = provider._gen_contents(record)
+        contents = provider._gen_data(record)
         self.assertEquals({
-            'content': u'www.unit.tests.',
+            'content': 'www.unit.tests.',
             'name': 'unit.tests',
+            'proxied': False,
             'ttl': 300,
             'type': 'CNAME'
         }, list(contents)[0])
 
+    def test_gen_key(self):
+        provider = CloudflareProvider('test', 'email', 'token')
+
+        for expected, data in (
+            ('foo.bar.com.', {
+                'content': 'foo.bar.com.',
+                'type': 'CNAME',
+            }),
+            ('10 foo.bar.com.', {
+                'content': 'foo.bar.com.',
+                'priority': 10,
+                'type': 'MX',
+            }),
+            ('0 tag some-value', {
+                'data': {
+                    'flags': 0,
+                    'tag': 'tag',
+                    'value': 'some-value',
+                },
+                'type': 'CAA',
+            }),
+            ('42 100 thing-were-pointed.at 101', {
+                'data': {
+                    'port': 42,
+                    'priority': 100,
+                    'target': 'thing-were-pointed.at',
+                    'weight': 101,
+                },
+                'type': 'SRV',
+            }),
+        ):
+            self.assertEqual(expected, provider._gen_key(data))
+
     def test_cdn(self):
         provider = CloudflareProvider('test', 'email', 'token', True)
 
@@ -689,3 +768,386 @@ class TestCloudflareProvider(TestCase):
 
         plan = provider.plan(wanted)
         self.assertEquals(False, hasattr(plan, 'changes'))
+
+    def test_unproxiabletype_recordfor_returnsrecordwithnocloudflare(self):
+        provider = CloudflareProvider('test', 'email', 'token')
+        name = "unit.tests"
+        _type = "NS"
+        zone_records = [
+            {
+                "id": "fc12ab34cd5611334422ab3322997654",
+                "type": _type,
+                "name": name,
+                "content": "ns2.foo.bar",
+                "proxiable": True,
+                "proxied": False,
+                "ttl": 300,
+                "locked": False,
+                "zone_id": "ff12ab34cd5611334422ab3322997650",
+                "zone_name": "unit.tests",
+                "modified_on": "2017-03-11T18:01:43.420689Z",
+                "created_on": "2017-03-11T18:01:43.420689Z",
+                "meta": {
+                    "auto_added": False
+                }
+            }
+        ]
+        provider.zone_records = Mock(return_value=zone_records)
+        zone = Zone('unit.tests.', [])
+        provider.populate(zone)
+
+        record = provider._record_for(zone, name, _type, zone_records, False)
+
+        self.assertFalse('cloudflare' in record._octodns)
+
+    def test_proxiabletype_recordfor_retrecordwithcloudflareunproxied(self):
+        provider = CloudflareProvider('test', 'email', 'token')
+        name = "multi.unit.tests"
+        _type = "AAAA"
+        zone_records = [
+            {
+                "id": "fc12ab34cd5611334422ab3322997642",
+                "type": _type,
+                "name": name,
+                "content": "::1",
+                "proxiable": True,
+                "proxied": False,
+                "ttl": 300,
+                "locked": False,
+                "zone_id": "ff12ab34cd5611334422ab3322997650",
+                "zone_name": "unit.tests",
+                "modified_on": "2017-03-11T18:01:43.420689Z",
+                "created_on": "2017-03-11T18:01:43.420689Z",
+                "meta": {
+                    "auto_added": False
+                }
+            }
+        ]
+        provider.zone_records = Mock(return_value=zone_records)
+        zone = Zone('unit.tests.', [])
+        provider.populate(zone)
+
+        record = provider._record_for(zone, name, _type, zone_records, False)
+
+        self.assertFalse(record._octodns['cloudflare']['proxied'])
+
+    def test_proxiabletype_recordfor_returnsrecordwithcloudflareproxied(self):
+        provider = CloudflareProvider('test', 'email', 'token')
+        name = "multi.unit.tests"
+        _type = "AAAA"
+        zone_records = [
+            {
+                "id": "fc12ab34cd5611334422ab3322997642",
+                "type": _type,
+                "name": name,
+                "content": "::1",
+                "proxiable": True,
+                "proxied": True,
+                "ttl": 300,
+                "locked": False,
+                "zone_id": "ff12ab34cd5611334422ab3322997650",
+                "zone_name": "unit.tests",
+                "modified_on": "2017-03-11T18:01:43.420689Z",
+                "created_on": "2017-03-11T18:01:43.420689Z",
+                "meta": {
+                    "auto_added": False
+                }
+            }
+        ]
+        provider.zone_records = Mock(return_value=zone_records)
+        zone = Zone('unit.tests.', [])
+        provider.populate(zone)
+
+        record = provider._record_for(zone, name, _type, zone_records, False)
+
+        self.assertTrue(record._octodns['cloudflare']['proxied'])
+
+    def test_proxiedrecordandnewttl_includechange_returnsfalse(self):
+        provider = CloudflareProvider('test', 'email', 'token')
+        zone = Zone('unit.tests.', [])
+        existing = set_record_proxied_flag(
+            Record.new(zone, 'a', {
+                'ttl': 1,
+                'type': 'A',
+                'values': ['1.1.1.1', '2.2.2.2']
+            }), True
+        )
+        new = Record.new(zone, 'a', {
+            'ttl': 300,
+            'type': 'A',
+            'values': ['1.1.1.1', '2.2.2.2']
+        })
+        change = Update(existing, new)
+
+        include_change = provider._include_change(change)
+
+        self.assertFalse(include_change)
+
+    def test_unproxiabletype_gendata_returnsnoproxied(self):
+        provider = CloudflareProvider('test', 'email', 'token')
+        zone = Zone('unit.tests.', [])
+        record = Record.new(zone, 'a', {
+            'ttl': 3600,
+            'type': 'NS',
+            'value': 'ns1.unit.tests.'
+        })
+
+        data = provider._gen_data(record).next()
+
+        self.assertFalse('proxied' in data)
+
+    def test_proxiabletype_gendata_returnsunproxied(self):
+        provider = CloudflareProvider('test', 'email', 'token')
+        zone = Zone('unit.tests.', [])
+        record = set_record_proxied_flag(
+            Record.new(zone, 'a', {
+                'ttl': 300,
+                'type': 'A',
+                'value': '1.2.3.4'
+            }), False
+        )
+
+        data = provider._gen_data(record).next()
+
+        self.assertFalse(data['proxied'])
+
+    def test_proxiabletype_gendata_returnsproxied(self):
+        provider = CloudflareProvider('test', 'email', 'token')
+        zone = Zone('unit.tests.', [])
+        record = set_record_proxied_flag(
+            Record.new(zone, 'a', {
+                'ttl': 300,
+                'type': 'A',
+                'value': '1.2.3.4'
+            }), True
+        )
+
+        data = provider._gen_data(record).next()
+
+        self.assertTrue(data['proxied'])
+
+    def test_createrecord_extrachanges_returnsemptylist(self):
+        provider = CloudflareProvider('test', 'email', 'token')
+        provider.zone_records = Mock(return_value=[])
+        existing = Zone('unit.tests.', [])
+        provider.populate(existing)
+        provider.zone_records = Mock(return_value=[
+            {
+                "id": "fc12ab34cd5611334422ab3322997642",
+                "type": "CNAME",
+                "name": "a.unit.tests",
+                "content": "www.unit.tests",
+                "proxiable": True,
+                "proxied": True,
+                "ttl": 300,
+                "locked": False,
+                "zone_id": "ff12ab34cd5611334422ab3322997650",
+                "zone_name": "unit.tests",
+                "modified_on": "2017-03-11T18:01:43.420689Z",
+                "created_on": "2017-03-11T18:01:43.420689Z",
+                "meta": {
+                    "auto_added": False
+                }
+            }
+        ])
+        desired = Zone('unit.tests.', [])
+        provider.populate(desired)
+        changes = existing.changes(desired, provider)
+
+        extra_changes = provider._extra_changes(existing, desired, changes)
+
+        self.assertFalse(extra_changes)
+
+    def test_updaterecord_extrachanges_returnsemptylist(self):
+        provider = CloudflareProvider('test', 'email', 'token')
+        provider.zone_records = Mock(return_value=[
+            {
+                "id": "fc12ab34cd5611334422ab3322997642",
+                "type": "CNAME",
+                "name": "a.unit.tests",
+                "content": "www.unit.tests",
+                "proxiable": True,
+                "proxied": True,
+                "ttl": 120,
+                "locked": False,
+                "zone_id": "ff12ab34cd5611334422ab3322997650",
+                "zone_name": "unit.tests",
+                "modified_on": "2017-03-11T18:01:43.420689Z",
+                "created_on": "2017-03-11T18:01:43.420689Z",
+                "meta": {
+                    "auto_added": False
+                }
+            }
+        ])
+        existing = Zone('unit.tests.', [])
+        provider.populate(existing)
+        provider.zone_records = Mock(return_value=[
+            {
+                "id": "fc12ab34cd5611334422ab3322997642",
+                "type": "CNAME",
+                "name": "a.unit.tests",
+                "content": "www.unit.tests",
+                "proxiable": True,
+                "proxied": True,
+                "ttl": 300,
+                "locked": False,
+                "zone_id": "ff12ab34cd5611334422ab3322997650",
+                "zone_name": "unit.tests",
+                "modified_on": "2017-03-11T18:01:43.420689Z",
+                "created_on": "2017-03-11T18:01:43.420689Z",
+                "meta": {
+                    "auto_added": False
+                }
+            }
+        ])
+        desired = Zone('unit.tests.', [])
+        provider.populate(desired)
+        changes = existing.changes(desired, provider)
+
+        extra_changes = provider._extra_changes(existing, desired, changes)
+
+        self.assertFalse(extra_changes)
+
+    def test_deleterecord_extrachanges_returnsemptylist(self):
+        provider = CloudflareProvider('test', 'email', 'token')
+        provider.zone_records = Mock(return_value=[
+            {
+                "id": "fc12ab34cd5611334422ab3322997642",
+                "type": "CNAME",
+                "name": "a.unit.tests",
+                "content": "www.unit.tests",
+                "proxiable": True,
+                "proxied": True,
+                "ttl": 300,
+                "locked": False,
+                "zone_id": "ff12ab34cd5611334422ab3322997650",
+                "zone_name": "unit.tests",
+                "modified_on": "2017-03-11T18:01:43.420689Z",
+                "created_on": "2017-03-11T18:01:43.420689Z",
+                "meta": {
+                    "auto_added": False
+                }
+            }
+        ])
+        existing = Zone('unit.tests.', [])
+        provider.populate(existing)
+        provider.zone_records = Mock(return_value=[])
+        desired = Zone('unit.tests.', [])
+        provider.populate(desired)
+        changes = existing.changes(desired, provider)
+
+        extra_changes = provider._extra_changes(existing, desired, changes)
+
+        self.assertFalse(extra_changes)
+
+    def test_proxify_extrachanges_returnsupdatelist(self):
+        provider = CloudflareProvider('test', 'email', 'token')
+        provider.zone_records = Mock(return_value=[
+            {
+                "id": "fc12ab34cd5611334422ab3322997642",
+                "type": "CNAME",
+                "name": "a.unit.tests",
+                "content": "www.unit.tests",
+                "proxiable": True,
+                "proxied": False,
+                "ttl": 300,
+                "locked": False,
+                "zone_id": "ff12ab34cd5611334422ab3322997650",
+                "zone_name": "unit.tests",
+                "modified_on": "2017-03-11T18:01:43.420689Z",
+                "created_on": "2017-03-11T18:01:43.420689Z",
+                "meta": {
+                    "auto_added": False
+                }
+            }
+        ])
+        existing = Zone('unit.tests.', [])
+        provider.populate(existing)
+        provider.zone_records = Mock(return_value=[
+            {
+                "id": "fc12ab34cd5611334422ab3322997642",
+                "type": "CNAME",
+                "name": "a.unit.tests",
+                "content": "www.unit.tests",
+                "proxiable": True,
+                "proxied": True,
+                "ttl": 300,
+                "locked": False,
+                "zone_id": "ff12ab34cd5611334422ab3322997650",
+                "zone_name": "unit.tests",
+                "modified_on": "2017-03-11T18:01:43.420689Z",
+                "created_on": "2017-03-11T18:01:43.420689Z",
+                "meta": {
+                    "auto_added": False
+                }
+            }
+        ])
+        desired = Zone('unit.tests.', [])
+        provider.populate(desired)
+        changes = existing.changes(desired, provider)
+
+        extra_changes = provider._extra_changes(existing, desired, changes)
+
+        self.assertEquals(1, len(extra_changes))
+        self.assertFalse(
+            extra_changes[0].existing._octodns['cloudflare']['proxied']
+        )
+        self.assertTrue(
+            extra_changes[0].new._octodns['cloudflare']['proxied']
+        )
+
+    def test_unproxify_extrachanges_returnsupdatelist(self):
+        provider = CloudflareProvider('test', 'email', 'token')
+        provider.zone_records = Mock(return_value=[
+            {
+                "id": "fc12ab34cd5611334422ab3322997642",
+                "type": "CNAME",
+                "name": "a.unit.tests",
+                "content": "www.unit.tests",
+                "proxiable": True,
+                "proxied": True,
+                "ttl": 300,
+                "locked": False,
+                "zone_id": "ff12ab34cd5611334422ab3322997650",
+                "zone_name": "unit.tests",
+                "modified_on": "2017-03-11T18:01:43.420689Z",
+                "created_on": "2017-03-11T18:01:43.420689Z",
+                "meta": {
+                    "auto_added": False
+                }
+            }
+        ])
+        existing = Zone('unit.tests.', [])
+        provider.populate(existing)
+        provider.zone_records = Mock(return_value=[
+            {
+                "id": "fc12ab34cd5611334422ab3322997642",
+                "type": "CNAME",
+                "name": "a.unit.tests",
+                "content": "www.unit.tests",
+                "proxiable": True,
+                "proxied": False,
+                "ttl": 300,
+                "locked": False,
+                "zone_id": "ff12ab34cd5611334422ab3322997650",
+                "zone_name": "unit.tests",
+                "modified_on": "2017-03-11T18:01:43.420689Z",
+                "created_on": "2017-03-11T18:01:43.420689Z",
+                "meta": {
+                    "auto_added": False
+                }
+            }
+        ])
+        desired = Zone('unit.tests.', [])
+        provider.populate(desired)
+        changes = existing.changes(desired, provider)
+
+        extra_changes = provider._extra_changes(existing, desired, changes)
+
+        self.assertEquals(1, len(extra_changes))
+        self.assertTrue(
+            extra_changes[0].existing._octodns['cloudflare']['proxied']
+        )
+        self.assertFalse(
+            extra_changes[0].new._octodns['cloudflare']['proxied']
+        )

tests/test_octodns_provider_yaml.py

@@ -60,6 +60,12 @@ class TestYamlProvider(TestCase):
             # There should be no changes after the round trip
             reloaded = Zone('unit.tests.', [])
             target.populate(reloaded)
+            self.assertDictEqual(
+                {'included': ['test']},
+                filter(
+                    lambda x: x.name == 'included', reloaded.records
+                )[0]._octodns)
+
             self.assertFalse(zone.changes(reloaded, target=source))
 
             # A 2nd sync should still create everything

tests/test_octodns_source_axfr.py

@@ -11,14 +11,15 @@ from dns.exception import DNSException
 from mock import patch
 from unittest import TestCase
 
-from octodns.source.axfr import AxfrSource, AxfrSourceZoneTransferFailed
+from octodns.source.axfr import AxfrSource, AxfrSourceZoneTransferFailed, \
+    ZoneFileSource, ZoneFileSourceLoadFailure
 from octodns.zone import Zone
 
 
 class TestAxfrSource(TestCase):
     source = AxfrSource('test', 'localhost')
 
-    forward_zonefile = dns.zone.from_file('./tests/zones/unit.tests.db',
+    forward_zonefile = dns.zone.from_file('./tests/zones/unit.tests.',
                                           'unit.tests', relativize=False)
 
     @patch('dns.zone.from_xfr')
@@ -38,3 +39,33 @@ class TestAxfrSource(TestCase):
             self.source.populate(zone)
         self.assertEquals('Unable to Perform Zone Transfer',
                           ctx.exception.message)
+
+
+class TestZoneFileSource(TestCase):
+    source = ZoneFileSource('test', './tests/zones')
+
+    def test_populate(self):
+        # Valid zone file in directory
+        valid = Zone('unit.tests.', [])
+        self.source.populate(valid)
+        self.assertEquals(11, len(valid.records))
+
+        # 2nd populate does not read file again
+        again = Zone('unit.tests.', [])
+        self.source.populate(again)
+        self.assertEquals(11, len(again.records))
+
+        # bust the cache
+        del self.source._zone_records[valid.name]
+
+        # No zone file in directory
+        missing = Zone('missing.zone.', [])
+        self.source.populate(missing)
+        self.assertEquals(0, len(missing.records))
+
+        # Zone file is not valid
+        with self.assertRaises(ZoneFileSourceLoadFailure) as ctx:
+            zone = Zone('invalid.zone.', [])
+            self.source.populate(zone)
+        self.assertEquals('The DNS zone has no NS RRset at its origin.',
+                          ctx.exception.message)

tests/zones/invalid.zone.

@@ -0,0 +1,8 @@
+$ORIGIN invalid.zone.
+@           IN	SOA	ns1.invalid.zone. root.invalid.zone. (
+                        2018071501		; Serial
+                        3600    ; Refresh (1 hour)
+                        600     ; Retry (10 minutes)
+                        604800  ; Expire (1 week)
+                        3600    ; NXDOMAIN ttl (1 hour)
+                    )