Merge pull request #264 from begincalendar/add-cloudflare-proxied-support

Add ability to manage Cloudflare proxy flag

octodns/provider/cloudflare.py

@@ -6,6 +6,7 @@ from __future__ import absolute_import, division, print_function, \
     unicode_literals
 
 from collections import defaultdict
+from copy import deepcopy
 from logging import getLogger
 from requests import Session
 
@@ -27,6 +28,9 @@ class CloudflareAuthenticationError(CloudflareError):
         CloudflareError.__init__(self, data)
 
 
+_PROXIABLE_RECORD_TYPES = {'A', 'AAAA', 'CNAME'}
+
+
 class CloudflareProvider(BaseProvider):
     '''
     Cloudflare DNS provider
@@ -43,6 +47,16 @@ class CloudflareProvider(BaseProvider):
         #
         # See: https://support.cloudflare.com/hc/en-us/articles/115000830351
         cdn: false
+
+    Note: The "proxied" flag of "A", "AAAA" and "CNAME" records can be managed
+          via the YAML provider like so:
+              name:
+                octodons:
+                  cloudflare:
+                    proxied: true
+                ttl: 120
+                type: A
+                value: 1.2.3.4
     '''
     SUPPORTS_GEO = False
     SUPPORTS = set(('ALIAS', 'A', 'AAAA', 'CAA', 'CNAME', 'MX', 'NS', 'SRV',
@@ -221,7 +235,14 @@ class CloudflareProvider(BaseProvider):
             data_for = getattr(self, '_data_for_{}'.format(_type))
             data = data_for(_type, records)
 
-        return Record.new(zone, name, data, source=self, lenient=lenient)
+        record = Record.new(zone, name, data, source=self, lenient=lenient)
+
+        if _type in _PROXIABLE_RECORD_TYPES:
+            record._octodns['cloudflare'] = {
+                'proxied': records[0].get('proxied', False)
+            }
+
+        return record
 
     def populate(self, zone, target=False, lenient=False):
         self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name,
@@ -260,8 +281,18 @@ class CloudflareProvider(BaseProvider):
 
     def _include_change(self, change):
         if isinstance(change, Update):
-            existing = change.existing.data
             new = change.new.data
+
+            # Cloudflare manages TTL of proxied records, so we should exclude
+            # TTL from the comparison (to prevent false-positives).
+            if self._record_is_proxied(change.existing):
+                existing = deepcopy(change.existing.data)
+                existing.update({
+                    'ttl': new['ttl']
+                })
+            else:
+                existing = change.existing.data
+
             new['ttl'] = max(self.MIN_TTL, new['ttl'])
             if new == existing:
                 return False
@@ -322,6 +353,12 @@ class CloudflareProvider(BaseProvider):
                 }
             }
 
+    def _record_is_proxied(self, record):
+        return (
+            not self.cdn and
+            record._octodns.get('cloudflare', {}).get('proxied', False)
+        )
+
     def _gen_data(self, record):
         name = record.fqdn[:-1]
         _type = record._type
@@ -338,6 +375,12 @@ class CloudflareProvider(BaseProvider):
                 'type': _type,
                 'ttl': ttl,
             })
+
+            if _type in _PROXIABLE_RECORD_TYPES:
+                content.update({
+                    'proxied': self._record_is_proxied(record)
+                })
+
             yield content
 
     def _gen_key(self, data):
@@ -512,3 +555,23 @@ class CloudflareProvider(BaseProvider):
 
         # clear the cache
         self._zone_records.pop(name, None)
+
+    def _extra_changes(self, existing, desired, changes):
+        extra_changes = []
+
+        existing_records = {r: r for r in existing.records}
+        changed_records = {c.record for c in changes}
+
+        for desired_record in desired.records:
+            if desired_record not in existing.records:  # Will be created
+                continue
+            elif desired_record in changed_records:  # Already being updated
+                continue
+
+            existing_record = existing_records[desired_record]
+
+            if (self._record_is_proxied(existing_record) !=
+                    self._record_is_proxied(desired_record)):
+                extra_changes.append(Update(existing_record, desired_record))
+
+        return extra_changes

tests/test_octodns_provider_cloudflare.py

@@ -18,6 +18,17 @@ from octodns.provider.yaml import YamlProvider
 from octodns.zone import Zone
 
 
+def set_record_proxied_flag(record, proxied):
+    try:
+        record._octodns['cloudflare']['proxied'] = proxied
+    except KeyError:
+        record._octodns['cloudflare'] = {
+            'proxied': proxied
+        }
+
+    return record
+
+
 class TestCloudflareProvider(TestCase):
     expected = Zone('unit.tests.', [])
     source = YamlProvider('test', join(dirname(__file__), 'config'))
@@ -294,6 +305,7 @@ class TestCloudflareProvider(TestCase):
                      'content': '3.2.3.4',
                      'type': 'A',
                      'name': 'ttl.unit.tests',
+                     'proxied': False,
                      'ttl': 300
                  }),
             call('DELETE', '/zones/ff12ab34cd5611334422ab3322997650/'
@@ -386,6 +398,7 @@ class TestCloudflareProvider(TestCase):
                 'content': '4.4.4.4',
                 'type': 'A',
                 'name': 'a.unit.tests',
+                'proxied': False,
                 'ttl': 300
             }),
             call('PUT', '/zones/42/dns_records/'
@@ -393,6 +406,7 @@ class TestCloudflareProvider(TestCase):
                      'content': '2.2.2.2',
                      'type': 'A',
                      'name': 'a.unit.tests',
+                     'proxied': False,
                      'ttl': 300
                  }),
             call('PUT', '/zones/42/dns_records/'
@@ -400,6 +414,7 @@ class TestCloudflareProvider(TestCase):
                      'content': '3.3.3.3',
                      'type': 'A',
                      'name': 'a.unit.tests',
+                     'proxied': False,
                      'ttl': 300
                  }),
         ])
@@ -532,6 +547,7 @@ class TestCloudflareProvider(TestCase):
         self.assertEquals({
             'content': 'www.unit.tests.',
             'name': 'unit.tests',
+            'proxied': False,
             'ttl': 300,
             'type': 'CNAME'
         }, list(contents)[0])
@@ -752,3 +768,386 @@ class TestCloudflareProvider(TestCase):
 
         plan = provider.plan(wanted)
         self.assertEquals(False, hasattr(plan, 'changes'))
+
+    def test_unproxiabletype_recordfor_returnsrecordwithnocloudflare(self):
+        provider = CloudflareProvider('test', 'email', 'token')
+        name = "unit.tests"
+        _type = "NS"
+        zone_records = [
+            {
+                "id": "fc12ab34cd5611334422ab3322997654",
+                "type": _type,
+                "name": name,
+                "content": "ns2.foo.bar",
+                "proxiable": True,
+                "proxied": False,
+                "ttl": 300,
+                "locked": False,
+                "zone_id": "ff12ab34cd5611334422ab3322997650",
+                "zone_name": "unit.tests",
+                "modified_on": "2017-03-11T18:01:43.420689Z",
+                "created_on": "2017-03-11T18:01:43.420689Z",
+                "meta": {
+                    "auto_added": False
+                }
+            }
+        ]
+        provider.zone_records = Mock(return_value=zone_records)
+        zone = Zone('unit.tests.', [])
+        provider.populate(zone)
+
+        record = provider._record_for(zone, name, _type, zone_records, False)
+
+        self.assertFalse('cloudflare' in record._octodns)
+
+    def test_proxiabletype_recordfor_retrecordwithcloudflareunproxied(self):
+        provider = CloudflareProvider('test', 'email', 'token')
+        name = "multi.unit.tests"
+        _type = "AAAA"
+        zone_records = [
+            {
+                "id": "fc12ab34cd5611334422ab3322997642",
+                "type": _type,
+                "name": name,
+                "content": "::1",
+                "proxiable": True,
+                "proxied": False,
+                "ttl": 300,
+                "locked": False,
+                "zone_id": "ff12ab34cd5611334422ab3322997650",
+                "zone_name": "unit.tests",
+                "modified_on": "2017-03-11T18:01:43.420689Z",
+                "created_on": "2017-03-11T18:01:43.420689Z",
+                "meta": {
+                    "auto_added": False
+                }
+            }
+        ]
+        provider.zone_records = Mock(return_value=zone_records)
+        zone = Zone('unit.tests.', [])
+        provider.populate(zone)
+
+        record = provider._record_for(zone, name, _type, zone_records, False)
+
+        self.assertFalse(record._octodns['cloudflare']['proxied'])
+
+    def test_proxiabletype_recordfor_returnsrecordwithcloudflareproxied(self):
+        provider = CloudflareProvider('test', 'email', 'token')
+        name = "multi.unit.tests"
+        _type = "AAAA"
+        zone_records = [
+            {
+                "id": "fc12ab34cd5611334422ab3322997642",
+                "type": _type,
+                "name": name,
+                "content": "::1",
+                "proxiable": True,
+                "proxied": True,
+                "ttl": 300,
+                "locked": False,
+                "zone_id": "ff12ab34cd5611334422ab3322997650",
+                "zone_name": "unit.tests",
+                "modified_on": "2017-03-11T18:01:43.420689Z",
+                "created_on": "2017-03-11T18:01:43.420689Z",
+                "meta": {
+                    "auto_added": False
+                }
+            }
+        ]
+        provider.zone_records = Mock(return_value=zone_records)
+        zone = Zone('unit.tests.', [])
+        provider.populate(zone)
+
+        record = provider._record_for(zone, name, _type, zone_records, False)
+
+        self.assertTrue(record._octodns['cloudflare']['proxied'])
+
+    def test_proxiedrecordandnewttl_includechange_returnsfalse(self):
+        provider = CloudflareProvider('test', 'email', 'token')
+        zone = Zone('unit.tests.', [])
+        existing = set_record_proxied_flag(
+            Record.new(zone, 'a', {
+                'ttl': 1,
+                'type': 'A',
+                'values': ['1.1.1.1', '2.2.2.2']
+            }), True
+        )
+        new = Record.new(zone, 'a', {
+            'ttl': 300,
+            'type': 'A',
+            'values': ['1.1.1.1', '2.2.2.2']
+        })
+        change = Update(existing, new)
+
+        include_change = provider._include_change(change)
+
+        self.assertFalse(include_change)
+
+    def test_unproxiabletype_gendata_returnsnoproxied(self):
+        provider = CloudflareProvider('test', 'email', 'token')
+        zone = Zone('unit.tests.', [])
+        record = Record.new(zone, 'a', {
+            'ttl': 3600,
+            'type': 'NS',
+            'value': 'ns1.unit.tests.'
+        })
+
+        data = provider._gen_data(record).next()
+
+        self.assertFalse('proxied' in data)
+
+    def test_proxiabletype_gendata_returnsunproxied(self):
+        provider = CloudflareProvider('test', 'email', 'token')
+        zone = Zone('unit.tests.', [])
+        record = set_record_proxied_flag(
+            Record.new(zone, 'a', {
+                'ttl': 300,
+                'type': 'A',
+                'value': '1.2.3.4'
+            }), False
+        )
+
+        data = provider._gen_data(record).next()
+
+        self.assertFalse(data['proxied'])
+
+    def test_proxiabletype_gendata_returnsproxied(self):
+        provider = CloudflareProvider('test', 'email', 'token')
+        zone = Zone('unit.tests.', [])
+        record = set_record_proxied_flag(
+            Record.new(zone, 'a', {
+                'ttl': 300,
+                'type': 'A',
+                'value': '1.2.3.4'
+            }), True
+        )
+
+        data = provider._gen_data(record).next()
+
+        self.assertTrue(data['proxied'])
+
+    def test_createrecord_extrachanges_returnsemptylist(self):
+        provider = CloudflareProvider('test', 'email', 'token')
+        provider.zone_records = Mock(return_value=[])
+        existing = Zone('unit.tests.', [])
+        provider.populate(existing)
+        provider.zone_records = Mock(return_value=[
+            {
+                "id": "fc12ab34cd5611334422ab3322997642",
+                "type": "CNAME",
+                "name": "a.unit.tests",
+                "content": "www.unit.tests",
+                "proxiable": True,
+                "proxied": True,
+                "ttl": 300,
+                "locked": False,
+                "zone_id": "ff12ab34cd5611334422ab3322997650",
+                "zone_name": "unit.tests",
+                "modified_on": "2017-03-11T18:01:43.420689Z",
+                "created_on": "2017-03-11T18:01:43.420689Z",
+                "meta": {
+                    "auto_added": False
+                }
+            }
+        ])
+        desired = Zone('unit.tests.', [])
+        provider.populate(desired)
+        changes = existing.changes(desired, provider)
+
+        extra_changes = provider._extra_changes(existing, desired, changes)
+
+        self.assertFalse(extra_changes)
+
+    def test_updaterecord_extrachanges_returnsemptylist(self):
+        provider = CloudflareProvider('test', 'email', 'token')
+        provider.zone_records = Mock(return_value=[
+            {
+                "id": "fc12ab34cd5611334422ab3322997642",
+                "type": "CNAME",
+                "name": "a.unit.tests",
+                "content": "www.unit.tests",
+                "proxiable": True,
+                "proxied": True,
+                "ttl": 120,
+                "locked": False,
+                "zone_id": "ff12ab34cd5611334422ab3322997650",
+                "zone_name": "unit.tests",
+                "modified_on": "2017-03-11T18:01:43.420689Z",
+                "created_on": "2017-03-11T18:01:43.420689Z",
+                "meta": {
+                    "auto_added": False
+                }
+            }
+        ])
+        existing = Zone('unit.tests.', [])
+        provider.populate(existing)
+        provider.zone_records = Mock(return_value=[
+            {
+                "id": "fc12ab34cd5611334422ab3322997642",
+                "type": "CNAME",
+                "name": "a.unit.tests",
+                "content": "www.unit.tests",
+                "proxiable": True,
+                "proxied": True,
+                "ttl": 300,
+                "locked": False,
+                "zone_id": "ff12ab34cd5611334422ab3322997650",
+                "zone_name": "unit.tests",
+                "modified_on": "2017-03-11T18:01:43.420689Z",
+                "created_on": "2017-03-11T18:01:43.420689Z",
+                "meta": {
+                    "auto_added": False
+                }
+            }
+        ])
+        desired = Zone('unit.tests.', [])
+        provider.populate(desired)
+        changes = existing.changes(desired, provider)
+
+        extra_changes = provider._extra_changes(existing, desired, changes)
+
+        self.assertFalse(extra_changes)
+
+    def test_deleterecord_extrachanges_returnsemptylist(self):
+        provider = CloudflareProvider('test', 'email', 'token')
+        provider.zone_records = Mock(return_value=[
+            {
+                "id": "fc12ab34cd5611334422ab3322997642",
+                "type": "CNAME",
+                "name": "a.unit.tests",
+                "content": "www.unit.tests",
+                "proxiable": True,
+                "proxied": True,
+                "ttl": 300,
+                "locked": False,
+                "zone_id": "ff12ab34cd5611334422ab3322997650",
+                "zone_name": "unit.tests",
+                "modified_on": "2017-03-11T18:01:43.420689Z",
+                "created_on": "2017-03-11T18:01:43.420689Z",
+                "meta": {
+                    "auto_added": False
+                }
+            }
+        ])
+        existing = Zone('unit.tests.', [])
+        provider.populate(existing)
+        provider.zone_records = Mock(return_value=[])
+        desired = Zone('unit.tests.', [])
+        provider.populate(desired)
+        changes = existing.changes(desired, provider)
+
+        extra_changes = provider._extra_changes(existing, desired, changes)
+
+        self.assertFalse(extra_changes)
+
+    def test_proxify_extrachanges_returnsupdatelist(self):
+        provider = CloudflareProvider('test', 'email', 'token')
+        provider.zone_records = Mock(return_value=[
+            {
+                "id": "fc12ab34cd5611334422ab3322997642",
+                "type": "CNAME",
+                "name": "a.unit.tests",
+                "content": "www.unit.tests",
+                "proxiable": True,
+                "proxied": False,
+                "ttl": 300,
+                "locked": False,
+                "zone_id": "ff12ab34cd5611334422ab3322997650",
+                "zone_name": "unit.tests",
+                "modified_on": "2017-03-11T18:01:43.420689Z",
+                "created_on": "2017-03-11T18:01:43.420689Z",
+                "meta": {
+                    "auto_added": False
+                }
+            }
+        ])
+        existing = Zone('unit.tests.', [])
+        provider.populate(existing)
+        provider.zone_records = Mock(return_value=[
+            {
+                "id": "fc12ab34cd5611334422ab3322997642",
+                "type": "CNAME",
+                "name": "a.unit.tests",
+                "content": "www.unit.tests",
+                "proxiable": True,
+                "proxied": True,
+                "ttl": 300,
+                "locked": False,
+                "zone_id": "ff12ab34cd5611334422ab3322997650",
+                "zone_name": "unit.tests",
+                "modified_on": "2017-03-11T18:01:43.420689Z",
+                "created_on": "2017-03-11T18:01:43.420689Z",
+                "meta": {
+                    "auto_added": False
+                }
+            }
+        ])
+        desired = Zone('unit.tests.', [])
+        provider.populate(desired)
+        changes = existing.changes(desired, provider)
+
+        extra_changes = provider._extra_changes(existing, desired, changes)
+
+        self.assertEquals(1, len(extra_changes))
+        self.assertFalse(
+            extra_changes[0].existing._octodns['cloudflare']['proxied']
+        )
+        self.assertTrue(
+            extra_changes[0].new._octodns['cloudflare']['proxied']
+        )
+
+    def test_unproxify_extrachanges_returnsupdatelist(self):
+        provider = CloudflareProvider('test', 'email', 'token')
+        provider.zone_records = Mock(return_value=[
+            {
+                "id": "fc12ab34cd5611334422ab3322997642",
+                "type": "CNAME",
+                "name": "a.unit.tests",
+                "content": "www.unit.tests",
+                "proxiable": True,
+                "proxied": True,
+                "ttl": 300,
+                "locked": False,
+                "zone_id": "ff12ab34cd5611334422ab3322997650",
+                "zone_name": "unit.tests",
+                "modified_on": "2017-03-11T18:01:43.420689Z",
+                "created_on": "2017-03-11T18:01:43.420689Z",
+                "meta": {
+                    "auto_added": False
+                }
+            }
+        ])
+        existing = Zone('unit.tests.', [])
+        provider.populate(existing)
+        provider.zone_records = Mock(return_value=[
+            {
+                "id": "fc12ab34cd5611334422ab3322997642",
+                "type": "CNAME",
+                "name": "a.unit.tests",
+                "content": "www.unit.tests",
+                "proxiable": True,
+                "proxied": False,
+                "ttl": 300,
+                "locked": False,
+                "zone_id": "ff12ab34cd5611334422ab3322997650",
+                "zone_name": "unit.tests",
+                "modified_on": "2017-03-11T18:01:43.420689Z",
+                "created_on": "2017-03-11T18:01:43.420689Z",
+                "meta": {
+                    "auto_added": False
+                }
+            }
+        ])
+        desired = Zone('unit.tests.', [])
+        provider.populate(desired)
+        changes = existing.changes(desired, provider)
+
+        extra_changes = provider._extra_changes(existing, desired, changes)
+
+        self.assertEquals(1, len(extra_changes))
+        self.assertTrue(
+            extra_changes[0].existing._octodns['cloudflare']['proxied']
+        )
+        self.assertFalse(
+            extra_changes[0].new._octodns['cloudflare']['proxied']
+        )