Merge remote-tracking branch 'origin/master' into ns1-configure-monitors

README.md

@@ -188,7 +188,7 @@ The above command pulled the existing data out of Route53 and placed the results
 | [GoogleCloudProvider](/octodns/provider/googlecloud.py) | google-cloud-dns | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT  | No | |
 | [MythicBeastsProvider](/octodns/provider/mythicbeasts.py) | Mythic Beasts | A, AAAA, ALIAS, CNAME, MX, NS, SRV, SSHFP, CAA, TXT | No | |
 | [Ns1Provider](/octodns/provider/ns1.py) | ns1-python | All | Yes | No CNAME support, missing `NA` geo target |
-| [OVH](/octodns/provider/ovh.py) | ovh | A, AAAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, SSHFP, TXT, DKIM | No | |
+| [OVH](/octodns/provider/ovh.py) | ovh | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, SSHFP, TXT, DKIM | No | |
 | [PowerDnsProvider](/octodns/provider/powerdns.py) | | All | No | |
 | [Rackspace](/octodns/provider/rackspace.py) | | A, AAAA, ALIAS, CNAME, MX, NS, PTR, SPF, TXT | No |  |
 | [Route53](/octodns/provider/route53.py) | boto3 | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | Both | CNAME health checks don't support a Host header |

octodns/provider/ovh.py

@@ -40,8 +40,8 @@ class OvhProvider(BaseProvider):
 
     # This variable is also used in populate method to filter which OVH record
     # types are supported by octodns
-    SUPPORTS = set(('A', 'AAAA', 'CNAME', 'DKIM', 'MX', 'NAPTR', 'NS', 'PTR',
-                    'SPF', 'SRV', 'SSHFP', 'TXT'))
+    SUPPORTS = set(('A', 'AAAA', 'CAA', 'CNAME', 'DKIM', 'MX', 'NAPTR', 'NS',
+                    'PTR', 'SPF', 'SRV', 'SSHFP', 'TXT'))
 
     def __init__(self, id, endpoint, application_key, application_secret,
                  consumer_key, *args, **kwargs):
@@ -139,6 +139,22 @@ class OvhProvider(BaseProvider):
             'value': record['target']
         }
 
+    @staticmethod
+    def _data_for_CAA(_type, records):
+        values = []
+        for record in records:
+            flags, tag, value = record['target'].split(' ', 2)
+            values.append({
+                'flags': flags,
+                'tag': tag,
+                'value': value[1:-1]
+            })
+        return {
+            'ttl': records[0]['ttl'],
+            'type': _type,
+            'values': values
+        }
+
     @staticmethod
     def _data_for_MX(_type, records):
         values = []
@@ -244,6 +260,17 @@ class OvhProvider(BaseProvider):
             'fieldType': record._type
         }
 
+    @staticmethod
+    def _params_for_CAA(record):
+        for value in record.values:
+            yield {
+                'target': '{} {} "{}"'.format(value.flags, value.tag,
+                                              value.value),
+                'subDomain': record.name,
+                'ttl': record.ttl,
+                'fieldType': record._type
+            }
+
     @staticmethod
     def _params_for_MX(record):
         for value in record.values:

octodns/provider/yaml.py

@@ -28,7 +28,79 @@ class YamlProvider(BaseProvider):
         default_ttl: 3600
         # Whether or not to enforce sorting order on the yaml config
         # (optional, default True)
-        enforce_order: True
+        enforce_order: true
+        # Whether duplicate records should replace rather than error
+        # (optiona, default False)
+        populate_should_replace: false
+
+    Overriding values can be accomplished using multiple yaml providers in the
+    `sources` list where subsequent providers have `populate_should_replace`
+    set to `true`. An example use of this would be a zone that you want to push
+    to external DNS providers and internally, but you want to modify some of
+    the records in the internal version.
+
+    config/octodns.com.yaml
+    ---
+    other:
+      type: A
+      values:
+        - 192.30.252.115
+        - 192.30.252.116
+    www:
+      type: A
+      values:
+        - 192.30.252.113
+        - 192.30.252.114
+
+
+    internal/octodns.com.yaml
+    ---
+    'www':
+      type: A
+      values:
+        - 10.0.0.12
+        - 10.0.0.13
+
+    external.yaml
+    ---
+    providers:
+      config:
+        class: octodns.provider.yaml.YamlProvider
+        directory: ./config
+
+    zones:
+
+      octodns.com.:
+        sources:
+          - config
+        targets:
+          - route53
+
+    internal.yaml
+    ---
+    providers:
+      config:
+        class: octodns.provider.yaml.YamlProvider
+        directory: ./config
+
+      internal:
+        class: octodns.provider.yaml.YamlProvider
+        directory: ./internal
+        populate_should_replace: true
+
+    zones:
+
+      octodns.com.:
+        sources:
+          - config
+          - internal
+        targets:
+          - pdns
+
+    You can then sync our records eternally with `--config-file=external.yaml`
+    and internally (with the custom overrides) with
+    `--config-file=internal.yaml`
+
     '''
     SUPPORTS_GEO = True
     SUPPORTS_DYNAMIC = True
@@ -36,16 +108,18 @@ class YamlProvider(BaseProvider):
                     'PTR', 'SSHFP', 'SPF', 'SRV', 'TXT'))
 
     def __init__(self, id, directory, default_ttl=3600, enforce_order=True,
-                 *args, **kwargs):
+                 populate_should_replace=False, *args, **kwargs):
         self.log = logging.getLogger('{}[{}]'.format(
             self.__class__.__name__, id))
         self.log.debug('__init__: id=%s, directory=%s, default_ttl=%d, '
-                       'enforce_order=%d', id, directory, default_ttl,
-                       enforce_order)
+                       'enforce_order=%d, populate_should_replace=%d',
+                       id, directory, default_ttl, enforce_order,
+                       populate_should_replace)
         super(YamlProvider, self).__init__(id, *args, **kwargs)
         self.directory = directory
         self.default_ttl = default_ttl
         self.enforce_order = enforce_order
+        self.populate_should_replace = populate_should_replace
 
     def _populate_from_file(self, filename, zone, lenient):
         with open(filename, 'r') as fh:
@@ -59,9 +133,10 @@ class YamlProvider(BaseProvider):
                             d['ttl'] = self.default_ttl
                         record = Record.new(zone, name, d, source=self,
                                             lenient=lenient)
-                        zone.add_record(record, lenient=lenient)
-            self.log.debug(
-                '_populate_from_file: successfully loaded "%s"', filename)
+                        zone.add_record(record, lenient=lenient,
+                                        replace=self.populate_should_replace)
+            self.log.debug('_populate_from_file: successfully loaded "%s"',
+                           filename)
 
     def populate(self, zone, target=False, lenient=False):
         self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name,

requirements.txt

@@ -1,14 +1,14 @@
 PyYaml==5.3
 azure-common==1.1.24
 azure-mgmt-dns==3.0.0
-boto3==1.11.0
-botocore==1.14.0
+boto3==1.11.9
+botocore==1.14.9
 dnspython==1.16.0
 docutils==0.16
 dyn==1.8.1
 edgegrid-python==1.1.1
 futures==3.2.0; python_version < '3.0'
-google-cloud-core==1.1.0
+google-cloud-core==1.2.0
 google-cloud-dns==0.31.0
 ipaddress==1.0.23
 jmespath==0.9.4
@@ -20,7 +20,7 @@ pycountry-convert==0.7.2
 pycountry==19.8.18
 python-dateutil==2.8.1
 requests==2.22.0
-s3transfer==0.3.0
+s3transfer==0.3.2
 setuptools==44.0.0
-six==1.13.0
+six==1.14.0
 transip==2.0.0

tests/config/override/dynamic.tests.yaml

@@ -0,0 +1,13 @@
+---
+# Replace 'a' with a generic record
+a:
+  type: A
+  values:
+    - 4.4.4.4
+    - 5.5.5.5
+# Add another record
+added:
+  type: A
+  values:
+    - 6.6.6.6
+    - 7.7.7.7

tests/test_octodns_provider_ovh.py

@@ -279,6 +279,24 @@ class TestOvhProvider(TestCase):
         'id': 18
     })
 
+    # CAA
+    api_record.append({
+        'fieldType': 'CAA',
+        'ttl': 1600,
+        'target': '0 issue "ca.unit.tests"',
+        'subDomain': 'caa',
+        'id': 19
+    })
+    expected.add(Record.new(zone, 'caa', {
+        'ttl': 1600,
+        'type': 'CAA',
+        'values': [{
+            'flags': 0,
+            'tag': 'issue',
+            'value': 'ca.unit.tests'
+        }]
+    }))
+
     valid_dkim = [valid_dkim_key,
                   'v=DKIM1 \\; %s' % valid_dkim_key,
                   'h=sha256 \\; %s' % valid_dkim_key,
@@ -404,6 +422,9 @@ class TestOvhProvider(TestCase):
                     call('/domain/zone/unit.tests/record', fieldType='SRV',
                          subDomain='_srv._tcp',
                          target='40 50 60 foo-2.unit.tests.', ttl=800),
+                    call('/domain/zone/unit.tests/record', fieldType='CAA',
+                         subDomain='caa', target='0 issue "ca.unit.tests"',
+                         ttl=1600),
                     call('/domain/zone/unit.tests/record', fieldType='DKIM',
                          subDomain='dkim',
                          target='p=MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCxLaG'

tests/test_octodns_provider_yaml.py

@@ -370,3 +370,36 @@ class TestSplitYamlProvider(TestCase):
             source.populate(zone)
         self.assertEquals('Record www.sub.unit.tests. is under a managed '
                           'subzone', text_type(ctx.exception))
+
+
+class TestOverridingYamlProvider(TestCase):
+
+    def test_provider(self):
+        config = join(dirname(__file__), 'config')
+        override_config = join(dirname(__file__), 'config', 'override')
+        base = YamlProvider('base', config, populate_should_replace=False)
+        override = YamlProvider('test', override_config,
+                                populate_should_replace=True)
+
+        zone = Zone('dynamic.tests.', [])
+
+        # Load the base, should see the 5 records
+        base.populate(zone)
+        got = {r.name: r for r in zone.records}
+        self.assertEquals(5, len(got))
+        # We get the "dynamic" A from the bae config
+        self.assertTrue('dynamic' in got['a'].data)
+        # No added
+        self.assertFalse('added' in got)
+
+        # Load the overrides, should replace one and add 1
+        override.populate(zone)
+        got = {r.name: r for r in zone.records}
+        self.assertEquals(6, len(got))
+        # 'a' was replaced with a generic record
+        self.assertEquals({
+            'ttl': 3600,
+            'values': ['4.4.4.4', '5.5.5.5']
+        }, got['a'].data)
+        # And we have the new one
+        self.assertTrue('added' in got)