Merge pull request #243 from github/lenient-add-record

Lenient add record

CHANGELOG.md

@@ -1,3 +1,9 @@
+## v0.9.2 - Unreleased
+
+* Add lenient support to Zone.add_record, allows populate from providers that
+  have allowed/created invalid data and situations where a sub-zone is being
+  extracted from a parent, but the records still exist in the remote provider.
+
 ## v0.9.1 - 2018-05-21 - Going backwards with setup.py
 
 ### NOTICE

octodns/provider/azuredns.py

@@ -345,7 +345,7 @@ class AzureProvider(BaseProvider):
                 data['type'] = typ
                 data['ttl'] = azrecord.ttl
                 record = Record.new(zone, record_name, data, source=self)
-                zone.add_record(record)
+                zone.add_record(record, lenient=lenient)
 
         self.log.info('populate: found %s records, exists=%s',
                       len(zone.records) - before, exists)

octodns/provider/cloudflare.py

@@ -253,7 +253,7 @@ class CloudflareProvider(BaseProvider):
                         self.log.info('CDN rewrite %s already in zone', name)
                         continue
 
-                    zone.add_record(record)
+                    zone.add_record(record, lenient=lenient)
 
         self.log.info('populate:   found %s records, exists=%s',
                       len(zone.records) - before, exists)

octodns/provider/digitalocean.py

@@ -230,7 +230,7 @@ class DigitalOceanProvider(BaseProvider):
                 data_for = getattr(self, '_data_for_{}'.format(_type))
                 record = Record.new(zone, name, data_for(_type, records),
                                     source=self, lenient=lenient)
-                zone.add_record(record)
+                zone.add_record(record, lenient=lenient)
 
         exists = zone.name in self._zone_records
         self.log.info('populate:   found %s records, exists=%s',

octodns/provider/dnsimple.py

@@ -270,7 +270,7 @@ class DnsimpleProvider(BaseProvider):
                 data_for = getattr(self, '_data_for_{}'.format(_type))
                 record = Record.new(zone, name, data_for(_type, records),
                                     source=self, lenient=lenient)
-                zone.add_record(record)
+                zone.add_record(record, lenient=lenient)
 
         exists = zone.name in self._zone_records
         self.log.info('populate:   found %s records, exists=%s',

octodns/provider/dnsmadeeasy.py

@@ -263,7 +263,7 @@ class DnsMadeEasyProvider(BaseProvider):
                 data_for = getattr(self, '_data_for_{}'.format(_type))
                 record = Record.new(zone, name, data_for(_type, records),
                                     source=self, lenient=lenient)
-                zone.add_record(record)
+                zone.add_record(record, lenient=lenient)
 
         exists = zone.name in self._zone_records
         self.log.info('populate:   found %s records, exists=%s',

octodns/provider/dyn.py

@@ -399,7 +399,7 @@ class DynProvider(BaseProvider):
 
         return self._traffic_directors
 
-    def _populate_traffic_directors(self, zone):
+    def _populate_traffic_directors(self, zone, lenient):
         self.log.debug('_populate_traffic_directors: zone=%s', zone.name)
         td_records = set()
         for fqdn, types in self.traffic_directors.items():
@@ -444,7 +444,7 @@ class DynProvider(BaseProvider):
 
                 name = zone.hostname_from_fqdn(fqdn)
                 record = Record.new(zone, name, data, source=self)
-                zone.add_record(record)
+                zone.add_record(record, lenient=lenient)
                 td_records.add(record)
 
         return td_records
@@ -460,7 +460,7 @@ class DynProvider(BaseProvider):
 
         td_records = set()
         if self.traffic_directors_enabled:
-            td_records = self._populate_traffic_directors(zone)
+            td_records = self._populate_traffic_directors(zone, lenient)
             exists = True
 
         dyn_zone = _CachingDynZone.get(zone.name[:-1])
@@ -483,7 +483,7 @@ class DynProvider(BaseProvider):
                     record = Record.new(zone, name, data, source=self,
                                         lenient=lenient)
                     if record not in td_records:
-                        zone.add_record(record)
+                        zone.add_record(record, lenient=lenient)
 
         self.log.info('populate:   found %s records, exists=%s',
                       len(zone.records) - before, exists)

octodns/provider/googlecloud.py

@@ -230,7 +230,7 @@ class GoogleCloudProvider(BaseProvider):
                 self.log.debug('populate: adding record {} records: {!s}'
                                .format(record_name, data))
                 record = Record.new(zone, record_name, data, source=self)
-                zone.add_record(record)
+                zone.add_record(record, lenient=lenient)
 
         self.log.info('populate: found %s records, exists=%s',
                       len(zone.records) - before, exists)

octodns/provider/ns1.py

@@ -211,7 +211,7 @@ class Ns1Provider(BaseProvider):
             record = Record.new(zone, name, data_for(_type, record),
                                 source=self, lenient=lenient)
             zone_hash[(_type, name)] = record
-        [zone.add_record(r) for r in zone_hash.values()]
+        [zone.add_record(r, lenient=lenient) for r in zone_hash.values()]
         self.log.info('populate:   found %s records, exists=%s',
                       len(zone.records) - before, exists)
         return exists

octodns/provider/ovh.py

@@ -82,7 +82,7 @@ class OvhProvider(BaseProvider):
                 data_for = getattr(self, '_data_for_{}'.format(_type))
                 record = Record.new(zone, name, data_for(_type, records),
                                     source=self, lenient=lenient)
-                zone.add_record(record)
+                zone.add_record(record, lenient=lenient)
 
         self.log.info('populate:   found %s records, exists=%s',
                       len(zone.records) - before, exists)

octodns/provider/powerdns.py

@@ -199,7 +199,7 @@ class PowerDnsBaseProvider(BaseProvider):
                 record_name = zone.hostname_from_fqdn(rrset['name'])
                 record = Record.new(zone, record_name, data_for(rrset),
                                     source=self, lenient=lenient)
-                zone.add_record(record)
+                zone.add_record(record, lenient=lenient)
 
         self.log.info('populate:   found %s records, exists=%s',
                       len(zone.records) - before, exists)

octodns/provider/rackspace.py

@@ -215,7 +215,7 @@ class RackspaceProvider(BaseProvider):
                     record = Record.new(zone, record_name,
                                         data_for(record_set),
                                         source=self)
-                    zone.add_record(record)
+                    zone.add_record(record, lenient=lenient)
 
         self.log.info('populate:   found %s records, exists=True',
                       len(zone.records) - before)

octodns/provider/route53.py

@@ -489,7 +489,7 @@ class Route53Provider(BaseProvider):
                         data = data[0]
                     record = Record.new(zone, name, data, source=self,
                                         lenient=lenient)
-                    zone.add_record(record)
+                    zone.add_record(record, lenient=lenient)
 
         self.log.info('populate:   found %s records, exists=%s',
                       len(zone.records) - before, exists)

octodns/provider/yaml.py

@@ -67,7 +67,7 @@ class YamlProvider(BaseProvider):
                             d['ttl'] = self.default_ttl
                         record = Record.new(zone, name, d, source=self,
                                             lenient=lenient)
-                        zone.add_record(record)
+                        zone.add_record(record, lenient=lenient)
 
         self.log.info('populate:   found %s records, exists=False',
                       len(zone.records) - before)

octodns/source/tinydns.py

@@ -134,7 +134,7 @@ class TinyDnsBaseSource(BaseSource):
                     record = Record.new(zone, name, data, source=self,
                                         lenient=lenient)
                     try:
-                        zone.add_record(record)
+                        zone.add_record(record, lenient=lenient)
                     except SubzoneRecordException:
                         self.log.debug('_populate_normal: skipping subzone '
                                        'record=%s', record)
@@ -175,7 +175,7 @@ class TinyDnsBaseSource(BaseSource):
                     'value': value
                 }, source=self, lenient=lenient)
                 try:
-                    zone.add_record(record)
+                    zone.add_record(record, lenient=lenient)
                 except DuplicateRecordException:
                     self.log.warn('Duplicate PTR record for {}, '
                                   'skipping'.format(addr))

octodns/zone.py

@@ -56,11 +56,11 @@ class Zone(object):
     def hostname_from_fqdn(self, fqdn):
         return self._name_re.sub('', fqdn)
 
-    def add_record(self, record, replace=False):
+    def add_record(self, record, replace=False, lenient=False):
         name = record.name
         last = name.split('.')[-1]
 
-        if last in self.sub_zones:
+        if not lenient and last in self.sub_zones:
             if name != last:
                 # it's a record for something under a sub-zone
                 raise SubzoneRecordException('Record {} is under a '
@@ -82,8 +82,8 @@ class Zone(object):
             raise DuplicateRecordException('Duplicate record {}, type {}'
                                            .format(record.fqdn,
                                                    record._type))
-        elif ((record._type == 'CNAME' and len(node) > 0) or
-              ('CNAME' in map(lambda r: r._type, node))):
+        elif not lenient and (((record._type == 'CNAME' and len(node) > 0) or
+                               ('CNAME' in map(lambda r: r._type, node)))):
             # We're adding a CNAME to existing records or adding to an existing
             # CNAME
             raise InvalidNodeException('Invalid state, CNAME at {} cannot '

tests/test_octodns_provider_base.py

@@ -61,7 +61,7 @@ class TestBaseProvider(TestCase):
         class HasSupportsGeo(HasLog):
             SUPPORTS_GEO = False
 
-        zone = Zone('unit.tests.', [])
+        zone = Zone('unit.tests.', ['sub'])
         with self.assertRaises(NotImplementedError) as ctx:
             HasSupportsGeo('hassupportsgeo').populate(zone)
         self.assertEquals('Abstract base class, SUPPORTS property missing',
@@ -81,12 +81,17 @@ class TestBaseProvider(TestCase):
                     'ttl': 60,
                     'type': 'A',
                     'value': '2.3.4.5'
-                }))
+                }), lenient=lenient)
                 zone.add_record(Record.new(zone, 'going', {
                     'ttl': 60,
                     'type': 'A',
                     'value': '3.4.5.6'
-                }))
+                }), lenient=lenient)
+                zone.add_record(Record.new(zone, 'foo.sub', {
+                    'ttl': 61,
+                    'type': 'A',
+                    'value': '4.5.6.7'
+                }), lenient=lenient)
 
         zone.add_record(Record.new(zone, '', {
             'ttl': 60,
@@ -98,7 +103,7 @@ class TestBaseProvider(TestCase):
                         .supports(list(zone.records)[0]))
 
         plan = HasPopulate('haspopulate').plan(zone)
-        self.assertEquals(2, len(plan.changes))
+        self.assertEquals(3, len(plan.changes))
 
         with self.assertRaises(NotImplementedError) as ctx:
             HasPopulate('haspopulate').apply(plan)

tests/test_octodns_zone.py

@@ -139,9 +139,9 @@ class TestZone(TestCase):
         self.assertTrue('missing ending dot' in ctx.exception.message)
 
     def test_sub_zones(self):
-        zone = Zone('unit.tests.', set(['sub', 'barred']))
 
         # NS for exactly the sub is allowed
+        zone = Zone('unit.tests.', set(['sub', 'barred']))
         record = Record.new(zone, 'sub', {
             'ttl': 3600,
             'type': 'NS',
@@ -151,6 +151,7 @@ class TestZone(TestCase):
         self.assertEquals(set([record]), zone.records)
 
         # non-NS for exactly the sub is rejected
+        zone = Zone('unit.tests.', set(['sub', 'barred']))
         record = Record.new(zone, 'sub', {
             'ttl': 3600,
             'type': 'A',
@@ -159,8 +160,12 @@ class TestZone(TestCase):
         with self.assertRaises(SubzoneRecordException) as ctx:
             zone.add_record(record)
         self.assertTrue('not of type NS', ctx.exception.message)
+        # Can add it w/lenient
+        zone.add_record(record, lenient=True)
+        self.assertEquals(set([record]), zone.records)
 
         # NS for something below the sub is rejected
+        zone = Zone('unit.tests.', set(['sub', 'barred']))
         record = Record.new(zone, 'foo.sub', {
             'ttl': 3600,
             'type': 'NS',
@@ -169,8 +174,12 @@ class TestZone(TestCase):
         with self.assertRaises(SubzoneRecordException) as ctx:
             zone.add_record(record)
         self.assertTrue('under a managed sub-zone', ctx.exception.message)
+        # Can add it w/lenient
+        zone.add_record(record, lenient=True)
+        self.assertEquals(set([record]), zone.records)
 
         # A for something below the sub is rejected
+        zone = Zone('unit.tests.', set(['sub', 'barred']))
         record = Record.new(zone, 'foo.bar.sub', {
             'ttl': 3600,
             'type': 'A',
@@ -179,6 +188,9 @@ class TestZone(TestCase):
         with self.assertRaises(SubzoneRecordException) as ctx:
             zone.add_record(record)
         self.assertTrue('under a managed sub-zone', ctx.exception.message)
+        # Can add it w/lenient
+        zone.add_record(record, lenient=True)
+        self.assertEquals(set([record]), zone.records)
 
     def test_ignored_records(self):
         zone_normal = Zone('unit.tests.', [])
@@ -230,12 +242,18 @@ class TestZone(TestCase):
         zone.add_record(a)
         with self.assertRaises(InvalidNodeException):
             zone.add_record(cname)
+        self.assertEquals(set([a]), zone.records)
+        zone.add_record(cname, lenient=True)
+        self.assertEquals(set([a, cname]), zone.records)
 
         # add a to cname
         zone = Zone('unit.tests.', [])
         zone.add_record(cname)
         with self.assertRaises(InvalidNodeException):
             zone.add_record(a)
+        self.assertEquals(set([cname]), zone.records)
+        zone.add_record(a, lenient=True)
+        self.assertEquals(set([a, cname]), zone.records)
 
     def test_excluded_records(self):
         zone_normal = Zone('unit.tests.', [])