Merge branch 'master' into patch-1

CHANGELOG.md

@@ -1,4 +1,10 @@
-## v0.9.1 - UNRELEASED
+## v0.9.2 - Unreleased
+
+* Add lenient support to Zone.add_record, allows populate from providers that
+  have allowed/created invalid data and situations where a sub-zone is being
+  extracted from a parent, but the records still exist in the remote provider.
+
+## v0.9.1 - 2018-05-21 - Going backwards with setup.py
 
 ### NOTICE
 
@@ -11,6 +17,10 @@ all health checks are passing before the first sync with `--doit`. See
 
 * Major update to geo healthchecks to allow configuring host (header), path,
   protocol, and port [#67](https://github.com/github/octodns/pull/67)
+* SSHFP algorithm type 4
+* NS1 and DNSimple support skipping unsupported record types
+* Revert back to old style setup.py & requirements.txt, setup.cfg was
+  causing too much pita
 
 ## v0.9.0 - 2018-03-26 - Way too long since we last met
 

README.md

@@ -98,7 +98,7 @@ There will be other logging information presented on the screen, but successful
 
 ### Making changes
 
-**WARNING**: OctoDNS assumes ownership of any domain you point it to. When you tell it to act it will do whatever is necessary to try and match up states including deleting any unexpected records. Be careful when playing around with OctoDNS. It's best to experiment with a fake zone or one without any data that matters until your comfortable with the system.
+**WARNING**: OctoDNS assumes ownership of any domain you point it to. When you tell it to act it will do whatever is necessary to try and match up states including deleting any unexpected records. Be careful when playing around with OctoDNS. It's best to experiment with a fake zone or one without any data that matters until you're comfortable with the system.
 
 Now it's time to tell OctoDNS to make things happen. We'll invoke it again with the same options and add a `--doit` on the end to tell it this time we actually want it to try and make the specified changes.
 

octodns/__init__.py

@@ -3,4 +3,4 @@
 from __future__ import absolute_import, division, print_function, \
     unicode_literals
 
-__VERSION__ = '0.9.0'
+__VERSION__ = '0.9.1'

octodns/manager.py

@@ -65,6 +65,11 @@ class MainThreadExecutor(object):
 class Manager(object):
     log = logging.getLogger('Manager')
 
+    @classmethod
+    def _plan_keyer(cls, p):
+        plan = p[1]
+        return len(plan.changes[0].record.zone.name) if plan.changes else 0
+
     def __init__(self, config_file, max_workers=None, include_meta=False):
         self.log.info('__init__: config_file=%s', config_file)
 
@@ -288,6 +293,13 @@ class Manager(object):
         # plan pairs.
         plans = [p for f in futures for p in f.result()]
 
+        # Best effort sort plans children first so that we create/update
+        # children zones before parents which should allow us to more safely
+        # extract things into sub-zones. Combining a child back into a parent
+        # can't really be done all that safely in general so we'll optimize for
+        # this direction.
+        plans.sort(key=self._plan_keyer, reverse=True)
+
         for output in self.plan_outputs.values():
             output.run(plans=plans, log=self.log)
 

octodns/provider/azuredns.py

@@ -345,7 +345,7 @@ class AzureProvider(BaseProvider):
                 data['type'] = typ
                 data['ttl'] = azrecord.ttl
                 record = Record.new(zone, record_name, data, source=self)
-                zone.add_record(record)
+                zone.add_record(record, lenient=lenient)
 
         self.log.info('populate: found %s records, exists=%s',
                       len(zone.records) - before, exists)

octodns/provider/base.py

@@ -17,7 +17,7 @@ class BaseProvider(BaseSource):
                  delete_pcent_threshold=Plan.MAX_SAFE_DELETE_PCENT):
         super(BaseProvider, self).__init__(id)
         self.log.debug('__init__: id=%s, apply_disabled=%s, '
-                       'update_pcent_threshold=%.2f'
+                       'update_pcent_threshold=%.2f, '
                        'delete_pcent_threshold=%.2f',
                        id,
                        apply_disabled,

octodns/provider/cloudflare.py

@@ -253,7 +253,7 @@ class CloudflareProvider(BaseProvider):
                         self.log.info('CDN rewrite %s already in zone', name)
                         continue
 
-                    zone.add_record(record)
+                    zone.add_record(record, lenient=lenient)
 
         self.log.info('populate:   found %s records, exists=%s',
                       len(zone.records) - before, exists)

octodns/provider/digitalocean.py

@@ -230,7 +230,7 @@ class DigitalOceanProvider(BaseProvider):
                 data_for = getattr(self, '_data_for_{}'.format(_type))
                 record = Record.new(zone, name, data_for(_type, records),
                                     source=self, lenient=lenient)
-                zone.add_record(record)
+                zone.add_record(record, lenient=lenient)
 
         exists = zone.name in self._zone_records
         self.log.info('populate:   found %s records, exists=%s',

octodns/provider/dnsimple.py

@@ -270,7 +270,7 @@ class DnsimpleProvider(BaseProvider):
                 data_for = getattr(self, '_data_for_{}'.format(_type))
                 record = Record.new(zone, name, data_for(_type, records),
                                     source=self, lenient=lenient)
-                zone.add_record(record)
+                zone.add_record(record, lenient=lenient)
 
         exists = zone.name in self._zone_records
         self.log.info('populate:   found %s records, exists=%s',

octodns/provider/dnsmadeeasy.py

@@ -263,7 +263,7 @@ class DnsMadeEasyProvider(BaseProvider):
                 data_for = getattr(self, '_data_for_{}'.format(_type))
                 record = Record.new(zone, name, data_for(_type, records),
                                     source=self, lenient=lenient)
-                zone.add_record(record)
+                zone.add_record(record, lenient=lenient)
 
         exists = zone.name in self._zone_records
         self.log.info('populate:   found %s records, exists=%s',

octodns/provider/dyn.py

@@ -399,7 +399,7 @@ class DynProvider(BaseProvider):
 
         return self._traffic_directors
 
-    def _populate_traffic_directors(self, zone):
+    def _populate_traffic_directors(self, zone, lenient):
         self.log.debug('_populate_traffic_directors: zone=%s', zone.name)
         td_records = set()
         for fqdn, types in self.traffic_directors.items():
@@ -444,7 +444,7 @@ class DynProvider(BaseProvider):
 
                 name = zone.hostname_from_fqdn(fqdn)
                 record = Record.new(zone, name, data, source=self)
-                zone.add_record(record)
+                zone.add_record(record, lenient=lenient)
                 td_records.add(record)
 
         return td_records
@@ -460,7 +460,7 @@ class DynProvider(BaseProvider):
 
         td_records = set()
         if self.traffic_directors_enabled:
-            td_records = self._populate_traffic_directors(zone)
+            td_records = self._populate_traffic_directors(zone, lenient)
             exists = True
 
         dyn_zone = _CachingDynZone.get(zone.name[:-1])
@@ -483,7 +483,7 @@ class DynProvider(BaseProvider):
                     record = Record.new(zone, name, data, source=self,
                                         lenient=lenient)
                     if record not in td_records:
-                        zone.add_record(record)
+                        zone.add_record(record, lenient=lenient)
 
         self.log.info('populate:   found %s records, exists=%s',
                       len(zone.records) - before, exists)

octodns/provider/googlecloud.py

@@ -230,7 +230,7 @@ class GoogleCloudProvider(BaseProvider):
                 self.log.debug('populate: adding record {} records: {!s}'
                                .format(record_name, data))
                 record = Record.new(zone, record_name, data, source=self)
-                zone.add_record(record)
+                zone.add_record(record, lenient=lenient)
 
         self.log.info('populate: found %s records, exists=%s',
                       len(zone.records) - before, exists)

octodns/provider/ns1.py

@@ -189,6 +189,15 @@ class Ns1Provider(BaseProvider):
         try:
             nsone_zone = self._client.loadZone(zone.name[:-1])
             records = nsone_zone.data['records']
+
+            # change answers for certain types to always be absolute
+            for record in records:
+                if record['type'] in ['ALIAS', 'CNAME', 'MX', 'NS', 'PTR',
+                                      'SRV']:
+                    for i, a in enumerate(record['short_answers']):
+                        if not a.endswith('.'):
+                            record['short_answers'][i] = '{}.'.format(a)
+
             geo_records = nsone_zone.search(has_geo=True)
             exists = True
         except ResourceException as e:
@@ -211,7 +220,7 @@ class Ns1Provider(BaseProvider):
             record = Record.new(zone, name, data_for(_type, record),
                                 source=self, lenient=lenient)
             zone_hash[(_type, name)] = record
-        [zone.add_record(r) for r in zone_hash.values()]
+        [zone.add_record(r, lenient=lenient) for r in zone_hash.values()]
         self.log.info('populate:   found %s records, exists=%s',
                       len(zone.records) - before, exists)
         return exists

octodns/provider/ovh.py

@@ -82,7 +82,7 @@ class OvhProvider(BaseProvider):
                 data_for = getattr(self, '_data_for_{}'.format(_type))
                 record = Record.new(zone, name, data_for(_type, records),
                                     source=self, lenient=lenient)
-                zone.add_record(record)
+                zone.add_record(record, lenient=lenient)
 
         self.log.info('populate:   found %s records, exists=%s',
                       len(zone.records) - before, exists)

octodns/provider/powerdns.py

@@ -199,7 +199,7 @@ class PowerDnsBaseProvider(BaseProvider):
                 record_name = zone.hostname_from_fqdn(rrset['name'])
                 record = Record.new(zone, record_name, data_for(rrset),
                                     source=self, lenient=lenient)
-                zone.add_record(record)
+                zone.add_record(record, lenient=lenient)
 
         self.log.info('populate:   found %s records, exists=%s',
                       len(zone.records) - before, exists)

octodns/provider/rackspace.py

@@ -215,7 +215,7 @@ class RackspaceProvider(BaseProvider):
                     record = Record.new(zone, record_name,
                                         data_for(record_set),
                                         source=self)
-                    zone.add_record(record)
+                    zone.add_record(record, lenient=lenient)
 
         self.log.info('populate:   found %s records, exists=True',
                       len(zone.records) - before)

octodns/provider/route53.py

@@ -217,11 +217,14 @@ class Route53Provider(BaseProvider):
 
     route53:
         class: octodns.provider.route53.Route53Provider
-        # The AWS access key id (required)
+        # The AWS access key id
         access_key_id:
-        # The AWS secret access key (required)
+        # The AWS secret access key
         secret_access_key:
 
+    Alternatively, you may leave out access_key_id and secret_access_key,
+    this will result in boto3 deciding authentication dynamically.
+
     In general the account used will need full permissions on Route53.
     '''
     SUPPORTS_GEO = True
@@ -232,12 +235,14 @@ class Route53Provider(BaseProvider):
     # health check config.
     HEALTH_CHECK_VERSION = '0001'
 
-    def __init__(self, id, access_key_id, secret_access_key, max_changes=1000,
-                 client_max_attempts=None, *args, **kwargs):
+    def __init__(self, id, access_key_id=None, secret_access_key=None,
+                 max_changes=1000, client_max_attempts=None, *args, **kwargs):
         self.max_changes = max_changes
+        _msg = 'access_key_id={}, secret_access_key=***'.format(access_key_id)
+        if access_key_id is None and secret_access_key is None:
+            _msg = 'auth=fallback'
         self.log = logging.getLogger('Route53Provider[{}]'.format(id))
-        self.log.debug('__init__: id=%s, access_key_id=%s, '
-                       'secret_access_key=***', id, access_key_id)
+        self.log.debug('__init__: id=%s, %s', id, _msg)
         super(Route53Provider, self).__init__(id, *args, **kwargs)
 
         config = None
@@ -246,9 +251,12 @@ class Route53Provider(BaseProvider):
                           client_max_attempts)
             config = Config(retries={'max_attempts': client_max_attempts})
 
-        self._conn = client('route53', aws_access_key_id=access_key_id,
-                            aws_secret_access_key=secret_access_key,
-                            config=config)
+        if access_key_id is None and secret_access_key is None:
+            self._conn = client('route53', config=config)
+        else:
+            self._conn = client('route53', aws_access_key_id=access_key_id,
+                                aws_secret_access_key=secret_access_key,
+                                config=config)
 
         self._r53_zones = None
         self._r53_rrsets = {}
@@ -489,7 +497,7 @@ class Route53Provider(BaseProvider):
                         data = data[0]
                     record = Record.new(zone, name, data, source=self,
                                         lenient=lenient)
-                    zone.add_record(record)
+                    zone.add_record(record, lenient=lenient)
 
         self.log.info('populate:   found %s records, exists=%s',
                       len(zone.records) - before, exists)

octodns/provider/yaml.py

@@ -67,7 +67,7 @@ class YamlProvider(BaseProvider):
                             d['ttl'] = self.default_ttl
                         record = Record.new(zone, name, d, source=self,
                                             lenient=lenient)
-                        zone.add_record(record)
+                        zone.add_record(record, lenient=lenient)
 
         self.log.info('populate:   found %s records, exists=False',
                       len(zone.records) - before)

octodns/source/tinydns.py

@@ -134,7 +134,7 @@ class TinyDnsBaseSource(BaseSource):
                     record = Record.new(zone, name, data, source=self,
                                         lenient=lenient)
                     try:
-                        zone.add_record(record)
+                        zone.add_record(record, lenient=lenient)
                     except SubzoneRecordException:
                         self.log.debug('_populate_normal: skipping subzone '
                                        'record=%s', record)
@@ -175,7 +175,7 @@ class TinyDnsBaseSource(BaseSource):
                     'value': value
                 }, source=self, lenient=lenient)
                 try:
-                    zone.add_record(record)
+                    zone.add_record(record, lenient=lenient)
                 except DuplicateRecordException:
                     self.log.warn('Duplicate PTR record for {}, '
                                   'skipping'.format(addr))

octodns/zone.py

@@ -56,11 +56,11 @@ class Zone(object):
     def hostname_from_fqdn(self, fqdn):
         return self._name_re.sub('', fqdn)
 
-    def add_record(self, record, replace=False):
+    def add_record(self, record, replace=False, lenient=False):
         name = record.name
         last = name.split('.')[-1]
 
-        if last in self.sub_zones:
+        if not lenient and last in self.sub_zones:
             if name != last:
                 # it's a record for something under a sub-zone
                 raise SubzoneRecordException('Record {} is under a '
@@ -82,8 +82,8 @@ class Zone(object):
             raise DuplicateRecordException('Duplicate record {}, type {}'
                                            .format(record.fqdn,
                                                    record._type))
-        elif ((record._type == 'CNAME' and len(node) > 0) or
-              ('CNAME' in map(lambda r: r._type, node))):
+        elif not lenient and (((record._type == 'CNAME' and len(node) > 0) or
+                               ('CNAME' in map(lambda r: r._type, node)))):
             # We're adding a CNAME to existing records or adding to an existing
             # CNAME
             raise InvalidNodeException('Invalid state, CNAME at {} cannot '

tests/test_octodns_provider_base.py

@@ -61,7 +61,7 @@ class TestBaseProvider(TestCase):
         class HasSupportsGeo(HasLog):
             SUPPORTS_GEO = False
 
-        zone = Zone('unit.tests.', [])
+        zone = Zone('unit.tests.', ['sub'])
         with self.assertRaises(NotImplementedError) as ctx:
             HasSupportsGeo('hassupportsgeo').populate(zone)
         self.assertEquals('Abstract base class, SUPPORTS property missing',
@@ -81,12 +81,17 @@ class TestBaseProvider(TestCase):
                     'ttl': 60,
                     'type': 'A',
                     'value': '2.3.4.5'
-                }))
+                }), lenient=lenient)
                 zone.add_record(Record.new(zone, 'going', {
                     'ttl': 60,
                     'type': 'A',
                     'value': '3.4.5.6'
-                }))
+                }), lenient=lenient)
+                zone.add_record(Record.new(zone, 'foo.sub', {
+                    'ttl': 61,
+                    'type': 'A',
+                    'value': '4.5.6.7'
+                }), lenient=lenient)
 
         zone.add_record(Record.new(zone, '', {
             'ttl': 60,
@@ -98,7 +103,7 @@ class TestBaseProvider(TestCase):
                         .supports(list(zone.records)[0]))
 
         plan = HasPopulate('haspopulate').plan(zone)
-        self.assertEquals(2, len(plan.changes))
+        self.assertEquals(3, len(plan.changes))
 
         with self.assertRaises(NotImplementedError) as ctx:
             HasPopulate('haspopulate').apply(plan)

tests/test_octodns_provider_ns1.py

@@ -133,12 +133,12 @@ class TestNs1Provider(TestCase):
     }, {
         'type': 'CNAME',
         'ttl': 34,
-        'short_answers': ['foo.unit.tests.'],
+        'short_answers': ['foo.unit.tests'],
         'domain': 'cname.unit.tests.',
     }, {
         'type': 'MX',
         'ttl': 35,
-        'short_answers': ['10 mx1.unit.tests.', '20 mx2.unit.tests.'],
+        'short_answers': ['10 mx1.unit.tests.', '20 mx2.unit.tests'],
         'domain': 'unit.tests.',
     }, {
         'type': 'NAPTR',
@@ -151,18 +151,18 @@ class TestNs1Provider(TestCase):
     }, {
         'type': 'NS',
         'ttl': 37,
-        'short_answers': ['ns1.unit.tests.', 'ns2.unit.tests.'],
+        'short_answers': ['ns1.unit.tests.', 'ns2.unit.tests'],
         'domain': 'unit.tests.',
     }, {
         'type': 'SRV',
         'ttl': 38,
         'short_answers': ['12 30 30 foo-2.unit.tests.',
-                          '10 20 30 foo-1.unit.tests.'],
+                          '10 20 30 foo-1.unit.tests'],
         'domain': '_srv._tcp.unit.tests.',
     }, {
         'type': 'NS',
         'ttl': 39,
-        'short_answers': ['ns3.unit.tests.', 'ns4.unit.tests.'],
+        'short_answers': ['ns3.unit.tests.', 'ns4.unit.tests'],
         'domain': 'sub.unit.tests.',
     }, {
         'type': 'CAA',

tests/test_octodns_provider_route53.py

@@ -167,6 +167,23 @@ class TestRoute53Provider(TestCase):
 
         return (provider, stubber)
 
+    def _get_stubbed_fallback_auth_provider(self):
+        provider = Route53Provider('test')
+
+        # Use the stubber
+        stubber = Stubber(provider._conn)
+        stubber.activate()
+
+        return (provider, stubber)
+
+    def test_populate_with_fallback(self):
+        provider, stubber = self._get_stubbed_fallback_auth_provider()
+
+        got = Zone('unit.tests.', [])
+        with self.assertRaises(ClientError):
+            stubber.add_client_error('list_hosted_zones')
+            provider.populate(got)
+
     def test_populate(self):
         provider, stubber = self._get_stubbed_provider()
 

tests/test_octodns_zone.py

@@ -139,9 +139,9 @@ class TestZone(TestCase):
         self.assertTrue('missing ending dot' in ctx.exception.message)
 
     def test_sub_zones(self):
-        zone = Zone('unit.tests.', set(['sub', 'barred']))
 
         # NS for exactly the sub is allowed
+        zone = Zone('unit.tests.', set(['sub', 'barred']))
         record = Record.new(zone, 'sub', {
             'ttl': 3600,
             'type': 'NS',
@@ -151,6 +151,7 @@ class TestZone(TestCase):
         self.assertEquals(set([record]), zone.records)
 
         # non-NS for exactly the sub is rejected
+        zone = Zone('unit.tests.', set(['sub', 'barred']))
         record = Record.new(zone, 'sub', {
             'ttl': 3600,
             'type': 'A',
@@ -159,8 +160,12 @@ class TestZone(TestCase):
         with self.assertRaises(SubzoneRecordException) as ctx:
             zone.add_record(record)
         self.assertTrue('not of type NS', ctx.exception.message)
+        # Can add it w/lenient
+        zone.add_record(record, lenient=True)
+        self.assertEquals(set([record]), zone.records)
 
         # NS for something below the sub is rejected
+        zone = Zone('unit.tests.', set(['sub', 'barred']))
         record = Record.new(zone, 'foo.sub', {
             'ttl': 3600,
             'type': 'NS',
@@ -169,8 +174,12 @@ class TestZone(TestCase):
         with self.assertRaises(SubzoneRecordException) as ctx:
             zone.add_record(record)
         self.assertTrue('under a managed sub-zone', ctx.exception.message)
+        # Can add it w/lenient
+        zone.add_record(record, lenient=True)
+        self.assertEquals(set([record]), zone.records)
 
         # A for something below the sub is rejected
+        zone = Zone('unit.tests.', set(['sub', 'barred']))
         record = Record.new(zone, 'foo.bar.sub', {
             'ttl': 3600,
             'type': 'A',
@@ -179,6 +188,9 @@ class TestZone(TestCase):
         with self.assertRaises(SubzoneRecordException) as ctx:
             zone.add_record(record)
         self.assertTrue('under a managed sub-zone', ctx.exception.message)
+        # Can add it w/lenient
+        zone.add_record(record, lenient=True)
+        self.assertEquals(set([record]), zone.records)
 
     def test_ignored_records(self):
         zone_normal = Zone('unit.tests.', [])
@@ -230,12 +242,18 @@ class TestZone(TestCase):
         zone.add_record(a)
         with self.assertRaises(InvalidNodeException):
             zone.add_record(cname)
+        self.assertEquals(set([a]), zone.records)
+        zone.add_record(cname, lenient=True)
+        self.assertEquals(set([a, cname]), zone.records)
 
         # add a to cname
         zone = Zone('unit.tests.', [])
         zone.add_record(cname)
         with self.assertRaises(InvalidNodeException):
             zone.add_record(a)
+        self.assertEquals(set([cname]), zone.records)
+        zone.add_record(a, lenient=True)
+        self.assertEquals(set([a, cname]), zone.records)
 
     def test_excluded_records(self):
         zone_normal = Zone('unit.tests.', [])