Merge branch 'master' into master

README.md

@@ -187,6 +187,7 @@ The above command pulled the existing data out of Route53 and placed the results
 | [DnsimpleProvider](/octodns/provider/dnsimple.py) | | All | No | CAA tags restricted |
 | [DynProvider](/octodns/provider/dyn.py) | dyn | All | Both | |
 | [EtcHostsProvider](/octodns/provider/etc_hosts.py) | | A, AAAA, ALIAS, CNAME | No | |
+| [EnvVarSource](/octodns/source/envvar.py) | | TXT | No | read-only environment variable injection |
 | [GoogleCloudProvider](/octodns/provider/googlecloud.py) | google-cloud-dns | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT  | No | |
 | [MythicBeastsProvider](/octodns/provider/mythicbeasts.py) | Mythic Beasts | A, AAAA, ALIAS, CNAME, MX, NS, SRV, SSHFP, CAA, TXT | No | |
 | [Ns1Provider](/octodns/provider/ns1.py) | ns1-python | All | Yes | No CNAME support, missing `NA` geo target |

octodns/provider/cloudflare.py

@@ -58,6 +58,10 @@ class CloudflareProvider(BaseProvider):
         retry_count: 4
         # Optional. Default: 300. Number of seconds to wait before retrying.
         retry_period: 300
+        # Optional. Default: 50. Number of zones per page.
+        zones_per_page: 50
+        # Optional. Default: 100. Number of dns records per page.
+        records_per_page: 100
 
     Note: The "proxied" flag of "A", "AAAA" and "CNAME" records can be managed
           via the YAML provider like so:
@@ -78,7 +82,8 @@ class CloudflareProvider(BaseProvider):
     TIMEOUT = 15
 
     def __init__(self, id, email=None, token=None, cdn=False, retry_count=4,
-                 retry_period=300, *args, **kwargs):
+                 retry_period=300, zones_per_page=50, records_per_page=100,
+                 *args, **kwargs):
         self.log = getLogger('CloudflareProvider[{}]'.format(id))
         self.log.debug('__init__: id=%s, email=%s, token=***, cdn=%s', id,
                        email, cdn)
@@ -99,6 +104,8 @@ class CloudflareProvider(BaseProvider):
         self.cdn = cdn
         self.retry_count = retry_count
         self.retry_period = retry_period
+        self.zones_per_page = zones_per_page
+        self.records_per_page = records_per_page
         self._sess = sess
 
         self._zones = None
@@ -142,7 +149,10 @@ class CloudflareProvider(BaseProvider):
             zones = []
             while page:
                 resp = self._try_request('GET', '/zones',
-                                         params={'page': page})
+                                         params={
+                                                'page': page,
+                                                'per_page': self.zones_per_page
+                                         })
                 zones += resp['result']
                 info = resp['result_info']
                 if info['count'] > 0 and info['count'] == info['per_page']:
@@ -251,7 +261,8 @@ class CloudflareProvider(BaseProvider):
             path = '/zones/{}/dns_records'.format(zone_id)
             page = 1
             while page:
-                resp = self._try_request('GET', path, params={'page': page})
+                resp = self._try_request('GET', path, params={'page': page,
+                                         'per_page': self.records_per_page})
                 records += resp['result']
                 info = resp['result_info']
                 if info['count'] > 0 and info['count'] == info['per_page']:

octodns/record/__init__.py

@@ -1209,7 +1209,7 @@ class SrvValue(EqualityTupleMixin):
 class SrvRecord(_ValuesMixin, Record):
     _type = 'SRV'
     _value_type = SrvValue
-    _name_re = re.compile(r'^_[^\.]+\.[^\.]+')
+    _name_re = re.compile(r'^(\*|_[^\.]+)\.[^\.]+')
 
     @classmethod
     def validate(cls, name, fqdn, data):

octodns/source/envvar.py

@@ -0,0 +1,100 @@
+
+import logging
+import os
+
+from ..record import Record
+from .base import BaseSource
+
+
+class EnvVarSourceException(Exception):
+    pass
+
+
+class EnvironmentVariableNotFoundException(EnvVarSourceException):
+    def __init__(self, data):
+        super(EnvironmentVariableNotFoundException, self).__init__(
+            'Unknown environment variable {}'.format(data))
+
+
+class EnvVarSource(BaseSource):
+    '''
+    This source allows for environment variables to be embedded at octodns
+    execution time into zones.  Intended to capture artifacts of deployment to
+    facilitate operational objectives.
+
+    The TXT record generated will only have a single value.
+
+    The record name cannot conflict with any other co-existing sources.  If
+    this occurs, an exception will be thrown.
+
+    Possible use cases include:
+     - Embedding a version number into a TXT record to monitor update
+       propagation across authoritative providers.
+     - Capturing identifying information about the deployment process to
+       record where and when the zone was updated.
+
+    version:
+        class: octodns.source.envvar.EnvVarSource
+        # The environment variable in question, in this example the username
+        # currently executing octodns
+        variable: USER
+        # The TXT record name to embed the value found at the above
+        # environment variable
+        name: deployuser
+        # The TTL of the TXT record (optional, default 60)
+        ttl: 3600
+
+    This source is then combined with other sources in the octodns config
+    file:
+
+    zones:
+      netflix.com.:
+        sources:
+          - yaml
+          - version
+        targets:
+          - ultra
+          - ns1
+    '''
+    SUPPORTS_GEO = False
+    SUPPORTS_DYNAMIC = False
+    SUPPORTS = set(('TXT'))
+
+    DEFAULT_TTL = 60
+
+    def __init__(self, id, variable, name, ttl=DEFAULT_TTL):
+        self.log = logging.getLogger('{}[{}]'.format(
+            self.__class__.__name__, id))
+        self.log.debug('__init__: id=%s, variable=%s, name=%s, '
+                       'ttl=%d', id, variable, name, ttl)
+        super(EnvVarSource, self).__init__(id)
+        self.envvar = variable
+        self.name = name
+        self.ttl = ttl
+
+    def _read_variable(self):
+        value = os.environ.get(self.envvar)
+        if value is None:
+            raise EnvironmentVariableNotFoundException(self.envvar)
+
+        self.log.debug('_read_variable: successfully loaded var=%s val=%s',
+                       self.envvar, value)
+        return value
+
+    def populate(self, zone, target=False, lenient=False):
+        self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name,
+                       target, lenient)
+
+        before = len(zone.records)
+
+        value = self._read_variable()
+
+        # We don't need to worry about conflicting records here because the
+        # manager will deconflict sources on our behalf.
+        payload = {'ttl': self.ttl, 'type': 'TXT', 'values': [value]}
+        record = Record.new(zone, self.name, payload, source=self,
+                            lenient=lenient)
+        zone.add_record(record, lenient=lenient)
+
+        self.log.info('populate:   found %s records, exists=False',
+                      len(zone.records) - before)

requirements.txt

@@ -7,10 +7,10 @@ dnspython==1.16.0
 docutils==0.16
 dyn==1.8.1
 edgegrid-python==1.1.1
-futures==3.2.0; python_version < '3.0'
+futures==3.2.0; python_version < '3.2'
 google-cloud-core==1.3.0
 google-cloud-dns==0.32.0
-ipaddress==1.0.23
+ipaddress==1.0.23; python_version < '3.3'
 jmespath==0.10.0
 msrestazure==0.6.4
 natsort==6.2.1

setup.py

@@ -69,7 +69,7 @@ setup(
         'PyYaml>=4.2b1',
         'dnspython>=1.15.0',
         'futures>=3.2.0; python_version<"3.2"',
-        'ipaddress>=1.0.22',
+        'ipaddress>=1.0.22; python_version<"3.3"',
         'natsort>=5.5.0',
         'pycountry>=19.8.18',
         'pycountry-convert>=0.7.2',

tests/test_octodns_provider_cloudflare.py

@@ -426,7 +426,7 @@ class TestCloudflareProvider(TestCase):
         # get the list of zones, create a zone, add some records, update
         # something, and delete something
         provider._request.assert_has_calls([
-            call('GET', '/zones', params={'page': 1}),
+            call('GET', '/zones', params={'page': 1, 'per_page': 50}),
             call('POST', '/zones', data={
                 'jump_start': False,
                 'name': 'unit.tests'
@@ -531,7 +531,7 @@ class TestCloudflareProvider(TestCase):
 
         # Get zones, create zone, create a record, delete a record
         provider._request.assert_has_calls([
-            call('GET', '/zones', params={'page': 1}),
+            call('GET', '/zones', params={'page': 1, 'per_page': 50}),
             call('POST', '/zones', data={
                 'jump_start': False,
                 'name': 'unit.tests'
@@ -1302,7 +1302,8 @@ class TestCloudflareProvider(TestCase):
         provider._request.side_effect = [result]
         self.assertEquals([], provider.zone_records(zone))
         provider._request.assert_has_calls([call('GET', '/zones',
-                                           params={'page': 1})])
+                                           params={'page': 1,
+                                                   'per_page': 50})])
 
         # One retry required
         provider._zones = None
@@ -1313,7 +1314,8 @@ class TestCloudflareProvider(TestCase):
         ]
         self.assertEquals([], provider.zone_records(zone))
         provider._request.assert_has_calls([call('GET', '/zones',
-                                           params={'page': 1})])
+                                           params={'page': 1,
+                                                   'per_page': 50})])
 
         # Two retries required
         provider._zones = None
@@ -1325,7 +1327,8 @@ class TestCloudflareProvider(TestCase):
         ]
         self.assertEquals([], provider.zone_records(zone))
         provider._request.assert_has_calls([call('GET', '/zones',
-                                           params={'page': 1})])
+                                           params={'page': 1,
+                                                   'per_page': 50})])
 
         # # Exhaust our retries
         provider._zones = None

tests/test_octodns_record.py

@@ -2155,6 +2155,18 @@ class TestRecordValidation(TestCase):
             }
         })
 
+        # permit wildcard entries
+        Record.new(self.zone, '*._tcp', {
+            'type': 'SRV',
+            'ttl': 600,
+            'value': {
+                'priority': 1,
+                'weight': 2,
+                'port': 3,
+                'target': 'food.bar.baz.'
+            }
+        })
+
         # invalid name
         with self.assertRaises(ValidationError) as ctx:
             Record.new(self.zone, 'neup', {

tests/test_octodns_source_envvar.py

@@ -0,0 +1,41 @@
+from six import text_type
+from unittest import TestCase
+from unittest.mock import patch
+
+from octodns.source.envvar import EnvVarSource
+from octodns.source.envvar import EnvironmentVariableNotFoundException
+from octodns.zone import Zone
+
+
+class TestEnvVarSource(TestCase):
+
+    def test_read_variable(self):
+        envvar = 'OCTODNS_TEST_ENVIRONMENT_VARIABLE'
+        source = EnvVarSource('testid', envvar, 'recordname', ttl=120)
+        with self.assertRaises(EnvironmentVariableNotFoundException) as ctx:
+            source._read_variable()
+        msg = 'Unknown environment variable {}'.format(envvar)
+        self.assertEquals(msg, text_type(ctx.exception))
+
+        with patch.dict('os.environ', {envvar: 'testvalue'}):
+            value = source._read_variable()
+        self.assertEquals(value, 'testvalue')
+
+    def test_populate(self):
+        envvar = 'TEST_VAR'
+        value = 'somevalue'
+        name = 'testrecord'
+        zone_name = 'unit.tests.'
+        source = EnvVarSource('testid', envvar, name)
+        zone = Zone(zone_name, [])
+
+        with patch.dict('os.environ', {envvar: value}):
+            source.populate(zone)
+
+        self.assertEquals(1, len(zone.records))
+        record = list(zone.records)[0]
+        self.assertEquals(name, record.name)
+        self.assertEquals('{}.{}'.format(name, zone_name), record.fqdn)
+        self.assertEquals('TXT', record._type)
+        self.assertEquals(1, len(record.values))
+        self.assertEquals(value, record.values[0])