diff --git a/README.md b/README.md index d4e7171..995776a 100644 --- a/README.md +++ b/README.md @@ -187,6 +187,7 @@ The above command pulled the existing data out of Route53 and placed the results | [DnsimpleProvider](/octodns/provider/dnsimple.py) | | All | No | CAA tags restricted | | [DynProvider](/octodns/provider/dyn.py) | dyn | All | Both | | | [EtcHostsProvider](/octodns/provider/etc_hosts.py) | | A, AAAA, ALIAS, CNAME | No | | +| [EnvVarSource](/octodns/source/envvar.py) | | TXT | No | read-only environment variable injection | | [GoogleCloudProvider](/octodns/provider/googlecloud.py) | google-cloud-dns | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | No | | | [MythicBeastsProvider](/octodns/provider/mythicbeasts.py) | Mythic Beasts | A, AAAA, ALIAS, CNAME, MX, NS, SRV, SSHFP, CAA, TXT | No | | | [Ns1Provider](/octodns/provider/ns1.py) | ns1-python | All | Yes | No CNAME support, missing `NA` geo target | diff --git a/octodns/provider/cloudflare.py b/octodns/provider/cloudflare.py index 698fbee..db937e5 100644 --- a/octodns/provider/cloudflare.py +++ b/octodns/provider/cloudflare.py @@ -58,6 +58,10 @@ class CloudflareProvider(BaseProvider): retry_count: 4 # Optional. Default: 300. Number of seconds to wait before retrying. retry_period: 300 + # Optional. Default: 50. Number of zones per page. + zones_per_page: 50 + # Optional. Default: 100. Number of dns records per page. + records_per_page: 100 Note: The "proxied" flag of "A", "AAAA" and "CNAME" records can be managed via the YAML provider like so: @@ -78,7 +82,8 @@ class CloudflareProvider(BaseProvider): TIMEOUT = 15 def __init__(self, id, email=None, token=None, cdn=False, retry_count=4, - retry_period=300, *args, **kwargs): + retry_period=300, zones_per_page=50, records_per_page=100, + *args, **kwargs): self.log = getLogger('CloudflareProvider[{}]'.format(id)) self.log.debug('__init__: id=%s, email=%s, token=***, cdn=%s', id, email, cdn) @@ -99,6 +104,8 @@ class CloudflareProvider(BaseProvider): self.cdn = cdn self.retry_count = retry_count self.retry_period = retry_period + self.zones_per_page = zones_per_page + self.records_per_page = records_per_page self._sess = sess self._zones = None @@ -142,7 +149,10 @@ class CloudflareProvider(BaseProvider): zones = [] while page: resp = self._try_request('GET', '/zones', - params={'page': page}) + params={ + 'page': page, + 'per_page': self.zones_per_page + }) zones += resp['result'] info = resp['result_info'] if info['count'] > 0 and info['count'] == info['per_page']: @@ -251,7 +261,8 @@ class CloudflareProvider(BaseProvider): path = '/zones/{}/dns_records'.format(zone_id) page = 1 while page: - resp = self._try_request('GET', path, params={'page': page}) + resp = self._try_request('GET', path, params={'page': page, + 'per_page': self.records_per_page}) records += resp['result'] info = resp['result_info'] if info['count'] > 0 and info['count'] == info['per_page']: diff --git a/octodns/record/__init__.py b/octodns/record/__init__.py index 04eb2da..849e035 100644 --- a/octodns/record/__init__.py +++ b/octodns/record/__init__.py @@ -1209,7 +1209,7 @@ class SrvValue(EqualityTupleMixin): class SrvRecord(_ValuesMixin, Record): _type = 'SRV' _value_type = SrvValue - _name_re = re.compile(r'^_[^\.]+\.[^\.]+') + _name_re = re.compile(r'^(\*|_[^\.]+)\.[^\.]+') @classmethod def validate(cls, name, fqdn, data): diff --git a/octodns/source/envvar.py b/octodns/source/envvar.py new file mode 100644 index 0000000..adf267a --- /dev/null +++ b/octodns/source/envvar.py @@ -0,0 +1,100 @@ + +import logging +import os + +from ..record import Record +from .base import BaseSource + + +class EnvVarSourceException(Exception): + pass + + +class EnvironmentVariableNotFoundException(EnvVarSourceException): + def __init__(self, data): + super(EnvironmentVariableNotFoundException, self).__init__( + 'Unknown environment variable {}'.format(data)) + + +class EnvVarSource(BaseSource): + ''' + This source allows for environment variables to be embedded at octodns + execution time into zones. Intended to capture artifacts of deployment to + facilitate operational objectives. + + The TXT record generated will only have a single value. + + The record name cannot conflict with any other co-existing sources. If + this occurs, an exception will be thrown. + + Possible use cases include: + - Embedding a version number into a TXT record to monitor update + propagation across authoritative providers. + - Capturing identifying information about the deployment process to + record where and when the zone was updated. + + version: + class: octodns.source.envvar.EnvVarSource + # The environment variable in question, in this example the username + # currently executing octodns + variable: USER + # The TXT record name to embed the value found at the above + # environment variable + name: deployuser + # The TTL of the TXT record (optional, default 60) + ttl: 3600 + + This source is then combined with other sources in the octodns config + file: + + zones: + netflix.com.: + sources: + - yaml + - version + targets: + - ultra + - ns1 + ''' + SUPPORTS_GEO = False + SUPPORTS_DYNAMIC = False + SUPPORTS = set(('TXT')) + + DEFAULT_TTL = 60 + + def __init__(self, id, variable, name, ttl=DEFAULT_TTL): + self.log = logging.getLogger('{}[{}]'.format( + self.__class__.__name__, id)) + self.log.debug('__init__: id=%s, variable=%s, name=%s, ' + 'ttl=%d', id, variable, name, ttl) + super(EnvVarSource, self).__init__(id) + self.envvar = variable + self.name = name + self.ttl = ttl + + def _read_variable(self): + value = os.environ.get(self.envvar) + if value is None: + raise EnvironmentVariableNotFoundException(self.envvar) + + self.log.debug('_read_variable: successfully loaded var=%s val=%s', + self.envvar, value) + return value + + def populate(self, zone, target=False, lenient=False): + self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, + target, lenient) + + before = len(zone.records) + + value = self._read_variable() + + # We don't need to worry about conflicting records here because the + # manager will deconflict sources on our behalf. + payload = {'ttl': self.ttl, 'type': 'TXT', 'values': [value]} + record = Record.new(zone, self.name, payload, source=self, + lenient=lenient) + zone.add_record(record, lenient=lenient) + + self.log.info('populate: found %s records, exists=False', + len(zone.records) - before) diff --git a/requirements.txt b/requirements.txt index 0c6cc97..dd1643f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,10 +7,10 @@ dnspython==1.16.0 docutils==0.16 dyn==1.8.1 edgegrid-python==1.1.1 -futures==3.2.0; python_version < '3.0' +futures==3.2.0; python_version < '3.2' google-cloud-core==1.3.0 google-cloud-dns==0.32.0 -ipaddress==1.0.23 +ipaddress==1.0.23; python_version < '3.3' jmespath==0.10.0 msrestazure==0.6.4 natsort==6.2.1 diff --git a/setup.py b/setup.py index c56aa82..9394e7f 100644 --- a/setup.py +++ b/setup.py @@ -69,7 +69,7 @@ setup( 'PyYaml>=4.2b1', 'dnspython>=1.15.0', 'futures>=3.2.0; python_version<"3.2"', - 'ipaddress>=1.0.22', + 'ipaddress>=1.0.22; python_version<"3.3"', 'natsort>=5.5.0', 'pycountry>=19.8.18', 'pycountry-convert>=0.7.2', diff --git a/tests/test_octodns_provider_cloudflare.py b/tests/test_octodns_provider_cloudflare.py index 08608ea..735d95c 100644 --- a/tests/test_octodns_provider_cloudflare.py +++ b/tests/test_octodns_provider_cloudflare.py @@ -426,7 +426,7 @@ class TestCloudflareProvider(TestCase): # get the list of zones, create a zone, add some records, update # something, and delete something provider._request.assert_has_calls([ - call('GET', '/zones', params={'page': 1}), + call('GET', '/zones', params={'page': 1, 'per_page': 50}), call('POST', '/zones', data={ 'jump_start': False, 'name': 'unit.tests' @@ -531,7 +531,7 @@ class TestCloudflareProvider(TestCase): # Get zones, create zone, create a record, delete a record provider._request.assert_has_calls([ - call('GET', '/zones', params={'page': 1}), + call('GET', '/zones', params={'page': 1, 'per_page': 50}), call('POST', '/zones', data={ 'jump_start': False, 'name': 'unit.tests' @@ -1302,7 +1302,8 @@ class TestCloudflareProvider(TestCase): provider._request.side_effect = [result] self.assertEquals([], provider.zone_records(zone)) provider._request.assert_has_calls([call('GET', '/zones', - params={'page': 1})]) + params={'page': 1, + 'per_page': 50})]) # One retry required provider._zones = None @@ -1313,7 +1314,8 @@ class TestCloudflareProvider(TestCase): ] self.assertEquals([], provider.zone_records(zone)) provider._request.assert_has_calls([call('GET', '/zones', - params={'page': 1})]) + params={'page': 1, + 'per_page': 50})]) # Two retries required provider._zones = None @@ -1325,7 +1327,8 @@ class TestCloudflareProvider(TestCase): ] self.assertEquals([], provider.zone_records(zone)) provider._request.assert_has_calls([call('GET', '/zones', - params={'page': 1})]) + params={'page': 1, + 'per_page': 50})]) # # Exhaust our retries provider._zones = None diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index e2917b3..08a3e7a 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -2155,6 +2155,18 @@ class TestRecordValidation(TestCase): } }) + # permit wildcard entries + Record.new(self.zone, '*._tcp', { + 'type': 'SRV', + 'ttl': 600, + 'value': { + 'priority': 1, + 'weight': 2, + 'port': 3, + 'target': 'food.bar.baz.' + } + }) + # invalid name with self.assertRaises(ValidationError) as ctx: Record.new(self.zone, 'neup', { diff --git a/tests/test_octodns_source_envvar.py b/tests/test_octodns_source_envvar.py new file mode 100644 index 0000000..0714883 --- /dev/null +++ b/tests/test_octodns_source_envvar.py @@ -0,0 +1,41 @@ +from six import text_type +from unittest import TestCase +from unittest.mock import patch + +from octodns.source.envvar import EnvVarSource +from octodns.source.envvar import EnvironmentVariableNotFoundException +from octodns.zone import Zone + + +class TestEnvVarSource(TestCase): + + def test_read_variable(self): + envvar = 'OCTODNS_TEST_ENVIRONMENT_VARIABLE' + source = EnvVarSource('testid', envvar, 'recordname', ttl=120) + with self.assertRaises(EnvironmentVariableNotFoundException) as ctx: + source._read_variable() + msg = 'Unknown environment variable {}'.format(envvar) + self.assertEquals(msg, text_type(ctx.exception)) + + with patch.dict('os.environ', {envvar: 'testvalue'}): + value = source._read_variable() + self.assertEquals(value, 'testvalue') + + def test_populate(self): + envvar = 'TEST_VAR' + value = 'somevalue' + name = 'testrecord' + zone_name = 'unit.tests.' + source = EnvVarSource('testid', envvar, name) + zone = Zone(zone_name, []) + + with patch.dict('os.environ', {envvar: value}): + source.populate(zone) + + self.assertEquals(1, len(zone.records)) + record = list(zone.records)[0] + self.assertEquals(name, record.name) + self.assertEquals('{}.{}'.format(name, zone_name), record.fqdn) + self.assertEquals('TXT', record._type) + self.assertEquals(1, len(record.values)) + self.assertEquals(value, record.values[0])